diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 13572e95421f0a..a5b42ffcf76b85 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,3 +10,8 @@ updates: directory: "/lib/spack/docs" schedule: interval: "daily" + # Requirements to run style checks + - package-ecosystem: "pip" + directory: "/.github/workflows/style" + schedule: + interval: "daily" diff --git a/.github/workflows/audit.yaml b/.github/workflows/audit.yaml index 9ea27e2834166d..749e6f526aa4b1 100644 --- a/.github/workflows/audit.yaml +++ b/.github/workflows/audit.yaml @@ -17,10 +17,13 @@ concurrency: jobs: # Run audits on all the packages in the built-in repository package-audits: - runs-on: ubuntu-latest + runs-on: ${{ matrix.operating_system }} + strategy: + matrix: + operating_system: ["ubuntu-latest", "macos-latest"] steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v2 - - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # @v2 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2 with: python-version: ${{inputs.python_version}} - name: Install Python packages @@ -31,6 +34,7 @@ jobs: run: | . share/spack/setup-env.sh coverage run $(which spack) audit packages + coverage run $(which spack) audit externals coverage combine coverage xml - name: Package audits (without coverage) @@ -38,7 +42,8 @@ jobs: run: | . share/spack/setup-env.sh $(which spack) audit packages + $(which spack) audit externals - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # @v2.1.0 if: ${{ inputs.with_coverage == 'true' }} with: - flags: unittests,linux,audits + flags: unittests,audits diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 884c1bd45a83ea..fd863b6abb8726 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -24,7 +24,7 @@ jobs: make patch unzip which xz python3 python3-devel tree \ cmake bison bison-devel libstdc++-static - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - name: Setup non-root user @@ -42,8 +42,8 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | source share/spack/setup-env.sh + spack bootstrap disable github-actions-v0.5 spack bootstrap disable github-actions-v0.4 - spack bootstrap disable github-actions-v0.3 spack external find cmake bison spack -d solve zlib tree ~/.spack/bootstrap/store/ @@ -62,7 +62,7 @@ jobs: make patch unzip xz-utils python3 python3-dev tree \ cmake bison - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - name: Setup non-root user @@ -80,8 +80,8 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | source share/spack/setup-env.sh + spack bootstrap disable github-actions-v0.5 spack bootstrap disable github-actions-v0.4 - spack bootstrap disable github-actions-v0.3 spack external find cmake bison spack -d solve zlib tree ~/.spack/bootstrap/store/ @@ -99,7 +99,7 @@ jobs: bzip2 curl file g++ gcc gfortran git gnupg2 gzip \ make patch unzip xz-utils python3 python3-dev tree - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - name: Setup non-root user @@ -133,7 +133,7 @@ jobs: make patch unzip which xz python3 python3-devel tree \ cmake bison - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - name: Setup repo @@ -145,8 +145,8 @@ jobs: - name: Bootstrap clingo run: | source share/spack/setup-env.sh + spack bootstrap disable github-actions-v0.5 spack bootstrap disable github-actions-v0.4 - spack bootstrap disable github-actions-v0.3 spack external find cmake bison spack -d solve zlib tree ~/.spack/bootstrap/store/ @@ -158,13 +158,16 @@ jobs: run: | brew install cmake bison@2.7 tree - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2 + with: + python-version: "3.12" - name: Bootstrap clingo run: | source share/spack/setup-env.sh export PATH=/usr/local/opt/bison@2.7/bin:$PATH + spack bootstrap disable github-actions-v0.5 spack bootstrap disable github-actions-v0.4 - spack bootstrap disable github-actions-v0.3 spack external find --not-buildable cmake bison spack -d solve zlib tree ~/.spack/bootstrap/store/ @@ -179,11 +182,11 @@ jobs: run: | brew install tree - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Bootstrap clingo run: | set -ex - for ver in '3.6' '3.7' '3.8' '3.9' '3.10' ; do + for ver in '3.7' '3.8' '3.9' '3.10' '3.11' ; do not_found=1 ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)" echo "Testing $ver_dir" @@ -204,7 +207,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - name: Setup repo @@ -214,7 +217,7 @@ jobs: - name: Bootstrap clingo run: | set -ex - for ver in '3.6' '3.7' '3.8' '3.9' '3.10' ; do + for ver in '3.7' '3.8' '3.9' '3.10' '3.11' ; do not_found=1 ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)" echo "Testing $ver_dir" @@ -247,7 +250,7 @@ jobs: bzip2 curl file g++ gcc patchelf gfortran git gzip \ make patch unzip xz-utils python3 python3-dev tree - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - name: Setup non-root user @@ -265,6 +268,7 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | source share/spack/setup-env.sh + spack bootstrap disable github-actions-v0.4 spack bootstrap disable spack-install spack -d gpg list tree ~/.spack/bootstrap/store/ @@ -283,7 +287,7 @@ jobs: make patch unzip xz-utils python3 python3-dev tree \ gawk - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - name: Setup non-root user @@ -302,8 +306,8 @@ jobs: run: | source share/spack/setup-env.sh spack solve zlib + spack bootstrap disable github-actions-v0.5 spack bootstrap disable github-actions-v0.4 - spack bootstrap disable github-actions-v0.3 spack -d gpg list tree ~/.spack/bootstrap/store/ @@ -316,10 +320,11 @@ jobs: # Remove GnuPG since we want to bootstrap it sudo rm -rf /usr/local/bin/gpg - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Bootstrap GnuPG run: | source share/spack/setup-env.sh + spack bootstrap disable github-actions-v0.4 spack bootstrap disable spack-install spack -d gpg list tree ~/.spack/bootstrap/store/ @@ -333,13 +338,13 @@ jobs: # Remove GnuPG since we want to bootstrap it sudo rm -rf /usr/local/bin/gpg - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Bootstrap GnuPG run: | source share/spack/setup-env.sh spack solve zlib + spack bootstrap disable github-actions-v0.5 spack bootstrap disable github-actions-v0.4 - spack bootstrap disable github-actions-v0.3 spack -d gpg list tree ~/.spack/bootstrap/store/ diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index c4969189564322..880cf9c64495f5 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -38,12 +38,11 @@ jobs: # Meaning of the various items in the matrix list # 0: Container name (e.g. ubuntu-bionic) # 1: Platforms to build for - # 2: Base image (e.g. ubuntu:18.04) + # 2: Base image (e.g. ubuntu:22.04) dockerfile: [[amazon-linux, 'linux/amd64,linux/arm64', 'amazonlinux:2'], [centos7, 'linux/amd64,linux/arm64,linux/ppc64le', 'centos:7'], [centos-stream, 'linux/amd64,linux/arm64,linux/ppc64le', 'centos:stream'], [leap15, 'linux/amd64,linux/arm64,linux/ppc64le', 'opensuse/leap:15'], - [ubuntu-bionic, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:18.04'], [ubuntu-focal, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:20.04'], [ubuntu-jammy, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:22.04'], [almalinux8, 'linux/amd64,linux/arm64,linux/ppc64le', 'almalinux:8'], @@ -56,20 +55,22 @@ jobs: if: github.repository == 'spack/spack' steps: - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v2 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2 - - name: Set Container Tag Normal (Nightly) - run: | - container="${{ matrix.dockerfile[0] }}:latest" - echo "container=${container}" >> $GITHUB_ENV - echo "versioned=${container}" >> $GITHUB_ENV - - # On a new release create a container with the same tag as the release. - - name: Set Container Tag on Release - if: github.event_name == 'release' - run: | - versioned="${{matrix.dockerfile[0]}}:${GITHUB_REF##*/}" - echo "versioned=${versioned}" >> $GITHUB_ENV + - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 + id: docker_meta + with: + images: | + ghcr.io/${{ github.repository_owner }}/${{ matrix.dockerfile[0] }} + ${{ github.repository_owner }}/${{ matrix.dockerfile[0] }} + tags: | + type=schedule,pattern=nightly + type=schedule,pattern=develop + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=ref,event=branch + type=ref,event=pr - name: Generate the Dockerfile env: @@ -86,19 +87,19 @@ jobs: fi - name: Upload Dockerfile - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 with: name: dockerfiles path: dockerfiles - name: Set up QEMU - uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # @v1 + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@16c0bc4a6e6ada2cfd8afd41d22d95379cf7c32a # @v1 + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 - name: Log in to GitHub Container Registry - uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # @v1 + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d with: registry: ghcr.io username: ${{ github.actor }} @@ -106,21 +107,18 @@ jobs: - name: Log in to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # @v1 + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build & Deploy ${{ matrix.dockerfile[0] }} - uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # @v2 + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 with: context: dockerfiles/${{ matrix.dockerfile[0] }} platforms: ${{ matrix.dockerfile[1] }} push: ${{ github.event_name != 'pull_request' }} cache-from: type=gha cache-to: type=gha,mode=max - tags: | - spack/${{ env.container }} - spack/${{ env.versioned }} - ghcr.io/spack/${{ env.container }} - ghcr.io/spack/${{ env.versioned }} + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5885b54fd4a176..047109ca76e70e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -35,7 +35,7 @@ jobs: core: ${{ steps.filter.outputs.core }} packages: ${{ steps.filter.outputs.packages }} steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v2 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2 if: ${{ github.event_name == 'push' }} with: fetch-depth: 0 diff --git a/.github/workflows/nightly-win-builds.yml b/.github/workflows/nightly-win-builds.yml index b62416101966d4..511316a2a35504 100644 --- a/.github/workflows/nightly-win-builds.yml +++ b/.github/workflows/nightly-win-builds.yml @@ -14,10 +14,10 @@ jobs: build-paraview-deps: runs-on: windows-latest steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 with: python-version: 3.9 - name: Install Python packages diff --git a/.github/workflows/style/requirements.txt b/.github/workflows/style/requirements.txt new file mode 100644 index 00000000000000..0822ba39339737 --- /dev/null +++ b/.github/workflows/style/requirements.txt @@ -0,0 +1,7 @@ +black==23.10.1 +clingo==5.6.2 +flake8==6.1.0 +isort==5.12.0 +mypy==1.6.1 +types-six==1.16.21.9 +vermin==1.5.2 diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 8cd8b2279931f5..ec2b5aa52c2764 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -27,7 +27,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] concretizer: ['clingo'] on_develop: - ${{ github.ref == 'refs/heads/develop' }} @@ -57,12 +57,16 @@ jobs: os: ubuntu-latest concretizer: 'clingo' on_develop: false + - python-version: '3.11' + os: ubuntu-latest + concretizer: 'clingo' + on_develop: false steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v2 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2 with: fetch-depth: 0 - - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # @v2 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2 with: python-version: ${{ matrix.python-version }} - name: Install System packages @@ -110,10 +114,10 @@ jobs: shell: runs-on: ubuntu-latest steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v2 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2 with: fetch-depth: 0 - - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # @v2 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2 with: python-version: '3.10' - name: Install System packages @@ -153,7 +157,7 @@ jobs: dnf install -y \ bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \ make patch tcl unzip which xz - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v2 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2 - name: Setup repo and non-root user run: | git --version @@ -177,10 +181,10 @@ jobs: clingo-cffi: runs-on: ubuntu-latest steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v2 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2 with: fetch-depth: 0 - - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # @v2 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2 with: python-version: '3.10' - name: Install System packages @@ -190,6 +194,7 @@ jobs: - name: Install Python packages run: | pip install --upgrade pip setuptools pytest coverage[toml] pytest-cov clingo pytest-xdist + pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click" "black" - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -213,12 +218,12 @@ jobs: runs-on: macos-latest strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v2 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2 with: fetch-depth: 0 - - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # @v2 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2 with: python-version: ${{ matrix.python-version }} - name: Install Python packages diff --git a/.github/workflows/valid-style.yml b/.github/workflows/valid-style.yml index 1ef1a3bc9ce015..80aa5b2a4f1ff1 100644 --- a/.github/workflows/valid-style.yml +++ b/.github/workflows/valid-style.yml @@ -27,15 +27,15 @@ jobs: validate: runs-on: ubuntu-latest steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v2 - - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # @v2 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 with: python-version: '3.11' cache: 'pip' - name: Install Python Packages run: | - pip install --upgrade pip - pip install --upgrade vermin + pip install --upgrade pip setuptools + pip install -r .github/workflows/style/requirements.txt - name: vermin (Spack's Core) run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/ - name: vermin (Repositories) @@ -44,16 +44,17 @@ jobs: style: runs-on: ubuntu-latest steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v2 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # @v2 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 with: python-version: '3.11' cache: 'pip' - name: Install Python packages run: | - python3 -m pip install --upgrade pip setuptools types-six black==23.1.0 mypy isort clingo flake8 + pip install --upgrade pip setuptools + pip install -r .github/workflows/style/requirements.txt - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -77,7 +78,7 @@ jobs: dnf install -y \ bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \ make patch tcl unzip which xz - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # @v2 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2 - name: Setup repo and non-root user run: | git --version diff --git a/.github/workflows/windows_python.yml b/.github/workflows/windows_python.yml index bfba155caf4bdb..137c00a9bdbc70 100644 --- a/.github/workflows/windows_python.yml +++ b/.github/workflows/windows_python.yml @@ -15,10 +15,10 @@ jobs: unit-tests: runs-on: windows-latest steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 with: python-version: 3.9 - name: Install Python packages @@ -39,10 +39,10 @@ jobs: unit-tests-cmd: runs-on: windows-latest steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 with: python-version: 3.9 - name: Install Python packages @@ -63,10 +63,10 @@ jobs: build-abseil: runs-on: windows-latest steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: fetch-depth: 0 - - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 with: python-version: 3.9 - name: Install Python packages @@ -75,6 +75,5 @@ jobs: - name: Build Test run: | spack compiler find - spack external find cmake - spack external find ninja + spack -d external find cmake ninja spack -d install abseil-cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b46827b79e849..9ec04198b22912 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,338 @@ +# v0.21.0 (2023-11-11) + +`v0.21.0` is a major feature release. + +## Features in this release + +1. **Better error messages with condition chaining** + + In v0.18, we added better error messages that could tell you what problem happened, + but they couldn't tell you *why* it happened. `0.21` adds *condition chaining* to the + solver, and Spack can now trace back through the conditions that led to an error and + build a tree of causes potential causes and where they came from. For example: + + ```console + $ spack solve hdf5 ^cmake@3.0.1 + ==> Error: concretization failed for the following reasons: + + 1. Cannot satisfy 'cmake@3.0.1' + 2. Cannot satisfy 'cmake@3.0.1' + required because hdf5 ^cmake@3.0.1 requested from CLI + 3. Cannot satisfy 'cmake@3.18:' and 'cmake@3.0.1 + required because hdf5 ^cmake@3.0.1 requested from CLI + required because hdf5 depends on cmake@3.18: when @1.13: + required because hdf5 ^cmake@3.0.1 requested from CLI + 4. Cannot satisfy 'cmake@3.12:' and 'cmake@3.0.1 + required because hdf5 depends on cmake@3.12: + required because hdf5 ^cmake@3.0.1 requested from CLI + required because hdf5 ^cmake@3.0.1 requested from CLI + ``` + + More details in #40173. + +2. **OCI build caches** + + You can now use an arbitrary [OCI](https://opencontainers.org) registry as a build + cache: + + ```console + $ spack mirror add my_registry oci://user/image # Dockerhub + $ spack mirror add my_registry oci://ghcr.io/haampie/spack-test # GHCR + $ spack mirror set --push --oci-username ... --oci-password ... my_registry # set login creds + $ spack buildcache push my_registry [specs...] + ``` + + And you can optionally add a base image to get *runnable* images: + + ```console + $ spack buildcache push --base-image ubuntu:23.04 my_registry python + Pushed ... as [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack + + $ docker run --rm -it [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack + ``` + + This creates a container image from the Spack installations on the host system, + without the need to run `spack install` from a `Dockerfile` or `sif` file. It also + addresses the inconvenience of losing binaries of dependencies when `RUN spack + install` fails inside `docker build`. + + Further, the container image layers and build cache tarballs are the same files. This + means that `spack install` and `docker pull` use the exact same underlying binaries. + If you previously used `spack install` inside of `docker build`, this feature helps + you save storage by a factor two. + + More details in #38358. + +3. **Multiple versions of build dependencies** + + Increasingly, complex package builds require multiple versions of some build + dependencies. For example, Python packages frequently require very specific versions + of `setuptools`, `cython`, and sometimes different physics packages require different + versions of Python to build. The concretizer enforced that every solve was *unified*, + i.e., that there only be one version of every package. The concretizer now supports + "duplicate" nodes for *build dependencies*, but enforces unification through + transitive link and run dependencies. This will allow it to better resolve complex + dependency graphs in ecosystems like Python, and it also gets us very close to + modeling compilers as proper dependencies. + + This change required a major overhaul of the concretizer, as well as a number of + performance optimizations. See #38447, #39621. + +4. **Cherry-picking virtual dependencies** + + You can now select only a subset of virtual dependencies from a spec that may provide + more. For example, if you want `mpich` to be your `mpi` provider, you can be explicit + by writing: + + ``` + hdf5 ^[virtuals=mpi] mpich + ``` + + Or, if you want to use, e.g., `intel-parallel-studio` for `blas` along with an external + `lapack` like `openblas`, you could write: + + ``` + strumpack ^[virtuals=mpi] intel-parallel-studio+mkl ^[virtuals=lapack] openblas + ``` + + The `virtuals=mpi` is an edge attribute, and dependency edges in Spack graphs now + track which virtuals they satisfied. More details in #17229 and #35322. + + Note for packaging: in Spack 0.21 `spec.satisfies("^virtual")` is true if and only if + the package specifies `depends_on("virtual")`. This is different from Spack 0.20, + where depending on a provider implied depending on the virtual provided. See #41002 + for an example where `^mkl` was being used to test for several `mkl` providers in a + package that did not depend on `mkl`. + +5. **License directive** + + Spack packages can now have license metadata, with the new `license()` directive: + + ```python + license("Apache-2.0") + ``` + + Licenses use [SPDX identifiers](https://spdx.org/licenses), and you can use SPDX + expressions to combine them: + + ```python + license("Apache-2.0 OR MIT") + ``` + + Like other directives in Spack, it's conditional, so you can handle complex cases like + Spack itself: + + ```python + license("LGPL-2.1", when="@:0.11") + license("Apache-2.0 OR MIT", when="@0.12:") + ``` + + More details in #39346, #40598. + +6. **`spack deconcretize` command** + + We are getting close to having a `spack update` command for environments, but we're + not quite there yet. This is the next best thing. `spack deconcretize` gives you + control over what you want to update in an already concrete environment. If you have + an environment built with, say, `meson`, and you want to update your `meson` version, + you can run: + + ```console + spack deconcretize meson + ``` + + and have everything that depends on `meson` rebuilt the next time you run `spack + concretize`. In a future Spack version, we'll handle all of this in a single command, + but for now you can use this to drop bits of your lockfile and resolve your + dependencies again. More in #38803. + +7. **UI Improvements** + + The venerable `spack info` command was looking shabby compared to the rest of Spack's + UI, so we reworked it to have a bit more flair. `spack info` now makes much better + use of terminal space and shows variants, their values, and their descriptions much + more clearly. Conditional variants are grouped separately so you can more easily + understand how packages are structured. More in #40998. + + `spack checksum` now allows you to filter versions from your editor, or by version + range. It also notifies you about potential download URL changes. See #40403. + +8. **Environments can include definitions** + + Spack did not previously support using `include:` with The + [definitions](https://spack.readthedocs.io/en/latest/environments.html#spec-list-references) + section of an environment, but now it does. You can use this to curate lists of specs + and more easily reuse them across environments. See #33960. + +9. **Aliases** + + You can now add aliases to Spack commands in `config.yaml`, e.g. this might enshrine + your favorite args to `spack find` as `spack f`: + + ```yaml + config: + aliases: + f: find -lv + ``` + + See #17229. + +10. **Improved autoloading of modules** + + Spack 0.20 was the first release to enable autoloading of direct dependencies in + module files. + + The downside of this was that `module avail` and `module load` tab completion would + show users too many modules to choose from, and many users disabled generating + modules for dependencies through `exclude_implicits: true`. Further, it was + necessary to keep hashes in module names to avoid file name clashes. + + In this release, you can start using `hide_implicits: true` instead, which exposes + only explicitly installed packages to the user, while still autoloading + dependencies. On top of that, you can safely use `hash_length: 0`, as this config + now only applies to the modules exposed to the user -- you don't have to worry about + file name clashes for hidden dependencies. + + Note: for `tcl` this feature requires Modules 4.7 or higher + +11. **Updated container labeling** + + Nightly Docker images from the `develop` branch will now be tagged as `:develop` and + `:nightly`. The `:latest` tag is no longer associated with `:develop`, but with the + latest stable release. Releases will be tagged with `:{major}`, `:{major}.{minor}` + and `:{major}.{minor}.{patch}`. `ubuntu:18.04` has also been removed from the list of + generated Docker images, as it is no longer supported. See #40593. + +## Other new commands and directives + +* `spack env activate` without arguments now loads a `default` environment that you do + not have to create (#40756). +* `spack find -H` / `--hashes`: a new shortcut for piping `spack find` output to + other commands (#38663) +* Add `spack checksum --verify`, fix `--add` (#38458) +* New `default_args` context manager factors out common args for directives (#39964) +* `spack compiler find --[no]-mixed-toolchain` lets you easily mix `clang` and + `gfortran` on Linux (#40902) + +## Performance improvements + +* `spack external find` execution is now much faster (#39843) +* `spack location -i` now much faster on success (#40898) +* Drop redundant rpaths post install (#38976) +* ASP-based solver: avoid cycles in clingo using hidden directive (#40720) +* Fix multiple quadratic complexity issues in environments (#38771) + +## Other new features of note + +* archspec: update to v0.2.2, support for Sapphire Rapids, Power10, Neoverse V2 (#40917) +* Propagate variants across nodes that don't have that variant (#38512) +* Implement fish completion (#29549) +* Can now distinguish between source/binary mirror; don't ping mirror.spack.io as much (#34523) +* Improve status reporting on install (add [n/total] display) (#37903) + +## Windows + +This release has the best Windows support of any Spack release yet, with numerous +improvements and much larger swaths of tests passing: + +* MSVC and SDK improvements (#37711, #37930, #38500, #39823, #39180) +* Windows external finding: update default paths; treat .bat as executable on Windows (#39850) +* Windows decompression: fix removal of intermediate file (#38958) +* Windows: executable/path handling (#37762) +* Windows build systems: use ninja and enable tests (#33589) +* Windows testing (#36970, #36972, #36973, #36840, #36977, #36792, #36834, #34696, #36971) +* Windows PowerShell support (#39118, #37951) +* Windows symlinking and libraries (#39933, #38599, #34701, #38578, #34701) + +## Notable refactors +* User-specified flags take precedence over others in Spack compiler wrappers (#37376) +* Improve setup of build, run, and test environments (#35737, #40916) +* `make` is no longer a required system dependency of Spack (#40380) +* Support Python 3.12 (#40404, #40155, #40153) +* docs: Replace package list with packages.spack.io (#40251) +* Drop Python 2 constructs in Spack (#38720, #38718, #38703) + +## Binary cache and stack updates +* e4s arm stack: duplicate and target neoverse v1 (#40369) +* Add macOS ML CI stacks (#36586) +* E4S Cray CI Stack (#37837) +* e4s cray: expand spec list (#38947) +* e4s cray sles ci: expand spec list (#39081) + +## Removals, deprecations, and syntax changes +* ASP: targets, compilers and providers soft-preferences are only global (#31261) +* Parser: fix ambiguity with whitespace in version ranges (#40344) +* Module file generation is disabled by default; you'll need to enable it to use it (#37258) +* Remove deprecated "extra_instructions" option for containers (#40365) +* Stand-alone test feature deprecation postponed to v0.22 (#40600) +* buildcache push: make `--allow-root` the default and deprecate the option (#38878) + +## Notable Bugfixes +* Bugfix: propagation of multivalued variants (#39833) +* Allow `/` in git versions (#39398) +* Fetch & patch: actually acquire stage lock, and many more issues (#38903) +* Environment/depfile: better escaping of targets with Git versions (#37560) +* Prevent "spack external find" to error out on wrong permissions (#38755) +* lmod: allow core compiler to be specified with a version range (#37789) + +## Spack community stats + +* 7,469 total packages, 303 new since `v0.20.0` + * 150 new Python packages + * 34 new R packages +* 353 people contributed to this release + * 336 committers to packages + * 65 committers to core + + +# v0.20.3 (2023-10-31) + +## Bugfixes + +- Fix a bug where `spack mirror set-url` would drop configured connection info (reverts #34210) +- Fix a minor issue with package hash computation for Python 3.12 (#40328) + + +# v0.20.2 (2023-10-03) + +## Features in this release + +Spack now supports Python 3.12 (#40155) + +## Bugfixes + +- Improve escaping in Tcl module files (#38375) +- Make repo cache work on repositories with zero mtime (#39214) +- Ignore errors for newer, incompatible buildcache version (#40279) +- Print an error when git is required, but missing (#40254) +- Ensure missing build dependencies get installed when using `spack install --overwrite` (#40252) +- Fix an issue where Spack freezes when the build process unexpectedly exits (#39015) +- Fix a bug where installation failures cause an unrelated `NameError` to be thrown (#39017) +- Fix an issue where Spack package versions would be incorrectly derived from git tags (#39414) +- Fix a bug triggered when file locking fails internally (#39188) +- Prevent "spack external find" to error out when a directory cannot be accessed (#38755) +- Fix multiple performance regressions in environments (#38771) +- Add more ignored modules to `pyproject.toml` for `mypy` (#38769) + + +# v0.20.1 (2023-07-10) + +## Spack Bugfixes + +- Spec removed from an environment where not actually removed if `--force` was not given (#37877) +- Speed-up module file generation (#37739) +- Hotfix for a few recipes that treat CMake as a link dependency (#35816) +- Fix re-running stand-alone test a second time, which was getting a trailing spurious failure (#37840) +- Fixed reading JSON manifest on Cray, reporting non-concrete specs (#37909) +- Fixed a few bugs when generating Dockerfiles from Spack (#37766,#37769) +- Fixed a few long-standing bugs when generating module files (#36678,#38347,#38465,#38455) +- Fixed issues with building Python extensions using an external Python (#38186) +- Fixed compiler removal from command line (#38057) +- Show external status as [e] (#33792) +- Backported `archspec` fixes (#37793) +- Improved a few error messages (#37791) + + # v0.20.0 (2023-05-21) `v0.20.0` is a major feature release. diff --git a/CITATION.cff b/CITATION.cff index 4ae54a57df4165..16f42d01651f19 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -27,12 +27,53 @@ # And here's the CITATION.cff format: # cff-version: 1.2.0 +type: software message: "If you are referencing Spack in a publication, please cite the paper below." +title: "The Spack Package Manager: Bringing Order to HPC Software Chaos" +abstract: >- + Large HPC centers spend considerable time supporting software for thousands of users, but the complexity of HPC software is quickly outpacing the capabilities of existing software management tools. + Scientific applications require specific versions of compilers, MPI, and other dependency libraries, so using a single, standard software stack is infeasible. + However, managing many configurations is difficult because the configuration space is combinatorial in size. + We introduce Spack, a tool used at Lawrence Livermore National Laboratory to manage this complexity. + Spack provides a novel, re- cursive specification syntax to invoke parametric builds of packages and dependencies. + It allows any number of builds to coexist on the same system, and it ensures that installed packages can find their dependencies, regardless of the environment. + We show through real-world use cases that Spack supports diverse and demanding applications, bringing order to HPC software chaos. preferred-citation: + title: "The Spack Package Manager: Bringing Order to HPC Software Chaos" type: conference-paper - doi: "10.1145/2807591.2807623" - url: "https://github.com/spack/spack" + url: "https://tgamblin.github.io/pubs/spack-sc15.pdf" authors: + - family-names: "Gamblin" + given-names: "Todd" + - family-names: "LeGendre" + given-names: "Matthew" + - family-names: "Collette" + given-names: "Michael R." + - family-names: "Lee" + given-names: "Gregory L." + - family-names: "Moody" + given-names: "Adam" + - family-names: "de Supinski" + given-names: "Bronis R." + - family-names: "Futral" + given-names: "Scott" + conference: + name: "Supercomputing 2015 (SC’15)" + city: "Austin" + region: "Texas" + country: "US" + date-start: 2015-11-15 + date-end: 2015-11-20 + month: 11 + year: 2015 + identifiers: + - description: "The concept DOI of the work." + type: doi + value: 10.1145/2807591.2807623 + - description: "The DOE Document Release Number of the work" + type: other + value: "LLNL-CONF-669890" +authors: - family-names: "Gamblin" given-names: "Todd" - family-names: "LeGendre" @@ -47,12 +88,3 @@ preferred-citation: given-names: "Bronis R." - family-names: "Futral" given-names: "Scott" - title: "The Spack Package Manager: Bringing Order to HPC Software Chaos" - conference: - name: "Supercomputing 2015 (SC’15)" - city: "Austin" - region: "Texas" - country: "USA" - month: November 15-20 - year: 2015 - notes: LLNL-CONF-669890 diff --git a/README.md b/README.md index cf4b413af8b9e0..d048140ec1badc 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ [![Read the Docs](https://readthedocs.org/projects/spack/badge/?version=latest)](https://spack.readthedocs.io) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Slack](https://slack.spack.io/badge.svg)](https://slack.spack.io) +[![Matrix](https://img.shields.io/matrix/spack-space%3Amatrix.org?label=Matrix)](https://matrix.to/#/#spack-space:matrix.org) Spack is a multi-platform package manager that builds and installs multiple versions and configurations of software. It works on Linux, @@ -62,7 +63,10 @@ Resources: * **Slack workspace**: [spackpm.slack.com](https://spackpm.slack.com). To get an invitation, visit [slack.spack.io](https://slack.spack.io). -* [**Github Discussions**](https://github.com/spack/spack/discussions): not just for discussions, also Q&A. +* **Matrix space**: [#spack-space:matrix.org](https://matrix.to/#/#spack-space:matrix.org): + [bridged](https://github.com/matrix-org/matrix-appservice-slack#matrix-appservice-slack) to Slack. +* [**Github Discussions**](https://github.com/spack/spack/discussions): + not just for discussions, but also Q&A. * **Mailing list**: [groups.google.com/d/forum/spack](https://groups.google.com/d/forum/spack) * **Twitter**: [@spackpm](https://twitter.com/spackpm). Be sure to `@mention` us! diff --git a/SECURITY.md b/SECURITY.md index 3a8381140c8875..c0cb40cbde931d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,24 +2,26 @@ ## Supported Versions -We provide security updates for the following releases. +We provide security updates for `develop` and for the last two +stable (`0.x`) release series of Spack. Security updates will be +made available as patch (`0.x.1`, `0.x.2`, etc.) releases. + For more on Spack's release structure, see [`README.md`](https://github.com/spack/spack#releases). +## Reporting a Vulnerability -| Version | Supported | -| ------- | ------------------ | -| develop | :white_check_mark: | -| 0.19.x | :white_check_mark: | -| 0.18.x | :white_check_mark: | +You can report a vulnerability using GitHub's private reporting +feature: -## Reporting a Vulnerability +1. Go to [github.com/spack/spack/security](https://github.com/spack/spack/security). +2. Click "Report a vulnerability" in the upper right corner of that page. +3. Fill out the form and submit your draft security advisory. -To report a vulnerability or other security -issue, email maintainers@spack.io. +More details are available in +[GitHub's docs](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability). -You can expect to hear back within two days. -If your security issue is accepted, we will do -our best to release a fix within a week. If -fixing the issue will take longer than this, -we will discuss timeline options with you. +You can expect to hear back about security issues within two days. +If your security issue is accepted, we will do our best to release +a fix within a week. If fixing the issue will take longer than +this, we will discuss timeline options with you. diff --git a/bin/spack.bat b/bin/spack.bat index 9aff863a905276..b067256cf90ef7 100644 --- a/bin/spack.bat +++ b/bin/spack.bat @@ -14,7 +14,7 @@ :: @echo off -set spack=%SPACK_ROOT%\bin\spack +set spack="%SPACK_ROOT%"\bin\spack ::####################################################################### :: This is a wrapper around the spack command that forwards calls to @@ -51,65 +51,43 @@ setlocal enabledelayedexpansion :: subcommands will never start with '-' :: everything after the subcommand is an arg -:: we cannot allow batch "for" loop to directly process CL args -:: a number of batch reserved characters are commonly passed to -:: spack and allowing batch's "for" method to process the raw inputs -:: results in a large number of formatting issues -:: instead, treat the entire CLI as one string -:: and split by space manually -:: capture cl args in variable named cl_args -set cl_args=%* + :process_cl_args -rem tokens=1* returns the first processed token produced -rem by tokenizing the input string cl_args on spaces into -rem the named variable %%g -rem While this make look like a for loop, it only -rem executes a single time for each of the cl args -rem the actual iterative loop is performed by the -rem goto process_cl_args stanza -rem we are simply leveraging the "for" method's string -rem tokenization -for /f "tokens=1*" %%g in ("%cl_args%") do ( - set t=%%~g - rem remainder of string is composed into %%h - rem these are the cl args yet to be processed - rem assign cl_args var to only the args to be processed - rem effectively discarding the current arg %%g - rem this will be nul when we have no further tokens to process - set cl_args=%%h - rem process the first space delineated cl arg - rem of this iteration - if "!t:~0,1!" == "-" ( - if defined _sp_subcommand ( - rem We already have a subcommand, processing args now - if not defined _sp_args ( - set "_sp_args=!t!" - ) else ( - set "_sp_args=!_sp_args! !t!" - ) - ) else ( - if not defined _sp_flags ( - set "_sp_flags=!t!" - shift - ) else ( - set "_sp_flags=!_sp_flags! !t!" - shift - ) - ) - ) else if not defined _sp_subcommand ( - set "_sp_subcommand=!t!" - shift - ) else ( +rem Set first cl argument (denoted by %1) to be processed +set t=%1 +rem shift moves all cl positional arguments left by one +rem meaning %2 is now %1, this allows us to iterate over each +rem argument +shift +rem assign next "first" cl argument to cl_args, will be null when +rem there are now further arguments to process +set cl_args=%1 +if "!t:~0,1!" == "-" ( + if defined _sp_subcommand ( + rem We already have a subcommand, processing args now if not defined _sp_args ( set "_sp_args=!t!" - shift ) else ( set "_sp_args=!_sp_args! !t!" - shift + ) + ) else ( + if not defined _sp_flags ( + set "_sp_flags=!t!" + ) else ( + set "_sp_flags=!_sp_flags! !t!" ) ) +) else if not defined _sp_subcommand ( + set "_sp_subcommand=!t!" +) else ( + if not defined _sp_args ( + set "_sp_args=!t!" + ) else ( + set "_sp_args=!_sp_args! !t!" + ) ) -rem if this is not nil, we have more tokens to process + +rem if this is not nu;ll, we have more tokens to process rem start above process again with remaining unprocessed cl args if defined cl_args goto :process_cl_args diff --git a/bin/spack.ps1 b/bin/spack.ps1 index 39fe0167ca7899..e08f1fc53d7204 100644 --- a/bin/spack.ps1 +++ b/bin/spack.ps1 @@ -39,12 +39,26 @@ function Read-SpackArgs { return $SpackCMD_params, $SpackSubCommand, $SpackSubCommandArgs } +function Set-SpackEnv { + # This method is responsible + # for processing the return from $(spack ) + # which are returned as System.Object[]'s containing + # a list of env commands + # Invoke-Expression can only handle one command at a time + # so we iterate over the list to invoke the env modification + # expressions one at a time + foreach($envop in $args[0]){ + Invoke-Expression $envop + } +} + + function Invoke-SpackCD { if (Compare-CommonArgs $SpackSubCommandArgs) { - python $Env:SPACK_ROOT/bin/spack cd -h + python "$Env:SPACK_ROOT/bin/spack" cd -h } else { - $LOC = $(python $Env:SPACK_ROOT/bin/spack location $SpackSubCommandArgs) + $LOC = $(python "$Env:SPACK_ROOT/bin/spack" location $SpackSubCommandArgs) if (($NULL -ne $LOC)){ if ( Test-Path -Path $LOC){ Set-Location $LOC @@ -61,7 +75,7 @@ function Invoke-SpackCD { function Invoke-SpackEnv { if (Compare-CommonArgs $SpackSubCommandArgs[0]) { - python $Env:SPACK_ROOT/bin/spack env -h + python "$Env:SPACK_ROOT/bin/spack" env -h } else { $SubCommandSubCommand = $SpackSubCommandArgs[0] @@ -69,46 +83,46 @@ function Invoke-SpackEnv { switch ($SubCommandSubCommand) { "activate" { if (Compare-CommonArgs $SubCommandSubCommandArgs) { - python $Env:SPACK_ROOT/bin/spack env activate $SubCommandSubCommandArgs + python "$Env:SPACK_ROOT/bin/spack" env activate $SubCommandSubCommandArgs } elseif ([bool]($SubCommandSubCommandArgs.Where({$_ -eq "--pwsh"}))) { - python $Env:SPACK_ROOT/bin/spack env activate $SubCommandSubCommandArgs + python "$Env:SPACK_ROOT/bin/spack" env activate $SubCommandSubCommandArgs } elseif (!$SubCommandSubCommandArgs) { - python $Env:SPACK_ROOT/bin/spack env activate $SubCommandSubCommandArgs + python "$Env:SPACK_ROOT/bin/spack" env activate $SubCommandSubCommandArgs } else { - $SpackEnv = $(python $Env:SPACK_ROOT/bin/spack $SpackCMD_params env activate "--pwsh" $SubCommandSubCommandArgs) - $ExecutionContext.InvokeCommand($SpackEnv) + $SpackEnv = $(python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params env activate "--pwsh" $SubCommandSubCommandArgs) + Set-SpackEnv $SpackEnv } } "deactivate" { if ([bool]($SubCommandSubCommandArgs.Where({$_ -eq "--pwsh"}))) { - python $Env:SPACK_ROOT/bin/spack env deactivate $SubCommandSubCommandArgs + python"$Env:SPACK_ROOT/bin/spack" env deactivate $SubCommandSubCommandArgs } elseif($SubCommandSubCommandArgs) { - python $Env:SPACK_ROOT/bin/spack env deactivate -h + python "$Env:SPACK_ROOT/bin/spack" env deactivate -h } else { - $SpackEnv = $(python $Env:SPACK_ROOT/bin/spack $SpackCMD_params env deactivate --pwsh) - $ExecutionContext.InvokeCommand($SpackEnv) + $SpackEnv = $(python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params env deactivate "--pwsh") + Set-SpackEnv $SpackEnv } } - default {python $Env:SPACK_ROOT/bin/spack $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs} + default {python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs} } } } function Invoke-SpackLoad { if (Compare-CommonArgs $SpackSubCommandArgs) { - python $Env:SPACK_ROOT/bin/spack $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs + python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs } elseif ([bool]($SpackSubCommandArgs.Where({($_ -eq "--pwsh") -or ($_ -eq "--list")}))) { - python $Env:SPACK_ROOT/bin/spack $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs + python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs } else { - $SpackEnv = $(python $Env:SPACK_ROOT/bin/spack $SpackCMD_params $SpackSubCommand "--pwsh" $SpackSubCommandArgs) - $ExecutionContext.InvokeCommand($SpackEnv) + $SpackEnv = $(python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand "--pwsh" $SpackSubCommandArgs) + Set-SpackEnv $SpackEnv } } @@ -116,7 +130,7 @@ function Invoke-SpackLoad { $SpackCMD_params, $SpackSubCommand, $SpackSubCommandArgs = Read-SpackArgs $args if (Compare-CommonArgs $SpackCMD_params) { - python $Env:SPACK_ROOT/bin/spack $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs + python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs exit $LASTEXITCODE } @@ -128,5 +142,5 @@ switch($SpackSubCommand) "env" {Invoke-SpackEnv} "load" {Invoke-SpackLoad} "unload" {Invoke-SpackLoad} - default {python $Env:SPACK_ROOT/bin/spack $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs} + default {python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs} } diff --git a/etc/spack/defaults/bootstrap.yaml b/etc/spack/defaults/bootstrap.yaml index 464994d171e4b3..6f2dbe171c5f60 100644 --- a/etc/spack/defaults/bootstrap.yaml +++ b/etc/spack/defaults/bootstrap.yaml @@ -9,15 +9,15 @@ bootstrap: # may not be able to bootstrap all the software that Spack needs, # depending on its type. sources: + - name: 'github-actions-v0.5' + metadata: $spack/share/spack/bootstrap/github-actions-v0.5 - name: 'github-actions-v0.4' metadata: $spack/share/spack/bootstrap/github-actions-v0.4 - - name: 'github-actions-v0.3' - metadata: $spack/share/spack/bootstrap/github-actions-v0.3 - name: 'spack-install' metadata: $spack/share/spack/bootstrap/spack-install trusted: # By default we trust bootstrapping from sources and from binaries # produced on Github via the workflow + github-actions-v0.5: true github-actions-v0.4: true - github-actions-v0.3: true spack-install: true diff --git a/etc/spack/defaults/concretizer.yaml b/etc/spack/defaults/concretizer.yaml index caf500eb1f8e1b..edefa552cee227 100644 --- a/etc/spack/defaults/concretizer.yaml +++ b/etc/spack/defaults/concretizer.yaml @@ -36,3 +36,9 @@ concretizer: # on each root spec, allowing different versions and variants of the same package in # an environment. unify: true + # Option to deal with possible duplicate nodes (i.e. different nodes from the same package) in the DAG. + duplicates: + # "none": allows a single node for any package in the DAG. + # "minimal": allows the duplication of 'build-tools' nodes only (e.g. py-setuptools, cmake etc.) + # "full" (experimental): allows separation of the entire build-tool stack (e.g. the entire "cmake" subDAG) + strategy: minimal diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 0e845c5a19292d..9b58822ee4c7f6 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -219,10 +219,11 @@ config: # manipulation by unprivileged user (e.g. AFS) allow_sgid: true - # Whether to set the terminal title to display status information during - # building and installing packages. This gives information about Spack's - # current progress as well as the current and total number of packages. - terminal_title: false + # Whether to show status information during building and installing packages. + # This gives information about Spack's current progress as well as the current + # and total number of packages. Information is shown both in the terminal + # title and inline. + install_status: true # Number of seconds a buildcache's index.json is cached locally before probing # for updates, within a single Spack invocation. Defaults to 10 minutes. @@ -231,3 +232,11 @@ config: flags: # Whether to keep -Werror flags active in package builds. keep_werror: 'none' + + # A mapping of aliases that can be used to define new commands. For instance, + # `sp: spec -I` will define a new command `sp` that will execute `spec` with + # the `-I` argument. Aliases cannot override existing commands. + aliases: + concretise: concretize + containerise: containerize + rm: remove diff --git a/etc/spack/defaults/mirrors.yaml b/etc/spack/defaults/mirrors.yaml index 4db4f7dedb758e..0891ae4504ec0f 100644 --- a/etc/spack/defaults/mirrors.yaml +++ b/etc/spack/defaults/mirrors.yaml @@ -1,2 +1,4 @@ mirrors: - spack-public: https://mirror.spack.io + spack-public: + binary: false + url: https://mirror.spack.io diff --git a/etc/spack/defaults/packages.yaml b/etc/spack/defaults/packages.yaml index 22b766264c205e..24d19ac4cbbd73 100644 --- a/etc/spack/defaults/packages.yaml +++ b/etc/spack/defaults/packages.yaml @@ -49,6 +49,7 @@ packages: pbs: [openpbs, torque] pil: [py-pillow] pkgconfig: [pkgconf, pkg-config] + qmake: [qt-base, qt] rpc: [libtirpc] scalapack: [netlib-scalapack, amdscalapack] sycl: [hipsycl] @@ -59,6 +60,7 @@ packages: xxd: [xxd-standalone, vim] yacc: [bison, byacc] ziglang: [zig] + zlib-api: [zlib-ng+compat, zlib] permissions: read: world write: user diff --git a/lib/jcsda-emc/spack-stack/stack/stack_env.py b/lib/jcsda-emc/spack-stack/stack/stack_env.py index abd0dd5ae2892c..795587c6f2cc57 100644 --- a/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -220,7 +220,7 @@ def write(self): # Precedence order (high to low) is original spack.yaml, # then common configs, then site configs. original_sections = {} - for key in spack.config.section_schemas.keys(): + for key in spack.config.SECTION_SCHEMAS.keys(): section = spack.config.get(key, scope=env_scope) if section: original_sections[key] = copy.deepcopy(section) @@ -294,7 +294,7 @@ def write(self): # Merge the original spack.yaml template back in # so it has the higest precedence - for section in spack.config.section_schemas.keys(): + for section in spack.config.SECTION_SCHEMAS.keys(): original = original_sections.get(section, {}) existing = spack.config.get(section, scope=env_scope) new = spack.config.merge_yaml(existing, original) diff --git a/lib/spack/docs/.gitignore b/lib/spack/docs/.gitignore index d481aa0923b9db..b349291a8a598b 100644 --- a/lib/spack/docs/.gitignore +++ b/lib/spack/docs/.gitignore @@ -1,4 +1,3 @@ -package_list.html command_index.rst spack*.rst llnl*.rst diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index af6d2dab91a8c9..52054a9405653e 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -45,7 +45,8 @@ Listing available packages To install software with Spack, you need to know what software is available. You can see a list of available package names at the -:ref:`package-list` webpage, or using the ``spack list`` command. +`packages.spack.io `_ website, or +using the ``spack list`` command. .. _cmd-spack-list: @@ -60,7 +61,7 @@ can install: :ellipsis: 10 There are thousands of them, so we've truncated the output above, but you -can find a :ref:`full list here `. +can find a `full list here `_. Packages are listed by name in alphabetical order. A pattern to match with no wildcards, ``*`` or ``?``, will be treated as though it started and ended with @@ -1525,6 +1526,30 @@ any MPI implementation will do. If another package depends on error. Likewise, if you try to plug in some package that doesn't provide MPI, Spack will raise an error. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Explicit binding of virtual dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are packages that provide more than just one virtual dependency. When interacting with them, users +might want to utilize just a subset of what they could provide, and use other providers for virtuals they +need. + +It is possible to be more explicit and tell Spack which dependency should provide which virtual, using a +special syntax: + +.. code-block:: console + + $ spack spec strumpack ^[virtuals=mpi] intel-parallel-studio+mkl ^[virtuals=lapack] openblas + +Concretizing the spec above produces the following DAG: + +.. figure:: images/strumpack_virtuals.svg + :scale: 60 % + :align: center + +where ``intel-parallel-studio`` *could* provide ``mpi``, ``lapack``, and ``blas`` but is used only for the former. The ``lapack`` +and ``blas`` dependencies are satisfied by ``openblas``. + ^^^^^^^^^^^^^^^^^^^^^^^^ Specifying Specs by Hash ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/binary_caches.rst b/lib/spack/docs/binary_caches.rst index 61cf53cafa7bc6..eeb6c4b783c8d1 100644 --- a/lib/spack/docs/binary_caches.rst +++ b/lib/spack/docs/binary_caches.rst @@ -48,14 +48,10 @@ Here is an example where a build cache is created in a local directory named .. code-block:: console - $ spack buildcache push --allow-root ./spack-cache ninja + $ spack buildcache push ./spack-cache ninja ==> Pushing binary packages to file:///home/spackuser/spack/spack-cache/build_cache -Not that ``ninja`` must be installed locally for this to work. - -We're using the ``--allow-root`` flag to tell Spack that is OK when any of -the binaries we're pushing contain references to the local Spack install -directory. +Note that ``ninja`` must be installed locally for this to work. Once you have a build cache, you can add it as a mirror, discussed next. @@ -147,7 +143,7 @@ and then install from it exclusively, you would do: $ spack mirror add E4S https://cache.e4s.io $ spack buildcache keys --install --trust - $ spack install --use-buildache only + $ spack install --use-buildcache only We use ``--install`` and ``--trust`` to say that we are installing keys to our keyring, and trusting all downloaded keys. @@ -159,16 +155,182 @@ List of popular build caches * `Extreme-scale Scientific Software Stack (E4S) `_: `build cache `_ - ---------- Relocation ---------- -Initial build and later installation do not necessarily happen at the same -location. Spack provides a relocation capability and corrects for RPATHs and -non-relocatable scripts. However, many packages compile paths into binary -artifacts directly. In such cases, the build instructions of this package would -need to be adjusted for better re-locatability. +When using buildcaches across different machines, it is likely that the install +root will be different from the one used to build the binaries. + +To address this issue, Spack automatically relocates all paths encoded in binaries +and scripts to their new location upon install. + +Note that there are some cases where this is not possible: if binaries are built in +a relatively short path, and then installed to a longer path, there may not be enough +space in the binary to encode the new path. In this case, Spack will fail to install +the package from the build cache, and a source build is required. + +To reduce the likelihood of this happening, it is highly recommended to add padding to +the install root during the build, as specified in the :ref:`config ` +section of the configuration: + +.. code-block:: yaml + + config: + install_tree: + root: /opt/spack + padded_length: 128 + + + +----------------------------------------- +OCI / Docker V2 registries as build cache +----------------------------------------- + +Spack can also use OCI or Docker V2 registries such as Dockerhub, Quay.io, +Github Packages, GitLab Container Registry, JFrog Artifactory, and others +as build caches. This is a convenient way to share binaries using public +infrastructure, or to cache Spack built binaries in Github Actions and +GitLab CI. + +To get started, configure an OCI mirror using ``oci://`` as the scheme, +and optionally specify a username and password (or personal access token): + +.. code-block:: console + + $ spack mirror add --oci-username username --oci-password password my_registry oci://example.com/my_image + +Spack follows the naming conventions of Docker, with Dockerhub as the default +registry. To use Dockerhub, you can omit the registry domain: + +.. code-block:: console + + $ spack mirror add --oci-username username --oci-password password my_registry oci://username/my_image + +From here, you can use the mirror as any other build cache: + +.. code-block:: console + + $ spack buildcache push my_registry # push to the registry + $ spack install # install from the registry + +A unique feature of buildcaches on top of OCI registries is that it's incredibly +easy to generate get a runnable container image with the binaries installed. This +is a great way to make applications available to users without requiring them to +install Spack -- all you need is Docker, Podman or any other OCI-compatible container +runtime. + +To produce container images, all you need to do is add the ``--base-image`` flag +when pushing to the build cache: + +.. code-block:: console + + $ spack buildcache push --base-image ubuntu:20.04 my_registry ninja + Pushed to example.com/my_image:ninja-1.11.1-yxferyhmrjkosgta5ei6b4lqf6bxbscz.spack + + $ docker run -it example.com/my_image:ninja-1.11.1-yxferyhmrjkosgta5ei6b4lqf6bxbscz.spack + root@e4c2b6f6b3f4:/# ninja --version + 1.11.1 + +If ``--base-image`` is not specified, distroless images are produced. In practice, +you won't be able to run these as containers, since they don't come with libc and +other system dependencies. However, they are still compatible with tools like +``skopeo``, ``podman``, and ``docker`` for pulling and pushing. + +.. note:: + The docker ``overlayfs2`` storage driver is limited to 128 layers, above which a + ``max depth exceeded`` error may be produced when pulling the image. There + are `alternative drivers `_. + +------------------------------------ +Spack build cache for GitHub Actions +------------------------------------ + +To significantly speed up Spack in GitHub Actions, binaries can be cached in +GitHub Packages. This service is an OCI registry that can be linked to a GitHub +repository. + +A typical workflow is to include a ``spack.yaml`` environment in your repository +that specifies the packages to install, the target architecture, and the build +cache to use under ``mirrors``: + +.. code-block:: yaml + + spack: + specs: + - python@3.11 + config: + install_tree: + root: /opt/spack + padded_length: 128 + packages: + all: + require: target=x86_64_v2 + mirrors: + local-buildcache: oci://ghcr.io// + +A GitHub action can then be used to install the packages and push them to the +build cache: + +.. code-block:: yaml + + name: Install Spack packages + + on: push + + env: + SPACK_COLOR: always + + jobs: + example: + runs-on: ubuntu-22.04 + permissions: + packages: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Checkout Spack + uses: actions/checkout@v3 + with: + repository: spack/spack + path: spack + + - name: Setup Spack + run: echo "$PWD/spack/bin" >> "$GITHUB_PATH" + + - name: Concretize + run: spack -e . concretize + + - name: Install + run: spack -e . install --no-check-signature + + - name: Run tests + run: ./my_view/bin/python3 -c 'print("hello world")' + + - name: Push to buildcache + run: | + spack -e . mirror set --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + spack -e . buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache + if: ${{ !cancelled() }} + +The first time this action runs, it will build the packages from source and +push them to the build cache. Subsequent runs will pull the binaries from the +build cache. The concretizer will ensure that prebuilt binaries are favored +over source builds. + +The build cache entries appear in the GitHub Packages section of your repository, +and contain instructions for pulling and running them with ``docker`` or ``podman``. + + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Using Spack's public build cache for GitHub Actions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Spack offers a public build cache for GitHub Actions with a set of common packages, +which lets you get started quickly. See the following resources for more information: + +* `spack/github-actions-buildcache `_ .. _cmd-spack-buildcache: diff --git a/lib/spack/docs/bootstrapping.rst b/lib/spack/docs/bootstrapping.rst index dec548e9c6c314..8eca492413c998 100644 --- a/lib/spack/docs/bootstrapping.rst +++ b/lib/spack/docs/bootstrapping.rst @@ -32,9 +32,14 @@ can't be found. You can readily check if any prerequisite for using Spack is mis Spack will take care of bootstrapping any missing dependency marked as [B]. Dependencies marked as [-] are instead required to be found on the system. + % echo $? + 1 + In the case of the output shown above Spack detected that both ``clingo`` and ``gnupg`` are missing and it's giving detailed information on why they are needed and whether -they can be bootstrapped. Running a command that concretize a spec, like: +they can be bootstrapped. The return code of this command summarizes the results, if any +dependencies are missing the return code is ``1``, otherwise ``0``. Running a command that +concretizes a spec, like: .. code-block:: console @@ -44,7 +49,7 @@ they can be bootstrapped. Running a command that concretize a spec, like: ==> Installing "clingo-bootstrap@spack%apple-clang@12.0.0~docs~ipo+python build_type=Release arch=darwin-catalina-x86_64" from a buildcache [ ... ] -triggers the bootstrapping of clingo from pre-built binaries as expected. +automatically triggers the bootstrapping of clingo from pre-built binaries as expected. Users can also bootstrap all the dependencies needed by Spack in a single command, which might be useful to setup containers or other similar environments: diff --git a/lib/spack/docs/build_settings.rst b/lib/spack/docs/build_settings.rst index 0bbd27e8c32258..38fe2fb2c06d76 100644 --- a/lib/spack/docs/build_settings.rst +++ b/lib/spack/docs/build_settings.rst @@ -3,6 +3,103 @@ SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _concretizer-options: + +========================================== +Concretization Settings (concretizer.yaml) +========================================== + +The ``concretizer.yaml`` configuration file allows to customize aspects of the +algorithm used to select the dependencies you install. The default configuration +is the following: + +.. literalinclude:: _spack_root/etc/spack/defaults/concretizer.yaml + :language: yaml + +-------------------------------- +Reuse already installed packages +-------------------------------- + +The ``reuse`` attribute controls whether Spack will prefer to use installed packages (``true``), or +whether it will do a "fresh" installation and prefer the latest settings from +``package.py`` files and ``packages.yaml`` (``false``). +You can use: + +.. code-block:: console + + % spack install --reuse + +to enable reuse for a single installation, and you can use: + +.. code-block:: console + + spack install --fresh + +to do a fresh install if ``reuse`` is enabled by default. +``reuse: true`` is the default. + +------------------------------------------ +Selection of the target microarchitectures +------------------------------------------ + +The options under the ``targets`` attribute control which targets are considered during a solve. +Currently the options in this section are only configurable from the ``concretizer.yaml`` file +and there are no corresponding command line arguments to enable them for a single solve. + +The ``granularity`` option can take two possible values: ``microarchitectures`` and ``generic``. +If set to: + +.. code-block:: yaml + + concretizer: + targets: + granularity: microarchitectures + +Spack will consider all the microarchitectures known to ``archspec`` to label nodes for +compatibility. If instead the option is set to: + +.. code-block:: yaml + + concretizer: + targets: + granularity: generic + +Spack will consider only generic microarchitectures. For instance, when running on an +Haswell node, Spack will consider ``haswell`` as the best target in the former case and +``x86_64_v3`` as the best target in the latter case. + +The ``host_compatible`` option is a Boolean option that determines whether or not the +microarchitectures considered during the solve are constrained to be compatible with the +host Spack is currently running on. For instance, if this option is set to ``true``, a +user cannot concretize for ``target=icelake`` while running on an Haswell node. + +--------------- +Duplicate nodes +--------------- + +The ``duplicates`` attribute controls whether the DAG can contain multiple configurations of +the same package. This is mainly relevant for build dependencies, which may have their version +pinned by some nodes, and thus be required at different versions by different nodes in the same +DAG. + +The ``strategy`` option controls how the solver deals with duplicates. If the value is ``none``, +then a single configuration per package is allowed in the DAG. This means, for instance, that only +a single ``cmake`` or a single ``py-setuptools`` version is allowed. The result would be a slightly +faster concretization, at the expense of making a few specs unsolvable. + +If the value is ``minimal`` Spack will allow packages tagged as ``build-tools`` to have duplicates. +This allows, for instance, to concretize specs whose nodes require different, and incompatible, ranges +of some build tool. For instance, in the figure below the latest `py-shapely` requires a newer `py-setuptools`, +while `py-numpy` still needs an older version: + +.. figure:: images/shapely_duplicates.svg + :scale: 70 % + :align: center + +Up to Spack v0.20 ``duplicates:strategy:none`` was the default (and only) behavior. From Spack v0.21 the +default behavior is ``duplicates:strategy:minimal``. + .. _build-settings: ================================ @@ -232,76 +329,6 @@ Specific limitations include: then Spack will not add a new external entry (``spack config blame packages`` can help locate all external entries). -.. _concretizer-options: - ----------------------- -Concretizer options ----------------------- - -``packages.yaml`` gives the concretizer preferences for specific packages, -but you can also use ``concretizer.yaml`` to customize aspects of the -algorithm it uses to select the dependencies you install: - -.. literalinclude:: _spack_root/etc/spack/defaults/concretizer.yaml - :language: yaml - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Reuse already installed packages -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The ``reuse`` attribute controls whether Spack will prefer to use installed packages (``true``), or -whether it will do a "fresh" installation and prefer the latest settings from -``package.py`` files and ``packages.yaml`` (``false``). -You can use: - -.. code-block:: console - - % spack install --reuse - -to enable reuse for a single installation, and you can use: - -.. code-block:: console - - spack install --fresh - -to do a fresh install if ``reuse`` is enabled by default. -``reuse: true`` is the default. - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Selection of the target microarchitectures -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The options under the ``targets`` attribute control which targets are considered during a solve. -Currently the options in this section are only configurable from the ``concretizer.yaml`` file -and there are no corresponding command line arguments to enable them for a single solve. - -The ``granularity`` option can take two possible values: ``microarchitectures`` and ``generic``. -If set to: - -.. code-block:: yaml - - concretizer: - targets: - granularity: microarchitectures - -Spack will consider all the microarchitectures known to ``archspec`` to label nodes for -compatibility. If instead the option is set to: - -.. code-block:: yaml - - concretizer: - targets: - granularity: generic - -Spack will consider only generic microarchitectures. For instance, when running on an -Haswell node, Spack will consider ``haswell`` as the best target in the former case and -``x86_64_v3`` as the best target in the latter case. - -The ``host_compatible`` option is a Boolean option that determines whether or not the -microarchitectures considered during the solve are constrained to be compatible with the -host Spack is currently running on. For instance, if this option is set to ``true``, a -user cannot concretize for ``target=icelake`` while running on an Haswell node. - .. _package-requirements: -------------------- @@ -499,56 +526,52 @@ Package Preferences In some cases package requirements can be too strong, and package preferences are the better option. Package preferences do not impose constraints on packages for particular versions or variants values, -they rather only set defaults -- the concretizer is free to change -them if it must due to other constraints. Also note that package -preferences are of lower priority than reuse of already installed -packages. +they rather only set defaults. The concretizer is free to change +them if it must, due to other constraints, and also prefers reusing +installed packages over building new ones that are a better match for +preferences. -Here's an example ``packages.yaml`` file that sets preferred packages: +Most package preferences (``compilers``, ``target`` and ``providers``) +can only be set globally under the ``all`` section of ``packages.yaml``: + +.. code-block:: yaml + + packages: + all: + compiler: [gcc@12.2.0, clang@12:, oneapi@2023:] + target: [x86_64_v3] + providers: + mpi: [mvapich2, mpich, openmpi] + +These preferences override Spack's default and effectively reorder priorities +when looking for the best compiler, target or virtual package provider. Each +preference takes an ordered list of spec constraints, with earlier entries in +the list being preferred over later entries. + +In the example above all packages prefer to be compiled with ``gcc@12.2.0``, +to target the ``x86_64_v3`` microarchitecture and to use ``mvapich2`` if they +depend on ``mpi``. + +The ``variants`` and ``version`` preferences can be set under +package specific sections of the ``packages.yaml`` file: .. code-block:: yaml packages: opencv: - compiler: [gcc@4.9] variants: +debug gperftools: version: [2.2, 2.4, 2.3] - all: - compiler: [gcc@4.4.7, 'gcc@4.6:', intel, clang, pgi] - target: [sandybridge] - providers: - mpi: [mvapich2, mpich, openmpi] -At a high level, this example is specifying how packages are preferably -concretized. The opencv package should prefer using GCC 4.9 and -be built with debug options. The gperftools package should prefer version -2.2 over 2.4. Every package on the system should prefer mvapich2 for -its MPI and GCC 4.4.7 (except for opencv, which overrides this by preferring GCC 4.9). -These options are used to fill in implicit defaults. Any of them can be overwritten -on the command line if explicitly requested. - -Package preferences accept the follow keys or components under -the specific package (or ``all``) section: ``compiler``, ``variants``, -``version``, ``providers``, and ``target``. Each component has an -ordered list of spec ``constraints``, with earlier entries in the -list being preferred over later entries. - -Sometimes a package installation may have constraints that forbid -the first concretization rule, in which case Spack will use the first -legal concretization rule. Going back to the example, if a user -requests gperftools 2.3 or later, then Spack will install version 2.4 -as the 2.4 version of gperftools is preferred over 2.3. - -An explicit concretization rule in the preferred section will always -take preference over unlisted concretizations. In the above example, -xlc isn't listed in the compiler list. Every listed compiler from -gcc to pgi will thus be preferred over the xlc compiler. - -The syntax for the ``provider`` section differs slightly from other -concretization rules. A provider lists a value that packages may -``depends_on`` (e.g, MPI) and a list of rules for fulfilling that -dependency. +In this case, the preference for ``opencv`` is to build with debug options, while +``gperftools`` prefers version 2.2 over 2.4. + +Any preference can be overwritten on the command line if explicitly requested. + +Preferences cannot overcome explicit constraints, as they only set a preferred +ordering among homogeneous attribute values. Going back to the example, if +``gperftools@2.3:`` was requested, then Spack will install version 2.4 +since the most preferred version 2.2 is prohibited by the version constraint. .. _package_permissions: diff --git a/lib/spack/docs/build_systems/autotoolspackage.rst b/lib/spack/docs/build_systems/autotoolspackage.rst index abf25f149bc59a..8b8ccb8f35c1c7 100644 --- a/lib/spack/docs/build_systems/autotoolspackage.rst +++ b/lib/spack/docs/build_systems/autotoolspackage.rst @@ -127,9 +127,9 @@ check out a commit from the ``master`` branch, you would want to add: .. code-block:: python - depends_on('autoconf', type='build', when='@master') - depends_on('automake', type='build', when='@master') - depends_on('libtool', type='build', when='@master') + depends_on("autoconf", type="build", when="@master") + depends_on("automake", type="build", when="@master") + depends_on("libtool", type="build", when="@master") It is typically redundant to list the ``m4`` macro processor package as a dependency, since ``autoconf`` already depends on it. @@ -145,7 +145,7 @@ example, the ``bash`` shell is used to run the ``autogen.sh`` script. .. code-block:: python def autoreconf(self, spec, prefix): - which('bash')('autogen.sh') + which("bash")("autogen.sh") """"""""""""""""""""""""""""""""""""""" patching configure or Makefile.in files @@ -186,9 +186,9 @@ To opt out of this feature, use the following setting: To enable it conditionally on different architectures, define a property and make the package depend on ``gnuconfig`` as a build dependency: -.. code-block +.. code-block:: python - depends_on('gnuconfig', when='@1.0:') + depends_on("gnuconfig", when="@1.0:") @property def patch_config_files(self): @@ -230,7 +230,7 @@ version, this can be done like so: @property def force_autoreconf(self): - return self.version == Version('1.2.3') + return self.version == Version("1.2.3") ^^^^^^^^^^^^^^^^^^^^^^^ Finding configure flags @@ -278,13 +278,22 @@ function like so: def configure_args(self): args = [] - if '+mpi' in self.spec: - args.append('--enable-mpi') + if self.spec.satisfies("+mpi"): + args.append("--enable-mpi") else: - args.append('--disable-mpi') + args.append("--disable-mpi") return args + +Alternatively, you can use the :ref:`enable_or_disable ` helper: + +.. code-block:: python + + def configure_args(self): + return [self.enable_or_disable("mpi")] + + Note that we are explicitly disabling MPI support if it is not requested. This is important, as many Autotools packages will enable options by default if the dependencies are found, and disable them @@ -295,9 +304,11 @@ and `here ` ensure that common build options +are consistent across the packages supporting them. :ref:`Conflicts +and requirements ` prevent attempts to build with known +bugs or limitations. + +For example, if ``MyBundlePackage`` is known to only build on ``linux``, +it could use the ``require`` directive as follows: + +.. code-block:: python + + require("platform=linux", msg="MyBundlePackage only builds on linux") + +Spack has a number of built-in bundle packages, such as: + +* `AmdAocl `_ +* `EcpProxyApps `_ +* `Libc `_ +* `Xsdk `_ + +where ``Xsdk`` also inherits from ``CudaPackage`` and ``RocmPackage`` and +``Libc`` is a virtual bundle package for the C standard library. ^^^^^^^^ diff --git a/lib/spack/docs/build_systems/cachedcmakepackage.rst b/lib/spack/docs/build_systems/cachedcmakepackage.rst index f16499267952d7..7d0b8ff7e8d0c5 100644 --- a/lib/spack/docs/build_systems/cachedcmakepackage.rst +++ b/lib/spack/docs/build_systems/cachedcmakepackage.rst @@ -87,7 +87,7 @@ A typical usage of these methods may look something like this: .. code-block:: python - def initconfig_mpi_entries(self) + def initconfig_mpi_entries(self): # Get existing MPI configurations entries = super(self, Foo).initconfig_mpi_entries() @@ -95,25 +95,25 @@ A typical usage of these methods may look something like this: # This spec has an MPI variant, and we need to enable MPI when it is on. # This hypothetical package controls MPI with the ``FOO_MPI`` option to # cmake. - if '+mpi' in self.spec: - entries.append(cmake_cache_option('FOO_MPI', True, "enable mpi")) + if self.spec.satisfies("+mpi"): + entries.append(cmake_cache_option("FOO_MPI", True, "enable mpi")) else: - entries.append(cmake_cache_option('FOO_MPI', False, "disable mpi")) + entries.append(cmake_cache_option("FOO_MPI", False, "disable mpi")) def initconfig_package_entries(self): # Package specific options entries = [] - entries.append('#Entries for build options') + entries.append("#Entries for build options") - bar_on = '+bar' in self.spec - entries.append(cmake_cache_option('FOO_BAR', bar_on, 'toggle bar')) + bar_on = self.spec.satisfies("+bar") + entries.append(cmake_cache_option("FOO_BAR", bar_on, "toggle bar")) - entries.append('#Entries for dependencies') + entries.append("#Entries for dependencies") - if self.spec['blas'].name == 'baz': # baz is our blas provider - entries.append(cmake_cache_string('FOO_BLAS', 'baz', 'Use baz')) - entries.append(cmake_cache_path('BAZ_PREFIX', self.spec['baz'].prefix)) + if self.spec["blas"].name == "baz": # baz is our blas provider + entries.append(cmake_cache_string("FOO_BLAS", "baz", "Use baz")) + entries.append(cmake_cache_path("BAZ_PREFIX", self.spec["baz"].prefix)) ^^^^^^^^^^^^^^^^^^^^^^ External documentation diff --git a/lib/spack/docs/build_systems/cudapackage.rst b/lib/spack/docs/build_systems/cudapackage.rst index 15fd6d7f029d52..79c98dd52c94f1 100644 --- a/lib/spack/docs/build_systems/cudapackage.rst +++ b/lib/spack/docs/build_systems/cudapackage.rst @@ -54,8 +54,8 @@ to terminate such build attempts with a suitable message: .. code-block:: python - conflicts('cuda_arch=none', when='+cuda', - msg='CUDA architecture is required') + conflicts("cuda_arch=none", when="+cuda", + msg="CUDA architecture is required") Similarly, if your software does not support all versions of the property, you could add ``conflicts`` to your package for those versions. For example, @@ -66,13 +66,13 @@ custom message should a user attempt such a build: .. code-block:: python unsupported_cuda_archs = [ - '10', '11', '12', '13', - '20', '21', - '30', '32', '35', '37' + "10", "11", "12", "13", + "20", "21", + "30", "32", "35", "37" ] for value in unsupported_cuda_archs: - conflicts('cuda_arch={0}'.format(value), when='+cuda', - msg='CUDA architecture {0} is not supported'.format(value)) + conflicts(f"cuda_arch={value}", when="+cuda", + msg=f"CUDA architecture {value} is not supported") ^^^^^^^ Methods @@ -107,16 +107,16 @@ class of your package. For example, you can add it to your spec = self.spec args = [] ... - if '+cuda' in spec: + if spec.satisfies("+cuda"): # Set up the cuda macros needed by the build - args.append('-DWITH_CUDA=ON') - cuda_arch_list = spec.variants['cuda_arch'].value + args.append("-DWITH_CUDA=ON") + cuda_arch_list = spec.variants["cuda_arch"].value cuda_arch = cuda_arch_list[0] - if cuda_arch != 'none': - args.append('-DCUDA_FLAGS=-arch=sm_{0}'.format(cuda_arch)) + if cuda_arch != "none": + args.append(f"-DCUDA_FLAGS=-arch=sm_{cuda_arch}") else: # Ensure build with cuda is disabled - args.append('-DWITH_CUDA=OFF') + args.append("-DWITH_CUDA=OFF") ... return args @@ -125,7 +125,7 @@ You will need to customize options as needed for your build. This example also illustrates how to check for the ``cuda`` variant using ``self.spec`` and how to retrieve the ``cuda_arch`` variant's value, which -is a list, using ``self.spec.variants['cuda_arch'].value``. +is a list, using ``self.spec.variants["cuda_arch"].value``. With over 70 packages using ``CudaPackage`` as of January 2021 there are lots of examples to choose from to get more ideas for using this package. diff --git a/lib/spack/docs/build_systems/custompackage.rst b/lib/spack/docs/build_systems/custompackage.rst index 4979f458ba94b4..5175a0df4df703 100644 --- a/lib/spack/docs/build_systems/custompackage.rst +++ b/lib/spack/docs/build_systems/custompackage.rst @@ -57,13 +57,13 @@ If you look at the ``perl`` package, you'll see: .. code-block:: python - phases = ['configure', 'build', 'install'] + phases = ["configure", "build", "install"] Similarly, ``cmake`` defines: .. code-block:: python - phases = ['bootstrap', 'build', 'install'] + phases = ["bootstrap", "build", "install"] If we look at the ``cmake`` example, this tells Spack's ``PackageBase`` class to run the ``bootstrap``, ``build``, and ``install`` functions @@ -78,7 +78,7 @@ If we look at ``perl``, we see that it defines a ``configure`` method: .. code-block:: python def configure(self, spec, prefix): - configure = Executable('./Configure') + configure = Executable("./Configure") configure(*self.configure_args()) There is also a corresponding ``configure_args`` function that handles @@ -92,7 +92,7 @@ phases are pretty simple: make() def install(self, spec, prefix): - make('install') + make("install") The ``cmake`` package looks very similar, but with a ``bootstrap`` function instead of ``configure``: @@ -100,14 +100,14 @@ function instead of ``configure``: .. code-block:: python def bootstrap(self, spec, prefix): - bootstrap = Executable('./bootstrap') + bootstrap = Executable("./bootstrap") bootstrap(*self.bootstrap_args()) def build(self, spec, prefix): make() def install(self, spec, prefix): - make('install') + make("install") Again, there is a ``boostrap_args`` function that determines the correct bootstrap flags to use. @@ -128,16 +128,16 @@ before or after a particular phase. For example, in ``perl``, we see: .. code-block:: python - @run_after('install') + @run_after("install") def install_cpanm(self): spec = self.spec - if '+cpanm' in spec: - with working_dir(join_path('cpanm', 'cpanm')): - perl = spec['perl'].command - perl('Makefile.PL') + if spec.satisfies("+cpanm"): + with working_dir(join_path("cpanm", "cpanm")): + perl = spec["perl"].command + perl("Makefile.PL") make() - make('install') + make("install") This extra step automatically installs ``cpanm`` in addition to the base Perl installation. @@ -174,10 +174,10 @@ In the ``perl`` package, we can see: .. code-block:: python - @run_after('build') + @run_after("build") @on_package_attributes(run_tests=True) def test(self): - make('test') + make("test") As you can guess, this runs ``make test`` *after* building the package, if and only if testing is requested. Again, this is not specific to @@ -189,7 +189,7 @@ custom build systems, it can be added to existing build systems as well. .. code-block:: python - @run_after('install') + @run_after("install") @on_package_attributes(run_tests=True) works as expected. However, if you reverse the ordering: @@ -197,7 +197,7 @@ custom build systems, it can be added to existing build systems as well. .. code-block:: python @on_package_attributes(run_tests=True) - @run_after('install') + @run_after("install") the tests will always be run regardless of whether or not ``--test=root`` is requested. See https://github.com/spack/spack/issues/3833 diff --git a/lib/spack/docs/build_systems/inteloneapipackage.rst b/lib/spack/docs/build_systems/inteloneapipackage.rst index 102e26f4fbc91e..e9fd26690f9ddb 100644 --- a/lib/spack/docs/build_systems/inteloneapipackage.rst +++ b/lib/spack/docs/build_systems/inteloneapipackage.rst @@ -25,8 +25,8 @@ use Spack to build packages with the tools. The Spack Python class ``IntelOneapiPackage`` is a base class that is used by ``IntelOneapiCompilers``, ``IntelOneapiMkl``, ``IntelOneapiTbb`` and other classes to implement the oneAPI -packages. See the :ref:`package-list` for the full list of available -oneAPI packages or use:: +packages. Search for ``oneAPI`` at ``_ for the full +list of available oneAPI packages, or use:: spack list -d oneAPI @@ -104,11 +104,13 @@ Clone `spack-configs `_ repo and activat `Intel oneAPI CPU environment `_ contains applications tested and validated by Intel, this list is constantly extended. And currently it supports: +- `Devito `_ - `GROMACS `_ - `HPCG `_ - `HPL `_ - `LAMMPS `_ - `OpenFOAM `_ +- `Quantum Espresso `_ - `STREAM `_ - `WRF `_ diff --git a/lib/spack/docs/build_systems/makefilepackage.rst b/lib/spack/docs/build_systems/makefilepackage.rst index 66f54a1c4bb97a..af027aab1c69c7 100644 --- a/lib/spack/docs/build_systems/makefilepackage.rst +++ b/lib/spack/docs/build_systems/makefilepackage.rst @@ -59,7 +59,7 @@ using GNU Make, you should add a dependency on ``gmake``: .. code-block:: python - depends_on('gmake', type='build') + depends_on("gmake", type="build") ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -93,8 +93,8 @@ there are any other variables you need to set, you can do this in the .. code-block:: python def edit(self, spec, prefix): - env['PREFIX'] = prefix - env['BLASLIB'] = spec['blas'].libs.ld_flags + env["PREFIX"] = prefix + env["BLASLIB"] = spec["blas"].libs.ld_flags `cbench `_ @@ -113,7 +113,7 @@ you can do this like so: .. code-block:: python - build_targets = ['CC=cc'] + build_targets = ["CC=cc"] If you do need access to the spec, you can create a property like so: @@ -125,8 +125,8 @@ If you do need access to the spec, you can create a property like so: spec = self.spec return [ - 'CC=cc', - 'BLASLIB={0}'.format(spec['blas'].libs.ld_flags), + "CC=cc", + f"BLASLIB={spec['blas'].libs.ld_flags}", ] @@ -145,12 +145,12 @@ and a ``filter_file`` method to help with this. For example: .. code-block:: python def edit(self, spec, prefix): - makefile = FileFilter('Makefile') + makefile = FileFilter("Makefile") - makefile.filter(r'^\s*CC\s*=.*', 'CC = ' + spack_cc) - makefile.filter(r'^\s*CXX\s*=.*', 'CXX = ' + spack_cxx) - makefile.filter(r'^\s*F77\s*=.*', 'F77 = ' + spack_f77) - makefile.filter(r'^\s*FC\s*=.*', 'FC = ' + spack_fc) + makefile.filter(r"^\s*CC\s*=.*", f"CC = {spack_cc}") + makefile.filter(r"^\s*CXX\s*=.*", f"CXX = {spack_cxx}") + makefile.filter(r"^\s*F77\s*=.*", f"F77 = {spack_f77}") + makefile.filter(r"^\s*FC\s*=.*", f"FC = {spack_fc}") `stream `_ @@ -181,16 +181,16 @@ well for storing variables: def edit(self, spec, prefix): config = { - 'CC': 'cc', - 'MAKE': 'make', + "CC": "cc", + "MAKE": "make", } - if '+blas' in spec: - config['BLAS_LIBS'] = spec['blas'].libs.joined() + if spec.satisfies("+blas"): + config["BLAS_LIBS"] = spec["blas"].libs.joined() - with open('make.inc', 'w') as inc: + with open("make.inc", "w") as inc: for key in config: - inc.write('{0} = {1}\n'.format(key, config[key])) + inc.write(f"{key} = {config[key]}\n") `elk `_ @@ -204,14 +204,14 @@ them in a list: def edit(self, spec, prefix): config = [ - 'INSTALL_DIR = {0}'.format(prefix), - 'INCLUDE_DIR = $(INSTALL_DIR)/include', - 'LIBRARY_DIR = $(INSTALL_DIR)/lib', + f"INSTALL_DIR = {prefix}", + "INCLUDE_DIR = $(INSTALL_DIR)/include", + "LIBRARY_DIR = $(INSTALL_DIR)/lib", ] - with open('make.inc', 'w') as inc: + with open("make.inc", "w") as inc: for var in config: - inc.write('{0}\n'.format(var)) + inc.write(f"{var}\n") `hpl `_ @@ -284,7 +284,7 @@ can tell Spack where to locate it like so: .. code-block:: python - build_directory = 'src' + build_directory = "src" ^^^^^^^^^^^^^^^^^^^ @@ -299,8 +299,8 @@ install the package: def install(self, spec, prefix): mkdir(prefix.bin) - install('foo', prefix.bin) - install_tree('lib', prefix.lib) + install("foo", prefix.bin) + install_tree("lib", prefix.lib) ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/build_systems/pythonpackage.rst b/lib/spack/docs/build_systems/pythonpackage.rst index 17295a457fe139..168ff5dc88223c 100644 --- a/lib/spack/docs/build_systems/pythonpackage.rst +++ b/lib/spack/docs/build_systems/pythonpackage.rst @@ -152,16 +152,16 @@ set. Once set, ``pypi`` will be used to define the ``homepage``, .. code-block:: python - homepage = 'https://pypi.org/project/setuptools/' - url = 'https://pypi.org/packages/source/s/setuptools/setuptools-49.2.0.zip' - list_url = 'https://pypi.org/simple/setuptools/' + homepage = "https://pypi.org/project/setuptools/" + url = "https://pypi.org/packages/source/s/setuptools/setuptools-49.2.0.zip" + list_url = "https://pypi.org/simple/setuptools/" is equivalent to: .. code-block:: python - pypi = 'setuptools/setuptools-49.2.0.zip' + pypi = "setuptools/setuptools-49.2.0.zip" If a package has a different homepage listed on PyPI, you can @@ -208,7 +208,7 @@ dependencies to your package: .. code-block:: python - depends_on('py-setuptools@42:', type='build') + depends_on("py-setuptools@42:", type="build") Note that ``py-wheel`` is already listed as a build dependency in the @@ -232,7 +232,7 @@ Look for dependencies under the following keys: * ``dependencies`` under ``[project]`` These packages are required for building and installation. You can - add them with ``type=('build', 'run')``. + add them with ``type=("build", "run")``. * ``[project.optional-dependencies]`` @@ -279,12 +279,12 @@ distutils library, and has almost the exact same API. In addition to * ``setup_requires`` These packages are usually only needed at build-time, so you can - add them with ``type='build'``. + add them with ``type="build"``. * ``install_requires`` These packages are required for building and installation. You can - add them with ``type=('build', 'run')``. + add them with ``type=("build", "run")``. * ``extras_require`` @@ -296,7 +296,7 @@ distutils library, and has almost the exact same API. In addition to These are packages that are required to run the unit tests for the package. These dependencies can be specified using the - ``type='test'`` dependency type. However, the PyPI tarballs rarely + ``type="test"`` dependency type. However, the PyPI tarballs rarely contain unit tests, so there is usually no reason to add these. See https://setuptools.pypa.io/en/latest/userguide/dependency_management.html @@ -321,7 +321,7 @@ older versions of flit may use the following keys: * ``requires`` under ``[tool.flit.metadata]`` These packages are required for building and installation. You can - add them with ``type=('build', 'run')``. + add them with ``type=("build", "run")``. * ``[tool.flit.metadata.requires-extra]`` @@ -434,12 +434,12 @@ the BLAS/LAPACK library you want pkg-config to search for: .. code-block:: python - depends_on('py-pip@22.1:', type='build') + depends_on("py-pip@22.1:", type="build") def config_settings(self, spec, prefix): return { - 'blas': spec['blas'].libs.names[0], - 'lapack': spec['lapack'].libs.names[0], + "blas": spec["blas"].libs.names[0], + "lapack": spec["lapack"].libs.names[0], } @@ -463,10 +463,10 @@ has an optional dependency on ``libyaml`` that can be enabled like so: def global_options(self, spec, prefix): options = [] - if '+libyaml' in spec: - options.append('--with-libyaml') + if spec.satisfies("+libyaml"): + options.append("--with-libyaml") else: - options.append('--without-libyaml') + options.append("--without-libyaml") return options @@ -492,10 +492,10 @@ allows you to specify the directories to search for ``libyaml``: def install_options(self, spec, prefix): options = [] - if '+libyaml' in spec: + if spec.satisfies("+libyaml"): options.extend([ - spec['libyaml'].libs.search_flags, - spec['libyaml'].headers.include_flags, + spec["libyaml"].libs.search_flags, + spec["libyaml"].headers.include_flags, ]) return options @@ -556,7 +556,7 @@ detected are wrong, you can provide the names yourself by overriding .. code-block:: python - import_modules = ['six'] + import_modules = ["six"] Sometimes the list of module names to import depends on how the @@ -571,9 +571,9 @@ This can be expressed like so: @property def import_modules(self): - modules = ['yaml'] - if '+libyaml' in self.spec: - modules.append('yaml.cyaml') + modules = ["yaml"] + if self.spec.satisfies("+libyaml"): + modules.append("yaml.cyaml") return modules @@ -586,14 +586,14 @@ Instead of defining the ``import_modules`` explicitly, only the subset of module names to be skipped can be defined by using ``skip_modules``. If a defined module has submodules, they are skipped as well, e.g., in case the ``plotting`` modules should be excluded from the -automatically detected ``import_modules`` ``['nilearn', 'nilearn.surface', -'nilearn.plotting', 'nilearn.plotting.data']`` set: +automatically detected ``import_modules`` ``["nilearn", "nilearn.surface", +"nilearn.plotting", "nilearn.plotting.data"]`` set: .. code-block:: python - skip_modules = ['nilearn.plotting'] + skip_modules = ["nilearn.plotting"] -This will set ``import_modules`` to ``['nilearn', 'nilearn.surface']`` +This will set ``import_modules`` to ``["nilearn", "nilearn.surface"]`` Import tests can be run during the installation using ``spack install --test=root`` or at any time after the installation using @@ -612,11 +612,11 @@ after the ``install`` phase: .. code-block:: python - @run_after('install') + @run_after("install") @on_package_attributes(run_tests=True) def install_test(self): - with working_dir('spack-test', create=True): - python('-c', 'import numpy; numpy.test("full", verbose=2)') + with working_dir("spack-test", create=True): + python("-c", "import numpy; numpy.test('full', verbose=2)") when testing is enabled during the installation (i.e., ``spack install @@ -638,7 +638,7 @@ provides Python bindings in a ``python`` directory, you can use: .. code-block:: python - build_directory = 'python' + build_directory = "python" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/build_systems/rocmpackage.rst b/lib/spack/docs/build_systems/rocmpackage.rst index 636e5b812623f2..8f90794dfb7df5 100644 --- a/lib/spack/docs/build_systems/rocmpackage.rst +++ b/lib/spack/docs/build_systems/rocmpackage.rst @@ -81,28 +81,27 @@ class of your package. For example, you can add it to your class MyRocmPackage(CMakePackage, ROCmPackage): ... # Ensure +rocm and amdgpu_targets are passed to dependencies - depends_on('mydeppackage', when='+rocm') + depends_on("mydeppackage", when="+rocm") for val in ROCmPackage.amdgpu_targets: - depends_on('mydeppackage amdgpu_target={0}'.format(val), - when='amdgpu_target={0}'.format(val)) + depends_on(f"mydeppackage amdgpu_target={val}", + when=f"amdgpu_target={val}") ... def cmake_args(self): spec = self.spec args = [] ... - if '+rocm' in spec: + if spec.satisfies("+rocm"): # Set up the hip macros needed by the build args.extend([ - '-DENABLE_HIP=ON', - '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix)]) - rocm_archs = spec.variants['amdgpu_target'].value - if 'none' not in rocm_archs: - args.append('-DHIP_HIPCC_FLAGS=--amdgpu-target={0}' - .format(",".join(rocm_archs))) + "-DENABLE_HIP=ON", + f"-DHIP_ROOT_DIR={spec['hip'].prefix}"]) + rocm_archs = spec.variants["amdgpu_target"].value + if "none" not in rocm_archs: + args.append(f"-DHIP_HIPCC_FLAGS=--amdgpu-target={','.join(rocm_archs}") else: # Ensure build with hip is disabled - args.append('-DENABLE_HIP=OFF') + args.append("-DENABLE_HIP=OFF") ... return args ... @@ -114,7 +113,7 @@ build. This example also illustrates how to check for the ``rocm`` variant using ``self.spec`` and how to retrieve the ``amdgpu_target`` variant's value -using ``self.spec.variants['amdgpu_target'].value``. +using ``self.spec.variants["amdgpu_target"].value``. All five packages using ``ROCmPackage`` as of January 2021 also use the :ref:`CudaPackage `. So it is worth looking at those packages diff --git a/lib/spack/docs/build_systems/sconspackage.rst b/lib/spack/docs/build_systems/sconspackage.rst index 18002586a06c75..a17e1271b86d3b 100644 --- a/lib/spack/docs/build_systems/sconspackage.rst +++ b/lib/spack/docs/build_systems/sconspackage.rst @@ -57,7 +57,7 @@ overridden like so: .. code-block:: python def test(self): - scons('check') + scons("check") ^^^^^^^^^^^^^^^ @@ -88,7 +88,7 @@ base class already contains: .. code-block:: python - depends_on('scons', type='build') + depends_on("scons", type="build") If you want to specify a particular version requirement, you can override @@ -96,7 +96,7 @@ this in your package: .. code-block:: python - depends_on('scons@2.3.0:', type='build') + depends_on("scons@2.3.0:", type="build") ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -238,14 +238,14 @@ the package build phase. This is done by overriding ``build_args`` like so: def build_args(self, spec, prefix): args = [ - 'PREFIX={0}'.format(prefix), - 'ZLIB={0}'.format(spec['zlib'].prefix), + f"PREFIX={prefix}", + f"ZLIB={spec['zlib'].prefix}", ] - if '+debug' in spec: - args.append('DEBUG=yes') + if spec.satisfies("+debug"): + args.append("DEBUG=yes") else: - args.append('DEBUG=no') + args.append("DEBUG=no") return args @@ -275,8 +275,8 @@ environment variables. For example, cantera has the following option: * env_vars: [ string ] Environment variables to propagate through to SCons. Either the string "all" or a comma separated list of variable names, e.g. - 'LD_LIBRARY_PATH,HOME'. - - default: 'LD_LIBRARY_PATH,PYTHONPATH' + "LD_LIBRARY_PATH,HOME". + - default: "LD_LIBRARY_PATH,PYTHONPATH" In the case of cantera, using ``env_vars=all`` allows us to use diff --git a/lib/spack/docs/build_systems/sippackage.rst b/lib/spack/docs/build_systems/sippackage.rst index 710334f283b23b..a4f52bf18648c4 100644 --- a/lib/spack/docs/build_systems/sippackage.rst +++ b/lib/spack/docs/build_systems/sippackage.rst @@ -32,7 +32,7 @@ By default, these phases run: .. code-block:: console - $ python configure.py --bindir ... --destdir ... + $ sip-build --verbose --target-dir ... $ make $ make install @@ -41,30 +41,30 @@ By default, these phases run: Important files ^^^^^^^^^^^^^^^ -Each SIP package comes with a custom ``configure.py`` build script, -written in Python. This script contains instructions to build the project. +Each SIP package comes with a custom configuration file written in Python. +For newer packages, this is called ``project.py``, while in older packages, +it may be called ``configure.py``. This script contains instructions to build +the project. ^^^^^^^^^^^^^^^^^^^^^^^^^ Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ -``SIPPackage`` requires several dependencies. Python is needed to run -the ``configure.py`` build script, and to run the resulting Python -libraries. Qt is needed to provide the ``qmake`` command. SIP is also -needed to build the package. All of these dependencies are automatically -added via the base class +``SIPPackage`` requires several dependencies. Python and SIP are needed at build-time +to run the aforementioned configure script. Python is also needed at run-time to +actually use the installed Python library. And as we are building Python bindings +for C/C++ libraries, Python is also needed as a link dependency. All of these +dependencies are automatically added via the base class. .. code-block:: python - extends('python') + extends("python", type=("build", "link", "run")) + depends_on("py-sip", type="build") - depends_on('qt', type='build') - depends_on('py-sip', type='build') - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Passing arguments to ``configure.py`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Passing arguments to ``sip-build`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Each phase comes with a ```` function that can be used to pass arguments to that particular phase. For example, if you need to pass @@ -73,10 +73,10 @@ arguments to the configure phase, you can use: .. code-block:: python def configure_args(self): - return ['--no-python-dbus'] + return ["--no-python-dbus"] -A list of valid options can be found by running ``python configure.py --help``. +A list of valid options can be found by running ``sip-build --help``. ^^^^^^^ Testing diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index 34710ea73a7b6c..250a600e7f75ec 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -48,9 +48,6 @@ os.environ["COLIFY_SIZE"] = "25x120" os.environ["COLUMNS"] = "120" -# Generate full package list if needed -subprocess.call(["spack", "list", "--format=html", "--update=package_list.html"]) - # Generate a command index if an update is needed subprocess.call( [ @@ -207,6 +204,7 @@ def setup(sphinx): ("py:class", "clingo.Control"), ("py:class", "six.moves.urllib.parse.ParseResult"), ("py:class", "TextIO"), + ("py:class", "hashlib._Hash"), # Spack classes that are private and we don't want to expose ("py:class", "spack.provider_index._IndexBase"), ("py:class", "spack.repo._PrependFileLoader"), @@ -214,6 +212,8 @@ def setup(sphinx): # Spack classes that intersphinx is unable to resolve ("py:class", "spack.version.StandardVersion"), ("py:class", "spack.spec.DependencySpec"), + ("py:class", "spack.spec.InstallStatus"), + ("py:class", "spack.spec.SpecfileReaderBase"), ("py:class", "spack.install_test.Pb"), ] diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index b1e7a1d249bcbb..d54977bebab21c 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -292,14 +292,29 @@ It is also worth noting that: non_bindable_shared_objects = ["libinterface.so"] ---------------------- -``terminal_title`` +``install_status`` ---------------------- -By setting this option to ``true``, Spack will update the terminal's title to -provide information about its current progress as well as the current and -total package numbers. +When set to ``true``, Spack will show information about its current progress +as well as the current and total package numbers. Progress is shown both +in the terminal title and inline. Setting it to ``false`` will not show any +progress information. To work properly, this requires your terminal to reset its title after Spack has finished its work, otherwise Spack's status information will remain in the terminal's title indefinitely. Most terminals should already be set up this way and clear Spack's status information. + +----------- +``aliases`` +----------- + +Aliases can be used to define new Spack commands. They can be either shortcuts +for longer commands or include specific arguments for convenience. For instance, +if users want to use ``spack install``'s ``-v`` argument all the time, they can +create a new alias called ``inst`` that will always call ``install -v``: + +.. code-block:: yaml + + aliases: + inst: install -v diff --git a/lib/spack/docs/containers.rst b/lib/spack/docs/containers.rst index ec9c02635ceb37..64ca1df926bbec 100644 --- a/lib/spack/docs/containers.rst +++ b/lib/spack/docs/containers.rst @@ -212,18 +212,12 @@ under the ``container`` attribute of environments: final: - libgomp - # Extra instructions - extra_instructions: - final: | - RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ "' >> ~/.bashrc - # Labels for the image labels: app: "gromacs" mpi: "mpich" -A detailed description of the options available can be found in the -:ref:`container_config_options` section. +A detailed description of the options available can be found in the :ref:`container_config_options` section. ------------------- Setting Base Images @@ -525,6 +519,13 @@ the example below: COPY data /share/myapp/data {% endblock %} +The Dockerfile is generated by running: + +.. code-block:: console + + $ spack -e /opt/environment containerize + +Note that the environment must be active for spack to read the template. The recipe that gets generated contains the two extra instruction that we added in our template extension: .. code-block:: Dockerfile diff --git a/lib/spack/docs/contribution_guide.rst b/lib/spack/docs/contribution_guide.rst index 8933f590d7ca06..ec9234aa461ab5 100644 --- a/lib/spack/docs/contribution_guide.rst +++ b/lib/spack/docs/contribution_guide.rst @@ -310,53 +310,11 @@ Once all of the dependencies are installed, you can try building the documentati $ make clean $ make -If you see any warning or error messages, you will have to correct those before -your PR is accepted. - -If you are editing the documentation, you should obviously be running the -documentation tests. But even if you are simply adding a new package, your -changes could cause the documentation tests to fail: - -.. code-block:: console - - package_list.rst:8745: WARNING: Block quote ends without a blank line; unexpected unindent. - -At first, this error message will mean nothing to you, since you didn't edit -that file. Until you look at line 8745 of the file in question: - -.. code-block:: rst - - Description: - NetCDF is a set of software libraries and self-describing, machine- - independent data formats that support the creation, access, and sharing - of array-oriented scientific data. - -Our documentation includes :ref:`a list of all Spack packages `. -If you add a new package, its docstring is added to this page. The problem in -this case was that the docstring looked like: - -.. code-block:: python - - class Netcdf(Package): - """ - NetCDF is a set of software libraries and self-describing, - machine-independent data formats that support the creation, - access, and sharing of array-oriented scientific data. - """ - -Docstrings cannot start with a newline character, or else Sphinx will complain. -Instead, they should look like: - -.. code-block:: python - - class Netcdf(Package): - """NetCDF is a set of software libraries and self-describing, - machine-independent data formats that support the creation, - access, and sharing of array-oriented scientific data.""" - -Documentation changes can result in much more obfuscated warning messages. -If you don't understand what they mean, feel free to ask when you submit -your PR. +If you see any warning or error messages, you will have to correct those before your PR +is accepted. If you are editing the documentation, you should be running the +documentation tests to make sure there are no errors. Documentation changes can result +in some obfuscated warning messages. If you don't understand what they mean, feel free +to ask when you submit your PR. -------- Coverage diff --git a/lib/spack/docs/gpu_configuration.rst b/lib/spack/docs/gpu_configuration.rst new file mode 100644 index 00000000000000..1f807bb3b8a82e --- /dev/null +++ b/lib/spack/docs/gpu_configuration.rst @@ -0,0 +1,113 @@ +.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +========================== +Using External GPU Support +========================== + +Many packages come with a ``+cuda`` or ``+rocm`` variant. With no added +configuration Spack will download and install the needed components. +It may be preferable to use existing system support: the following sections +help with using a system installation of GPU libraries. + +----------------------------------- +Using an External ROCm Installation +----------------------------------- + +Spack breaks down ROCm into many separate component packages. The following +is an example ``packages.yaml`` that organizes a consistent set of ROCm +components for use by dependent packages: + +.. code-block:: yaml + + packages: + all: + compiler: [rocmcc@=5.3.0] + variants: amdgpu_target=gfx90a + hip: + buildable: false + externals: + - spec: hip@5.3.0 + prefix: /opt/rocm-5.3.0/hip + hsa-rocr-dev: + buildable: false + externals: + - spec: hsa-rocr-dev@5.3.0 + prefix: /opt/rocm-5.3.0/ + llvm-amdgpu: + buildable: false + externals: + - spec: llvm-amdgpu@5.3.0 + prefix: /opt/rocm-5.3.0/llvm/ + comgr: + buildable: false + externals: + - spec: comgr@5.3.0 + prefix: /opt/rocm-5.3.0/ + hipsparse: + buildable: false + externals: + - spec: hipsparse@5.3.0 + prefix: /opt/rocm-5.3.0/ + hipblas: + buildable: false + externals: + - spec: hipblas@5.3.0 + prefix: /opt/rocm-5.3.0/ + rocblas: + buildable: false + externals: + - spec: rocblas@5.3.0 + prefix: /opt/rocm-5.3.0/ + rocprim: + buildable: false + externals: + - spec: rocprim@5.3.0 + prefix: /opt/rocm-5.3.0/rocprim/ + +This is in combination with the following compiler definition: + +.. code-block:: yaml + + compilers: + - compiler: + spec: rocmcc@=5.3.0 + paths: + cc: /opt/rocm-5.3.0/bin/amdclang + cxx: /opt/rocm-5.3.0/bin/amdclang++ + f77: null + fc: /opt/rocm-5.3.0/bin/amdflang + operating_system: rhel8 + target: x86_64 + +This includes the following considerations: + +- Each of the listed externals specifies ``buildable: false`` to force Spack + to use only the externals we defined. +- ``spack external find`` can automatically locate some of the ``hip``/``rocm`` + packages, but not all of them, and furthermore not in a manner that + guarantees a complementary set if multiple ROCm installations are available. +- The ``prefix`` is the same for several components, but note that others + require listing one of the subdirectories as a prefix. + +----------------------------------- +Using an External CUDA Installation +----------------------------------- + +CUDA is split into fewer components and is simpler to specify: + +.. code-block:: yaml + + packages: + all: + variants: + - cuda_arch=70 + cuda: + buildable: false + externals: + - spec: cuda@11.0.2 + prefix: /opt/cuda/cuda-11.0.2/ + +where ``/opt/cuda/cuda-11.0.2/lib/`` contains ``libcudart.so``. diff --git a/lib/spack/docs/images/setup_env.png b/lib/spack/docs/images/setup_env.png new file mode 100644 index 00000000000000..4b16cac281f981 Binary files /dev/null and b/lib/spack/docs/images/setup_env.png differ diff --git a/lib/spack/docs/images/shapely_duplicates.svg b/lib/spack/docs/images/shapely_duplicates.svg new file mode 100644 index 00000000000000..912f03b2e526cd --- /dev/null +++ b/lib/spack/docs/images/shapely_duplicates.svg @@ -0,0 +1,2784 @@ + + + + + + + + + + image/svg+xml + + + + + + + + G + + + + bqm4trdmbbqhrthe6flwnxp57cfbbser + + nghttp2@1.52.0/bqm4trd + + + + hsp7usvecwby6o6kszujxywbux5f5qc4 + + pkgconf@1.9.5/hsp7usv + + + + bqm4trdmbbqhrthe6flwnxp57cfbbser->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + s3mykqnlex5ygursynhv4cfu4p4jcp5c + + diffutils@3.9/s3mykqn + + + + gpd7yevon44acblslmgorfsxufgk3nhz + + libiconv@1.17/gpd7yev + + + + s3mykqnlex5ygursynhv4cfu4p4jcp5c->gpd7yevon44acblslmgorfsxufgk3nhz + + + + + + + al63766ivhemwb3bxsklvqmhdptf34fn + + geos@3.12.0/al63766 + + + + ma6zn6mykr7xe226v2hvu4ye7jltnddb + + cmake@3.27.4/ma6zn6m + + + + al63766ivhemwb3bxsklvqmhdptf34fn->ma6zn6mykr7xe226v2hvu4ye7jltnddb + + + + + + revhbmcsddofjb7jt3fql7fawtxjihvc + + ninja@1.11.1/revhbmc + + + + al63766ivhemwb3bxsklvqmhdptf34fn->revhbmcsddofjb7jt3fql7fawtxjihvc + + + + + + 7vxac6cvfyqggxsvd7votisi72rdfvoh + + ca-certificates-mozilla@2023-05-30/7vxac6c + + + + wuse2zg2p4ujfbsks4znlwyqumsa476w + + py-cython@0.29.36/wuse2zg + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + python@3.11.4/7ftqkn3 + + + + wuse2zg2p4ujfbsks4znlwyqumsa476w->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + 33tg442mk3uy52ocdgd7uxbusdtkozlq + + py-pip@23.1.2/33tg442 + + + + wuse2zg2p4ujfbsks4znlwyqumsa476w->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + isprdjk4hdva3owdr6bgzavgaqzyjwyj + + py-wheel@0.37.1/isprdjk + + + + wuse2zg2p4ujfbsks4znlwyqumsa476w->isprdjk4hdva3owdr6bgzavgaqzyjwyj + + + + + + esl2253adih4qsbluhmzdtsxfrws4fnt + + py-setuptools@59.4.0/esl2253 + + + + wuse2zg2p4ujfbsks4znlwyqumsa476w->esl2253adih4qsbluhmzdtsxfrws4fnt + + + + + + + e3xjka5zk6vtoen2oexuzxyorp6um5rv + + openssl@3.1.3/e3xjka5 + + + + e3xjka5zk6vtoen2oexuzxyorp6um5rv->7vxac6cvfyqggxsvd7votisi72rdfvoh + + + + + + ez3cm4rogbx7at45wfi6gquti6fbo3zz + + zlib-ng@2.1.3/ez3cm4r + + + + e3xjka5zk6vtoen2oexuzxyorp6um5rv->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + 7bvgd7zcvk3hglqgbqczma5h4urvrdjb + + perl@5.38.0/7bvgd7z + + + + e3xjka5zk6vtoen2oexuzxyorp6um5rv->7bvgd7zcvk3hglqgbqczma5h4urvrdjb + + + + + + ys6bcgmvdayitnod74ppxvzbn75e7227 + + py-shapely@2.0.1/ys6bcgm + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->al63766ivhemwb3bxsklvqmhdptf34fn + + + + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->wuse2zg2p4ujfbsks4znlwyqumsa476w + + + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p + + py-numpy@1.25.2/ca3noh6 + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->ca3noh6upxuh3hdx2lnrsdvw7blgcj5p + + + + + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->isprdjk4hdva3owdr6bgzavgaqzyjwyj + + + + + + 2ok2ozl5i2qphhfsbxkdtq3iezemvpsv + + py-setuptools@68.0.0/2ok2ozl + + + + ys6bcgmvdayitnod74ppxvzbn75e7227->2ok2ozl5i2qphhfsbxkdtq3iezemvpsv + + + + + + aoucvoqqeft4hsw3poydbf4mvong4nry + + ncurses@6.4/aoucvoq + + + + aoucvoqqeft4hsw3poydbf4mvong4nry->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + 7aawlyt3hu24znvpgwedu2s3jmz46dkn + + xz@5.4.1/7aawlyt + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->e3xjka5zk6vtoen2oexuzxyorp6um5rv + + + + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->aoucvoqqeft4hsw3poydbf4mvong4nry + + + + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->7aawlyt3hu24znvpgwedu2s3jmz46dkn + + + + + + + ygkrrpeszr4j377qqtqqecmwt27pm2ho + + expat@2.5.0/ygkrrpe + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->ygkrrpeszr4j377qqtqqecmwt27pm2ho + + + + + + + qlqyzklm3yyv6tkqgnj4tzoy7g72ejyu + + sqlite@3.42.0/qlqyzkl + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->qlqyzklm3yyv6tkqgnj4tzoy7g72ejyu + + + + + + + ihtvssgtl7yz2wj7wdla4hsi7nqfny42 + + util-linux-uuid@2.38.1/ihtvssg + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->ihtvssgtl7yz2wj7wdla4hsi7nqfny42 + + + + + + + bvcsrijbs7lp5jvlyooahoxc3zfapwfp + + gdbm@1.23/bvcsrij + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->bvcsrijbs7lp5jvlyooahoxc3zfapwfp + + + + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv + + gettext@0.21.1/3o2rmrx + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->3o2rmrxpwkmmetxmzvba6sizei5womzv + + + + + + + bcjm3vxlgrjgewpdakhpfea3y2kzcspe + + bzip2@1.0.8/bcjm3vx + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->bcjm3vxlgrjgewpdakhpfea3y2kzcspe + + + + + + + 7pjirtey2xqww2bbkil3yj3mtmasruaw + + readline@8.2/7pjirte + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->7pjirtey2xqww2bbkil3yj3mtmasruaw + + + + + + + d24pqmu7ayswej2jfwwcgnw26t4gatgv + + libxcrypt@4.4.35/d24pqmu + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->d24pqmu7ayswej2jfwwcgnw26t4gatgv + + + + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + rei73bcylffduxjtuwt5sbibc2cbvuyt + + libffi@3.4.4/rei73bc + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->rei73bcylffduxjtuwt5sbibc2cbvuyt + + + + + + + 7ftqkn35sy5bmqv3wui3ap3gubqyu4f4->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + mof23fyk5qdmzll42yrtvvjeafzn45rl + + libbsd@0.11.7/mof23fy + + + + ygkrrpeszr4j377qqtqqecmwt27pm2ho->mof23fyk5qdmzll42yrtvvjeafzn45rl + + + + + + + pbpdelsw4pyldezsnide5zcc4ym5rrzg + + re2c@2.2/pbpdels + + + + qlqyzklm3yyv6tkqgnj4tzoy7g72ejyu->7pjirtey2xqww2bbkil3yj3mtmasruaw + + + + + + + qlqyzklm3yyv6tkqgnj4tzoy7g72ejyu->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + 33tg442mk3uy52ocdgd7uxbusdtkozlq->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + ihtvssgtl7yz2wj7wdla4hsi7nqfny42->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->wuse2zg2p4ujfbsks4znlwyqumsa476w + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->isprdjk4hdva3owdr6bgzavgaqzyjwyj + + + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->esl2253adih4qsbluhmzdtsxfrws4fnt + + + + + + + buscwcl7gy7xqmrsmtewcustpjoa3jy6 + + openblas@0.3.24/buscwcl + + + + ca3noh6upxuh3hdx2lnrsdvw7blgcj5p->buscwcl7gy7xqmrsmtewcustpjoa3jy6 + + + + + + + bvcsrijbs7lp5jvlyooahoxc3zfapwfp->7pjirtey2xqww2bbkil3yj3mtmasruaw + + + + + + + isprdjk4hdva3owdr6bgzavgaqzyjwyj->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + isprdjk4hdva3owdr6bgzavgaqzyjwyj->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->aoucvoqqeft4hsw3poydbf4mvong4nry + + + + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->7aawlyt3hu24znvpgwedu2s3jmz46dkn + + + + + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->gpd7yevon44acblslmgorfsxufgk3nhz + + + + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->bcjm3vxlgrjgewpdakhpfea3y2kzcspe + + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33 + + tar@1.34/jofugpd + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->jofugpdt2lki4tvw3xa56pxz4kzmjb33 + + + + + + + yry2pcjkl2hcfeexfi2yvnar2lyplbyg + + libxml2@2.10.3/yry2pcj + + + + 3o2rmrxpwkmmetxmzvba6sizei5womzv->yry2pcjkl2hcfeexfi2yvnar2lyplbyg + + + + + + + 2ok2ozl5i2qphhfsbxkdtq3iezemvpsv->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + 2ok2ozl5i2qphhfsbxkdtq3iezemvpsv->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + ma7u5unvr5auweq7clkgz75hca33j6eb + + pigz@2.7/ma7u5un + + + + ma7u5unvr5auweq7clkgz75hca33j6eb->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + ma6zn6mykr7xe226v2hvu4ye7jltnddb->aoucvoqqeft4hsw3poydbf4mvong4nry + + + + + + + omxtm4xh3xbta4le4ehihd26gi3qn2hc + + curl@8.1.2/omxtm4x + + + + ma6zn6mykr7xe226v2hvu4ye7jltnddb->omxtm4xh3xbta4le4ehihd26gi3qn2hc + + + + + + + ma6zn6mykr7xe226v2hvu4ye7jltnddb->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + bcjm3vxlgrjgewpdakhpfea3y2kzcspe->s3mykqnlex5ygursynhv4cfu4p4jcp5c + + + + + + 7pjirtey2xqww2bbkil3yj3mtmasruaw->aoucvoqqeft4hsw3poydbf4mvong4nry + + + + + + + esl2253adih4qsbluhmzdtsxfrws4fnt->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + + esl2253adih4qsbluhmzdtsxfrws4fnt->33tg442mk3uy52ocdgd7uxbusdtkozlq + + + + + + r3mipc2ezzxleb6g3yjy2rgio44tpsnr + + libmd@1.0.4/r3mipc2 + + + + tolbgopadusf5fpqzmhm7qfsnhpluyvv + + zstd@1.5.5/tolbgop + + + + omxtm4xh3xbta4le4ehihd26gi3qn2hc->bqm4trdmbbqhrthe6flwnxp57cfbbser + + + + + + + omxtm4xh3xbta4le4ehihd26gi3qn2hc->e3xjka5zk6vtoen2oexuzxyorp6um5rv + + + + + + + omxtm4xh3xbta4le4ehihd26gi3qn2hc->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + omxtm4xh3xbta4le4ehihd26gi3qn2hc->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + revhbmcsddofjb7jt3fql7fawtxjihvc->7ftqkn35sy5bmqv3wui3ap3gubqyu4f4 + + + + + + revhbmcsddofjb7jt3fql7fawtxjihvc->pbpdelsw4pyldezsnide5zcc4ym5rrzg + + + + + + d24pqmu7ayswej2jfwwcgnw26t4gatgv->7bvgd7zcvk3hglqgbqczma5h4urvrdjb + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33->7aawlyt3hu24znvpgwedu2s3jmz46dkn + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33->gpd7yevon44acblslmgorfsxufgk3nhz + + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33->ma7u5unvr5auweq7clkgz75hca33j6eb + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33->bcjm3vxlgrjgewpdakhpfea3y2kzcspe + + + + + + jofugpdt2lki4tvw3xa56pxz4kzmjb33->tolbgopadusf5fpqzmhm7qfsnhpluyvv + + + + + + buscwcl7gy7xqmrsmtewcustpjoa3jy6->7bvgd7zcvk3hglqgbqczma5h4urvrdjb + + + + + + yry2pcjkl2hcfeexfi2yvnar2lyplbyg->7aawlyt3hu24znvpgwedu2s3jmz46dkn + + + + + + + yry2pcjkl2hcfeexfi2yvnar2lyplbyg->gpd7yevon44acblslmgorfsxufgk3nhz + + + + + + + yry2pcjkl2hcfeexfi2yvnar2lyplbyg->hsp7usvecwby6o6kszujxywbux5f5qc4 + + + + + + yry2pcjkl2hcfeexfi2yvnar2lyplbyg->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + lcvuenzomq3fdqabnz22ih3kpt4g2nyd + + berkeley-db@18.1.40/lcvuenz + + + + 7bvgd7zcvk3hglqgbqczma5h4urvrdjb->bvcsrijbs7lp5jvlyooahoxc3zfapwfp + + + + + + + 7bvgd7zcvk3hglqgbqczma5h4urvrdjb->bcjm3vxlgrjgewpdakhpfea3y2kzcspe + + + + + + + 7bvgd7zcvk3hglqgbqczma5h4urvrdjb->ez3cm4rogbx7at45wfi6gquti6fbo3zz + + + + + + + 7bvgd7zcvk3hglqgbqczma5h4urvrdjb->lcvuenzomq3fdqabnz22ih3kpt4g2nyd + + + + + + + mof23fyk5qdmzll42yrtvvjeafzn45rl->r3mipc2ezzxleb6g3yjy2rgio44tpsnr + + + + + + diff --git a/lib/spack/docs/images/strumpack_virtuals.svg b/lib/spack/docs/images/strumpack_virtuals.svg new file mode 100644 index 00000000000000..eb580f0a5805c7 --- /dev/null +++ b/lib/spack/docs/images/strumpack_virtuals.svg @@ -0,0 +1,534 @@ + + +G + + + +hkcrbrtf2qex6rvzuok5tzdrbam55pdn + +netlib-scalapack@2.2.0%gcc@9.4.0/hkcrbrt + + + +o524gebsxavobkte3k5fglgwnedfkadf + +openblas@0.3.21%gcc@9.4.0/o524geb + + + +hkcrbrtf2qex6rvzuok5tzdrbam55pdn->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas,lapack + + + +2w3nq3n3hcj2tqlvcpewsryamltlu5tw + +intel-parallel-studio@cluster.2020.4%gcc@9.4.0/2w3nq3n + + + +hkcrbrtf2qex6rvzuok5tzdrbam55pdn->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +gguve5icmo5e4cw5o3hvvfsxremc46if + +cmake@3.25.1%gcc@9.4.0/gguve5i + + + +hkcrbrtf2qex6rvzuok5tzdrbam55pdn->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +i4avrindvhcamhurzbfdaggbj2zgsrrh + +pkgconf@1.8.0%gcc@9.4.0/i4avrin + + + +ywrpvv2hgooeepdke33exkqrtdpd5gkl + +perl@5.36.0%gcc@9.4.0/ywrpvv2 + + + +h3ujmb3ts4kxxxv77knh2knuystuerbx + +bzip2@1.0.8%gcc@9.4.0/h3ujmb3 + + + +ywrpvv2hgooeepdke33exkqrtdpd5gkl->h3ujmb3ts4kxxxv77knh2knuystuerbx + + + + + + +uabgssx6lsgrevwbttslldnr5nzguprj + +gdbm@1.23%gcc@9.4.0/uabgssx + + + +ywrpvv2hgooeepdke33exkqrtdpd5gkl->uabgssx6lsgrevwbttslldnr5nzguprj + + + + + + +gkw4dg2p7rdnhru3m6lcnsjbzyr7g3hb + +berkeley-db@18.1.40%gcc@9.4.0/gkw4dg2 + + + +ywrpvv2hgooeepdke33exkqrtdpd5gkl->gkw4dg2p7rdnhru3m6lcnsjbzyr7g3hb + + + + + + +nizxi5u5bbrzhzwfy2qb7hatlhuswlrz + +zlib@1.2.13%gcc@9.4.0/nizxi5u + + + +ywrpvv2hgooeepdke33exkqrtdpd5gkl->nizxi5u5bbrzhzwfy2qb7hatlhuswlrz + + + + + + +idvshq5nqmygzd4uo62mdispwgxsw7id + +strumpack@7.0.1%gcc@9.4.0/idvshq5 + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->hkcrbrtf2qex6rvzuok5tzdrbam55pdn + + + +virtuals=scalapack + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas,lapack + + + +imopnxjmv7cwzyiecdw2saq42qvpnauh + +parmetis@4.0.3%gcc@9.4.0/imopnxj + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->imopnxjmv7cwzyiecdw2saq42qvpnauh + + + + + + +ern66gyp6qmhmpod4jaynxx4weoberfm + +metis@5.1.0%gcc@9.4.0/ern66gy + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->ern66gyp6qmhmpod4jaynxx4weoberfm + + + + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf + +butterflypack@2.2.2%gcc@9.4.0/nqiyrxl + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->nqiyrxlid6tikfpvoqdpvsjt5drs2obf + + + + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu + +slate@2022.07.00%gcc@9.4.0/4bu62ky + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->4bu62kyfuh4ikdkuyxfxjxanf7e7qopu + + + + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +7rzbmgoxhmm2jhellkgcjmn62uklf22x + +zfp@0.5.5%gcc@9.4.0/7rzbmgo + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->7rzbmgoxhmm2jhellkgcjmn62uklf22x + + + + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +mujlx42xgttdc6u6rmiftsktpsrcmpbs + +blaspp@2022.07.00%gcc@9.4.0/mujlx42 + + + +mujlx42xgttdc6u6rmiftsktpsrcmpbs->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas + + + +mujlx42xgttdc6u6rmiftsktpsrcmpbs->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +htzjns66gmq6pjofohp26djmjnpbegho + +patchelf@0.16.1%gcc@9.4.0/htzjns6 + + + +xm3ldz3y3msfdc3hzshvxpbpg5hnt6o6 + +diffutils@3.8%gcc@9.4.0/xm3ldz3 + + + +h3ujmb3ts4kxxxv77knh2knuystuerbx->xm3ldz3y3msfdc3hzshvxpbpg5hnt6o6 + + + + + +o524gebsxavobkte3k5fglgwnedfkadf->ywrpvv2hgooeepdke33exkqrtdpd5gkl + + + + + +4vsmjofkhntilgzh4zebluqak5mdsu3x + +ca-certificates-mozilla@2023-01-10%gcc@9.4.0/4vsmjof + + + +xiro2z6na56qdd4czjhj54eag3ekbiow + +lapackpp@2022.07.00%gcc@9.4.0/xiro2z6 + + + +xiro2z6na56qdd4czjhj54eag3ekbiow->mujlx42xgttdc6u6rmiftsktpsrcmpbs + + + + + + +xiro2z6na56qdd4czjhj54eag3ekbiow->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas,lapack + + + +xiro2z6na56qdd4czjhj54eag3ekbiow->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +j5rupoqliu7kasm6xndl7ui32wgawkru + +ncurses@6.4%gcc@9.4.0/j5rupoq + + + +j5rupoqliu7kasm6xndl7ui32wgawkru->i4avrindvhcamhurzbfdaggbj2zgsrrh + + +virtuals=pkgconfig + + + +imopnxjmv7cwzyiecdw2saq42qvpnauh->ern66gyp6qmhmpod4jaynxx4weoberfm + + + + + + +imopnxjmv7cwzyiecdw2saq42qvpnauh->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +imopnxjmv7cwzyiecdw2saq42qvpnauh->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +ern66gyp6qmhmpod4jaynxx4weoberfm->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->hkcrbrtf2qex6rvzuok5tzdrbam55pdn + + + +virtuals=scalapack + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas,lapack + + + +lfh3aovn65e66cs24qiehq3nd2ddojef + +arpack-ng@3.8.0%gcc@9.4.0/lfh3aov + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->lfh3aovn65e66cs24qiehq3nd2ddojef + + + + + + +57joith2sqq6sehge54vlloyolm36mdu + +sed@4.8%gcc@9.4.0/57joith + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->57joith2sqq6sehge54vlloyolm36mdu + + + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +ogcucq2eod3xusvvied5ol2iobui4nsb + +libiconv@1.17%gcc@9.4.0/ogcucq2 + + + +xm3ldz3y3msfdc3hzshvxpbpg5hnt6o6->ogcucq2eod3xusvvied5ol2iobui4nsb + + + +virtuals=iconv + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu->mujlx42xgttdc6u6rmiftsktpsrcmpbs + + + + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu->xiro2z6na56qdd4czjhj54eag3ekbiow + + + + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + + +5xerf6imlgo4xlubacr4mljacc3edexo + +openssl@1.1.1s%gcc@9.4.0/5xerf6i + + + +5xerf6imlgo4xlubacr4mljacc3edexo->ywrpvv2hgooeepdke33exkqrtdpd5gkl + + + + + +5xerf6imlgo4xlubacr4mljacc3edexo->4vsmjofkhntilgzh4zebluqak5mdsu3x + + + + + +5xerf6imlgo4xlubacr4mljacc3edexo->nizxi5u5bbrzhzwfy2qb7hatlhuswlrz + + + + + + +v32wejd4d5lc6uka4qlrogwh5xae2h3r + +readline@8.2%gcc@9.4.0/v32wejd + + + +uabgssx6lsgrevwbttslldnr5nzguprj->v32wejd4d5lc6uka4qlrogwh5xae2h3r + + + + + + +lfh3aovn65e66cs24qiehq3nd2ddojef->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas,lapack + + + +lfh3aovn65e66cs24qiehq3nd2ddojef->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +lfh3aovn65e66cs24qiehq3nd2ddojef->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +2w3nq3n3hcj2tqlvcpewsryamltlu5tw->htzjns66gmq6pjofohp26djmjnpbegho + + + + + +7rzbmgoxhmm2jhellkgcjmn62uklf22x->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +v32wejd4d5lc6uka4qlrogwh5xae2h3r->j5rupoqliu7kasm6xndl7ui32wgawkru + + + + + + +gguve5icmo5e4cw5o3hvvfsxremc46if->j5rupoqliu7kasm6xndl7ui32wgawkru + + + + + + +gguve5icmo5e4cw5o3hvvfsxremc46if->5xerf6imlgo4xlubacr4mljacc3edexo + + + + + + \ No newline at end of file diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 6a80aa1a2465c2..0dd27a2444516a 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -54,9 +54,16 @@ or refer to the full manual below. features getting_started basic_usage - Tutorial: Spack 101 replace_conda_homebrew +.. toctree:: + :maxdepth: 2 + :caption: Links + + Tutorial (spack-tutorial.rtfd.io) + Packages (packages.spack.io) + Binaries (binaries.spack.io) + .. toctree:: :maxdepth: 2 :caption: Reference @@ -72,11 +79,11 @@ or refer to the full manual below. repositories binary_caches command_index - package_list chain extensions pipelines signing + gpu_configuration .. toctree:: :maxdepth: 2 diff --git a/lib/spack/docs/module_file_support.rst b/lib/spack/docs/module_file_support.rst index 3d4e53915c390f..f6b292e7553dab 100644 --- a/lib/spack/docs/module_file_support.rst +++ b/lib/spack/docs/module_file_support.rst @@ -275,10 +275,12 @@ of the installed software. For instance, in the snippet below: set: BAR: 'bar' # This anonymous spec selects any package that - # depends on openmpi. The double colon at the + # depends on mpi. The double colon at the # end clears the set of rules that matched so far. - ^openmpi:: + ^mpi:: environment: + prepend_path: + PATH: '{^mpi.prefix}/bin' set: BAR: 'baz' # Selects any zlib package @@ -293,7 +295,9 @@ of the installed software. For instance, in the snippet below: - FOOBAR you are instructing Spack to set the environment variable ``BAR=bar`` for every module, -unless the associated spec satisfies ``^openmpi`` in which case ``BAR=baz``. +unless the associated spec satisfies the abstract dependency ``^mpi`` in which case +``BAR=baz``, and the directory containing the respective MPI executables is prepended +to the ``PATH`` variable. In addition in any spec that satisfies ``zlib`` the value ``foo`` will be prepended to ``LD_LIBRARY_PATH`` and in any spec that satisfies ``zlib%gcc@4.8`` the variable ``FOOBAR`` will be unset. @@ -396,28 +400,30 @@ that are already in the Lmod hierarchy. .. note:: - Tcl modules - Tcl modules also allow for explicit conflicts between modulefiles. + Tcl and Lua modules also allow for explicit conflicts between modulefiles. - .. code-block:: yaml + .. code-block:: yaml + + modules: + default: + enable: + - tcl + tcl: + projections: + all: '{name}/{version}-{compiler.name}-{compiler.version}' + all: + conflict: + - '{name}' + - 'intel/14.0.1' + + will create module files that will conflict with ``intel/14.0.1`` and with the + base directory of the same module, effectively preventing the possibility to + load two or more versions of the same software at the same time. The tokens + that are available for use in this directive are the same understood by the + :meth:`~spack.spec.Spec.format` method. - modules: - default: - enable: - - tcl - tcl: - projections: - all: '{name}/{version}-{compiler.name}-{compiler.version}' - all: - conflict: - - '{name}' - - 'intel/14.0.1' - - will create module files that will conflict with ``intel/14.0.1`` and with the - base directory of the same module, effectively preventing the possibility to - load two or more versions of the same software at the same time. The tokens - that are available for use in this directive are the same understood by - the :meth:`~spack.spec.Spec.format` method. + For Lmod and Environment Modules versions prior 4.2, it is important to + express the conflict on both modulefiles conflicting with each other. .. note:: @@ -513,11 +519,11 @@ inspections and customize them per-module-set. modules: prefix_inspections: - bin: + ./bin: - PATH - man: + ./man: - MANPATH - '': + ./: - CMAKE_PREFIX_PATH Prefix inspections are only applied if the relative path inside the @@ -573,7 +579,7 @@ the view. view_relative_modules: use_view: my_view prefix_inspections: - bin: + ./bin: - PATH view: my_view: diff --git a/lib/spack/docs/package_list.rst b/lib/spack/docs/package_list.rst deleted file mode 100644 index dfff0704608fb2..00000000000000 --- a/lib/spack/docs/package_list.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other - Spack Project Developers. See the top-level COPYRIGHT file for details. - - SPDX-License-Identifier: (Apache-2.0 OR MIT) - -.. _package-list: - -============ -Package List -============ - -This is a list of things you can install using Spack. It is -automatically generated based on the packages in this Spack -version. - -.. raw:: html - :file: package_list.html diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index fe1935cfb5c704..3dd1c7952d12e7 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -363,6 +363,42 @@ one of these:: If Spack finds none of these variables set, it will look for ``vim``, ``vi``, ``emacs``, ``nano``, and ``notepad``, in that order. +^^^^^^^^^^^^^^^^^ +Bundling software +^^^^^^^^^^^^^^^^^ + +If you have a collection of software expected to work well together with +no source code of its own, you can create a :ref:`BundlePackage `. +Examples where bundle packages can be useful include defining suites of +applications (e.g, `EcpProxyApps +`_), commonly used libraries +(e.g., `AmdAocl `_), +and software development kits (e.g., `EcpDataVisSdk `_). + +These versioned packages primarily consist of dependencies on the associated +software packages. They can include :ref:`variants ` to ensure +common build options are consistently applied to dependencies. Known build +failures, such as not building on a platform or when certain compilers or +variants are used, can be flagged with :ref:`conflicts `. +Build requirements, such as only building with specific compilers, can similarly +be flagged with :ref:`requires `. + +The ``spack create --template bundle`` command will create a skeleton +``BundlePackage`` ``package.py`` for you: + +.. code-block:: console + + $ spack create --template bundle --name coolsdk + +Now you can fill in the basic package documentation, version(s), and software +package dependencies along with any other relevant customizations. + +.. note:: + + Remember that bundle packages have no software of their own so there + is nothing to download. + + ^^^^^^^^^^^^^^^^^^^^^^^^^ Non-downloadable software ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -610,7 +646,16 @@ add a line like this in the package class: version("8.2.0", md5="1c9f62f0778697a09d36121ead88e08e") version("8.1.2", md5="d47dd09ed7ae6e7fd6f9a816d7f5fdf6") -Versions should be listed in descending order, from newest to oldest. +.. note:: + + By convention, we list versions in descending order, from newest to oldest. + +.. note:: + + :ref:`Bundle packages ` do not have source code so + there is nothing to fetch. Consequently, their version directives + consist solely of the version name (e.g., ``version("202309")``). + ^^^^^^^^^^^^^ Date Versions @@ -1504,7 +1549,7 @@ its value: def configure_args(self): ... - if "+shared" in self.spec: + if self.spec.satisfies("+shared"): extra_args.append("--enable-shared") else: extra_args.append("--disable-shared") @@ -1591,7 +1636,7 @@ Within a package recipe a multi-valued variant is tested using a ``key=value`` s .. code-block:: python - if "languages=jit" in spec: + if spec.satisfies("languages=jit"): options.append("--enable-host-shared") """"""""""""""""""""""""""""""""""""""""""" @@ -2243,7 +2288,7 @@ looks like this: url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" version("1.0.1h", md5="8d6d684a9430d5cc98a62a5d8fbda8cf") - depends_on("zlib") + depends_on("zlib-api") parallel = False @@ -2307,7 +2352,7 @@ the following at the command line of a bash shell: .. code-block:: console - $ for i in {1..12}; do nohup spack install -j 4 mpich@3.3.2 >> mpich_install.txt 2>&1 &; done + $ for i in {1..12}; do nohup spack install -j 4 mpich@3.3.2 >> mpich_install.txt 2>&1 & done .. note:: @@ -2512,9 +2557,10 @@ Conditional dependencies ^^^^^^^^^^^^^^^^^^^^^^^^ You may have a package that only requires a dependency under certain -conditions. For example, you may have a package that has optional MPI support, -- MPI is only a dependency when you want to enable MPI support for the -package. In that case, you could say something like: +conditions. For example, you may have a package with optional MPI support. +You would then provide a variant to reflect that the feature is optional +and specify the MPI dependency only applies when MPI support is enabled. +In that case, you could say something like: .. code-block:: python @@ -2522,13 +2568,39 @@ package. In that case, you could say something like: depends_on("mpi", when="+mpi") -``when`` can include constraints on the variant, version, compiler, etc. and -the :mod:`syntax` is the same as for Specs written on the command -line. -If a dependency/feature of a package isn't typically used, you can save time -by making it conditional (since Spack will not build the dependency unless it -is required for the Spec). +Suppose the above package also has, since version 3, optional `Trilinos` +support and you want them both to build either with or without MPI. Further +suppose you require a version of `Trilinos` no older than 12.6. In that case, +the `trilinos` variant and dependency directives would be: + +.. code-block:: python + + variant("trilinos", default=False, description="Enable Trilinos support") + + depends_on("trilinos@12.6:", when="@3: +trilinos") + depends_on("trilinos@12.6: +mpi", when="@3: +trilinos +mpi") + + +Alternatively, you could use the `when` context manager to equivalently specify +the `trilinos` variant dependencies as follows: + +.. code-block:: python + + with when("@3: +trilinos"): + depends_on("trilinos@12.6:") + depends_on("trilinos +mpi", when="+mpi") + + +The argument to ``when`` in either case can include any Spec constraints that +are supported on the command line using the same :ref:`syntax `. + +.. note:: + + If a dependency isn't typically used, you can save time by making it + conditional since Spack will not build the dependency unless it is + required for the Spec. + .. _dependency_dependency_patching: @@ -2616,60 +2688,6 @@ appear in the package file (or in this case, in the list). right version. If two packages depend on ``binutils`` patched *the same* way, they can both use a single installation of ``binutils``. -.. _setup-dependent-environment: - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Influence how dependents are built or run -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Spack provides a mechanism for dependencies to influence the -environment of their dependents by overriding the -:meth:`setup_dependent_run_environment ` -or the -:meth:`setup_dependent_build_environment ` -methods. -The Qt package, for instance, uses this call: - -.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/qt/package.py - :pyobject: Qt.setup_dependent_build_environment - :linenos: - -to set the ``QTDIR`` environment variable so that packages -that depend on a particular Qt installation will find it. -Another good example of how a dependency can influence -the build environment of dependents is the Python package: - -.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/python/package.py - :pyobject: Python.setup_dependent_build_environment - :linenos: - -In the method above it is ensured that any package that depends on Python -will have the ``PYTHONPATH``, ``PYTHONHOME`` and ``PATH`` environment -variables set appropriately before starting the installation. To make things -even simpler the ``python setup.py`` command is also inserted into the module -scope of dependents by overriding a third method called -:meth:`setup_dependent_package ` -: - -.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/python/package.py - :pyobject: Python.setup_dependent_package - :linenos: - -This allows most python packages to have a very simple install procedure, -like the following: - -.. code-block:: python - - def install(self, spec, prefix): - setup_py("install", "--prefix={0}".format(prefix)) - -Finally the Python package takes also care of the modifications to ``PYTHONPATH`` -to allow dependencies to run correctly: - -.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/python/package.py - :pyobject: Python.setup_dependent_run_environment - :linenos: - .. _packaging_conflicts: @@ -2678,7 +2696,7 @@ Conflicts and requirements -------------------------- Sometimes packages have known bugs, or limitations, that would prevent them -to build e.g. against other dependencies or with certain compilers. Spack +from building e.g. against other dependencies or with certain compilers. Spack makes it possible to express such constraints with the ``conflicts`` directive. Adding the following to a package: @@ -2814,6 +2832,70 @@ variant(s) are selected. This may be accomplished with conditional extends("python", when="+python") ... +.. _setup-environment: + +-------------------------------------------- +Runtime and build time environment variables +-------------------------------------------- + +Spack provides a few methods to help package authors set up the required environment variables for +their package. Environment variables typically depend on how the package is used: variables that +make sense during the build phase may not be needed at runtime, and vice versa. Further, sometimes +it makes sense to let a dependency set the environment variables for its dependents. To allow all +this, Spack provides four different methods that can be overridden in a package: + +1. :meth:`setup_build_environment ` +2. :meth:`setup_run_environment ` +3. :meth:`setup_dependent_build_environment ` +4. :meth:`setup_dependent_run_environment ` + +The Qt package, for instance, uses this call: + +.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/qt/package.py + :pyobject: Qt.setup_dependent_build_environment + :linenos: + +to set the ``QTDIR`` environment variable so that packages that depend on a particular Qt +installation will find it. + +The following diagram will give you an idea when each of these methods is called in a build +context: + +.. image:: images/setup_env.png + :align: center + +Notice that ``setup_dependent_run_environment`` can be called multiple times, once for each +dependent package, whereas ``setup_run_environment`` is called only once for the package itself. +This means that the former should only be used if the environment variables depend on the dependent +package, whereas the latter should be used if the environment variables depend only on the package +itself. + +-------------------------------- +Setting package module variables +-------------------------------- + +Apart from modifying environment variables of the dependent package, you can also define Python +variables to be used by the dependent. This is done by implementing +:meth:`setup_dependent_package `. An +example of this can be found in the ``Python`` package: + +.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/python/package.py + :pyobject: Python.setup_dependent_package + :linenos: + +This allows Python packages to directly use these variables: + +.. code-block:: python + + def install(self, spec, prefix): + ... + install("script.py", python_platlib) + +.. note:: + + We recommend using ``setup_dependent_package`` sparingly, as it is not always clear where + global variables are coming from when editing a ``package.py`` file. + ----- Views ----- @@ -2892,6 +2974,33 @@ The ``provides("mpi")`` call tells Spack that the ``mpich`` package can be used to satisfy the dependency of any package that ``depends_on("mpi")``. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Providing multiple virtuals simultaneously +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Packages can provide more than one virtual dependency. Sometimes, due to implementation details, +there are subsets of those virtuals that need to be provided together by the same package. + +A well-known example is ``openblas``, which provides both the ``lapack`` and ``blas`` API in a single ``libopenblas`` +library. A package that needs ``lapack`` and ``blas`` must either use ``openblas`` to provide both, or not use +``openblas`` at all. It cannot pick one or the other. + +To express this constraint in a package, the two virtual dependencies must be listed in the same ``provides`` directive: + +.. code-block:: python + + provides('blas', 'lapack') + +This makes it impossible to select ``openblas`` as a provider for one of the two +virtual dependencies and not for the other. If you try to, Spack will report an error: + +.. code-block:: console + + $ spack spec netlib-scalapack ^[virtuals=lapack] openblas ^[virtuals=blas] atlas + ==> Error: concretization failed for the following reasons: + + 1. Package 'openblas' needs to provide both 'lapack' and 'blas' together, but provides only 'lapack' + ^^^^^^^^^^^^^^^^^^^^ Versioned Interfaces ^^^^^^^^^^^^^^^^^^^^ @@ -3394,6 +3503,56 @@ is equivalent to: Constraints from nested context managers are also combined together, but they are rarely needed or recommended. +.. _default_args: + +------------------------ +Common default arguments +------------------------ + +Similarly, if directives have a common set of default arguments, you can +group them together in a ``with default_args()`` block: + +.. code-block:: python + + class PyExample(PythonPackage): + + with default_args(type=("build", "run")): + depends_on("py-foo") + depends_on("py-foo@2:", when="@2:") + depends_on("py-bar") + depends_on("py-bz") + +The above is short for: + +.. code-block:: python + + class PyExample(PythonPackage): + + depends_on("py-foo", type=("build", "run")) + depends_on("py-foo@2:", when="@2:", type=("build", "run")) + depends_on("py-bar", type=("build", "run")) + depends_on("py-bz", type=("build", "run")) + +.. note:: + + The ``with when()`` context manager is composable, while ``with default_args()`` + merely overrides the default. For example: + + .. code-block:: python + + with default_args(when="+feature"): + depends_on("foo") + depends_on("bar") + depends_on("baz", when="+baz") + + is equivalent to: + + .. code-block:: python + + depends_on("foo", when="+feature") + depends_on("bar", when="+feature") + depends_on("baz", when="+baz") # Note: not when="+feature+baz" + .. _install-method: ------------------ @@ -3456,7 +3615,7 @@ need to override methods like ``configure_args``: def configure_args(self): args = ["--enable-cxx"] + self.enable_or_disable("libs") - if "libs=static" in self.spec: + if self.spec.satisfies("libs=static"): args.append("--with-pic") return args @@ -3590,7 +3749,8 @@ regardless of the build system. The arguments for the phase are: The arguments ``spec`` and ``prefix`` are passed only for convenience, as they always correspond to ``self.spec`` and ``self.spec.prefix`` respectively. -If the ``package.py`` encodes builders explicitly, the signature for a phase changes slightly: +If the ``package.py`` has build instructions in a separate +:ref:`builder class `, the signature for a phase changes slightly: .. code-block:: python @@ -3600,56 +3760,6 @@ If the ``package.py`` encodes builders explicitly, the signature for a phase cha In this case the package is passed as the second argument, and ``self`` is the builder instance. -.. _multiple_build_systems: - -^^^^^^^^^^^^^^^^^^^^^^ -Multiple build systems -^^^^^^^^^^^^^^^^^^^^^^ - -There are cases where a software actively supports two build systems, or changes build systems -as it evolves, or needs different build systems on different platforms. Spack allows dealing with -these cases natively, if a recipe is written using builders explicitly. - -For instance, software that supports two build systems unconditionally should derive from -both ``*Package`` base classes, and declare the possible use of multiple build systems using -a directive: - -.. code-block:: python - - class ArpackNg(CMakePackage, AutotoolsPackage): - - build_system("cmake", "autotools", default="cmake") - -In this case the software can be built with both ``autotools`` and ``cmake``. Since the package -supports multiple build systems, it is necessary to declare which one is the default. The ``package.py`` -will likely contain some overriding of default builder methods: - -.. code-block:: python - - class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder): - def cmake_args(self): - pass - - class AutotoolsBuilder(spack.build_systems.autotools.AutotoolsBuilder): - def configure_args(self): - pass - -In more complex cases it might happen that the build system changes according to certain conditions, -for instance across versions. That can be expressed with conditional variant values: - -.. code-block:: python - - class ArpackNg(CMakePackage, AutotoolsPackage): - - build_system( - conditional("cmake", when="@0.64:"), - conditional("autotools", when="@:0.63"), - default="cmake", - ) - -In the example the directive impose a change from ``Autotools`` to ``CMake`` going -from ``v0.63`` to ``v0.64``. - ^^^^^^^^^^^^^^^^^^ Mixin base classes ^^^^^^^^^^^^^^^^^^ @@ -3696,6 +3806,106 @@ for instance: In the example above ``Cp2k`` inherits all the conflicts and variants that ``CudaPackage`` defines. +.. _multiple_build_systems: + +---------------------- +Multiple build systems +---------------------- + +There are cases where a package actively supports two build systems, or changes build systems +as it evolves, or needs different build systems on different platforms. Spack allows dealing with +these cases by splitting the build instructions into separate builder classes. + +For instance, software that supports two build systems unconditionally should derive from +both ``*Package`` base classes, and declare the possible use of multiple build systems using +a directive: + +.. code-block:: python + + class Example(CMakePackage, AutotoolsPackage): + + variant("my_feature", default=True) + + build_system("cmake", "autotools", default="cmake") + +In this case the software can be built with both ``autotools`` and ``cmake``. Since the package +supports multiple build systems, it is necessary to declare which one is the default. + +Additional build instructions are split into separate builder classes: + +.. code-block:: python + + class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder): + def cmake_args(self): + return [ + self.define_from_variant("MY_FEATURE", "my_feature") + ] + + class AutotoolsBuilder(spack.build_systems.autotools.AutotoolsBuilder): + def configure_args(self): + return self.with_or_without("my-feature", variant="my_feature") + +In this example, ``spack install example +feature build_sytem=cmake`` will +pick the ``CMakeBuilder`` and invoke ``cmake -DMY_FEATURE:BOOL=ON``. + +Similarly, ``spack install example +feature build_system=autotools`` will pick +the ``AutotoolsBuilder`` and invoke ``./configure --with-my-feature``. + +Dependencies are always specified in the package class. When some dependencies +depend on the choice of the build system, it is possible to use when conditions as +usual: + +.. code-block:: python + + class Example(CMakePackage, AutotoolsPackage): + + build_system("cmake", "autotools", default="cmake") + + # Runtime dependencies + depends_on("ncurses") + depends_on("libxml2") + + # Lowerbounds for cmake only apply when using cmake as the build system + with when("build_system=cmake"): + depends_on("cmake@3.18:", when="@2.0:", type="build") + depends_on("cmake@3:", type="build") + + # Specify extra build dependencies used only in the configure script + with when("build_system=autotools"): + depends_on("perl", type="build") + depends_on("pkgconfig", type="build") + +Very often projects switch from one build system to another, or add support +for a new build system from a certain version, which means that the choice +of the build system typically depends on a version range. Those situations can +be handled by using conditional values in the ``build_system`` directive: + +.. code-block:: python + + class Example(CMakePackage, AutotoolsPackage): + + build_system( + conditional("cmake", when="@0.64:"), + conditional("autotools", when="@:0.63"), + default="cmake", + ) + +In the example the directive impose a change from ``Autotools`` to ``CMake`` going +from ``v0.63`` to ``v0.64``. + +The ``build_system`` can be used as an ordinary variant, which also means that it can +be used in ``depends_on`` statements. This can be useful when a package *requires* that +its dependency has a CMake config file, meaning that the dependent can only build when the +dependency is built with CMake, and not Autotools. In that case, you can force the choice +of the build system in the dependent: + +.. code-block:: python + + class Dependent(CMakePackage): + + depends_on("example build_system=cmake") + + .. _install-environment: ----------------------- @@ -4268,7 +4478,7 @@ for supported features, for instance: .. code-block:: python - if "avx512" in spec.target: + if spec.satisfies("target=avx512"): args.append("--with-avx512") The snippet above will append the ``--with-avx512`` item to a list of arguments only if the corresponding @@ -4773,17 +4983,17 @@ For example, running: results in spack checking that the installation created the following **file**: -* ``self.prefix/bin/reframe`` +* ``self.prefix.bin.reframe`` and the following **directories**: -* ``self.prefix/bin`` -* ``self.prefix/config`` -* ``self.prefix/docs`` -* ``self.prefix/reframe`` -* ``self.prefix/tutorials`` -* ``self.prefix/unittests`` -* ``self.prefix/cscs-checks`` +* ``self.prefix.bin`` +* ``self.prefix.config`` +* ``self.prefix.docs`` +* ``self.prefix.reframe`` +* ``self.prefix.tutorials`` +* ``self.prefix.unittests`` +* ``self.prefix.cscs-checks`` If **any** of these paths are missing, then Spack considers the installation to have failed. @@ -4927,7 +5137,7 @@ installed executable. The check is implemented as follows: @on_package_attributes(run_tests=True) def check_list(self): with working_dir(self.stage.source_path): - reframe = Executable(join_path(self.prefix, "bin", "reframe")) + reframe = Executable(self.prefix.bin.reframe) reframe("-l") .. warning:: @@ -5147,8 +5357,8 @@ embedded test parts. for example in ["ex1", "ex2"]: with test_part( self, - "test_example_{0}".format(example), - purpose="run installed {0}".format(example), + f"test_example_{example}", + purpose=f"run installed {example}", ): exe = which(join_path(self.prefix.bin, example)) exe() @@ -5226,11 +5436,10 @@ Below illustrates using this feature to compile an example. ... cxx = which(os.environ["CXX"]) cxx( - "-L{0}".format(self.prefix.lib), - "-I{0}".format(self.prefix.include), - "{0}.cpp".format(exe), - "-o", - exe + f"-L{self.prefix.lib}", + f"-I{self.prefix.include}", + f"{exe}.cpp", + "-o", exe ) cxx_example = which(exe) cxx_example() @@ -5247,14 +5456,14 @@ Saving build-time files We highly recommend re-using build-time test sources and pared down input files for testing installed software. These files are easier to keep synchronized with software capabilities since they reside - within the software's repository. - + within the software's repository. + If that is not possible, you can add test-related files to the package repository (see :ref:`adding custom files `). It will be important to maintain them so they work across listed or supported versions of the package. -You can use the ``cache_extra_test_sources`` method to copy directories +You can use the ``cache_extra_test_sources`` helper to copy directories and or files from the source build stage directory to the package's installation directory. @@ -5262,10 +5471,15 @@ The signature for ``cache_extra_test_sources`` is: .. code-block:: python - def cache_extra_test_sources(self, srcs): + def cache_extra_test_sources(pkg, srcs): + +where each argument has the following meaning: + +* ``pkg`` is an instance of the package for the spec under test. + +* ``srcs`` is a string *or* a list of strings corresponding to the + paths of subdirectories and or files needed for stand-alone testing. -where ``srcs`` is a string *or* a list of strings corresponding to the -paths of subdirectories and or files needed for stand-alone testing. The paths must be relative to the staged source directory. Contents of subdirectories and files are copied to a special test cache subdirectory of the installation prefix. They are automatically copied to the appropriate @@ -5286,21 +5500,18 @@ and using ``foo.c`` in a test method is illustrated below. srcs = ["tests", join_path("examples", "foo.c"), join_path("examples", "bar.c")] - self.cache_extra_test_sources(srcs) + cache_extra_test_sources(self, srcs) def test_foo(self): exe = "foo" - src_dir = join_path( - self.test_suite.current_test_cache_dir, "examples" - ) + src_dir = self.test_suite.current_test_cache_dir.examples with working_dir(src_dir): cc = which(os.environ["CC"]) cc( - "-L{0}".format(self.prefix.lib), - "-I{0}".format(self.prefix.include), - "{0}.c".format(exe), - "-o", - exe + f"-L{self.prefix.lib}", + f"-I{self.prefix.include}", + f"{exe}.c", + "-o", exe ) foo = which(exe) foo() @@ -5326,9 +5537,9 @@ the files using the ``self.test_suite.current_test_cache_dir`` property. In our example above, test methods can use the following paths to reference the copy of each entry listed in ``srcs``, respectively: -* ``join_path(self.test_suite.current_test_cache_dir, "tests")`` -* ``join_path(self.test_suite.current_test_cache_dir, "examples", "foo.c")`` -* ``join_path(self.test_suite.current_test_cache_dir, "examples", "bar.c")`` +* ``self.test_suite.current_test_cache_dir.tests`` +* ``join_path(self.test_suite.current_test_cache_dir.examples, "foo.c")`` +* ``join_path(self.test_suite.current_test_cache_dir.examples, "bar.c")`` .. admonition:: Library packages should build stand-alone tests @@ -5347,7 +5558,7 @@ the copy of each entry listed in ``srcs``, respectively: If one or more of the copied files needs to be modified to reference the installed software, it is recommended that those changes be made to the cached files **once** in the ``copy_test_sources`` method and - ***after** the call to ``self.cache_extra_test_sources()``. This will + ***after** the call to ``cache_extra_test_sources()``. This will reduce the amount of unnecessary work in the test method **and** avoid problems testing in shared instances and facility deployments. @@ -5394,7 +5605,7 @@ property as shown below. """build and run custom-example""" data_dir = self.test_suite.current_test_data_dir exe = "custom-example" - src = datadir.join("{0}.cpp".format(exe)) + src = datadir.join(f"{exe}.cpp") ... # TODO: Build custom-example using src and exe ... @@ -5410,7 +5621,7 @@ Reading expected output from a file The helper function ``get_escaped_text_output`` is available for packages to retrieve and properly format the text from a file that contains the -expected output from running an executable that may contain special +expected output from running an executable that may contain special characters. The signature for ``get_escaped_text_output`` is: @@ -5444,7 +5655,7 @@ added to the package's ``test`` subdirectory. db_filename, ".dump", output=str.split, error=str.split ) for exp in expected: - assert re.search(exp, out), "Expected '{0}' in output".format(exp) + assert re.search(exp, out), f"Expected '{exp}' in output" If the file was instead copied from the ``tests`` subdirectory of the staged source code, the path would be obtained as shown below. @@ -5457,7 +5668,7 @@ source code, the path would be obtained as shown below. db_filename = test_cache_dir.join("packages.db") Alternatively, if the file was copied to the ``share/tests`` subdirectory -as part of the installation process, the test could access the path as +as part of the installation process, the test could access the path as follows: .. code-block:: python @@ -5494,9 +5705,12 @@ Invoking the method is the equivalent of: .. code-block:: python + errors = [] for check in expected: if not re.search(check, actual): - raise RuntimeError("Expected '{0}' in output '{1}'".format(check, actual)) + errors.append(f"Expected '{check}' in output '{actual}'") + if errors: + raise RuntimeError("\n ".join(errors)) .. _accessing-files: @@ -5536,7 +5750,7 @@ repository, and installation. - ``self.test_suite.test_dir_for_spec(self.spec)`` * - Current Spec's Build-time Files - ``self.test_suite.current_test_cache_dir`` - - ``join_path(self.test_suite.current_test_cache_dir, "examples", "foo.c")`` + - ``join_path(self.test_suite.current_test_cache_dir.examples, "foo.c")`` * - Current Spec's Custom Test Files - ``self.test_suite.current_test_data_dir`` - ``join_path(self.test_suite.current_test_data_dir, "hello.f90")`` @@ -5551,7 +5765,7 @@ Inheriting stand-alone tests Stand-alone tests defined in parent (.e.g., :ref:`build-systems`) and virtual (e.g., :ref:`virtual-dependencies`) packages are executed by packages that inherit from or provide interface implementations for those -packages, respectively. +packages, respectively. The table below summarizes the stand-alone tests that will be executed along with those implemented in the package itself. @@ -5621,7 +5835,7 @@ for ``openmpi``: SKIPPED: test_version_oshcc: oshcc is not installed ... ==> [2023-03-10-16:04:02.215227] Completed testing - ==> [2023-03-10-16:04:02.215597] + ==> [2023-03-10-16:04:02.215597] ======================== SUMMARY: openmpi-4.1.4-ubmrigj ======================== Openmpi::test_bin_mpirun .. PASSED Openmpi::test_bin_ompi_info .. PASSED @@ -6071,7 +6285,7 @@ in the extra attributes can implement this method like this: @classmethod def validate_detected_spec(cls, spec, extra_attributes): """Check that "compilers" is in the extra attributes.""" - msg = ("the extra attribute "compilers" must be set for " + msg = ("the extra attribute 'compilers' must be set for " "the detected spec '{0}'".format(spec)) assert "compilers" in extra_attributes, msg @@ -6147,7 +6361,100 @@ follows: "foo-package@{0}".format(version_str) ) -.. _package-lifecycle: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Add detection tests to packages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To ensure that software is detected correctly for multiple configurations +and on different systems users can write a ``detection_test.yaml`` file and +put it in the package directory alongside the ``package.py`` file. +This YAML file contains enough information for Spack to mock an environment +and try to check if the detection logic yields the results that are expected. + +As a general rule, attributes at the top-level of ``detection_test.yaml`` +represent search mechanisms and they each map to a list of tests that should confirm +the validity of the package's detection logic. + +The detection tests can be run with the following command: + +.. code-block:: console + + $ spack audit externals + +Errors that have been detected are reported to screen. + +"""""""""""""""""""""""""" +Tests for PATH inspections +"""""""""""""""""""""""""" + +Detection tests insisting on ``PATH`` inspections are listed under +the ``paths`` attribute: + +.. code-block:: yaml + + paths: + - layout: + - executables: + - "bin/clang-3.9" + - "bin/clang++-3.9" + script: | + echo "clang version 3.9.1-19ubuntu1 (tags/RELEASE_391/rc2)" + echo "Target: x86_64-pc-linux-gnu" + echo "Thread model: posix" + echo "InstalledDir: /usr/bin" + results: + - spec: 'llvm@3.9.1 +clang~lld~lldb' + +Each test is performed by first creating a temporary directory structure as +specified in the corresponding ``layout`` and by then running +package detection and checking that the outcome matches the expected +``results``. The exact details on how to specify both the ``layout`` and the +``results`` are reported in the table below: + +.. list-table:: Test based on PATH inspections + :header-rows: 1 + + * - Option Name + - Description + - Allowed Values + - Required Field + * - ``layout`` + - Specifies the filesystem tree used for the test + - List of objects + - Yes + * - ``layout:[0]:executables`` + - Relative paths for the mock executables to be created + - List of strings + - Yes + * - ``layout:[0]:script`` + - Mock logic for the executable + - Any valid shell script + - Yes + * - ``results`` + - List of expected results + - List of objects (empty if no result is expected) + - Yes + * - ``results:[0]:spec`` + - A spec that is expected from detection + - Any valid spec + - Yes + +""""""""""""""""""""""""""""""" +Reuse tests from other packages +""""""""""""""""""""""""""""""" + +When using a custom repository, it is possible to customize a package that already exists in ``builtin`` +and reuse its external tests. To do so, just write a ``detection_tests.yaml`` alongside the customized +``package.py`` with an ``includes`` attribute. For instance the ``detection_tests.yaml`` for +``myrepo.llvm`` might look like: + +.. code-block:: yaml + + includes: + - "builtin.llvm" + +This YAML file instructs Spack to run the detection tests defined in ``builtin.llvm`` in addition to +those locally defined in the file. ----------------------------- Style guidelines for packages @@ -6606,3 +6913,63 @@ To achieve backward compatibility with the single-class format Spack creates in Overall the role of the adapter is to route access to attributes of methods first through the ``*Package`` hierarchy, and then back to the base class builder. This is schematically shown in the diagram above, where the adapter role is to "emulate" a method resolution order like the one represented by the red arrows. + +------------------------------ +Specifying License Information +------------------------------ + +Most of the software in Spack is open source, and most open source software is released +under one or more `common open source licenses `_. +Specifying the license that a package is released under in a project's +`package.py` is good practice. To specify a license, find the `SPDX identifier +`_ for a project and then add it using the license +directive: + +.. code-block:: python + + license("") + +For example, the SPDX ID for the Apache Software License, version 2.0 is ``Apache-2.0``, +so you'd write: + +.. code-block:: python + + license("Apache-2.0") + +Or, for a dual-licensed package like Spack, you would use an `SPDX Expression +`_ with both of its +licenses: + +.. code-block:: python + + license("Apache-2.0 OR MIT") + +Note that specifying a license without a when clause makes it apply to all +versions and variants of the package, which might not actually be the case. +For example, a project might have switched licenses at some point or have +certain build configurations that include files that are licensed differently. +Spack itself used to be under the ``LGPL-2.1`` license, until it was relicensed +in version ``0.12`` in 2018. + +You can specify when a ``license()`` directive applies using with a ``when=`` +clause, just like other directives. For example, to specify that a specific +license identifier should only apply to versions up to ``0.11``, but another +license should apply for later versions, you could write: + +.. code-block:: python + + license("LGPL-2.1", when="@:0.11") + license("Apache-2.0 OR MIT", when="@0.12:") + +Note that unlike for most other directives, the ``when=`` constraints in the +``license()`` directive can't intersect. Spack needs to be able to resolve +exactly one license identifier expression for any given version. To specify +*multiple* licenses, use SPDX expressions and operators as above. The operators +you probably care most about are: + +* ``OR``: user chooses one license to adhere to; and +* ``AND``: user has to adhere to all the licenses. + +You may also care about `license exceptions +`_ that use the ``WITH`` operator, +e.g. ``Apache-2.0 WITH LLVM-exception``. diff --git a/lib/spack/docs/pipelines.rst b/lib/spack/docs/pipelines.rst index d594879aab51dd..4ebe90fb0b6de4 100644 --- a/lib/spack/docs/pipelines.rst +++ b/lib/spack/docs/pipelines.rst @@ -213,6 +213,16 @@ pipeline jobs. ``spack ci generate`` ^^^^^^^^^^^^^^^^^^^^^ +Throughout this documentation, references to the "mirror" mean the target +mirror which is checked for the presence of up-to-date specs, and where +any scheduled jobs should push built binary packages. In the past, this +defaulted to the mirror at index 0 in the mirror configs, and could be +overridden using the ``--buildcache-destination`` argument. Starting with +Spack 0.23, ``spack ci generate`` will require you to identify this mirror +by the name "buildcache-destination". While you can configure any number +of mirrors as sources for your pipelines, you will need to identify the +destination mirror by name. + Concretizes the specs in the active environment, stages them (as described in :ref:`staging_algorithm`), and writes the resulting ``.gitlab-ci.yml`` to disk. During concretization of the environment, ``spack ci generate`` also writes a diff --git a/lib/spack/docs/replace_conda_homebrew.rst b/lib/spack/docs/replace_conda_homebrew.rst index 42a3561300eab4..c0d2060c703b96 100644 --- a/lib/spack/docs/replace_conda_homebrew.rst +++ b/lib/spack/docs/replace_conda_homebrew.rst @@ -4,7 +4,7 @@ SPDX-License-Identifier: (Apache-2.0 OR MIT) ===================================== -Using Spack to Replace Homebrew/Conda +Spack for Homebrew/Conda Users ===================================== Spack is an incredibly powerful package manager, designed for supercomputers @@ -191,18 +191,18 @@ The ``--fresh`` flag tells Spack to use the latest version of every package where possible instead of trying to optimize for reuse of existing installed packages. -The ``--force`` flag in addition tells Spack to overwrite its previous -concretization decisions, allowing you to choose a new version of Python. -If any of the new packages like Bash are already installed, ``spack install`` +The ``--force`` flag in addition tells Spack to overwrite its previous +concretization decisions, allowing you to choose a new version of Python. +If any of the new packages like Bash are already installed, ``spack install`` won't re-install them, it will keep the symlinks in place. ----------------------------------- Updating & Cleaning Up Old Packages ----------------------------------- -If you're looking to mimic the behavior of Homebrew, you may also want to -clean up out-of-date packages from your environment after an upgrade. To -upgrade your entire software stack within an environment and clean up old +If you're looking to mimic the behavior of Homebrew, you may also want to +clean up out-of-date packages from your environment after an upgrade. To +upgrade your entire software stack within an environment and clean up old package versions, simply run the following commands: .. code-block:: console @@ -212,9 +212,9 @@ package versions, simply run the following commands: $ spack concretize --fresh --force $ spack install $ spack gc - -Running ``spack mark -i --all`` tells Spack to mark all of the existing -packages within an environment as "implicitly" installed. This tells + +Running ``spack mark -i --all`` tells Spack to mark all of the existing +packages within an environment as "implicitly" installed. This tells spack's garbage collection system that these packages should be cleaned up. Don't worry however, this will not remove your entire environment. @@ -223,8 +223,8 @@ a fresh concretization and will re-mark any packages that should remain installed as "explicitly" installed. **Note:** if you use multiple spack environments you should re-run ``spack install`` -in each of your environments prior to running ``spack gc`` to prevent spack -from uninstalling any shared packages that are no longer required by the +in each of your environments prior to running ``spack gc`` to prevent spack +from uninstalling any shared packages that are no longer required by the environment you just upgraded. -------------- diff --git a/lib/spack/docs/requirements.txt b/lib/spack/docs/requirements.txt index 9e09e31e995946..10e19f093e5eec 100644 --- a/lib/spack/docs/requirements.txt +++ b/lib/spack/docs/requirements.txt @@ -1,8 +1,13 @@ -sphinx==6.2.1 +sphinx==7.2.6 sphinxcontrib-programoutput==0.17 -sphinx_design==0.4.1 -sphinx-rtd-theme==1.2.2 -python-levenshtein==0.21.1 +sphinx_design==0.5.0 +sphinx-rtd-theme==1.3.0 +python-levenshtein==0.23.0 docutils==0.18.1 -pygments==2.15.1 -urllib3==2.0.3 +pygments==2.16.1 +urllib3==2.0.7 +pytest==7.4.3 +isort==5.12.0 +black==23.10.1 +flake8==6.1.0 +mypy==1.6.1 diff --git a/lib/spack/docs/signing.rst b/lib/spack/docs/signing.rst index f88b87f07e0fb4..dc44eaec1d8678 100644 --- a/lib/spack/docs/signing.rst +++ b/lib/spack/docs/signing.rst @@ -217,13 +217,7 @@ file would live in the ``build_cache`` directory of a binary mirror:: "binary_cache_checksum": { "hash_algorithm": "sha256", "hash": "4f1e46452c35a5e61bcacca205bae1bfcd60a83a399af201a29c95b7cc3e1423" - }, - - "buildinfo": { - "relative_prefix": - "linux-ubuntu18.04-haswell/gcc-7.5.0/zlib-1.2.12-llv2ysfdxnppzjrt5ldybb5c52qbmoow", - "relative_rpaths": false - } + } } -----BEGIN PGP SIGNATURE----- diff --git a/lib/spack/docs/tables/system_prerequisites.csv b/lib/spack/docs/tables/system_prerequisites.csv index 0bb82638eb9d70..7a72078cdd6a10 100644 --- a/lib/spack/docs/tables/system_prerequisites.csv +++ b/lib/spack/docs/tables/system_prerequisites.csv @@ -1,9 +1,7 @@ Name, Supported Versions, Notes, Requirement Reason -Python, 3.6--3.11, , Interpreter for Spack +Python, 3.6--3.12, , Interpreter for Spack C/C++ Compilers, , , Building software -make, , , Build software patch, , , Build software -bash, , , Compiler wrappers tar, , , Extract/create archives gzip, , , Compress/Decompress archives unzip, , , Compress/Decompress archives diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py index 7abfaa4a03884f..2e8bf3a4f8b80b 100644 --- a/lib/spack/external/__init__.py +++ b/lib/spack/external/__init__.py @@ -18,7 +18,7 @@ * Homepage: https://pypi.python.org/pypi/archspec * Usage: Labeling, comparison and detection of microarchitectures -* Version: 0.2.1 (commit 9e1117bd8a2f0581bced161f2a2e8d6294d0300b) +* Version: 0.2.2 (commit 1dc58a5776dd77e6fc6e4ba5626af5b1fb24996e) astunparse ---------------- diff --git a/lib/spack/external/archspec/__init__.py b/lib/spack/external/archspec/__init__.py index fbbab9f78a7486..22a430894b4af0 100644 --- a/lib/spack/external/archspec/__init__.py +++ b/lib/spack/external/archspec/__init__.py @@ -1,2 +1,2 @@ """Init file to avoid namespace packages""" -__version__ = "0.2.0" +__version__ = "0.2.2" diff --git a/lib/spack/external/archspec/cpu/microarchitecture.py b/lib/spack/external/archspec/cpu/microarchitecture.py index fdc7707fda2ff1..954242736bc704 100644 --- a/lib/spack/external/archspec/cpu/microarchitecture.py +++ b/lib/spack/external/archspec/cpu/microarchitecture.py @@ -79,14 +79,18 @@ def __init__(self, name, parents, vendor, features, compilers, generation=0): self.features = features self.compilers = compilers self.generation = generation + # Cache the ancestor computation + self._ancestors = None @property def ancestors(self): """All the ancestors of this microarchitecture.""" - value = self.parents[:] - for parent in self.parents: - value.extend(a for a in parent.ancestors if a not in value) - return value + if self._ancestors is None: + value = self.parents[:] + for parent in self.parents: + value.extend(a for a in parent.ancestors if a not in value) + self._ancestors = value + return self._ancestors def _to_set(self): """Returns a set of the nodes in this microarchitecture DAG.""" diff --git a/lib/spack/external/archspec/json/cpu/microarchitectures.json b/lib/spack/external/archspec/json/cpu/microarchitectures.json index a7e8e5d1caa180..1e77caba4aea00 100644 --- a/lib/spack/external/archspec/json/cpu/microarchitectures.json +++ b/lib/spack/external/archspec/json/cpu/microarchitectures.json @@ -145,6 +145,13 @@ "flags": "-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3" } ], + "intel": [ + { + "versions": "16.0:", + "name": "corei7", + "flags": "-march={name} -mtune=generic -mpopcnt" + } + ], "oneapi": [ { "versions": "2021.2.0:", @@ -217,6 +224,13 @@ "flags": "-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave" } ], + "intel": [ + { + "versions": "16.0:", + "name": "core-avx2", + "flags": "-march={name} -mtune={name} -fma -mf16c" + } + ], "oneapi": [ { "versions": "2021.2.0:", @@ -300,6 +314,13 @@ "flags": "-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave -mavx512f -mavx512bw -mavx512cd -mavx512dq -mavx512vl" } ], + "intel": [ + { + "versions": "16.0:", + "name": "skylake-avx512", + "flags": "-march={name} -mtune={name}" + } + ], "oneapi": [ { "versions": "2021.2.0:", @@ -1412,6 +1433,92 @@ ] } }, + "sapphirerapids": { + "from": [ + "icelake" + ], + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt", + "aes", + "pclmulqdq", + "avx", + "rdrand", + "f16c", + "movbe", + "fma", + "avx2", + "bmi1", + "bmi2", + "rdseed", + "adx", + "clflushopt", + "xsavec", + "xsaveopt", + "avx512f", + "avx512vl", + "avx512bw", + "avx512dq", + "avx512cd", + "avx512vbmi", + "avx512ifma", + "sha_ni", + "clwb", + "rdpid", + "gfni", + "avx512_vbmi2", + "avx512_vpopcntdq", + "avx512_bitalg", + "avx512_vnni", + "vpclmulqdq", + "vaes", + "avx512_bf16", + "cldemote", + "movdir64b", + "movdiri", + "pdcm", + "serialize", + "waitpkg" + ], + "compilers": { + "gcc": [ + { + "versions": "11.0:", + "flags": "-march={name} -mtune={name}" + } + ], + "clang": [ + { + "versions": "12.0:", + "flags": "-march={name} -mtune={name}" + } + ], + "intel": [ + { + "versions": "2021.2:", + "flags": "-march={name} -mtune={name}" + } + ], + "oneapi": [ + { + "versions": "2021.2:", + "flags": "-march={name} -mtune={name}" + } + ], + "dpcpp": [ + { + "versions": "2021.2:", + "flags": "-march={name} -mtune={name}" + } + ] + } + }, "k10": { "from": ["x86_64"], "vendor": "AuthenticAMD", @@ -2065,8 +2172,6 @@ "pku", "gfni", "flush_l1d", - "erms", - "avic", "avx512f", "avx512dq", "avx512ifma", @@ -2083,12 +2188,12 @@ "compilers": { "gcc": [ { - "versions": "10.3:13.0", + "versions": "10.3:12.2", "name": "znver3", "flags": "-march={name} -mtune={name} -mavx512f -mavx512dq -mavx512ifma -mavx512cd -mavx512bw -mavx512vl -mavx512vbmi -mavx512vbmi2 -mavx512vnni -mavx512bitalg" }, { - "versions": "13.1:", + "versions": "12.3:", "name": "znver4", "flags": "-march={name} -mtune={name}" } @@ -2213,6 +2318,26 @@ ] } }, + "power10": { + "from": ["power9"], + "vendor": "IBM", + "generation": 10, + "features": [], + "compilers": { + "gcc": [ + { + "versions": "11.1:", + "flags": "-mcpu={name} -mtune={name}" + } + ], + "clang": [ + { + "versions": "11.0:", + "flags": "-mcpu={name} -mtune={name}" + } + ] + } + }, "ppc64le": { "from": [], "vendor": "generic", @@ -2300,6 +2425,29 @@ ] } }, + "power10le": { + "from": ["power9le"], + "vendor": "IBM", + "generation": 10, + "features": [], + "compilers": { + "gcc": [ + { + "name": "power10", + "versions": "11.1:", + "flags": "-mcpu={name} -mtune={name}" + } + ], + "clang": [ + { + "versions": "11.0:", + "family": "ppc64le", + "name": "power10", + "flags": "-mcpu={name} -mtune={name}" + } + ] + } + }, "aarch64": { "from": [], "vendor": "generic", @@ -2487,6 +2635,37 @@ ] } }, + "armv9.0a": { + "from": ["armv8.5a"], + "vendor": "generic", + "features": [], + "compilers": { + "gcc": [ + { + "versions": "12:", + "flags": "-march=armv9-a -mtune=generic" + } + ], + "clang": [ + { + "versions": "14:", + "flags": "-march=armv9-a -mtune=generic" + } + ], + "apple-clang": [ + { + "versions": ":", + "flags": "-march=armv9-a -mtune=generic" + } + ], + "arm": [ + { + "versions": ":", + "flags": "-march=armv9-a -mtune=generic" + } + ] + } + }, "thunderx2": { "from": ["armv8.1a"], "vendor": "Cavium", @@ -2708,8 +2887,12 @@ ], "arm" : [ { - "versions": "20:", + "versions": "20:21.9", "flags" : "-march=armv8.2-a+fp16+rcpc+dotprod+crypto" + }, + { + "versions": "22:", + "flags" : "-mcpu=neoverse-n1" } ], "nvhpc" : [ @@ -2837,7 +3020,7 @@ }, { "versions": "22:", - "flags" : "-march=armv8.4-a+sve+ssbs+fp16+bf16+crypto+i8mm+rng" + "flags" : "-mcpu=neoverse-v1" } ], "nvhpc" : [ @@ -2849,6 +3032,126 @@ ] } }, + "neoverse_v2": { + "from": ["neoverse_n1", "armv9.0a"], + "vendor": "ARM", + "features": [ + "fp", + "asimd", + "evtstrm", + "aes", + "pmull", + "sha1", + "sha2", + "crc32", + "atomics", + "fphp", + "asimdhp", + "cpuid", + "asimdrdm", + "jscvt", + "fcma", + "lrcpc", + "dcpop", + "sha3", + "sm3", + "sm4", + "asimddp", + "sha512", + "sve", + "asimdfhm", + "dit", + "uscat", + "ilrcpc", + "flagm", + "ssbs", + "sb", + "paca", + "pacg", + "dcpodp", + "sve2", + "sveaes", + "svepmull", + "svebitperm", + "svesha3", + "svesm4", + "flagm2", + "frint", + "svei8mm", + "svebf16", + "i8mm", + "bf16", + "dgh", + "bti" + ], + "compilers" : { + "gcc": [ + { + "versions": "4.8:5.99", + "flags": "-march=armv8-a" + }, + { + "versions": "6:6.99", + "flags" : "-march=armv8.1-a" + }, + { + "versions": "7.0:7.99", + "flags" : "-march=armv8.2-a -mtune=cortex-a72" + }, + { + "versions": "8.0:8.99", + "flags" : "-march=armv8.4-a+sve -mtune=cortex-a72" + }, + { + "versions": "9.0:9.99", + "flags" : "-march=armv8.5-a+sve -mtune=cortex-a76" + }, + { + "versions": "10.0:11.99", + "flags" : "-march=armv8.5-a+sve+sve2+i8mm+bf16 -mtune=cortex-a77" + }, + { + "versions": "12.0:12.99", + "flags" : "-march=armv9-a+i8mm+bf16 -mtune=cortex-a710" + }, + { + "versions": "13.0:", + "flags" : "-mcpu=neoverse-v2" + } + ], + "clang" : [ + { + "versions": "9.0:10.99", + "flags" : "-march=armv8.5-a+sve" + }, + { + "versions": "11.0:13.99", + "flags" : "-march=armv8.5-a+sve+sve2+i8mm+bf16" + }, + { + "versions": "14.0:15.99", + "flags" : "-march=armv9-a+i8mm+bf16" + }, + { + "versions": "16.0:", + "flags" : "-mcpu=neoverse-v2" + } + ], + "arm" : [ + { + "versions": "23.04.0:", + "flags" : "-mcpu=neoverse-v2" + } + ], + "nvhpc" : [ + { + "versions": "23.3:", + "name": "neoverse-v2", + "flags": "-tp {name}" + } + ] + } + }, "m1": { "from": ["armv8.4a"], "vendor": "Apple", diff --git a/lib/spack/llnl/path.py b/lib/spack/llnl/path.py new file mode 100644 index 00000000000000..bef9e14adac4c2 --- /dev/null +++ b/lib/spack/llnl/path.py @@ -0,0 +1,105 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Path primitives that just require Python standard library.""" +import functools +import sys +from typing import List, Optional +from urllib.parse import urlparse + + +class Path: + """Enum to identify the path-style.""" + + unix: int = 0 + windows: int = 1 + platform_path: int = windows if sys.platform == "win32" else unix + + +def format_os_path(path: str, mode: int = Path.unix) -> str: + """Formats the input path to use consistent, platform specific separators. + + Absolute paths are converted between drive letters and a prepended '/' as per platform + requirement. + + Parameters: + path: the path to be normalized, must be a string or expose the replace method. + mode: the path file separator style to normalize the passed path to. + Default is unix style, i.e. '/' + """ + if not path: + return path + if mode == Path.windows: + path = path.replace("/", "\\") + else: + path = path.replace("\\", "/") + return path + + +def convert_to_posix_path(path: str) -> str: + """Converts the input path to POSIX style.""" + return format_os_path(path, mode=Path.unix) + + +def convert_to_windows_path(path: str) -> str: + """Converts the input path to Windows style.""" + return format_os_path(path, mode=Path.windows) + + +def convert_to_platform_path(path: str) -> str: + """Converts the input path to the current platform's native style.""" + return format_os_path(path, mode=Path.platform_path) + + +def path_to_os_path(*parameters: str) -> List[str]: + """Takes an arbitrary number of positional parameters, converts each argument of type + string to use a normalized filepath separator, and returns a list of all values. + """ + + def _is_url(path_or_url: str) -> bool: + if "\\" in path_or_url: + return False + url_tuple = urlparse(path_or_url) + return bool(url_tuple.scheme) and len(url_tuple.scheme) > 1 + + result = [] + for item in parameters: + if isinstance(item, str) and not _is_url(item): + item = convert_to_platform_path(item) + result.append(item) + return result + + +def system_path_filter(_func=None, arg_slice: Optional[slice] = None): + """Filters function arguments to account for platform path separators. + Optional slicing range can be specified to select specific arguments + + This decorator takes all (or a slice) of a method's positional arguments + and normalizes usage of filepath separators on a per platform basis. + + Note: `**kwargs`, urls, and any type that is not a string are ignored + so in such cases where path normalization is required, that should be + handled by calling path_to_os_path directly as needed. + + Parameters: + arg_slice: a slice object specifying the slice of arguments + in the decorated method over which filepath separators are + normalized + """ + + def holder_func(func): + @functools.wraps(func) + def path_filter_caller(*args, **kwargs): + args = list(args) + if arg_slice: + args[arg_slice] = path_to_os_path(*args[arg_slice]) + else: + args = path_to_os_path(*args) + return func(*args, **kwargs) + + return path_filter_caller + + if _func: + return holder_func(_func) + return holder_func diff --git a/lib/spack/llnl/string.py b/lib/spack/llnl/string.py new file mode 100644 index 00000000000000..a203be8d3468b6 --- /dev/null +++ b/lib/spack/llnl/string.py @@ -0,0 +1,67 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""String manipulation functions that do not have other dependencies than Python +standard library +""" +from typing import List, Optional + + +def comma_list(sequence: List[str], article: str = "") -> str: + if type(sequence) is not list: + sequence = list(sequence) + + if not sequence: + return "" + if len(sequence) == 1: + return sequence[0] + + out = ", ".join(str(s) for s in sequence[:-1]) + if len(sequence) != 2: + out += "," # oxford comma + out += " " + if article: + out += article + " " + out += str(sequence[-1]) + return out + + +def comma_or(sequence: List[str]) -> str: + """Return a string with all the elements of the input joined by comma, but the last + one (which is joined by 'or'). + """ + return comma_list(sequence, "or") + + +def comma_and(sequence: List[str]) -> str: + """Return a string with all the elements of the input joined by comma, but the last + one (which is joined by 'and'). + """ + return comma_list(sequence, "and") + + +def quote(sequence: List[str], q: str = "'") -> List[str]: + """Quotes each item in the input list with the quote character passed as second argument.""" + return [f"{q}{e}{q}" for e in sequence] + + +def plural(n: int, singular: str, plural: Optional[str] = None, show_n: bool = True) -> str: + """Pluralize word by adding an s if n != 1. + + Arguments: + n: number of things there are + singular: singular form of word + plural: optional plural form, for when it's not just singular + 's' + show_n: whether to include n in the result string (default True) + + Returns: + "1 thing" if n == 1 or "n things" if n != 1 + """ + number = f"{n} " if show_n else "" + if n == 1: + return f"{number}{singular}" + elif plural is not None: + return f"{number}{plural}" + else: + return f"{number}{singular}s" diff --git a/lib/spack/llnl/url.py b/lib/spack/llnl/url.py new file mode 100644 index 00000000000000..40e7606506c6d2 --- /dev/null +++ b/lib/spack/llnl/url.py @@ -0,0 +1,459 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""URL primitives that just require Python standard library.""" +import itertools +import os.path +import re +from typing import Optional, Set, Tuple +from urllib.parse import urlsplit, urlunsplit + +# Archive extensions allowed in Spack +PREFIX_EXTENSIONS = ("tar", "TAR") +EXTENSIONS = ("gz", "bz2", "xz", "Z") +NO_TAR_EXTENSIONS = ("zip", "tgz", "tbz2", "tbz", "txz") + +# Add PREFIX_EXTENSIONS and EXTENSIONS last so that .tar.gz is matched *before* .tar or .gz +ALLOWED_ARCHIVE_TYPES = ( + tuple(".".join(ext) for ext in itertools.product(PREFIX_EXTENSIONS, EXTENSIONS)) + + PREFIX_EXTENSIONS + + EXTENSIONS + + NO_TAR_EXTENSIONS +) +CONTRACTION_MAP = {"tgz": "tar.gz", "txz": "tar.xz", "tbz": "tar.bz2", "tbz2": "tar.bz2"} + + +def find_list_urls(url: str) -> Set[str]: + r"""Find good list URLs for the supplied URL. + + By default, returns the dirname of the archive path. + + Provides special treatment for the following websites, which have a + unique list URL different from the dirname of the download URL: + + ========= ======================================================= + GitHub https://github.com///releases + GitLab https://gitlab.\*///tags + BitBucket https://bitbucket.org///downloads/?tab=tags + CRAN https://\*.r-project.org/src/contrib/Archive/ + PyPI https://pypi.org/simple// + LuaRocks https://luarocks.org/modules// + ========= ======================================================= + + Note: this function is called by `spack versions`, `spack checksum`, + and `spack create`, but not by `spack fetch` or `spack install`. + + Parameters: + url (str): The download URL for the package + + Returns: + set: One or more list URLs for the package + """ + + url_types = [ + # GitHub + # e.g. https://github.com/llnl/callpath/archive/v1.0.1.tar.gz + (r"(.*github\.com/[^/]+/[^/]+)", lambda m: m.group(1) + "/releases"), + # GitLab API endpoint + # e.g. https://gitlab.dkrz.de/api/v4/projects/k202009%2Flibaec/repository/archive.tar.gz?sha=v1.0.2 + ( + r"(.*gitlab[^/]+)/api/v4/projects/([^/]+)%2F([^/]+)", + lambda m: m.group(1) + "/" + m.group(2) + "/" + m.group(3) + "/tags", + ), + # GitLab non-API endpoint + # e.g. https://gitlab.dkrz.de/k202009/libaec/uploads/631e85bcf877c2dcaca9b2e6d6526339/libaec-1.0.0.tar.gz + (r"(.*gitlab[^/]+/(?!api/v4/projects)[^/]+/[^/]+)", lambda m: m.group(1) + "/tags"), + # BitBucket + # e.g. https://bitbucket.org/eigen/eigen/get/3.3.3.tar.bz2 + (r"(.*bitbucket.org/[^/]+/[^/]+)", lambda m: m.group(1) + "/downloads/?tab=tags"), + # CRAN + # e.g. https://cran.r-project.org/src/contrib/Rcpp_0.12.9.tar.gz + # e.g. https://cloud.r-project.org/src/contrib/rgl_0.98.1.tar.gz + ( + r"(.*\.r-project\.org/src/contrib)/([^_]+)", + lambda m: m.group(1) + "/Archive/" + m.group(2), + ), + # PyPI + # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip + # e.g. https://pypi.io/packages/py2.py3/o/opencensus-context/opencensus_context-0.1.1-py2.py3-none-any.whl + ( + r"(?:pypi|pythonhosted)[^/]+/packages/[^/]+/./([^/]+)", + lambda m: "https://pypi.org/simple/" + m.group(1) + "/", + ), + # LuaRocks + # e.g. https://luarocks.org/manifests/gvvaughan/lpeg-1.0.2-1.src.rock + # e.g. https://luarocks.org/manifests/openresty/lua-cjson-2.1.0-1.src.rock + ( + r"luarocks[^/]+/(?:modules|manifests)/(?P[^/]+)/" + + r"(?P.+?)-[0-9.-]*\.src\.rock", + lambda m: "https://luarocks.org/modules/" + + m.group("org") + + "/" + + m.group("name") + + "/", + ), + ] + + list_urls = {os.path.dirname(url)} + + for pattern, fun in url_types: + match = re.search(pattern, url) + if match: + list_urls.add(fun(match)) + + return list_urls + + +def strip_query_and_fragment(url: str) -> Tuple[str, str]: + """Strips query and fragment from a url, then returns the base url and the suffix. + + Args: + url: URL to be stripped + + Raises: + ValueError: when there is any error parsing the URL + """ + components = urlsplit(url) + stripped = components[:3] + (None, None) + + query, frag = components[3:5] + suffix = "" + if query: + suffix += "?" + query + if frag: + suffix += "#" + frag + + return urlunsplit(stripped), suffix + + +SOURCEFORGE_RE = re.compile(r"(.*(?:sourceforge\.net|sf\.net)/.*)(/download)$") + + +def split_url_on_sourceforge_suffix(url: str) -> Tuple[str, ...]: + """If the input is a sourceforge URL, returns base URL and "/download" suffix. Otherwise, + returns the input URL and an empty string. + """ + match = SOURCEFORGE_RE.search(url) + if match is not None: + return match.groups() + return url, "" + + +def has_extension(path_or_url: str, ext: str) -> bool: + """Returns true if the extension in input is present in path, false otherwise.""" + prefix, _ = split_url_on_sourceforge_suffix(path_or_url) + if not ext.startswith(r"\."): + ext = rf"\.{ext}$" + + if re.search(ext, prefix): + return True + return False + + +def extension_from_path(path_or_url: Optional[str]) -> Optional[str]: + """Tries to match an allowed archive extension to the input. Returns the first match, + or None if no match was found. + + Raises: + ValueError: if the input is None + """ + if path_or_url is None: + raise ValueError("Can't call extension() on None") + + for t in ALLOWED_ARCHIVE_TYPES: + if has_extension(path_or_url, t): + return t + return None + + +def remove_extension(path_or_url: str, *, extension: str) -> str: + """Returns the input with the extension removed""" + suffix = rf"\.{extension}$" + return re.sub(suffix, "", path_or_url) + + +def check_and_remove_ext(path: str, *, extension: str) -> str: + """Returns the input path with the extension removed, if the extension is present in path. + Otherwise, returns the input unchanged. + """ + if not has_extension(path, extension): + return path + path, _ = split_url_on_sourceforge_suffix(path) + return remove_extension(path, extension=extension) + + +def strip_extension(path_or_url: str, *, extension: Optional[str] = None) -> str: + """If a path contains the extension in input, returns the path stripped of the extension. + Otherwise, returns the input path. + + If extension is None, attempts to strip any allowed extension from path. + """ + if extension is None: + for t in ALLOWED_ARCHIVE_TYPES: + if has_extension(path_or_url, ext=t): + extension = t + break + else: + return path_or_url + + return check_and_remove_ext(path_or_url, extension=extension) + + +def split_url_extension(url: str) -> Tuple[str, ...]: + """Some URLs have a query string, e.g.: + + 1. https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true + 2. http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz + 3. https://gitlab.kitware.com/vtk/vtk/repository/archive.tar.bz2?ref=v7.0.0 + + In (1), the query string needs to be stripped to get at the + extension, but in (2) & (3), the filename is IN a single final query + argument. + + This strips the URL into three pieces: ``prefix``, ``ext``, and ``suffix``. + The suffix contains anything that was stripped off the URL to + get at the file extension. In (1), it will be ``'?raw=true'``, but + in (2), it will be empty. In (3) the suffix is a parameter that follows + after the file extension, e.g.: + + 1. ``('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7', '.tgz', '?raw=true')`` + 2. ``('http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin', '.tar.gz', None)`` + 3. ``('https://gitlab.kitware.com/vtk/vtk/repository/archive', '.tar.bz2', '?ref=v7.0.0')`` + """ + # Strip off sourceforge download suffix. + # e.g. https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download + prefix, suffix = split_url_on_sourceforge_suffix(url) + + ext = extension_from_path(prefix) + if ext is not None: + prefix = strip_extension(prefix) + return prefix, ext, suffix + + try: + prefix, suf = strip_query_and_fragment(prefix) + except ValueError: + # FIXME: tty.debug("Got error parsing path %s" % path) + # Ignore URL parse errors here + return url, "" + + ext = extension_from_path(prefix) + prefix = strip_extension(prefix) + suffix = suf + suffix + if ext is None: + ext = "" + + return prefix, ext, suffix + + +def strip_version_suffixes(path_or_url: str) -> str: + """Some tarballs contain extraneous information after the version: + + * ``bowtie2-2.2.5-source`` + * ``libevent-2.0.21-stable`` + * ``cuda_8.0.44_linux.run`` + + These strings are not part of the version number and should be ignored. + This function strips those suffixes off and returns the remaining string. + The goal is that the version is always the last thing in ``path``: + + * ``bowtie2-2.2.5`` + * ``libevent-2.0.21`` + * ``cuda_8.0.44`` + + Args: + path_or_url: The filename or URL for the package + + Returns: + The ``path`` with any extraneous suffixes removed + """ + # NOTE: This could be done with complicated regexes in parse_version_offset + # NOTE: The problem is that we would have to add these regexes to the end + # NOTE: of every single version regex. Easier to just strip them off + # NOTE: permanently + + suffix_regexes = [ + # Download type + r"[Ii]nstall", + r"all", + r"code", + r"[Ss]ources?", + r"file", + r"full", + r"single", + r"with[a-zA-Z_-]+", + r"rock", + r"src(_0)?", + r"public", + r"bin", + r"binary", + r"run", + r"[Uu]niversal", + r"jar", + r"complete", + r"dynamic", + r"oss", + r"gem", + r"tar", + r"sh", + # Download version + r"release", + r"bin", + r"stable", + r"[Ff]inal", + r"rel", + r"orig", + r"dist", + r"\+", + # License + r"gpl", + # Arch + # Needs to come before and after OS, appears in both orders + r"ia32", + r"intel", + r"amd64", + r"linux64", + r"x64", + r"64bit", + r"x86[_-]64", + r"i586_64", + r"x86", + r"i[36]86", + r"ppc64(le)?", + r"armv?(7l|6l|64)", + # Other + r"cpp", + r"gtk", + r"incubating", + # OS + r"[Ll]inux(_64)?", + r"LINUX", + r"[Uu]ni?x", + r"[Ss]un[Oo][Ss]", + r"[Mm]ac[Oo][Ss][Xx]?", + r"[Oo][Ss][Xx]", + r"[Dd]arwin(64)?", + r"[Aa]pple", + r"[Ww]indows", + r"[Ww]in(64|32)?", + r"[Cc]ygwin(64|32)?", + r"[Mm]ingw", + r"centos", + # Arch + # Needs to come before and after OS, appears in both orders + r"ia32", + r"intel", + r"amd64", + r"linux64", + r"x64", + r"64bit", + r"x86[_-]64", + r"i586_64", + r"x86", + r"i[36]86", + r"ppc64(le)?", + r"armv?(7l|6l|64)?", + # PyPI + r"[._-]py[23].*\.whl", + r"[._-]cp[23].*\.whl", + r"[._-]win.*\.exe", + ] + + for regex in suffix_regexes: + # Remove the suffix from the end of the path + # This may be done multiple times + path_or_url = re.sub(r"[._-]?" + regex + "$", "", path_or_url) + + return path_or_url + + +def expand_contracted_extension(extension: str) -> str: + """Returns the expanded version of a known contracted extension. + + This function maps extensions like ".tgz" to ".tar.gz". On unknown extensions, + return the input unmodified. + """ + extension = extension.strip(".") + return CONTRACTION_MAP.get(extension, extension) + + +def expand_contracted_extension_in_path( + path_or_url: str, *, extension: Optional[str] = None +) -> str: + """Returns the input path or URL with any contraction extension expanded. + + Args: + path_or_url: path or URL to be expanded + extension: if specified, only attempt to expand that extension + """ + extension = extension or extension_from_path(path_or_url) + if extension is None: + return path_or_url + + expanded = expand_contracted_extension(extension) + if expanded != extension: + return re.sub(rf"{extension}", rf"{expanded}", path_or_url) + return path_or_url + + +def compression_ext_from_compressed_archive(extension: str) -> Optional[str]: + """Returns compression extension for a compressed archive""" + extension = expand_contracted_extension(extension) + for ext in [*EXTENSIONS]: + if ext in extension: + return ext + return None + + +def strip_compression_extension(path_or_url: str, ext: Optional[str] = None) -> str: + """Strips the compression extension from the input, and returns it. For instance, + "foo.tgz" becomes "foo.tar". + + If no extension is given, try a default list of extensions. + + Args: + path_or_url: input to be stripped + ext: if given, extension to be stripped + """ + if not extension_from_path(path_or_url): + return path_or_url + + expanded_path = expand_contracted_extension_in_path(path_or_url) + candidates = [ext] if ext is not None else EXTENSIONS + for current_extension in candidates: + modified_path = check_and_remove_ext(expanded_path, extension=current_extension) + if modified_path != expanded_path: + return modified_path + return expanded_path + + +def allowed_archive(path_or_url: str) -> bool: + """Returns true if the input is a valid archive, False otherwise.""" + return ( + False if not path_or_url else any(path_or_url.endswith(t) for t in ALLOWED_ARCHIVE_TYPES) + ) + + +def determine_url_file_extension(path: str) -> str: + """This returns the type of archive a URL refers to. This is + sometimes confusing because of URLs like: + + (1) https://github.com/petdance/ack/tarball/1.93_02 + + Where the URL doesn't actually contain the filename. We need + to know what type it is so that we can appropriately name files + in mirrors. + """ + match = re.search(r"github.com/.+/(zip|tar)ball/", path) + if match: + if match.group(1) == "zip": + return "zip" + elif match.group(1) == "tar": + return "tar.gz" + + prefix, ext, suffix = split_url_extension(path) + return ext diff --git a/lib/spack/llnl/util/argparsewriter.py b/lib/spack/llnl/util/argparsewriter.py index dfd602bb34fa4a..60a404abd438ff 100644 --- a/lib/spack/llnl/util/argparsewriter.py +++ b/lib/spack/llnl/util/argparsewriter.py @@ -9,7 +9,7 @@ import re import sys from argparse import ArgumentParser -from typing import IO, Optional, Sequence, Tuple +from typing import IO, Any, Iterable, List, Optional, Sequence, Tuple, Union class Command: @@ -25,9 +25,9 @@ def __init__( prog: str, description: Optional[str], usage: str, - positionals: Sequence[Tuple[str, str]], - optionals: Sequence[Tuple[Sequence[str], str, str]], - subcommands: Sequence[Tuple[ArgumentParser, str]], + positionals: List[Tuple[str, Optional[Iterable[Any]], Union[int, str, None], str]], + optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]], + subcommands: List[Tuple[ArgumentParser, str, str]], ) -> None: """Initialize a new Command instance. @@ -96,13 +96,30 @@ def parse(self, parser: ArgumentParser, prog: str) -> Command: if action.option_strings: flags = action.option_strings dest_flags = fmt._format_action_invocation(action) - help = self._expand_help(action) if action.help else "" - help = help.replace("\n", " ") - optionals.append((flags, dest_flags, help)) + nargs = action.nargs + help = ( + self._expand_help(action) + if action.help and action.help != argparse.SUPPRESS + else "" + ) + help = help.split("\n")[0] + + if action.choices is not None: + dest = [str(choice) for choice in action.choices] + else: + dest = [action.dest] + + optionals.append((flags, dest, dest_flags, nargs, help)) elif isinstance(action, argparse._SubParsersAction): for subaction in action._choices_actions: subparser = action._name_parser_map[subaction.dest] - subcommands.append((subparser, subaction.dest)) + help = ( + self._expand_help(subaction) + if subaction.help and action.help != argparse.SUPPRESS + else "" + ) + help = help.split("\n")[0] + subcommands.append((subparser, subaction.dest, help)) # Look for aliases of the form 'name (alias, ...)' if self.aliases and isinstance(subaction.metavar, str): @@ -111,12 +128,22 @@ def parse(self, parser: ArgumentParser, prog: str) -> Command: aliases = match.group(2).split(", ") for alias in aliases: subparser = action._name_parser_map[alias] - subcommands.append((subparser, alias)) + help = ( + self._expand_help(subaction) + if subaction.help and action.help != argparse.SUPPRESS + else "" + ) + help = help.split("\n")[0] + subcommands.append((subparser, alias, help)) else: args = fmt._format_action_invocation(action) - help = self._expand_help(action) if action.help else "" - help = help.replace("\n", " ") - positionals.append((args, help)) + help = ( + self._expand_help(action) + if action.help and action.help != argparse.SUPPRESS + else "" + ) + help = help.split("\n")[0] + positionals.append((args, action.choices, action.nargs, help)) return Command(prog, description, usage, positionals, optionals, subcommands) @@ -146,7 +173,7 @@ def _write(self, parser: ArgumentParser, prog: str, level: int = 0) -> None: cmd = self.parse(parser, prog) self.out.write(self.format(cmd)) - for subparser, prog in cmd.subcommands: + for subparser, prog, help in cmd.subcommands: self._write(subparser, prog, level=level + 1) def write(self, parser: ArgumentParser) -> None: @@ -205,13 +232,13 @@ def format(self, cmd: Command) -> str: if cmd.positionals: string.write(self.begin_positionals()) - for args, help in cmd.positionals: + for args, choices, nargs, help in cmd.positionals: string.write(self.positional(args, help)) string.write(self.end_positionals()) if cmd.optionals: string.write(self.begin_optionals()) - for flags, dest_flags, help in cmd.optionals: + for flags, dest, dest_flags, nargs, help in cmd.optionals: string.write(self.optional(dest_flags, help)) string.write(self.end_optionals()) @@ -338,7 +365,7 @@ def end_optionals(self) -> str: """ return "" - def begin_subcommands(self, subcommands: Sequence[Tuple[ArgumentParser, str]]) -> str: + def begin_subcommands(self, subcommands: List[Tuple[ArgumentParser, str, str]]) -> str: """Table with links to other subcommands. Arguments: @@ -355,114 +382,8 @@ def begin_subcommands(self, subcommands: Sequence[Tuple[ArgumentParser, str]]) - """ - for cmd, _ in subcommands: + for cmd, _, _ in subcommands: prog = re.sub(r"^[^ ]* ", "", cmd.prog) string += " * :ref:`{0} <{1}>`\n".format(prog, cmd.prog.replace(" ", "-")) return string + "\n" - - -class ArgparseCompletionWriter(ArgparseWriter): - """Write argparse output as shell programmable tab completion functions.""" - - def format(self, cmd: Command) -> str: - """Return the string representation of a single node in the parser tree. - - Args: - cmd: Parsed information about a command or subcommand. - - Returns: - String representation of this subcommand. - """ - - assert cmd.optionals # we should always at least have -h, --help - assert not (cmd.positionals and cmd.subcommands) # one or the other - - # We only care about the arguments/flags, not the help messages - positionals: Tuple[str, ...] = () - if cmd.positionals: - positionals, _ = zip(*cmd.positionals) - optionals, _, _ = zip(*cmd.optionals) - subcommands: Tuple[str, ...] = () - if cmd.subcommands: - _, subcommands = zip(*cmd.subcommands) - - # Flatten lists of lists - optionals = [x for xx in optionals for x in xx] - - return ( - self.start_function(cmd.prog) - + self.body(positionals, optionals, subcommands) - + self.end_function(cmd.prog) - ) - - def start_function(self, prog: str) -> str: - """Return the syntax needed to begin a function definition. - - Args: - prog: Program name. - - Returns: - Function definition beginning. - """ - name = prog.replace("-", "_").replace(" ", "_") - return "\n_{0}() {{".format(name) - - def end_function(self, prog: str) -> str: - """Return the syntax needed to end a function definition. - - Args: - prog: Program name - - Returns: - Function definition ending. - """ - return "}\n" - - def body( - self, positionals: Sequence[str], optionals: Sequence[str], subcommands: Sequence[str] - ) -> str: - """Return the body of the function. - - Args: - positionals: List of positional arguments. - optionals: List of optional arguments. - subcommands: List of subcommand parsers. - - Returns: - Function body. - """ - return "" - - def positionals(self, positionals: Sequence[str]) -> str: - """Return the syntax for reporting positional arguments. - - Args: - positionals: List of positional arguments. - - Returns: - Syntax for positional arguments. - """ - return "" - - def optionals(self, optionals: Sequence[str]) -> str: - """Return the syntax for reporting optional flags. - - Args: - optionals: List of optional arguments. - - Returns: - Syntax for optional flags. - """ - return "" - - def subcommands(self, subcommands: Sequence[str]) -> str: - """Return the syntax for reporting subcommands. - - Args: - subcommands: List of subcommand parsers. - - Returns: - Syntax for subcommand parsers - """ - return "" diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 3f36e82d5888da..47c66248b53aee 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -11,6 +11,7 @@ import itertools import numbers import os +import pathlib import posixpath import re import shutil @@ -18,14 +19,17 @@ import sys import tempfile from contextlib import contextmanager +from itertools import accumulate from typing import Callable, Iterable, List, Match, Optional, Tuple, Union +import llnl.util.symlink from llnl.util import tty from llnl.util.lang import dedupe, memoized -from llnl.util.symlink import islink, symlink +from llnl.util.symlink import islink, readlink, resolve_link_target_relative_to_the_link, symlink from spack.util.executable import Executable, which -from spack.util.path import path_to_os_path, system_path_filter + +from ..path import path_to_os_path, system_path_filter if sys.platform != "win32": import grp @@ -101,7 +105,7 @@ def _nop(args, ns=None, follow_symlinks=None): pass # follow symlinks (aka don't not follow symlinks) - follow = follow_symlinks or not (os.path.islink(src) and os.path.islink(dst)) + follow = follow_symlinks or not (islink(src) and islink(dst)) if follow: # use the real function if it exists def lookup(name): @@ -152,6 +156,37 @@ def lookup(name): shutil.copystat = copystat +def polite_path(components: Iterable[str]): + """ + Given a list of strings which are intended to be path components, + generate a path, and format each component to avoid generating extra + path entries. + + For example all "/", "\", and ":" characters will be replaced with + "_". Other characters like "=" will also be replaced. + """ + return os.path.join(*[polite_filename(x) for x in components]) + + +@memoized +def _polite_antipattern(): + # A regex of all the characters we don't want in a filename + return re.compile(r"[^A-Za-z0-9_.-]") + + +def polite_filename(filename: str) -> str: + """ + Replace generally problematic filename characters with underscores. + + This differs from sanitize_filename in that it is more aggressive in + changing characters in the name. For example it removes "=" which can + confuse path parsing in external tools. + """ + # This character set applies for both Windows and Linux. It does not + # account for reserved filenames in Windows. + return _polite_antipattern().sub("_", filename) + + def getuid(): if sys.platform == "win32": import ctypes @@ -169,7 +204,7 @@ def rename(src, dst): if sys.platform == "win32": # Windows path existence checks will sometimes fail on junctions/links/symlinks # so check for that case - if os.path.exists(dst) or os.path.islink(dst): + if os.path.exists(dst) or islink(dst): os.remove(dst) os.rename(src, dst) @@ -333,8 +368,7 @@ def groupid_to_group(x): if string: regex = re.escape(regex) - filenames = path_to_os_path(*filenames) - for filename in filenames: + for filename in path_to_os_path(*filenames): msg = 'FILTER FILE: {0} [replacing "{1}"]' tty.debug(msg.format(filename, regex)) @@ -566,7 +600,7 @@ def set_install_permissions(path): # If this points to a file maintained in a Spack prefix, it is assumed that # this function will be invoked on the target. If the file is outside a # Spack-maintained prefix, the permissions should not be modified. - if os.path.islink(path): + if islink(path): return if os.path.isdir(path): os.chmod(path, 0o755) @@ -635,7 +669,7 @@ def chmod_x(entry, perms): @system_path_filter def copy_mode(src, dest): """Set the mode of dest to that of src unless it is a link.""" - if os.path.islink(dest): + if islink(dest): return src_mode = os.stat(src).st_mode dest_mode = os.stat(dest).st_mode @@ -721,26 +755,12 @@ def install(src, dest): copy(src, dest, _permissions=True) -@system_path_filter -def resolve_link_target_relative_to_the_link(link): - """ - os.path.isdir uses os.path.exists, which for links will check - the existence of the link target. If the link target is relative to - the link, we need to construct a pathname that is valid from - our cwd (which may not be the same as the link's directory) - """ - target = os.readlink(link) - if os.path.isabs(target): - return target - link_dir = os.path.dirname(os.path.abspath(link)) - return os.path.join(link_dir, target) - - @system_path_filter def copy_tree( src: str, dest: str, symlinks: bool = True, + allow_broken_symlinks: bool = sys.platform != "win32", ignore: Optional[Callable[[str], bool]] = None, _permissions: bool = False, ): @@ -763,6 +783,8 @@ def copy_tree( src (str): the directory to copy dest (str): the destination directory symlinks (bool): whether or not to preserve symlinks + allow_broken_symlinks (bool): whether or not to allow broken (dangling) symlinks, + On Windows, setting this to True will raise an exception. Defaults to true on unix. ignore (typing.Callable): function indicating which files to ignore _permissions (bool): for internal use only @@ -770,6 +792,8 @@ def copy_tree( IOError: if *src* does not match any files or directories ValueError: if *src* is a parent directory of *dest* """ + if allow_broken_symlinks and sys.platform == "win32": + raise llnl.util.symlink.SymlinkError("Cannot allow broken symlinks on Windows!") if _permissions: tty.debug("Installing {0} to {1}".format(src, dest)) else: @@ -783,6 +807,11 @@ def copy_tree( if not files: raise IOError("No such file or directory: '{0}'".format(src)) + # For Windows hard-links and junctions, the source path must exist to make a symlink. Add + # all symlinks to this list while traversing the tree, then when finished, make all + # symlinks at the end. + links = [] + for src in files: abs_src = os.path.abspath(src) if not abs_src.endswith(os.path.sep): @@ -805,7 +834,7 @@ def copy_tree( ignore=ignore, follow_nonexisting=True, ): - if os.path.islink(s): + if islink(s): link_target = resolve_link_target_relative_to_the_link(s) if symlinks: target = os.readlink(s) @@ -819,7 +848,9 @@ def escaped_path(path): tty.debug("Redirecting link {0} to {1}".format(target, new_target)) target = new_target - symlink(target, d) + links.append((target, d, s)) + continue + elif os.path.isdir(link_target): mkdirp(d) else: @@ -834,9 +865,17 @@ def escaped_path(path): set_install_permissions(d) copy_mode(s, d) + for target, d, s in links: + symlink(target, d, allow_broken_symlinks=allow_broken_symlinks) + if _permissions: + set_install_permissions(d) + copy_mode(s, d) + @system_path_filter -def install_tree(src, dest, symlinks=True, ignore=None): +def install_tree( + src, dest, symlinks=True, ignore=None, allow_broken_symlinks=sys.platform != "win32" +): """Recursively install an entire directory tree rooted at *src*. Same as :py:func:`copy_tree` with the addition of setting proper @@ -847,12 +886,21 @@ def install_tree(src, dest, symlinks=True, ignore=None): dest (str): the destination directory symlinks (bool): whether or not to preserve symlinks ignore (typing.Callable): function indicating which files to ignore + allow_broken_symlinks (bool): whether or not to allow broken (dangling) symlinks, + On Windows, setting this to True will raise an exception. Raises: IOError: if *src* does not match any files or directories ValueError: if *src* is a parent directory of *dest* """ - copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True) + copy_tree( + src, + dest, + symlinks=symlinks, + allow_broken_symlinks=allow_broken_symlinks, + ignore=ignore, + _permissions=True, + ) @system_path_filter @@ -1256,7 +1304,12 @@ def traverse_tree( Keyword Arguments: order (str): Whether to do pre- or post-order traversal. Accepted values are 'pre' and 'post' - ignore (typing.Callable): function indicating which files to ignore + ignore (typing.Callable): function indicating which files to ignore. This will also + ignore symlinks if they point to an ignored file (regardless of whether the symlink + is explicitly ignored); note this only supports one layer of indirection (i.e. if + you have x -> y -> z, and z is ignored but x/y are not, then y would be ignored + but not x). To avoid this, make sure the ignore function also ignores the symlink + paths too. follow_nonexisting (bool): Whether to descend into directories in ``src`` that do not exit in ``dest``. Default is True follow_links (bool): Whether to descend into symlinks in ``src`` @@ -1283,11 +1336,24 @@ def traverse_tree( dest_child = os.path.join(dest_path, f) rel_child = os.path.join(rel_path, f) + # If the source path is a link and the link's source is ignored, then ignore the link too, + # but only do this if the ignore is defined. + if ignore is not None: + if islink(source_child) and not follow_links: + target = readlink(source_child) + all_parents = accumulate(target.split(os.sep), lambda x, y: os.path.join(x, y)) + if any(map(ignore, all_parents)): + tty.warn( + f"Skipping {source_path} because the source or a part of the source's " + f"path is included in the ignores." + ) + continue + # Treat as a directory # TODO: for symlinks, os.path.isdir looks for the link target. If the # target is relative to the link, then that may not resolve properly # relative to our cwd - see resolve_link_target_relative_to_the_link - if os.path.isdir(source_child) and (follow_links or not os.path.islink(source_child)): + if os.path.isdir(source_child) and (follow_links or not islink(source_child)): # When follow_nonexisting isn't set, don't descend into dirs # in source that do not exist in dest if follow_nonexisting or os.path.exists(dest_child): @@ -1313,7 +1379,11 @@ def traverse_tree( def lexists_islink_isdir(path): """Computes the tuple (lexists(path), islink(path), isdir(path)) in a minimal - number of stat calls.""" + number of stat calls on unix. Use os.path and symlink.islink methods for windows.""" + if sys.platform == "win32": + if not os.path.lexists(path): + return False, False, False + return os.path.lexists(path), islink(path), os.path.isdir(path) # First try to lstat, so we know if it's a link or not. try: lst = os.lstat(path) @@ -1528,7 +1598,7 @@ def remove_if_dead_link(path): Parameters: path (str): The potential dead link """ - if os.path.islink(path) and not os.path.exists(path): + if islink(path) and not os.path.exists(path): os.unlink(path) @@ -1587,7 +1657,7 @@ def remove_linked_tree(path): kwargs["onerror"] = readonly_file_handler(ignore_errors=True) if os.path.exists(path): - if os.path.islink(path): + if islink(path): shutil.rmtree(os.path.realpath(path), **kwargs) os.unlink(path) else: @@ -1754,9 +1824,14 @@ def find(root, files, recursive=True): files = [files] if recursive: - return _find_recursive(root, files) + tty.debug(f"Find (recursive): {root} {str(files)}") + result = _find_recursive(root, files) else: - return _find_non_recursive(root, files) + tty.debug(f"Find (not recursive): {root} {str(files)}") + result = _find_non_recursive(root, files) + + tty.debug(f"Find complete: {root} {str(files)}") + return result @system_path_filter @@ -2383,7 +2458,7 @@ def library_dependents(self): """ Set of directories where package binaries/libraries are located. """ - return set([self.pkg.prefix.bin]) | self._additional_library_dependents + return set([pathlib.Path(self.pkg.prefix.bin)]) | self._additional_library_dependents def add_library_dependent(self, *dest): """ @@ -2396,9 +2471,9 @@ def add_library_dependent(self, *dest): """ for pth in dest: if os.path.isfile(pth): - self._additional_library_dependents.add(os.path.dirname) + self._additional_library_dependents.add(pathlib.Path(pth).parent) else: - self._additional_library_dependents.add(pth) + self._additional_library_dependents.add(pathlib.Path(pth)) @property def rpaths(self): @@ -2411,7 +2486,7 @@ def rpaths(self): dependent_libs.extend(list(find_all_shared_libraries(path, recursive=True))) for extra_path in self._addl_rpaths: dependent_libs.extend(list(find_all_shared_libraries(extra_path, recursive=True))) - return set(dependent_libs) + return set([pathlib.Path(x) for x in dependent_libs]) def add_rpath(self, *paths): """ @@ -2427,7 +2502,7 @@ def add_rpath(self, *paths): """ self._addl_rpaths = self._addl_rpaths | set(paths) - def _link(self, path, dest_dir): + def _link(self, path: pathlib.Path, dest_dir: pathlib.Path): """Perform link step of simulated rpathing, installing simlinks of file in path to the dest_dir location. This method deliberately prevents @@ -2435,27 +2510,35 @@ def _link(self, path, dest_dir): This is because it is both meaningless from an rpath perspective, and will cause an error when Developer mode is not enabled""" - file_name = os.path.basename(path) - dest_file = os.path.join(dest_dir, file_name) - if os.path.exists(dest_dir) and not dest_file == path: + + def report_already_linked(): + # We have either already symlinked or we are encoutering a naming clash + # either way, we don't want to overwrite existing libraries + already_linked = islink(str(dest_file)) + tty.debug( + "Linking library %s to %s failed, " % (str(path), str(dest_file)) + + "already linked." + if already_linked + else "library with name %s already exists at location %s." + % (str(file_name), str(dest_dir)) + ) + + file_name = path.name + dest_file = dest_dir / file_name + if not dest_file.exists() and dest_dir.exists() and not dest_file == path: try: - symlink(path, dest_file) + symlink(str(path), str(dest_file)) # For py2 compatibility, we have to catch the specific Windows error code # associate with trying to create a file that already exists (winerror 183) + # Catch OSErrors missed by the SymlinkError checks except OSError as e: if sys.platform == "win32" and (e.winerror == 183 or e.errno == errno.EEXIST): - # We have either already symlinked or we are encoutering a naming clash - # either way, we don't want to overwrite existing libraries - already_linked = islink(dest_file) - tty.debug( - "Linking library %s to %s failed, " % (path, dest_file) + "already linked." - if already_linked - else "library with name %s already exists at location %s." - % (file_name, dest_dir) - ) - pass + report_already_linked() else: raise e + # catch errors we raise ourselves from Spack + except llnl.util.symlink.AlreadyExistsError: + report_already_linked() def establish_link(self): """ @@ -2688,7 +2771,7 @@ def remove_directory_contents(dir): """Remove all contents of a directory.""" if os.path.exists(dir): for entry in [os.path.join(dir, entry) for entry in os.listdir(dir)]: - if os.path.isfile(entry) or os.path.islink(entry): + if os.path.isfile(entry) or islink(entry): os.unlink(entry) else: shutil.rmtree(entry) diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index ffee4443dfe72a..607b093de8b406 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -821,7 +821,7 @@ def __getattr__(self, name): # 'instance'/'_instance' to be defined or it will enter an infinite # loop, so protect against that here. if name in ["_instance", "instance"]: - raise AttributeError() + raise AttributeError(f"cannot create {name}") return getattr(self.instance, name) def __getitem__(self, name): @@ -843,27 +843,6 @@ def __repr__(self): return repr(self.instance) -class LazyReference: - """Lazily evaluated reference to part of a singleton.""" - - def __init__(self, ref_function): - self.ref_function = ref_function - - def __getattr__(self, name): - if name == "ref_function": - raise AttributeError() - return getattr(self.ref_function(), name) - - def __getitem__(self, name): - return self.ref_function()[name] - - def __str__(self): - return str(self.ref_function()) - - def __repr__(self): - return repr(self.ref_function()) - - def load_module_from_file(module_name, module_path): """Loads a python module from the path of the corresponding file. diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index 2b9d2dfbf2decb..156300b891b528 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -9,11 +9,12 @@ import sys import time from datetime import datetime +from types import TracebackType +from typing import IO, Any, Callable, ContextManager, Dict, Generator, Optional, Tuple, Type, Union -import llnl.util.tty as tty -from llnl.util.lang import pretty_seconds +from llnl.util import lang, tty -import spack.util.string +from ..string import plural if sys.platform != "win32": import fcntl @@ -34,9 +35,12 @@ ] -#: A useful replacement for functions that should return True when not provided -#: for example. -true_fn = lambda: True +ReleaseFnType = Optional[Callable[[], bool]] + + +def true_fn() -> bool: + """A function that always returns True.""" + return True class OpenFile: @@ -48,7 +52,7 @@ class OpenFile: file descriptors as well in the future. """ - def __init__(self, fh): + def __init__(self, fh: IO) -> None: self.fh = fh self.refs = 0 @@ -78,11 +82,11 @@ class OpenFileTracker: work in Python and assume the GIL. """ - def __init__(self): + def __init__(self) -> None: """Create a new ``OpenFileTracker``.""" - self._descriptors = {} + self._descriptors: Dict[Any, OpenFile] = {} - def get_fh(self, path): + def get_fh(self, path: str) -> IO: """Get a filehandle for a lockfile. This routine will open writable files for read/write even if you're asking @@ -90,7 +94,7 @@ def get_fh(self, path): (write) lock later if requested. Arguments: - path (str): path to lock file we want a filehandle for + path: path to lock file we want a filehandle for """ # Open writable files as 'r+' so we can upgrade to write later os_mode, fh_mode = (os.O_RDWR | os.O_CREAT), "r+" @@ -139,7 +143,7 @@ def get_fh(self, path): def release_by_stat(self, stat): key = (stat.st_dev, stat.st_ino, os.getpid()) open_file = self._descriptors.get(key) - assert open_file, "Attempted to close non-existing inode: %s" % stat.st_inode + assert open_file, "Attempted to close non-existing inode: %s" % stat.st_ino open_file.refs -= 1 if not open_file.refs: @@ -157,7 +161,7 @@ def purge(self): #: Open file descriptors for locks in this process. Used to prevent one process #: from opening the sam file many times for different byte range locks -file_tracker = OpenFileTracker() +FILE_TRACKER = OpenFileTracker() def _attempts_str(wait_time, nattempts): @@ -165,8 +169,8 @@ def _attempts_str(wait_time, nattempts): if nattempts <= 1: return "" - attempts = spack.util.string.plural(nattempts, "attempt") - return " after {} and {}".format(pretty_seconds(wait_time), attempts) + attempts = plural(nattempts, "attempt") + return " after {} and {}".format(lang.pretty_seconds(wait_time), attempts) class LockType: @@ -188,7 +192,7 @@ def to_module(tid): return lock @staticmethod - def is_valid(op): + def is_valid(op: int) -> bool: return op == LockType.READ or op == LockType.WRITE @@ -207,7 +211,16 @@ class Lock: overlapping byte ranges in the same file). """ - def __init__(self, path, start=0, length=0, default_timeout=None, debug=False, desc=""): + def __init__( + self, + path: str, + *, + start: int = 0, + length: int = 0, + default_timeout: Optional[float] = None, + debug: bool = False, + desc: str = "", + ) -> None: """Construct a new lock on the file at ``path``. By default, the lock applies to the whole file. Optionally, @@ -220,17 +233,17 @@ def __init__(self, path, start=0, length=0, default_timeout=None, debug=False, d beginning of the file. Args: - path (str): path to the lock - start (int): optional byte offset at which the lock starts - length (int): optional number of bytes to lock - default_timeout (int): number of seconds to wait for lock attempts, + path: path to the lock + start: optional byte offset at which the lock starts + length: optional number of bytes to lock + default_timeout: seconds to wait for lock attempts, where None means to wait indefinitely - debug (bool): debug mode specific to locking - desc (str): optional debug message lock description, which is + debug: debug mode specific to locking + desc: optional debug message lock description, which is helpful for distinguishing between different Spack locks. """ self.path = path - self._file = None + self._file: Optional[IO] = None self._reads = 0 self._writes = 0 @@ -242,7 +255,7 @@ def __init__(self, path, start=0, length=0, default_timeout=None, debug=False, d self.debug = debug # optional debug description - self.desc = " ({0})".format(desc) if desc else "" + self.desc = f" ({desc})" if desc else "" # If the user doesn't set a default timeout, or if they choose # None, 0, etc. then lock attempts will not time out (unless the @@ -250,11 +263,15 @@ def __init__(self, path, start=0, length=0, default_timeout=None, debug=False, d self.default_timeout = default_timeout or None # PID and host of lock holder (only used in debug mode) - self.pid = self.old_pid = None - self.host = self.old_host = None + self.pid: Optional[int] = None + self.old_pid: Optional[int] = None + self.host: Optional[str] = None + self.old_host: Optional[str] = None @staticmethod - def _poll_interval_generator(_wait_times=None): + def _poll_interval_generator( + _wait_times: Optional[Tuple[float, float, float]] = None + ) -> Generator[float, None, None]: """This implements a backoff scheme for polling a contended resource by suggesting a succession of wait times between polls. @@ -277,21 +294,21 @@ def _poll_interval_generator(_wait_times=None): num_requests += 1 yield wait_time - def __repr__(self): + def __repr__(self) -> str: """Formal representation of the lock.""" rep = "{0}(".format(self.__class__.__name__) for attr, value in self.__dict__.items(): rep += "{0}={1}, ".format(attr, value.__repr__()) return "{0})".format(rep.strip(", ")) - def __str__(self): + def __str__(self) -> str: """Readable string (with key fields) of the lock.""" location = "{0}[{1}:{2}]".format(self.path, self._start, self._length) timeout = "timeout={0}".format(self.default_timeout) activity = "#reads={0}, #writes={1}".format(self._reads, self._writes) return "({0}, {1}, {2})".format(location, timeout, activity) - def _lock(self, op, timeout=None): + def _lock(self, op: int, timeout: Optional[float] = None) -> Tuple[float, int]: """This takes a lock using POSIX locks (``fcntl.lockf``). The lock is implemented as a spin lock using a nonblocking call @@ -310,7 +327,7 @@ def _lock(self, op, timeout=None): # Create file and parent directories if they don't exist. if self._file is None: self._ensure_parent_directory() - self._file = file_tracker.get_fh(self.path) + self._file = FILE_TRACKER.get_fh(self.path) if LockType.to_module(op) == fcntl.LOCK_EX and self._file.mode == "r": # Attempt to upgrade to write lock w/a read-only file. @@ -319,7 +336,7 @@ def _lock(self, op, timeout=None): self._log_debug( "{} locking [{}:{}]: timeout {}".format( - op_str.lower(), self._start, self._length, pretty_seconds(timeout or 0) + op_str.lower(), self._start, self._length, lang.pretty_seconds(timeout or 0) ) ) @@ -343,15 +360,20 @@ def _lock(self, op, timeout=None): total_wait_time = time.time() - start_time raise LockTimeoutError(op_str.lower(), self.path, total_wait_time, num_attempts) - def _poll_lock(self, op): + def _poll_lock(self, op: int) -> bool: """Attempt to acquire the lock in a non-blocking manner. Return whether the locking attempt succeeds """ + assert self._file is not None, "cannot poll a lock without the file being set" module_op = LockType.to_module(op) try: # Try to get the lock (will raise if not available.) fcntl.lockf( - self._file, module_op | fcntl.LOCK_NB, self._length, self._start, os.SEEK_SET + self._file.fileno(), + module_op | fcntl.LOCK_NB, + self._length, + self._start, + os.SEEK_SET, ) # help for debugging distributed locking @@ -377,7 +399,7 @@ def _poll_lock(self, op): return False - def _ensure_parent_directory(self): + def _ensure_parent_directory(self) -> str: parent = os.path.dirname(self.path) # relative paths to lockfiles in the current directory have no parent @@ -396,20 +418,22 @@ def _ensure_parent_directory(self): raise return parent - def _read_log_debug_data(self): + def _read_log_debug_data(self) -> None: """Read PID and host data out of the file if it is there.""" + assert self._file is not None, "cannot read debug log without the file being set" self.old_pid = self.pid self.old_host = self.host line = self._file.read() if line: pid, host = line.strip().split(",") - _, _, self.pid = pid.rpartition("=") + _, _, pid = pid.rpartition("=") _, _, self.host = host.rpartition("=") - self.pid = int(self.pid) + self.pid = int(pid) - def _write_log_debug_data(self): + def _write_log_debug_data(self) -> None: """Write PID and host data to the file, recording old values.""" + assert self._file is not None, "cannot write debug log without the file being set" self.old_pid = self.pid self.old_host = self.host @@ -423,20 +447,21 @@ def _write_log_debug_data(self): self._file.flush() os.fsync(self._file.fileno()) - def _unlock(self): + def _unlock(self) -> None: """Releases a lock using POSIX locks (``fcntl.lockf``) Releases the lock regardless of mode. Note that read locks may be masquerading as write locks, but this removes either. """ - fcntl.lockf(self._file, fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET) - file_tracker.release_by_fh(self._file) + assert self._file is not None, "cannot unlock without the file being set" + fcntl.lockf(self._file.fileno(), fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET) + FILE_TRACKER.release_by_fh(self._file) self._file = None self._reads = 0 self._writes = 0 - def acquire_read(self, timeout=None): + def acquire_read(self, timeout: Optional[float] = None) -> bool: """Acquires a recursive, shared lock for reading. Read and write locks can be acquired and released in arbitrary @@ -461,7 +486,7 @@ def acquire_read(self, timeout=None): self._reads += 1 return False - def acquire_write(self, timeout=None): + def acquire_write(self, timeout: Optional[float] = None) -> bool: """Acquires a recursive, exclusive lock for writing. Read and write locks can be acquired and released in arbitrary @@ -491,7 +516,7 @@ def acquire_write(self, timeout=None): self._writes += 1 return False - def is_write_locked(self): + def is_write_locked(self) -> bool: """Check if the file is write locked Return: @@ -508,7 +533,7 @@ def is_write_locked(self): return False - def downgrade_write_to_read(self, timeout=None): + def downgrade_write_to_read(self, timeout: Optional[float] = None) -> None: """ Downgrade from an exclusive write lock to a shared read. @@ -527,7 +552,7 @@ def downgrade_write_to_read(self, timeout=None): else: raise LockDowngradeError(self.path) - def upgrade_read_to_write(self, timeout=None): + def upgrade_read_to_write(self, timeout: Optional[float] = None) -> None: """ Attempts to upgrade from a shared read lock to an exclusive write. @@ -546,7 +571,7 @@ def upgrade_read_to_write(self, timeout=None): else: raise LockUpgradeError(self.path) - def release_read(self, release_fn=None): + def release_read(self, release_fn: ReleaseFnType = None) -> bool: """Releases a read lock. Arguments: @@ -582,7 +607,7 @@ def release_read(self, release_fn=None): self._reads -= 1 return False - def release_write(self, release_fn=None): + def release_write(self, release_fn: ReleaseFnType = None) -> bool: """Releases a write lock. Arguments: @@ -623,58 +648,58 @@ def release_write(self, release_fn=None): else: return False - def cleanup(self): + def cleanup(self) -> None: if self._reads == 0 and self._writes == 0: os.unlink(self.path) else: raise LockError("Attempting to cleanup active lock.") - def _get_counts_desc(self): + def _get_counts_desc(self) -> str: return ( "(reads {0}, writes {1})".format(self._reads, self._writes) if tty.is_verbose() else "" ) - def _log_acquired(self, locktype, wait_time, nattempts): + def _log_acquired(self, locktype, wait_time, nattempts) -> None: attempts_part = _attempts_str(wait_time, nattempts) now = datetime.now() desc = "Acquired at %s" % now.strftime("%H:%M:%S.%f") self._log_debug(self._status_msg(locktype, "{0}{1}".format(desc, attempts_part))) - def _log_acquiring(self, locktype): + def _log_acquiring(self, locktype) -> None: self._log_debug(self._status_msg(locktype, "Acquiring"), level=3) - def _log_debug(self, *args, **kwargs): + def _log_debug(self, *args, **kwargs) -> None: """Output lock debug messages.""" kwargs["level"] = kwargs.get("level", 2) tty.debug(*args, **kwargs) - def _log_downgraded(self, wait_time, nattempts): + def _log_downgraded(self, wait_time, nattempts) -> None: attempts_part = _attempts_str(wait_time, nattempts) now = datetime.now() desc = "Downgraded at %s" % now.strftime("%H:%M:%S.%f") self._log_debug(self._status_msg("READ LOCK", "{0}{1}".format(desc, attempts_part))) - def _log_downgrading(self): + def _log_downgrading(self) -> None: self._log_debug(self._status_msg("WRITE LOCK", "Downgrading"), level=3) - def _log_released(self, locktype): + def _log_released(self, locktype) -> None: now = datetime.now() desc = "Released at %s" % now.strftime("%H:%M:%S.%f") self._log_debug(self._status_msg(locktype, desc)) - def _log_releasing(self, locktype): + def _log_releasing(self, locktype) -> None: self._log_debug(self._status_msg(locktype, "Releasing"), level=3) - def _log_upgraded(self, wait_time, nattempts): + def _log_upgraded(self, wait_time, nattempts) -> None: attempts_part = _attempts_str(wait_time, nattempts) now = datetime.now() desc = "Upgraded at %s" % now.strftime("%H:%M:%S.%f") self._log_debug(self._status_msg("WRITE LOCK", "{0}{1}".format(desc, attempts_part))) - def _log_upgrading(self): + def _log_upgrading(self) -> None: self._log_debug(self._status_msg("READ LOCK", "Upgrading"), level=3) - def _status_msg(self, locktype, status): + def _status_msg(self, locktype: str, status: str) -> str: status_desc = "[{0}] {1}".format(status, self._get_counts_desc()) return "{0}{1.desc}: {1.path}[{1._start}:{1._length}] {2}".format( locktype, self, status_desc @@ -709,7 +734,13 @@ class LockTransaction: """ - def __init__(self, lock, acquire=None, release=None, timeout=None): + def __init__( + self, + lock: Lock, + acquire: Union[ReleaseFnType, ContextManager] = None, + release: Union[ReleaseFnType, ContextManager] = None, + timeout: Optional[float] = None, + ) -> None: self._lock = lock self._timeout = timeout self._acquire_fn = acquire @@ -724,15 +755,20 @@ def __enter__(self): else: return self._as - def __exit__(self, type, value, traceback): + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> bool: suppress = False def release_fn(): if self._release_fn is not None: - return self._release_fn(type, value, traceback) + return self._release_fn(exc_type, exc_value, traceback) if self._as and hasattr(self._as, "__exit__"): - if self._as.__exit__(type, value, traceback): + if self._as.__exit__(exc_type, exc_value, traceback): suppress = True if self._exit(release_fn): @@ -740,6 +776,12 @@ def release_fn(): return suppress + def _enter(self) -> bool: + return NotImplemented + + def _exit(self, release_fn: ReleaseFnType) -> bool: + return NotImplemented + class ReadTransaction(LockTransaction): """LockTransaction context manager that does a read and releases it.""" @@ -785,7 +827,7 @@ def __init__(self, lock_type, path, time, attempts): super().__init__( fmt.format( lock_type, - pretty_seconds(time), + lang.pretty_seconds(time), attempts, "attempt" if attempts == 1 else "attempts", path, diff --git a/lib/spack/llnl/util/symlink.py b/lib/spack/llnl/util/symlink.py index 69aacaf9f0bc26..4f4f965f139e59 100644 --- a/lib/spack/llnl/util/symlink.py +++ b/lib/spack/llnl/util/symlink.py @@ -2,77 +2,189 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import errno import os +import re import shutil +import subprocess import sys import tempfile -from os.path import exists, join -from llnl.util import lang +from llnl.util import lang, tty + +from ..path import system_path_filter if sys.platform == "win32": from win32file import CreateHardLink +is_windows = sys.platform == "win32" + -def symlink(real_path, link_path): +def symlink(source_path: str, link_path: str, allow_broken_symlinks: bool = not is_windows): """ - Create a symbolic link. + Create a link. + + On non-Windows and Windows with System Administrator + privleges this will be a normal symbolic link via + os.symlink. + + On Windows without privledges the link will be a + junction for a directory and a hardlink for a file. + On Windows the various link types are: + + Symbolic Link: A link to a file or directory on the + same or different volume (drive letter) or even to + a remote file or directory (using UNC in its path). + Need System Administrator privileges to make these. - On Windows, use junctions if os.symlink fails. + Hard Link: A link to a file on the same volume (drive + letter) only. Every file (file's data) has at least 1 + hard link (file's name). But when this method creates + a new hard link there will be 2. Deleting all hard + links effectively deletes the file. Don't need System + Administrator privileges. + + Junction: A link to a directory on the same or different + volume (drive letter) but not to a remote directory. Don't + need System Administrator privileges. + + Parameters: + source_path (str): The real file or directory that the link points to. + Must be absolute OR relative to the link. + link_path (str): The path where the link will exist. + allow_broken_symlinks (bool): On Linux or Mac, don't raise an exception if the source_path + doesn't exist. This will still raise an exception on Windows. """ - if sys.platform != "win32": - os.symlink(real_path, link_path) - elif _win32_can_symlink(): - # Windows requires target_is_directory=True when the target is a dir. - os.symlink(real_path, link_path, target_is_directory=os.path.isdir(real_path)) - else: - try: - # Try to use junctions - _win32_junction(real_path, link_path) - except OSError as e: - if e.errno == errno.EEXIST: - # EEXIST error indicates that file we're trying to "link" - # is already present, don't bother trying to copy which will also fail - # just raise - raise + source_path = os.path.normpath(source_path) + win_source_path = source_path + link_path = os.path.normpath(link_path) + + # Never allow broken links on Windows. + if sys.platform == "win32" and allow_broken_symlinks: + raise ValueError("allow_broken_symlinks parameter cannot be True on Windows.") + + if not allow_broken_symlinks: + # Perform basic checks to make sure symlinking will succeed + if os.path.lexists(link_path): + raise AlreadyExistsError( + f"Link path ({link_path}) already exists. Cannot create link." + ) + + if not os.path.exists(source_path): + if os.path.isabs(source_path) and not allow_broken_symlinks: + # An absolute source path that does not exist will result in a broken link. + raise SymlinkError( + f"Source path ({source_path}) is absolute but does not exist. Resulting " + f"link would be broken so not making link." + ) else: - # If all else fails, fall back to copying files - shutil.copyfile(real_path, link_path) + # os.symlink can create a link when the given source path is relative to + # the link path. Emulate this behavior and check to see if the source exists + # relative to the link path ahead of link creation to prevent broken + # links from being made. + link_parent_dir = os.path.dirname(link_path) + relative_path = os.path.join(link_parent_dir, source_path) + if os.path.exists(relative_path): + # In order to work on windows, the source path needs to be modified to be + # relative because hardlink/junction dont resolve relative paths the same + # way as os.symlink. This is ignored on other operating systems. + win_source_path = relative_path + elif not allow_broken_symlinks: + raise SymlinkError( + f"The source path ({source_path}) is not relative to the link path " + f"({link_path}). Resulting link would be broken so not making link." + ) + # Create the symlink + if sys.platform == "win32" and not _windows_can_symlink(): + _windows_create_link(win_source_path, link_path) + else: + os.symlink(source_path, link_path, target_is_directory=os.path.isdir(source_path)) -def islink(path): - return os.path.islink(path) or _win32_is_junction(path) +def islink(path: str) -> bool: + """Override os.islink to give correct answer for spack logic. -# '_win32' functions based on -# https://github.com/Erotemic/ubelt/blob/master/ubelt/util_links.py -def _win32_junction(path, link): - # junctions require absolute paths - if not os.path.isabs(link): - link = os.path.abspath(link) + For Non-Windows: a link can be determined with the os.path.islink method. + Windows-only methods will return false for other operating systems. - # os.symlink will fail if link exists, emulate the behavior here - if exists(link): - raise OSError(errno.EEXIST, "File exists: %s -> %s" % (link, path)) + For Windows: spack considers symlinks, hard links, and junctions to + all be links, so if any of those are True, return True. - if not os.path.isabs(path): - parent = os.path.join(link, os.pardir) - path = os.path.join(parent, path) - path = os.path.abspath(path) + Args: + path (str): path to check if it is a link. - CreateHardLink(link, path) + Returns: + bool - whether the path is any kind link or not. + """ + return any([os.path.islink(path), _windows_is_junction(path), _windows_is_hardlink(path)]) + + +def _windows_is_hardlink(path: str) -> bool: + """Determines if a path is a windows hard link. This is accomplished + by looking at the number of links using os.stat. A non-hard-linked file + will have a st_nlink value of 1, whereas a hard link will have a value + larger than 1. Note that both the original and hard-linked file will + return True because they share the same inode. + + Args: + path (str): Windows path to check for a hard link + + Returns: + bool - Whether the path is a hard link or not. + """ + if sys.platform != "win32" or os.path.islink(path) or not os.path.exists(path): + return False + + return os.stat(path).st_nlink > 1 + + +def _windows_is_junction(path: str) -> bool: + """Determines if a path is a windows junction. A junction can be + determined using a bitwise AND operation between the file's + attribute bitmask and the known junction bitmask (0x400). + + Args: + path (str): A non-file path + + Returns: + bool - whether the path is a junction or not. + """ + if sys.platform != "win32" or os.path.islink(path) or os.path.isfile(path): + return False + + import ctypes.wintypes + + get_file_attributes = ctypes.windll.kernel32.GetFileAttributesW # type: ignore[attr-defined] + get_file_attributes.argtypes = (ctypes.wintypes.LPWSTR,) + get_file_attributes.restype = ctypes.wintypes.DWORD + + invalid_file_attributes = 0xFFFFFFFF + reparse_point = 0x400 + file_attr = get_file_attributes(str(path)) + + if file_attr == invalid_file_attributes: + return False + + return file_attr & reparse_point > 0 @lang.memoized -def _win32_can_symlink(): +def _windows_can_symlink() -> bool: + """ + Determines if windows is able to make a symlink depending on + the system configuration and the level of the user's permissions. + """ + if sys.platform != "win32": + tty.warn("windows_can_symlink method can't be used on non-Windows OS.") + return False + tempdir = tempfile.mkdtemp() - dpath = join(tempdir, "dpath") - fpath = join(tempdir, "fpath.txt") + dpath = os.path.join(tempdir, "dpath") + fpath = os.path.join(tempdir, "fpath.txt") - dlink = join(tempdir, "dlink") - flink = join(tempdir, "flink.txt") + dlink = os.path.join(tempdir, "dlink") + flink = os.path.join(tempdir, "flink.txt") import llnl.util.filesystem as fs @@ -96,24 +208,140 @@ def _win32_can_symlink(): return can_symlink_directories and can_symlink_files -def _win32_is_junction(path): +def _windows_create_link(source: str, link: str): """ - Determines if a path is a win32 junction + Attempts to create a Hard Link or Junction as an alternative + to a symbolic link. This is called when symbolic links cannot + be created. """ - if os.path.islink(path): - return False + if sys.platform != "win32": + raise SymlinkError("windows_create_link method can't be used on non-Windows OS.") + elif os.path.isdir(source): + _windows_create_junction(source=source, link=link) + elif os.path.isfile(source): + _windows_create_hard_link(path=source, link=link) + else: + raise SymlinkError( + f"Cannot create link from {source}. It is neither a file nor a directory." + ) + + +def _windows_create_junction(source: str, link: str): + """Duly verify that the path and link are eligible to create a junction, + then create the junction. + """ + if sys.platform != "win32": + raise SymlinkError("windows_create_junction method can't be used on non-Windows OS.") + elif not os.path.exists(source): + raise SymlinkError("Source path does not exist, cannot create a junction.") + elif os.path.lexists(link): + raise AlreadyExistsError("Link path already exists, cannot create a junction.") + elif not os.path.isdir(source): + raise SymlinkError("Source path is not a directory, cannot create a junction.") + + import subprocess + + cmd = ["cmd", "/C", "mklink", "/J", link, source] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = proc.communicate() + tty.debug(out.decode()) + if proc.returncode != 0: + err = err.decode() + tty.error(err) + raise SymlinkError("Make junction command returned a non-zero return code.", err) + + +def _windows_create_hard_link(path: str, link: str): + """Duly verify that the path and link are eligible to create a hard + link, then create the hard link. + """ + if sys.platform != "win32": + raise SymlinkError("windows_create_hard_link method can't be used on non-Windows OS.") + elif not os.path.exists(path): + raise SymlinkError(f"File path {path} does not exist. Cannot create hard link.") + elif os.path.lexists(link): + raise AlreadyExistsError(f"Link path ({link}) already exists. Cannot create hard link.") + elif not os.path.isfile(path): + raise SymlinkError(f"File path ({link}) is not a file. Cannot create hard link.") + else: + tty.debug(f"Creating hard link {link} pointing to {path}") + CreateHardLink(link, path) - if sys.platform == "win32": - import ctypes.wintypes - GetFileAttributes = ctypes.windll.kernel32.GetFileAttributesW - GetFileAttributes.argtypes = (ctypes.wintypes.LPWSTR,) - GetFileAttributes.restype = ctypes.wintypes.DWORD +def readlink(path: str): + """Spack utility to override of os.readlink method to work cross platform""" + if _windows_is_hardlink(path): + return _windows_read_hard_link(path) + elif _windows_is_junction(path): + return _windows_read_junction(path) + else: + return os.readlink(path) + + +def _windows_read_hard_link(link: str) -> str: + """Find all of the files that point to the same inode as the link""" + if sys.platform != "win32": + raise SymlinkError("Can't read hard link on non-Windows OS.") + link = os.path.abspath(link) + fsutil_cmd = ["fsutil", "hardlink", "list", link] + proc = subprocess.Popen(fsutil_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + out, err = proc.communicate() + if proc.returncode != 0: + raise SymlinkError(f"An error occurred while reading hard link: {err.decode()}") + + # fsutil response does not include the drive name, so append it back to each linked file. + drive, link_tail = os.path.splitdrive(os.path.abspath(link)) + links = set([os.path.join(drive, p) for p in out.decode().splitlines()]) + links.remove(link) + if len(links) == 1: + return links.pop() + elif len(links) > 1: + # TODO: How best to handle the case where 3 or more paths point to a single inode? + raise SymlinkError(f"Found multiple paths pointing to the same inode {links}") + else: + raise SymlinkError("Cannot determine hard link source path.") + + +def _windows_read_junction(link: str): + """Find the path that a junction points to.""" + if sys.platform != "win32": + raise SymlinkError("Can't read junction on non-Windows OS.") - INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF - FILE_ATTRIBUTE_REPARSE_POINT = 0x400 + link = os.path.abspath(link) + link_basename = os.path.basename(link) + link_parent = os.path.dirname(link) + fsutil_cmd = ["dir", "/a:l", link_parent] + proc = subprocess.Popen(fsutil_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + out, err = proc.communicate() + if proc.returncode != 0: + raise SymlinkError(f"An error occurred while reading junction: {err.decode()}") + matches = re.search(rf"\s+{link_basename} \[(.*)]", out.decode()) + if matches: + return matches.group(1) + else: + raise SymlinkError("Could not find junction path.") + + +@system_path_filter +def resolve_link_target_relative_to_the_link(link): + """ + os.path.isdir uses os.path.exists, which for links will check + the existence of the link target. If the link target is relative to + the link, we need to construct a pathname that is valid from + our cwd (which may not be the same as the link's directory) + """ + target = readlink(link) + if os.path.isabs(target): + return target + link_dir = os.path.dirname(os.path.abspath(link)) + return os.path.join(link_dir, target) + + +class SymlinkError(RuntimeError): + """Exception class for errors raised while creating symlinks, + junctions and hard links + """ - res = GetFileAttributes(path) - return res != INVALID_FILE_ATTRIBUTES and bool(res & FILE_ATTRIBUTE_REPARSE_POINT) - return False +class AlreadyExistsError(SymlinkError): + """Link path already exists.""" diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index 05e81ef615ffda..ec7bd665374c08 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -12,6 +12,7 @@ import traceback from datetime import datetime from sys import platform as _platform +from typing import NoReturn if _platform != "win32": import fcntl @@ -210,6 +211,7 @@ def info(message, *args, **kwargs): stream.write(line + "\n") else: stream.write(indent + _output_filter(str(arg)) + "\n") + stream.flush() def verbose(message, *args, **kwargs): @@ -244,7 +246,7 @@ def warn(message, *args, **kwargs): info("Warning: " + str(message), *args, **kwargs) -def die(message, *args, **kwargs): +def die(message, *args, **kwargs) -> NoReturn: kwargs.setdefault("countback", 4) error(message, *args, **kwargs) sys.exit(1) diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index f352d33ae5f6c6..8f685442a2079f 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -780,7 +780,7 @@ def __enter__(self): raise RuntimeError("file argument must be set by __init__ ") # Open both write and reading on logfile - if type(self.logfile) == io.StringIO: + if isinstance(self.logfile, io.StringIO): self._ioflag = True # cannot have two streams on tempfile, so we must make our own sys.stdout = self.logfile diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 2fd13d0fe38b83..1fd96b7ef56e60 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) #: PEP440 canonical ... string -__version__ = "0.21.0.dev0" +__version__ = "0.21.0" spack_version = __version__ diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index c8111048dd9d95..dd12d6dbafe484 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -8,8 +8,8 @@ from llnl.util.lang import memoized import spack.spec +import spack.version from spack.compilers.clang import Clang -from spack.spec import CompilerSpec from spack.util.executable import Executable, ProcessError @@ -17,7 +17,9 @@ class ABI: """This class provides methods to test ABI compatibility between specs. The current implementation is rather rough and could be improved.""" - def architecture_compatible(self, target, constraint): + def architecture_compatible( + self, target: spack.spec.Spec, constraint: spack.spec.Spec + ) -> bool: """Return true if architecture of target spec is ABI compatible to the architecture of constraint spec. If either the target or constraint specs have no architecture, target is also defined @@ -34,7 +36,7 @@ def _gcc_get_libstdcxx_version(self, version): a compiler's libstdc++ or libgcc_s""" from spack.build_environment import dso_suffix - spec = CompilerSpec("gcc", version) + spec = spack.spec.CompilerSpec("gcc", version) compilers = spack.compilers.compilers_for_spec(spec) if not compilers: return None @@ -77,16 +79,20 @@ def _gcc_compiler_compare(self, pversion, cversion): return False return plib == clib - def _intel_compiler_compare(self, pversion, cversion): + def _intel_compiler_compare( + self, pversion: spack.version.ClosedOpenRange, cversion: spack.version.ClosedOpenRange + ) -> bool: """Returns true iff the intel version pversion and cversion are ABI compatible""" # Test major and minor versions. Ignore build version. - if len(pversion.version) < 2 or len(cversion.version) < 2: - return False - return pversion.version[:2] == cversion.version[:2] + pv = pversion.lo + cv = cversion.lo + return pv.up_to(2) == cv.up_to(2) - def compiler_compatible(self, parent, child, **kwargs): + def compiler_compatible( + self, parent: spack.spec.Spec, child: spack.spec.Spec, loose: bool = False + ) -> bool: """Return true if compilers for parent and child are ABI compatible.""" if not parent.compiler or not child.compiler: return True @@ -95,7 +101,7 @@ def compiler_compatible(self, parent, child, **kwargs): # Different compiler families are assumed ABI incompatible return False - if kwargs.get("loose", False): + if loose: return True # TODO: Can we move the specialized ABI matching stuff @@ -116,9 +122,10 @@ def compiler_compatible(self, parent, child, **kwargs): return True return False - def compatible(self, target, constraint, **kwargs): + def compatible( + self, target: spack.spec.Spec, constraint: spack.spec.Spec, loose: bool = False + ) -> bool: """Returns true if target spec is ABI compatible to constraint spec""" - loosematch = kwargs.get("loose", False) return self.architecture_compatible(target, constraint) and self.compiler_compatible( - target, constraint, loose=loosematch + target, constraint, loose=loose ) diff --git a/lib/spack/spack/audit.py b/lib/spack/spack/audit.py index d8d1875ab77c51..8b13ffc7cf72db 100644 --- a/lib/spack/spack/audit.py +++ b/lib/spack/spack/audit.py @@ -38,10 +38,13 @@ def _search_duplicate_compilers(error_cls): import ast import collections import collections.abc +import glob import inspect import itertools +import pathlib import pickle import re +import warnings from urllib.request import urlopen import llnl.util.lang @@ -286,7 +289,7 @@ def _check_build_test_callbacks(pkgs, error_cls): """Ensure stand-alone test method is not included in build-time callbacks""" errors = [] for pkg_name in pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) test_callbacks = getattr(pkg_cls, "build_time_test_callbacks", None) # TODO (post-34236): "test*"->"test_*" once remove deprecated methods @@ -304,33 +307,48 @@ def _check_build_test_callbacks(pkgs, error_cls): @package_directives def _check_patch_urls(pkgs, error_cls): - """Ensure that patches fetched from GitHub have stable sha256 hashes.""" + """Ensure that patches fetched from GitHub and GitLab have stable sha256 + hashes.""" github_patch_url_re = ( r"^https?://(?:patch-diff\.)?github(?:usercontent)?\.com/" - ".+/.+/(?:commit|pull)/[a-fA-F0-9]*.(?:patch|diff)" + r".+/.+/(?:commit|pull)/[a-fA-F0-9]+\.(?:patch|diff)" + ) + # Only .diff URLs have stable/full hashes: + # https://forum.gitlab.com/t/patches-with-full-index/29313 + gitlab_patch_url_re = ( + r"^https?://(?:.+)?gitlab(?:.+)/" + r".+/.+/-/(?:commit|merge_requests)/[a-fA-F0-9]+\.(?:patch|diff)" ) errors = [] for pkg_name in pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) for condition, patches in pkg_cls.patches.items(): for patch in patches: if not isinstance(patch, spack.patch.UrlPatch): continue - if not re.match(github_patch_url_re, patch.url): - continue - - full_index_arg = "?full_index=1" - if not patch.url.endswith(full_index_arg): - errors.append( - error_cls( - "patch URL in package {0} must end with {1}".format( - pkg_cls.name, full_index_arg - ), - [patch.url], + if re.match(github_patch_url_re, patch.url): + full_index_arg = "?full_index=1" + if not patch.url.endswith(full_index_arg): + errors.append( + error_cls( + "patch URL in package {0} must end with {1}".format( + pkg_cls.name, full_index_arg + ), + [patch.url], + ) + ) + elif re.match(gitlab_patch_url_re, patch.url): + if not patch.url.endswith(".diff"): + errors.append( + error_cls( + "patch URL in package {0} must end with .diff".format( + pkg_cls.name + ), + [patch.url], + ) ) - ) return errors @@ -342,7 +360,7 @@ def _search_for_reserved_attributes_names_in_packages(pkgs, error_cls): errors = [] for pkg_name in pkgs: name_definitions = collections.defaultdict(list) - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) for cls_item in inspect.getmro(pkg_cls): for name in RESERVED_NAMES: @@ -383,7 +401,7 @@ def _ensure_packages_are_pickeleable(pkgs, error_cls): """Ensure that package objects are pickleable""" errors = [] for pkg_name in pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) pkg = pkg_cls(spack.spec.Spec(pkg_name)) try: pickle.dumps(pkg) @@ -424,7 +442,7 @@ def _ensure_all_versions_can_produce_a_fetcher(pkgs, error_cls): """Ensure all versions in a package can produce a fetcher""" errors = [] for pkg_name in pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) pkg = pkg_cls(spack.spec.Spec(pkg_name)) try: spack.fetch_strategy.check_pkg_attributes(pkg) @@ -449,7 +467,7 @@ def _ensure_docstring_and_no_fixme(pkgs, error_cls): ] for pkg_name in pkgs: details = [] - filename = spack.repo.path.filename_for_package_name(pkg_name) + filename = spack.repo.PATH.filename_for_package_name(pkg_name) with open(filename, "r") as package_file: for i, line in enumerate(package_file): pattern = next((r for r in fixme_regexes if r.search(line)), None) @@ -461,7 +479,7 @@ def _ensure_docstring_and_no_fixme(pkgs, error_cls): error_msg = "Package '{}' contains boilerplate that need to be removed" errors.append(error_cls(error_msg.format(pkg_name), details)) - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) if not pkg_cls.__doc__: error_msg = "Package '{}' miss a docstring" errors.append(error_cls(error_msg.format(pkg_name), [])) @@ -474,7 +492,7 @@ def _ensure_all_packages_use_sha256_checksums(pkgs, error_cls): """Ensure no packages use md5 checksums""" errors = [] for pkg_name in pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) if pkg_cls.manual_download: continue @@ -511,7 +529,7 @@ def _ensure_env_methods_are_ported_to_builders(pkgs, error_cls): """Ensure that methods modifying the build environment are ported to builder classes.""" errors = [] for pkg_name in pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) buildsystem_variant, _ = pkg_cls.variants["build_system"] buildsystem_names = [getattr(x, "value", x) for x in buildsystem_variant.values] builder_cls_names = [spack.builder.BUILDER_CLS[x].__name__ for x in buildsystem_names] @@ -538,7 +556,7 @@ def _linting_package_file(pkgs, error_cls): """Check for correctness of links""" errors = [] for pkg_name in pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) # Does the homepage have http, and if so, does https work? if pkg_cls.homepage.startswith("http://"): @@ -562,7 +580,7 @@ def _unknown_variants_in_directives(pkgs, error_cls): """Report unknown or wrong variants in directives for this package""" errors = [] for pkg_name in pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) # Check "conflicts" directive for conflict, triggers in pkg_cls.conflicts.items(): @@ -628,15 +646,15 @@ def _unknown_variants_in_dependencies(pkgs, error_cls): """Report unknown dependencies and wrong variants for dependencies""" errors = [] for pkg_name in pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) - filename = spack.repo.path.filename_for_package_name(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) + filename = spack.repo.PATH.filename_for_package_name(pkg_name) for dependency_name, dependency_data in pkg_cls.dependencies.items(): # No need to analyze virtual packages - if spack.repo.path.is_virtual(dependency_name): + if spack.repo.PATH.is_virtual(dependency_name): continue try: - dependency_pkg_cls = spack.repo.path.get_pkg_class(dependency_name) + dependency_pkg_cls = spack.repo.PATH.get_pkg_class(dependency_name) except spack.repo.UnknownPackageError: # This dependency is completely missing, so report # and continue the analysis @@ -675,7 +693,7 @@ def _ensure_variant_defaults_are_parsable(pkgs, error_cls): """Ensures that variant defaults are present and parsable from cli""" errors = [] for pkg_name in pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) for variant_name, entry in pkg_cls.variants.items(): variant, _ = entry default_is_parsable = ( @@ -709,27 +727,53 @@ def _ensure_variant_defaults_are_parsable(pkgs, error_cls): return errors +@package_directives +def _ensure_variants_have_descriptions(pkgs, error_cls): + """Ensures that all variants have a description.""" + errors = [] + for pkg_name in pkgs: + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) + for variant_name, entry in pkg_cls.variants.items(): + variant, _ = entry + if not variant.description: + error_msg = "Variant '{}' in package '{}' is missing a description" + errors.append(error_cls(error_msg.format(variant_name, pkg_name), [])) + + return errors + + @package_directives def _version_constraints_are_satisfiable_by_some_version_in_repo(pkgs, error_cls): """Report if version constraints used in directives are not satisfiable""" errors = [] for pkg_name in pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) - filename = spack.repo.path.filename_for_package_name(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) + filename = spack.repo.PATH.filename_for_package_name(pkg_name) dependencies_to_check = [] for dependency_name, dependency_data in pkg_cls.dependencies.items(): # Skip virtual dependencies for the time being, check on # their versions can be added later - if spack.repo.path.is_virtual(dependency_name): + if spack.repo.PATH.is_virtual(dependency_name): continue dependencies_to_check.extend([edge.spec for edge in dependency_data.values()]) + host_architecture = spack.spec.ArchSpec.default_arch() for s in dependencies_to_check: dependency_pkg_cls = None try: - dependency_pkg_cls = spack.repo.path.get_pkg_class(s.name) - assert any(v.intersects(s.versions) for v in list(dependency_pkg_cls.versions)) + dependency_pkg_cls = spack.repo.PATH.get_pkg_class(s.name) + # Some packages have hacks that might cause failures on some platform + # Allow to explicitly set conditions to skip version checks in that case + skip_conditions = getattr(dependency_pkg_cls, "skip_version_audit", []) + skip_version_check = False + for condition in skip_conditions: + if host_architecture.satisfies(spack.spec.Spec(condition).architecture): + skip_version_check = True + break + assert skip_version_check or any( + v.intersects(s.versions) for v in list(dependency_pkg_cls.versions) + ) except Exception: summary = ( "{0}: dependency on {1} cannot be satisfied " "by known versions of {1.name}" @@ -761,7 +805,7 @@ def _analyze_variants_in_directive(pkg, constraint, directive, error_cls): except variant_exceptions as e: summary = pkg.name + ': wrong variant in "{0}" directive' summary = summary.format(directive) - filename = spack.repo.path.filename_for_package_name(pkg.name) + filename = spack.repo.PATH.filename_for_package_name(pkg.name) error_msg = str(e).strip() if isinstance(e, KeyError): @@ -772,3 +816,76 @@ def _analyze_variants_in_directive(pkg, constraint, directive, error_cls): errors.append(err) return errors + + +#: Sanity checks on package directives +external_detection = AuditClass( + group="externals", + tag="PKG-EXTERNALS", + description="Sanity checks for external software detection", + kwargs=("pkgs",), +) + + +def packages_with_detection_tests(): + """Return the list of packages with a corresponding detection_test.yaml file.""" + import spack.config + import spack.util.path + + to_be_tested = [] + for current_repo in spack.repo.PATH.repos: + namespace = current_repo.namespace + packages_dir = pathlib.PurePath(current_repo.packages_path) + pattern = packages_dir / "**" / "detection_test.yaml" + pkgs_with_tests = [ + f"{namespace}.{str(pathlib.PurePath(x).parent.name)}" for x in glob.glob(str(pattern)) + ] + to_be_tested.extend(pkgs_with_tests) + + return to_be_tested + + +@external_detection +def _test_detection_by_executable(pkgs, error_cls): + """Test drive external detection for packages""" + import spack.detection + + errors = [] + + # Filter the packages and retain only the ones with detection tests + pkgs_with_tests = packages_with_detection_tests() + selected_pkgs = [] + for current_package in pkgs_with_tests: + _, unqualified_name = spack.repo.partition_package_name(current_package) + # Check for both unqualified name and qualified name + if unqualified_name in pkgs or current_package in pkgs: + selected_pkgs.append(current_package) + selected_pkgs.sort() + + if not selected_pkgs: + summary = "No detection test to run" + details = [f' "{p}" has no detection test' for p in pkgs] + warnings.warn("\n".join([summary] + details)) + return errors + + for pkg_name in selected_pkgs: + for idx, test_runner in enumerate( + spack.detection.detection_tests(pkg_name, spack.repo.PATH) + ): + specs = test_runner.execute() + expected_specs = test_runner.expected_specs + + not_detected = set(expected_specs) - set(specs) + if not_detected: + summary = pkg_name + ": cannot detect some specs" + details = [f'"{s}" was not detected [test_id={idx}]' for s in sorted(not_detected)] + errors.append(error_cls(summary=summary, details=details)) + + not_expected = set(specs) - set(expected_specs) + if not_expected: + summary = pkg_name + ": detected unexpected specs" + msg = '"{0}" was detected, but was not expected [test_id={1}]' + details = [msg.format(s, idx) for s in sorted(not_expected)] + errors.append(error_cls(summary=summary, details=details)) + + return errors diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 53884a2912e50f..8cfb891640026c 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -5,12 +5,13 @@ import codecs import collections +import errno import hashlib import io import itertools import json -import multiprocessing.pool import os +import pathlib import re import shutil import sys @@ -24,7 +25,7 @@ import warnings from contextlib import closing, contextmanager from gzip import GzipFile -from typing import List, NamedTuple, Optional, Union +from typing import Dict, List, NamedTuple, Optional, Set, Tuple from urllib.error import HTTPError, URLError import llnl.util.filesystem as fsys @@ -32,12 +33,17 @@ import llnl.util.tty as tty from llnl.util.filesystem import BaseDirectoryVisitor, mkdirp, visit_directory_tree +import spack.caches import spack.cmd import spack.config as config import spack.database as spack_db +import spack.error import spack.hooks import spack.hooks.sbang import spack.mirror +import spack.oci.image +import spack.oci.oci +import spack.oci.opener import spack.platforms import spack.relocate as relocate import spack.repo @@ -47,18 +53,38 @@ import spack.util.crypto import spack.util.file_cache as file_cache import spack.util.gpg +import spack.util.path import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml +import spack.util.timer as timer import spack.util.url as url_util import spack.util.web as web_util from spack.caches import misc_cache_location +from spack.package_prefs import get_package_dir_permissions, get_package_group from spack.relocate_text import utf8_paths_to_single_binary_regex from spack.spec import Spec from spack.stage import Stage from spack.util.executable import which -_build_cache_relative_path = "build_cache" -_build_cache_keys_relative_path = "_pgp" +BUILD_CACHE_RELATIVE_PATH = "build_cache" +BUILD_CACHE_KEYS_RELATIVE_PATH = "_pgp" +CURRENT_BUILD_CACHE_LAYOUT_VERSION = 1 + + +class BuildCacheDatabase(spack_db.Database): + """A database for binary buildcaches. + + A database supports writing buildcache index files, in which case certain fields are not + needed in each install record, and no locking is required. To use this feature, it provides + ``lock_cfg=NO_LOCK``, and override the list of ``record_fields``. + """ + + record_fields = ("spec", "ref_count", "in_buildcache") + + def __init__(self, root): + super().__init__(root, lock_cfg=spack_db.NO_LOCK) + self._write_transaction_impl = llnl.util.lang.nullcontext + self._read_transaction_impl = llnl.util.lang.nullcontext class FetchCacheError(Exception): @@ -106,25 +132,25 @@ class BinaryCacheIndex: mean we should have paid the price to update the cache earlier? """ - def __init__(self, cache_root): - self._index_cache_root = cache_root + def __init__(self, cache_root: Optional[str] = None): + self._index_cache_root: str = cache_root or binary_index_location() # the key associated with the serialized _local_index_cache self._index_contents_key = "contents.json" # a FileCache instance storing copies of remote binary cache indices - self._index_file_cache = None + self._index_file_cache: Optional[file_cache.FileCache] = None # stores a map of mirror URL to index hash and cache key (index path) - self._local_index_cache = None + self._local_index_cache: Optional[dict] = None # hashes of remote indices already ingested into the concrete spec # cache (_mirrors_for_spec) - self._specs_already_associated = set() + self._specs_already_associated: Set[str] = set() # mapping from mirror urls to the time.time() of the last index fetch and a bool indicating # whether the fetch succeeded or not. - self._last_fetch_times = {} + self._last_fetch_times: Dict[str, float] = {} # _mirrors_for_spec is a dictionary mapping DAG hashes to lists of # entries indicating mirrors where that concrete spec can be found. @@ -134,7 +160,7 @@ def __init__(self, cache_root): # - the concrete spec itself, keyed by ``spec`` (including the # full hash, since the dag hash may match but we want to # use the updated source if available) - self._mirrors_for_spec = {} + self._mirrors_for_spec: Dict[str, dict] = {} def _init_local_index_cache(self): if not self._index_file_cache: @@ -190,8 +216,7 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_url): tmpdir = tempfile.mkdtemp() try: - db_root_dir = os.path.join(tmpdir, "db_root") - db = spack_db.Database(None, db_dir=db_root_dir, enable_transaction_locking=False) + db = BuildCacheDatabase(tmpdir) try: self._index_file_cache.init_entry(cache_key) @@ -199,11 +224,11 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_url): with self._index_file_cache.read_transaction(cache_key): db._read_from_file(cache_path) except spack_db.InvalidDatabaseVersionError as e: - msg = ( + tty.warn( f"you need a newer Spack version to read the buildcache index for the " f"following mirror: '{mirror_url}'. {e.database_version_message}" ) - raise BuildcacheIndexError(msg) from e + return spec_list = db.query_local(installed=False, in_buildcache=True) @@ -317,9 +342,9 @@ def update(self, with_cooldown=False): from each configured mirror and stored locally (both in memory and on disk under ``_index_cache_root``).""" self._init_local_index_cache() - - mirrors = spack.mirror.MirrorCollection() - configured_mirror_urls = [m.fetch_url for m in mirrors.values()] + configured_mirror_urls = [ + m.fetch_url for m in spack.mirror.MirrorCollection(binary=True).values() + ] items_to_remove = [] spec_cache_clear_needed = False spec_cache_regenerate_needed = not self._mirrors_for_spec @@ -454,14 +479,18 @@ def _fetch_and_cache_index(self, mirror_url, cache_entry={}): FetchIndexError """ # TODO: get rid of this request, handle 404 better - if not web_util.url_exists( - url_util.join(mirror_url, _build_cache_relative_path, "index.json") + scheme = urllib.parse.urlparse(mirror_url).scheme + + if scheme != "oci" and not web_util.url_exists( + url_util.join(mirror_url, BUILD_CACHE_RELATIVE_PATH, "index.json") ): return False - etag = cache_entry.get("etag", None) - if etag: - fetcher = EtagIndexFetcher(mirror_url, etag) + if scheme == "oci": + # TODO: Actually etag and OCI are not mutually exclusive... + fetcher = OCIIndexFetcher(mirror_url, cache_entry.get("index_hash", None)) + elif cache_entry.get("etag"): + fetcher = EtagIndexFetcher(mirror_url, cache_entry["etag"]) else: fetcher = DefaultIndexFetcher( mirror_url, local_hash=cache_entry.get("index_hash", None) @@ -502,15 +531,8 @@ def binary_index_location(): return spack.util.path.canonicalize_path(cache_root) -def _binary_index(): - """Get the singleton store instance.""" - return BinaryCacheIndex(binary_index_location()) - - -#: Singleton binary_index instance -binary_index: Union[BinaryCacheIndex, llnl.util.lang.Singleton] = llnl.util.lang.Singleton( - _binary_index -) +#: Default binary cache index instance +BINARY_INDEX: BinaryCacheIndex = llnl.util.lang.Singleton(BinaryCacheIndex) # type: ignore class NoOverwriteException(spack.error.SpackError): @@ -579,6 +601,10 @@ def __init__(self, msg): super().__init__(msg) +class InvalidMetadataFile(spack.error.SpackError): + pass + + class UnsignedPackageException(spack.error.SpackError): """ Raised if installation of unsigned package is attempted without @@ -593,11 +619,11 @@ def compute_hash(data): def build_cache_relative_path(): - return _build_cache_relative_path + return BUILD_CACHE_RELATIVE_PATH def build_cache_keys_relative_path(): - return _build_cache_keys_relative_path + return BUILD_CACHE_KEYS_RELATIVE_PATH def build_cache_prefix(prefix): @@ -605,22 +631,14 @@ def build_cache_prefix(prefix): def buildinfo_file_name(prefix): - """ - Filename of the binary package meta-data file - """ - name = os.path.join(prefix, ".spack/binary_distribution") - return name + """Filename of the binary package meta-data file""" + return os.path.join(prefix, ".spack", "binary_distribution") def read_buildinfo_file(prefix): - """ - Read buildinfo file - """ - filename = buildinfo_file_name(prefix) - with open(filename, "r") as inputfile: - content = inputfile.read() - buildinfo = syaml.load(content) - return buildinfo + """Read buildinfo file""" + with open(buildinfo_file_name(prefix), "r") as f: + return syaml.load(f) class BuildManifestVisitor(BaseDirectoryVisitor): @@ -630,8 +648,7 @@ class BuildManifestVisitor(BaseDirectoryVisitor): directories.""" def __init__(self): - # Save unique identifiers of files to avoid - # relocating hardlink files for each path. + # Save unique identifiers of hardlinks to avoid relocating them multiple times self.visited = set() # Lists of files we will check @@ -640,6 +657,8 @@ def __init__(self): def seen_before(self, root, rel_path): stat_result = os.lstat(os.path.join(root, rel_path)) + if stat_result.st_nlink == 1: + return False identifier = (stat_result.st_dev, stat_result.st_ino) if identifier in self.visited: return True @@ -703,7 +722,7 @@ def get_buildfile_manifest(spec): # look for them to decide if text file needs to be relocated or not prefixes = [d.prefix for d in spec.traverse(root=True, deptype="all") if not d.external] prefixes.append(spack.hooks.sbang.sbang_install_path()) - prefixes.append(str(spack.store.layout.root)) + prefixes.append(str(spack.store.STORE.layout.root)) # Create a giant regex that matches all prefixes regex = utf8_paths_to_single_binary_regex(prefixes) @@ -716,7 +735,7 @@ def get_buildfile_manifest(spec): for rel_path in visitor.symlinks: abs_path = os.path.join(root, rel_path) link = os.readlink(abs_path) - if os.path.isabs(link) and link.startswith(spack.store.layout.root): + if os.path.isabs(link) and link.startswith(spack.store.STORE.layout.root): data["link_to_relocate"].append(rel_path) # Non-symlinks. @@ -764,9 +783,9 @@ def get_buildinfo_dict(spec): return { "sbang_install_path": spack.hooks.sbang.sbang_install_path(), - "buildpath": spack.store.layout.root, + "buildpath": spack.store.STORE.layout.root, "spackprefix": spack.paths.prefix, - "relative_prefix": os.path.relpath(spec.prefix, spack.store.layout.root), + "relative_prefix": os.path.relpath(spec.prefix, spack.store.STORE.layout.root), "relocate_textfiles": manifest["text_to_relocate"], "relocate_binaries": manifest["binary_to_relocate"], "relocate_links": manifest["link_to_relocate"], @@ -780,11 +799,7 @@ def tarball_directory_name(spec): Return name of the tarball directory according to the convention -//-/ """ - return os.path.join( - str(spec.architecture), - f"{spec.compiler.name}-{spec.compiler.version}", - f"{spec.name}-{spec.version}", - ) + return spec.format_path("{architecture}/{compiler.name}-{compiler.version}/{name}-{version}") def tarball_name(spec, ext): @@ -792,10 +807,10 @@ def tarball_name(spec, ext): Return the name of the tarfile according to the convention --- """ - return ( - f"{spec.architecture}-{spec.compiler.name}-{spec.compiler.version}-" - f"{spec.name}-{spec.version}-{spec.dag_hash()}{ext}" + spec_formatted = spec.format_path( + "{architecture}-{compiler.name}-{compiler.version}-{name}-{version}-{hash}" ) + return f"{spec_formatted}{ext}" def tarball_path_name(spec, ext): @@ -806,18 +821,6 @@ def tarball_path_name(spec, ext): return os.path.join(tarball_directory_name(spec), tarball_name(spec, ext)) -def checksum_tarball(file): - # calculate sha256 hash of tar file - block_size = 65536 - hasher = hashlib.sha256() - with open(file, "rb") as tfile: - buf = tfile.read(block_size) - while len(buf) > 0: - hasher.update(buf) - buf = tfile.read(block_size) - return hasher.hexdigest() - - def select_signing_key(key=None): if key is None: keys = spack.util.gpg.signing_keys() @@ -860,32 +863,18 @@ def _read_specs_and_push_index(file_list, read_method, cache_prefix, db, temp_di db: A spack database used for adding specs and then writing the index. temp_dir (str): Location to write index.json and hash for pushing concurrency (int): Number of parallel processes to use when fetching - - Return: - None """ + for file in file_list: + contents = read_method(file) + # Need full spec.json name or this gets confused with index.json. + if file.endswith(".json.sig"): + specfile_json = Spec.extract_json_from_clearsig(contents) + fetched_spec = Spec.from_dict(specfile_json) + elif file.endswith(".json"): + fetched_spec = Spec.from_json(contents) + else: + continue - def _fetch_spec_from_mirror(spec_url): - spec_file_contents = read_method(spec_url) - - if spec_file_contents: - # Need full spec.json name or this gets confused with index.json. - if spec_url.endswith(".json.sig"): - specfile_json = Spec.extract_json_from_clearsig(spec_file_contents) - return Spec.from_dict(specfile_json) - if spec_url.endswith(".json"): - return Spec.from_json(spec_file_contents) - - tp = multiprocessing.pool.ThreadPool(processes=concurrency) - try: - fetched_specs = tp.map( - llnl.util.lang.star(_fetch_spec_from_mirror), [(f,) for f in file_list] - ) - finally: - tp.terminate() - tp.join() - - for fetched_spec in fetched_specs: db.add(fetched_spec, None) db.mark(fetched_spec, "in_buildcache", True) @@ -910,7 +899,7 @@ def _fetch_spec_from_mirror(spec_url): index_json_path, url_util.join(cache_prefix, "index.json"), keep_original=False, - extra_args={"ContentType": "application/json"}, + extra_args={"ContentType": "application/json", "CacheControl": "no-cache"}, ) # Push the hash @@ -918,7 +907,7 @@ def _fetch_spec_from_mirror(spec_url): index_hash_path, url_util.join(cache_prefix, "index.json.hash"), keep_original=False, - extra_args={"ContentType": "text/plain"}, + extra_args={"ContentType": "text/plain", "CacheControl": "no-cache"}, ) @@ -1059,13 +1048,10 @@ def generate_package_index(cache_prefix, concurrency=32): tty.debug("Retrieving spec descriptor files from {0} to build index".format(cache_prefix)) tmpdir = tempfile.mkdtemp() - db_root_dir = os.path.join(tmpdir, "db_root") - db = spack_db.Database( - None, - db_dir=db_root_dir, - enable_transaction_locking=False, - record_fields=["spec", "ref_count", "in_buildcache"], - ) + + db = BuildCacheDatabase(tmpdir) + db.root = None + db_root_dir = db.database_directory try: _read_specs_and_push_index(file_list, read_fn, cache_prefix, db, db_root_dir, concurrency) @@ -1151,64 +1137,196 @@ def gzip_compressed_tarfile(path): # compresslevel=6 gzip default: llvm takes 4mins, roughly 2.1GB # compresslevel=9 python default: llvm takes 12mins, roughly 2.1GB # So we follow gzip. - with open(path, "wb") as fileobj, closing( - GzipFile(filename="", mode="wb", compresslevel=6, mtime=0, fileobj=fileobj) - ) as gzip_file, tarfile.TarFile(name="", mode="w", fileobj=gzip_file) as tar: - yield tar + with open(path, "wb") as f, ChecksumWriter(f) as inner_checksum, closing( + GzipFile(filename="", mode="wb", compresslevel=6, mtime=0, fileobj=inner_checksum) + ) as gzip_file, ChecksumWriter(gzip_file) as outer_checksum, tarfile.TarFile( + name="", mode="w", fileobj=outer_checksum + ) as tar: + yield tar, inner_checksum, outer_checksum -def deterministic_tarinfo(tarinfo: tarfile.TarInfo): - # We only add files, symlinks, hardlinks, and directories - # No character devices, block devices and FIFOs should ever enter a tarball. - if tarinfo.isdev(): - return None +def _tarinfo_name(absolute_path: str, *, _path=pathlib.PurePath) -> str: + """Compute tarfile entry name as the relative path from the (system) root.""" + return _path(*_path(absolute_path).parts[1:]).as_posix() - # For distribution, it makes no sense to user/group data; since (a) they don't exist - # on other machines, and (b) they lead to surprises as `tar x` run as root will change - # ownership if it can. We want to extract as the current user. By setting owner to root, - # root will extract as root, and non-privileged user will extract as themselves. - tarinfo.uid = 0 - tarinfo.gid = 0 - tarinfo.uname = "" - tarinfo.gname = "" - # Reset mtime to epoch time, our prefixes are not truly immutable, so files may get - # touched; as long as the content does not change, this ensures we get stable tarballs. - tarinfo.mtime = 0 +def tarfile_of_spec_prefix(tar: tarfile.TarFile, prefix: str) -> None: + """Create a tarfile of an install prefix of a spec. Skips existing buildinfo file. + Only adds regular files, symlinks and dirs. Skips devices, fifos. Preserves hardlinks. + Normalizes permissions like git. Tar entries are added in depth-first pre-order, with + dir entries partitioned by file | dir, and sorted alphabetically, for reproducibility. + Partitioning ensures only one dir is in memory at a time, and sorting improves compression. - # Normalize mode - if tarinfo.isfile() or tarinfo.islnk(): - # If user can execute, use 0o755; else 0o644 - # This is to avoid potentially unsafe world writable & exeutable files that may get - # extracted when Python or tar is run with privileges - tarinfo.mode = 0o644 if tarinfo.mode & 0o100 == 0 else 0o755 - else: # symbolic link and directories - tarinfo.mode = 0o755 + Args: + tar: tarfile object to add files to + prefix: absolute install prefix of spec""" + if not os.path.isabs(prefix) or not os.path.isdir(prefix): + raise ValueError(f"prefix '{prefix}' must be an absolute path to a directory") + hardlink_to_tarinfo_name: Dict[Tuple[int, int], str] = dict() + stat_key = lambda stat: (stat.st_dev, stat.st_ino) + + try: # skip buildinfo file if it exists + files_to_skip = [stat_key(os.lstat(buildinfo_file_name(prefix)))] + except OSError: + files_to_skip = [] + + dir_stack = [prefix] + while dir_stack: + dir = dir_stack.pop() + + # Add the dir before its contents + dir_info = tarfile.TarInfo(_tarinfo_name(dir)) + dir_info.type = tarfile.DIRTYPE + dir_info.mode = 0o755 + tar.addfile(dir_info) + + # Sort by name: reproducible & improves compression + with os.scandir(dir) as it: + entries = sorted(it, key=lambda entry: entry.name) + + new_dirs = [] + for entry in entries: + if entry.is_dir(follow_symlinks=False): + new_dirs.append(entry.path) + continue + + file_info = tarfile.TarInfo(_tarinfo_name(entry.path)) - return tarinfo + s = entry.stat(follow_symlinks=False) + # Skip existing binary distribution files. + id = stat_key(s) + if id in files_to_skip: + continue + + # Normalize the mode + file_info.mode = 0o644 if s.st_mode & 0o100 == 0 else 0o755 + + if entry.is_symlink(): + file_info.type = tarfile.SYMTYPE + file_info.linkname = os.readlink(entry.path) + tar.addfile(file_info) + + elif entry.is_file(follow_symlinks=False): + # Deduplicate hardlinks + if s.st_nlink > 1: + if id in hardlink_to_tarinfo_name: + file_info.type = tarfile.LNKTYPE + file_info.linkname = hardlink_to_tarinfo_name[id] + tar.addfile(file_info) + continue + hardlink_to_tarinfo_name[id] = file_info.name -def tar_add_metadata(tar: tarfile.TarFile, path: str, data: dict): - # Serialize buildinfo for the tarball - bstring = syaml.dump(data, default_flow_style=True).encode("utf-8") - tarinfo = tarfile.TarInfo(name=path) - tarinfo.size = len(bstring) - tar.addfile(deterministic_tarinfo(tarinfo), io.BytesIO(bstring)) + # If file not yet seen, copy it. + file_info.type = tarfile.REGTYPE + file_info.size = s.st_size + with open(entry.path, "rb") as f: + tar.addfile(file_info, f) -def _do_create_tarball(tarfile_path, binaries_dir, pkg_dir, buildinfo): - with gzip_compressed_tarfile(tarfile_path) as tar: - tar.add(name=binaries_dir, arcname=pkg_dir, filter=deterministic_tarinfo) - tar_add_metadata(tar, buildinfo_file_name(pkg_dir), buildinfo) + dir_stack.extend(reversed(new_dirs)) # we pop, so reverse to stay alphabetical + + +class ChecksumWriter(io.BufferedIOBase): + """Checksum writer computes a checksum while writing to a file.""" + + myfileobj = None + + def __init__(self, fileobj, algorithm=hashlib.sha256): + self.fileobj = fileobj + self.hasher = algorithm() + self.length = 0 + + def hexdigest(self): + return self.hasher.hexdigest() + + def write(self, data): + if isinstance(data, (bytes, bytearray)): + length = len(data) + else: + data = memoryview(data) + length = data.nbytes + + if length > 0: + self.fileobj.write(data) + self.hasher.update(data) + + self.length += length + + return length + + def read(self, size=-1): + raise OSError(errno.EBADF, "read() on write-only object") + + def read1(self, size=-1): + raise OSError(errno.EBADF, "read1() on write-only object") + + def peek(self, n): + raise OSError(errno.EBADF, "peek() on write-only object") + + @property + def closed(self): + return self.fileobj is None + + def close(self): + fileobj = self.fileobj + if fileobj is None: + return + self.fileobj.close() + self.fileobj = None + + def flush(self): + self.fileobj.flush() + + def fileno(self): + return self.fileobj.fileno() + + def rewind(self): + raise OSError("Can't rewind while computing checksum") + + def readable(self): + return False + + def writable(self): + return True + + def seekable(self): + return True + + def tell(self): + return self.fileobj.tell() + + def seek(self, offset, whence=io.SEEK_SET): + # In principle forward seek is possible with b"0" padding, + # but this is not implemented. + if offset == 0 and whence == io.SEEK_CUR: + return + raise OSError("Can't seek while computing checksum") + + def readline(self, size=-1): + raise OSError(errno.EBADF, "readline() on write-only object") + + +def _do_create_tarball(tarfile_path: str, binaries_dir: str, buildinfo: dict): + with gzip_compressed_tarfile(tarfile_path) as (tar, inner_checksum, outer_checksum): + # Tarball the install prefix + tarfile_of_spec_prefix(tar, binaries_dir) + + # Serialize buildinfo for the tarball + bstring = syaml.dump(buildinfo, default_flow_style=True).encode("utf-8") + tarinfo = tarfile.TarInfo(name=_tarinfo_name(buildinfo_file_name(binaries_dir))) + tarinfo.type = tarfile.REGTYPE + tarinfo.size = len(bstring) + tarinfo.mode = 0o644 + tar.addfile(tarinfo, io.BytesIO(bstring)) + + return inner_checksum.hexdigest(), outer_checksum.hexdigest() class PushOptions(NamedTuple): #: Overwrite existing tarball/metadata files in buildcache force: bool = False - #: Allow absolute paths to package prefixes when creating a tarball - allow_root: bool = False - #: Regenerated indices after pushing regenerate_index: bool = False @@ -1253,7 +1371,7 @@ def _build_tarball_in_stage_dir(spec: Spec, out_url: str, stage_dir: str, option # without concretizing with the current spack packages # and preferences - spec_file = spack.store.layout.spec_file_path(spec) + spec_file = spack.store.STORE.layout.spec_file_path(spec) specfile_name = tarball_name(spec, ".spec.json") specfile_path = os.path.realpath(os.path.join(cache_prefix, specfile_name)) signed_specfile_path = "{0}.sig".format(specfile_path) @@ -1274,39 +1392,22 @@ def _build_tarball_in_stage_dir(spec: Spec, out_url: str, stage_dir: str, option ): raise NoOverwriteException(url_util.format(remote_specfile_path)) - pkg_dir = os.path.basename(spec.prefix.rstrip(os.path.sep)) - binaries_dir = spec.prefix # create info for later relocation and create tar buildinfo = get_buildinfo_dict(spec) - if not options.allow_root: - ensure_package_relocatable(buildinfo, binaries_dir) - - _do_create_tarball(tarfile_path, binaries_dir, pkg_dir, buildinfo) - - # get the sha256 checksum of the tarball - checksum = checksum_tarball(tarfile_path) + checksum, _ = _do_create_tarball(tarfile_path, binaries_dir, buildinfo) # add sha256 checksum to spec.json - with open(spec_file, "r") as inputfile: content = inputfile.read() if spec_file.endswith(".json"): spec_dict = sjson.load(content) else: raise ValueError("{0} not a valid spec file type".format(spec_file)) - spec_dict["buildcache_layout_version"] = 1 - bchecksum = {} - bchecksum["hash_algorithm"] = "sha256" - bchecksum["hash"] = checksum - spec_dict["binary_cache_checksum"] = bchecksum - # Add original install prefix relative to layout root to spec.json. - # This will be used to determine is the directory layout has changed. - buildinfo = {} - buildinfo["relative_prefix"] = os.path.relpath(spec.prefix, spack.store.layout.root) - spec_dict["buildinfo"] = buildinfo + spec_dict["buildcache_layout_version"] = CURRENT_BUILD_CACHE_LAYOUT_VERSION + spec_dict["binary_cache_checksum"] = {"hash_algorithm": "sha256", "hash": checksum} with open(specfile_path, "w") as outfile: # Note: when using gpg clear sign, we need to avoid long lines (19995 chars). @@ -1341,10 +1442,21 @@ def _build_tarball_in_stage_dir(spec: Spec, out_url: str, stage_dir: str, option return None +class NotInstalledError(spack.error.SpackError): + """Raised when a spec is not installed but picked to be packaged.""" + + def __init__(self, specs: List[Spec]): + super().__init__( + "Cannot push non-installed packages", + ", ".join(s.cformat("{name}{@version}{/hash:7}") for s in specs), + ) + + def specs_to_be_packaged( specs: List[Spec], root: bool = True, dependencies: bool = True ) -> List[Spec]: """Return the list of nodes to be packaged, given a list of specs. + Raises NotInstalledError if a spec is not installed but picked to be packaged. Args: specs: list of root specs to be processed @@ -1352,19 +1464,35 @@ def specs_to_be_packaged( dependencies: include the dependencies of each spec in the nodes """ + if not root and not dependencies: return [] - elif dependencies: - nodes = traverse.traverse_nodes(specs, root=root, deptype="all") - else: - nodes = set(specs) - # Limit to installed non-externals. - packageable = lambda n: not n.external and n.installed + # Filter packageable roots + with spack.store.STORE.db.read_transaction(): + if root: + # Error on uninstalled roots, when roots are requested + uninstalled_roots = list(s for s in specs if not s.installed) + if uninstalled_roots: + raise NotInstalledError(uninstalled_roots) + roots = specs + else: + roots = [] - # Mass install check - with spack.store.db.read_transaction(): - return list(filter(packageable, nodes)) + if dependencies: + # Error on uninstalled deps, when deps are requested + deps = list( + traverse.traverse_nodes( + specs, deptype="all", order="breadth", root=False, key=traverse.by_dag_hash + ) + ) + uninstalled_deps = list(s for s in deps if not s.installed) + if uninstalled_deps: + raise NotInstalledError(uninstalled_deps) + else: + deps = [] + + return [s for s in itertools.chain(roots, deps) if not s.external] def push(spec: Spec, mirror_url: str, options: PushOptions): @@ -1424,7 +1552,7 @@ def try_fetch(url_to_fetch): try: stage.fetch() - except web_util.FetchError: + except spack.error.FetchError: stage.destroy() return None @@ -1437,6 +1565,42 @@ def _delete_staged_downloads(download_result): download_result["specfile_stage"].destroy() +def _get_valid_spec_file(path: str, max_supported_layout: int) -> Tuple[Dict, int]: + """Read and validate a spec file, returning the spec dict with its layout version, or raising + InvalidMetadataFile if invalid.""" + try: + with open(path, "rb") as f: + binary_content = f.read() + except OSError: + raise InvalidMetadataFile(f"No such file: {path}") + + # In the future we may support transparently decompressing compressed spec files. + if binary_content[:2] == b"\x1f\x8b": + raise InvalidMetadataFile("Compressed spec files are not supported") + + try: + as_string = binary_content.decode("utf-8") + if path.endswith(".json.sig"): + spec_dict = Spec.extract_json_from_clearsig(as_string) + else: + spec_dict = json.loads(as_string) + except Exception as e: + raise InvalidMetadataFile(f"Could not parse {path} due to: {e}") from e + + # Ensure this version is not too new. + try: + layout_version = int(spec_dict.get("buildcache_layout_version", 0)) + except ValueError as e: + raise InvalidMetadataFile("Could not parse layout version") from e + + if layout_version > max_supported_layout: + raise InvalidMetadataFile( + f"Layout version {layout_version} is too new for this version of Spack" + ) + + return spec_dict, layout_version + + def download_tarball(spec, unsigned=False, mirrors_for_spec=None): """ Download binary tarball for given package into stage area, returning @@ -1465,14 +1629,13 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None): "signature_verified": "true-if-binary-pkg-was-already-verified" } """ - if not spack.mirror.MirrorCollection(): - tty.die("Please add a spack mirror to allow " + "download of pre-compiled packages.") + configured_mirrors = spack.mirror.MirrorCollection(binary=True).values() + if not configured_mirrors: + tty.die("Please add a spack mirror to allow download of pre-compiled packages.") tarball = tarball_path_name(spec, ".spack") specfile_prefix = tarball_name(spec, ".spec") - mirrors_to_try = [] - # Note on try_first and try_next: # mirrors_for_spec mostly likely came from spack caching remote # mirror indices locally and adding their specs to a local data @@ -1483,69 +1646,142 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None): # we need was in an un-indexed mirror. No need to check any # mirror for the spec twice though. try_first = [i["mirror_url"] for i in mirrors_for_spec] if mirrors_for_spec else [] - try_next = [ - i.fetch_url - for i in spack.mirror.MirrorCollection().values() - if i.fetch_url not in try_first - ] + try_next = [i.fetch_url for i in configured_mirrors if i.fetch_url not in try_first] - for url in try_first + try_next: - mirrors_to_try.append( - { - "specfile": url_util.join(url, _build_cache_relative_path, specfile_prefix), - "spackfile": url_util.join(url, _build_cache_relative_path, tarball), - } - ) + mirrors = try_first + try_next tried_to_verify_sigs = [] # Assumes we care more about finding a spec file by preferred ext # than by mirrory priority. This can be made less complicated as # we remove support for deprecated spec formats and buildcache layouts. - for ext in ["json.sig", "json"]: - for mirror_to_try in mirrors_to_try: - specfile_url = "{0}.{1}".format(mirror_to_try["specfile"], ext) - spackfile_url = mirror_to_try["spackfile"] - local_specfile_stage = try_fetch(specfile_url) - if local_specfile_stage: - local_specfile_path = local_specfile_stage.save_filename - signature_verified = False - - if ext.endswith(".sig") and not unsigned: - # If we found a signed specfile at the root, try to verify - # the signature immediately. We will not download the - # tarball if we could not verify the signature. - tried_to_verify_sigs.append(specfile_url) - signature_verified = try_verify(local_specfile_path) - if not signature_verified: - tty.warn("Failed to verify: {0}".format(specfile_url)) - - if unsigned or signature_verified or not ext.endswith(".sig"): - # We will download the tarball in one of three cases: - # 1. user asked for --no-check-signature - # 2. user didn't ask for --no-check-signature, but we - # found a spec.json.sig and verified the signature already - # 3. neither of the first two cases are true, but this file - # is *not* a signed json (not a spec.json.sig file). That - # means we already looked at all the mirrors and either didn't - # find any .sig files or couldn't verify any of them. But it - # is still possible to find an old style binary package where - # the signature is a detached .asc file in the outer archive - # of the tarball, and in that case, the only way to know is to - # download the tarball. This is a deprecated use case, so if - # something goes wrong during the extraction process (can't - # verify signature, checksum doesn't match) we will fail at - # that point instead of trying to download more tarballs from - # the remaining mirrors, looking for one we can use. - tarball_stage = try_fetch(spackfile_url) - if tarball_stage: - return { - "tarball_stage": tarball_stage, - "specfile_stage": local_specfile_stage, - "signature_verified": signature_verified, - } + for try_signed in (True, False): + for mirror in mirrors: + # If it's an OCI index, do things differently, since we cannot compose URLs. + parsed = urllib.parse.urlparse(mirror) + + # TODO: refactor this to some "nice" place. + if parsed.scheme == "oci": + ref = spack.oci.image.ImageReference.from_string(mirror[len("oci://") :]).with_tag( + spack.oci.image.default_tag(spec) + ) + + # Fetch the manifest + try: + response = spack.oci.opener.urlopen( + urllib.request.Request( + url=ref.manifest_url(), + headers={"Accept": "application/vnd.oci.image.manifest.v1+json"}, + ) + ) + except Exception: + continue + + # Download the config = spec.json and the relevant tarball + try: + manifest = json.loads(response.read()) + spec_digest = spack.oci.image.Digest.from_string(manifest["config"]["digest"]) + tarball_digest = spack.oci.image.Digest.from_string( + manifest["layers"][-1]["digest"] + ) + except Exception: + continue + + with spack.oci.oci.make_stage( + ref.blob_url(spec_digest), spec_digest, keep=True + ) as local_specfile_stage: + try: + local_specfile_stage.fetch() + local_specfile_stage.check() + try: + _get_valid_spec_file( + local_specfile_stage.save_filename, + CURRENT_BUILD_CACHE_LAYOUT_VERSION, + ) + except InvalidMetadataFile as e: + tty.warn( + f"Ignoring binary package for {spec.name}/{spec.dag_hash()[:7]} " + f"from {mirror} due to invalid metadata file: {e}" + ) + local_specfile_stage.destroy() + continue + except Exception: + continue + local_specfile_stage.cache_local() + + with spack.oci.oci.make_stage( + ref.blob_url(tarball_digest), tarball_digest, keep=True + ) as tarball_stage: + try: + tarball_stage.fetch() + tarball_stage.check() + except Exception: + continue + tarball_stage.cache_local() + + return { + "tarball_stage": tarball_stage, + "specfile_stage": local_specfile_stage, + "signature_verified": False, + } + + else: + ext = "json.sig" if try_signed else "json" + specfile_path = url_util.join(mirror, BUILD_CACHE_RELATIVE_PATH, specfile_prefix) + specfile_url = f"{specfile_path}.{ext}" + spackfile_url = url_util.join(mirror, BUILD_CACHE_RELATIVE_PATH, tarball) + local_specfile_stage = try_fetch(specfile_url) + if local_specfile_stage: + local_specfile_path = local_specfile_stage.save_filename + signature_verified = False - local_specfile_stage.destroy() + try: + _get_valid_spec_file( + local_specfile_path, CURRENT_BUILD_CACHE_LAYOUT_VERSION + ) + except InvalidMetadataFile as e: + tty.warn( + f"Ignoring binary package for {spec.name}/{spec.dag_hash()[:7]} " + f"from {mirror} due to invalid metadata file: {e}" + ) + local_specfile_stage.destroy() + continue + + if try_signed and not unsigned: + # If we found a signed specfile at the root, try to verify + # the signature immediately. We will not download the + # tarball if we could not verify the signature. + tried_to_verify_sigs.append(specfile_url) + signature_verified = try_verify(local_specfile_path) + if not signature_verified: + tty.warn("Failed to verify: {0}".format(specfile_url)) + + if unsigned or signature_verified or not try_signed: + # We will download the tarball in one of three cases: + # 1. user asked for --no-check-signature + # 2. user didn't ask for --no-check-signature, but we + # found a spec.json.sig and verified the signature already + # 3. neither of the first two cases are true, but this file + # is *not* a signed json (not a spec.json.sig file). That + # means we already looked at all the mirrors and either didn't + # find any .sig files or couldn't verify any of them. But it + # is still possible to find an old style binary package where + # the signature is a detached .asc file in the outer archive + # of the tarball, and in that case, the only way to know is to + # download the tarball. This is a deprecated use case, so if + # something goes wrong during the extraction process (can't + # verify signature, checksum doesn't match) we will fail at + # that point instead of trying to download more tarballs from + # the remaining mirrors, looking for one we can use. + tarball_stage = try_fetch(spackfile_url) + if tarball_stage: + return { + "tarball_stage": tarball_stage, + "specfile_stage": local_specfile_stage, + "signature_verified": signature_verified, + } + + local_specfile_stage.destroy() # Falling through the nested loops meeans we exhaustively searched # for all known kinds of spec files on all mirrors and did not find @@ -1565,12 +1801,6 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None): return None -def ensure_package_relocatable(buildinfo, binaries_dir): - """Check if package binaries are relocatable.""" - binaries = [os.path.join(binaries_dir, f) for f in buildinfo["relocate_binaries"]] - relocate.ensure_binaries_are_relocatable(binaries) - - def dedupe_hardlinks_if_necessary(root, buildinfo): """Updates a buildinfo dict for old archives that did not dedupe hardlinks. De-duping hardlinks is necessary @@ -1596,9 +1826,10 @@ def dedupe_hardlinks_if_necessary(root, buildinfo): for rel_path in buildinfo[key]: stat_result = os.lstat(os.path.join(root, rel_path)) identifier = (stat_result.st_dev, stat_result.st_ino) - if identifier in visited: - continue - visited.add(identifier) + if stat_result.st_nlink > 1: + if identifier in visited: + continue + visited.add(identifier) new_list.append(rel_path) buildinfo[key] = new_list @@ -1609,7 +1840,7 @@ def relocate_package(spec): """ workdir = str(spec.prefix) buildinfo = read_buildinfo_file(workdir) - new_layout_root = str(spack.store.layout.root) + new_layout_root = str(spack.store.STORE.layout.root) new_prefix = str(spec.prefix) new_rel_prefix = str(os.path.relpath(new_prefix, new_layout_root)) new_spack_prefix = str(spack.paths.prefix) @@ -1783,7 +2014,7 @@ def _extract_inner_tarball(spec, filename, extract_to, unsigned, remote_checksum ) # compute the sha256 checksum of the tarball - local_checksum = checksum_tarball(tarfile_path) + local_checksum = spack.util.crypto.checksum(hashlib.sha256, tarfile_path) expected = remote_checksum["hash"] # if the checksums don't match don't install @@ -1794,34 +2025,57 @@ def _extract_inner_tarball(spec, filename, extract_to, unsigned, remote_checksum return tarfile_path -def extract_tarball(spec, download_result, unsigned=False, force=False): +def _tar_strip_component(tar: tarfile.TarFile, prefix: str): + """Strip the top-level directory `prefix` from the member names in a tarfile.""" + # Including trailing /, otherwise we end up with absolute paths. + regex = re.compile(re.escape(prefix) + "/*") + + # Remove the top-level directory from the member (link)names. + # Note: when a tarfile is created, relative in-prefix symlinks are + # expanded to matching member names of tarfile entries. So, we have + # to ensure that those are updated too. + # Absolute symlinks are copied verbatim -- relocation should take care of + # them. + for m in tar.getmembers(): + result = regex.match(m.name) + assert result is not None + m.name = m.name[result.end() :] + if m.linkname: + result = regex.match(m.linkname) + if result: + m.linkname = m.linkname[result.end() :] + + +def extract_tarball(spec, download_result, unsigned=False, force=False, timer=timer.NULL_TIMER): """ extract binary tarball for given package into install area """ + timer.start("extract") if os.path.exists(spec.prefix): if force: shutil.rmtree(spec.prefix) else: raise NoOverwriteException(str(spec.prefix)) - specfile_path = download_result["specfile_stage"].save_filename - - with open(specfile_path, "r") as inputfile: - content = inputfile.read() - if specfile_path.endswith(".json.sig"): - spec_dict = Spec.extract_json_from_clearsig(content) - else: - spec_dict = sjson.load(content) + # Create the install prefix + fsys.mkdirp( + spec.prefix, + mode=get_package_dir_permissions(spec), + group=get_package_group(spec), + default_perms="parents", + ) + specfile_path = download_result["specfile_stage"].save_filename + spec_dict, layout_version = _get_valid_spec_file( + specfile_path, CURRENT_BUILD_CACHE_LAYOUT_VERSION + ) bchecksum = spec_dict["binary_cache_checksum"] + filename = download_result["tarball_stage"].save_filename signature_verified = download_result["signature_verified"] tmpdir = None - if ( - "buildcache_layout_version" not in spec_dict - or int(spec_dict["buildcache_layout_version"]) < 1 - ): + if layout_version == 0: # Handle the older buildcache layout where the .spack file # contains a spec json, maybe an .asc file (signature), # and another tarball containing the actual install tree. @@ -1832,7 +2086,7 @@ def extract_tarball(spec, download_result, unsigned=False, force=False): _delete_staged_downloads(download_result) shutil.rmtree(tmpdir) raise e - else: + elif layout_version == 1: # Newer buildcache layout: the .spack file contains just # in the install tree, the signature, if it exists, is # wrapped around the spec.json at the root. If sig verify @@ -1846,7 +2100,7 @@ def extract_tarball(spec, download_result, unsigned=False, force=False): ) # compute the sha256 checksum of the tarball - local_checksum = checksum_tarball(tarfile_path) + local_checksum = spack.util.crypto.checksum(hashlib.sha256, tarfile_path) expected = bchecksum["hash"] # if the checksums don't match don't install @@ -1856,57 +2110,59 @@ def extract_tarball(spec, download_result, unsigned=False, force=False): raise NoChecksumException( tarfile_path, size, contents, "sha256", expected, local_checksum ) - - new_relative_prefix = str(os.path.relpath(spec.prefix, spack.store.layout.root)) - # if the original relative prefix is in the spec file use it - buildinfo = spec_dict.get("buildinfo", {}) - old_relative_prefix = buildinfo.get("relative_prefix", new_relative_prefix) - rel = buildinfo.get("relative_rpaths") - info = "old relative prefix %s\nnew relative prefix %s\nrelative rpaths %s" - tty.debug(info % (old_relative_prefix, new_relative_prefix, rel), level=2) - - # Extract the tarball into the store root, presumably on the same filesystem. - # The directory created is the base directory name of the old prefix. - # Moving the old prefix name to the new prefix location should preserve - # hard links and symbolic links. - extract_tmp = os.path.join(spack.store.layout.root, ".tmp") - mkdirp(extract_tmp) - extracted_dir = os.path.join(extract_tmp, old_relative_prefix.split(os.path.sep)[-1]) - - with closing(tarfile.open(tarfile_path, "r")) as tar: - try: - tar.extractall(path=extract_tmp) - except Exception as e: - _delete_staged_downloads(download_result) - shutil.rmtree(extracted_dir) - raise e try: - shutil.move(extracted_dir, spec.prefix) - except Exception as e: + with closing(tarfile.open(tarfile_path, "r")) as tar: + # Remove install prefix from tarfil to extract directly into spec.prefix + _tar_strip_component(tar, prefix=_ensure_common_prefix(tar)) + tar.extractall(path=spec.prefix) + except Exception: + shutil.rmtree(spec.prefix, ignore_errors=True) _delete_staged_downloads(download_result) - shutil.rmtree(extracted_dir) - raise e + raise + os.remove(tarfile_path) os.remove(specfile_path) + timer.stop("extract") + timer.start("relocate") try: relocate_package(spec) except Exception as e: - shutil.rmtree(spec.prefix) + shutil.rmtree(spec.prefix, ignore_errors=True) raise e else: manifest_file = os.path.join( - spec.prefix, spack.store.layout.metadata_dir, spack.store.layout.manifest_file_name + spec.prefix, + spack.store.STORE.layout.metadata_dir, + spack.store.STORE.layout.manifest_file_name, ) if not os.path.exists(manifest_file): spec_id = spec.format("{name}/{hash:7}") tty.warn("No manifest file in tarball for spec %s" % spec_id) finally: if tmpdir: - shutil.rmtree(tmpdir) + shutil.rmtree(tmpdir, ignore_errors=True) if os.path.exists(filename): os.remove(filename) _delete_staged_downloads(download_result) + timer.stop("relocate") + + +def _ensure_common_prefix(tar: tarfile.TarFile) -> str: + # Get the shortest length directory. + common_prefix = min((e.name for e in tar.getmembers() if e.isdir()), key=len, default=None) + + if common_prefix is None: + raise ValueError("Tarball does not contain a common prefix") + + # Validate that each file starts with the prefix + for member in tar.getmembers(): + if not member.name.startswith(common_prefix): + raise ValueError( + f"Tarball contains file {member.name} outside of prefix {common_prefix}" + ) + + return common_prefix def install_root_node(spec, unsigned=False, force=False, sha256=None): @@ -1955,7 +2211,7 @@ def install_root_node(spec, unsigned=False, force=False, sha256=None): tty.msg('Installing "{0}" from a buildcache'.format(spec.format())) extract_tarball(spec, download_result, unsigned, force) spack.hooks.post_install(spec, False) - spack.store.db.add(spec, spack.store.layout) + spack.store.STORE.db.add(spec, spack.store.STORE.layout) def install_single_spec(spec, unsigned=False, force=False): @@ -1980,12 +2236,14 @@ def try_direct_fetch(spec, mirrors=None): specfile_is_signed = False found_specs = [] - for mirror in spack.mirror.MirrorCollection(mirrors=mirrors).values(): + binary_mirrors = spack.mirror.MirrorCollection(mirrors=mirrors, binary=True).values() + + for mirror in binary_mirrors: buildcache_fetch_url_json = url_util.join( - mirror.fetch_url, _build_cache_relative_path, specfile_name + mirror.fetch_url, BUILD_CACHE_RELATIVE_PATH, specfile_name ) buildcache_fetch_url_signed_json = url_util.join( - mirror.fetch_url, _build_cache_relative_path, signed_specfile_name + mirror.fetch_url, BUILD_CACHE_RELATIVE_PATH, signed_specfile_name ) try: _, _, fs = web_util.read_from_url(buildcache_fetch_url_signed_json) @@ -2043,11 +2301,11 @@ def get_mirrors_for_spec(spec=None, mirrors_to_check=None, index_only=False): if spec is None: return [] - if not spack.mirror.MirrorCollection(mirrors=mirrors_to_check): + if not spack.mirror.MirrorCollection(mirrors=mirrors_to_check, binary=True): tty.debug("No Spack mirrors are currently configured") return {} - results = binary_index.find_built_spec(spec, mirrors_to_check=mirrors_to_check) + results = BINARY_INDEX.find_built_spec(spec, mirrors_to_check=mirrors_to_check) # The index may be out-of-date. If we aren't only considering indices, try # to fetch directly since we know where the file should be. @@ -2056,7 +2314,7 @@ def get_mirrors_for_spec(spec=None, mirrors_to_check=None, index_only=False): # We found a spec by the direct fetch approach, we might as well # add it to our mapping. if results: - binary_index.update_spec(spec, results) + BINARY_INDEX.update_spec(spec, results) return results @@ -2072,17 +2330,17 @@ def update_cache_and_get_specs(): Throws: FetchCacheError """ - binary_index.update() - return binary_index.get_all_built_specs() + BINARY_INDEX.update() + return BINARY_INDEX.get_all_built_specs() def clear_spec_cache(): - binary_index.clear() + BINARY_INDEX.clear() def get_keys(install=False, trust=False, force=False, mirrors=None): """Get pgp public keys available on mirror with suffix .pub""" - mirror_collection = mirrors or spack.mirror.MirrorCollection() + mirror_collection = mirrors or spack.mirror.MirrorCollection(binary=True) if not mirror_collection: tty.die("Please add a spack mirror to allow " + "download of build caches.") @@ -2090,7 +2348,7 @@ def get_keys(install=False, trust=False, force=False, mirrors=None): for mirror in mirror_collection.values(): fetch_url = mirror.fetch_url keys_url = url_util.join( - fetch_url, _build_cache_relative_path, _build_cache_keys_relative_path + fetch_url, BUILD_CACHE_RELATIVE_PATH, BUILD_CACHE_KEYS_RELATIVE_PATH ) keys_index = url_util.join(keys_url, "index.json") @@ -2125,7 +2383,7 @@ def get_keys(install=False, trust=False, force=False, mirrors=None): if not os.path.exists(stage.save_filename): try: stage.fetch() - except web_util.FetchError: + except spack.error.FetchError: continue tty.debug("Found key {0}".format(fingerprint)) @@ -2155,7 +2413,7 @@ def push_keys(*mirrors, **kwargs): for mirror in mirrors: push_url = getattr(mirror, "push_url", mirror) keys_url = url_util.join( - push_url, _build_cache_relative_path, _build_cache_keys_relative_path + push_url, BUILD_CACHE_RELATIVE_PATH, BUILD_CACHE_KEYS_RELATIVE_PATH ) keys_local = url_util.local_file_path(keys_url) @@ -2243,7 +2501,7 @@ def check_specs_against_mirrors(mirrors, specs, output_file=None): """ rebuilds = {} - for mirror in spack.mirror.MirrorCollection(mirrors).values(): + for mirror in spack.mirror.MirrorCollection(mirrors, binary=True).values(): tty.debug("Checking for built specs at {0}".format(mirror.fetch_url)) rebuild_list = [] @@ -2277,7 +2535,7 @@ def _download_buildcache_entry(mirror_root, descriptions): try: stage.fetch() break - except web_util.FetchError as e: + except spack.error.FetchError as e: tty.debug(e) else: if fail_if_missing: @@ -2287,17 +2545,17 @@ def _download_buildcache_entry(mirror_root, descriptions): def download_buildcache_entry(file_descriptions, mirror_url=None): - if not mirror_url and not spack.mirror.MirrorCollection(): + if not mirror_url and not spack.mirror.MirrorCollection(binary=True): tty.die( "Please provide or add a spack mirror to allow " + "download of buildcache entries." ) if mirror_url: - mirror_root = os.path.join(mirror_url, _build_cache_relative_path) + mirror_root = os.path.join(mirror_url, BUILD_CACHE_RELATIVE_PATH) return _download_buildcache_entry(mirror_root, file_descriptions) - for mirror in spack.mirror.MirrorCollection().values(): - mirror_root = os.path.join(mirror.fetch_url, _build_cache_relative_path) + for mirror in spack.mirror.MirrorCollection(binary=True).values(): + mirror_root = os.path.join(mirror.fetch_url, BUILD_CACHE_RELATIVE_PATH) if _download_buildcache_entry(mirror_root, file_descriptions): return True @@ -2354,22 +2612,12 @@ def __init__(self, all_architectures): self.possible_specs = specs - def __call__(self, spec, **kwargs): + def __call__(self, spec: Spec, **kwargs): """ Args: - spec (str): The spec being searched for in its string representation or hash. + spec: The spec being searched for """ - matches = [] - if spec.startswith("/"): - # Matching a DAG hash - query_hash = spec.replace("/", "") - for candidate_spec in self.possible_specs: - if candidate_spec.dag_hash().startswith(query_hash): - matches.append(candidate_spec) - else: - # Matching a spec constraint - matches = [s for s in self.possible_specs if s.satisfies(spec)] - return matches + return [s for s in self.possible_specs if s.satisfies(spec)] class FetchIndexError(Exception): @@ -2398,7 +2646,7 @@ def __init__(self, url, local_hash, urlopen=web_util.urlopen): def get_remote_hash(self): # Failure to fetch index.json.hash is not fatal - url_index_hash = url_util.join(self.url, _build_cache_relative_path, "index.json.hash") + url_index_hash = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json.hash") try: response = self.urlopen(urllib.request.Request(url_index_hash, headers=self.headers)) except urllib.error.URLError: @@ -2410,7 +2658,7 @@ def get_remote_hash(self): return None return remote_hash.decode("utf-8") - def conditional_fetch(self): + def conditional_fetch(self) -> FetchIndexResult: # Do an intermediate fetch for the hash # and a conditional fetch for the contents @@ -2419,17 +2667,17 @@ def conditional_fetch(self): return FetchIndexResult(etag=None, hash=None, data=None, fresh=True) # Otherwise, download index.json - url_index = url_util.join(self.url, _build_cache_relative_path, "index.json") + url_index = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json") try: response = self.urlopen(urllib.request.Request(url_index, headers=self.headers)) except urllib.error.URLError as e: - raise FetchIndexError("Could not fetch index from {}".format(url_index), e) + raise FetchIndexError("Could not fetch index from {}".format(url_index), e) from e try: result = codecs.getreader("utf-8")(response).read() except ValueError as e: - return FetchCacheError("Remote index {} is invalid".format(url_index), e) + raise FetchIndexError("Remote index {} is invalid".format(url_index), e) from e computed_hash = compute_hash(result) @@ -2461,9 +2709,9 @@ def __init__(self, url, etag, urlopen=web_util.urlopen): self.etag = etag self.urlopen = urlopen - def conditional_fetch(self): + def conditional_fetch(self) -> FetchIndexResult: # Just do a conditional fetch immediately - url = url_util.join(self.url, _build_cache_relative_path, "index.json") + url = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json") headers = { "User-Agent": web_util.SPACK_USER_AGENT, "If-None-Match": '"{}"'.format(self.etag), @@ -2492,3 +2740,59 @@ def conditional_fetch(self): data=result, fresh=False, ) + + +class OCIIndexFetcher: + def __init__(self, url: str, local_hash, urlopen=None) -> None: + self.local_hash = local_hash + + # Remove oci:// prefix + assert url.startswith("oci://") + self.ref = spack.oci.image.ImageReference.from_string(url[6:]) + self.urlopen = urlopen or spack.oci.opener.urlopen + + def conditional_fetch(self) -> FetchIndexResult: + """Download an index from an OCI registry type mirror.""" + url_manifest = self.ref.with_tag(spack.oci.image.default_index_tag).manifest_url() + try: + response = self.urlopen( + urllib.request.Request( + url=url_manifest, + headers={"Accept": "application/vnd.oci.image.manifest.v1+json"}, + ) + ) + except urllib.error.URLError as e: + raise FetchIndexError( + "Could not fetch manifest from {}".format(url_manifest), e + ) from e + + try: + manifest = json.loads(response.read()) + except Exception as e: + raise FetchIndexError("Remote index {} is invalid".format(url_manifest), e) from e + + # Get first blob hash, which should be the index.json + try: + index_digest = spack.oci.image.Digest.from_string(manifest["layers"][0]["digest"]) + except Exception as e: + raise FetchIndexError("Remote index {} is invalid".format(url_manifest), e) from e + + # Fresh? + if index_digest.digest == self.local_hash: + return FetchIndexResult(etag=None, hash=None, data=None, fresh=True) + + # Otherwise fetch the blob / index.json + response = self.urlopen( + urllib.request.Request( + url=self.ref.blob_url(index_digest), + headers={"Accept": "application/vnd.oci.image.layer.v1.tar+gzip"}, + ) + ) + + result = codecs.getreader("utf-8")(response).read() + + # Make sure the blob we download has the advertised hash + if compute_hash(result) != index_digest.digest: + raise FetchIndexError(f"Remote index {url_manifest} is invalid") + + return FetchIndexResult(etag=None, hash=index_digest.digest, data=result, fresh=False) diff --git a/lib/spack/spack/bootstrap/__init__.py b/lib/spack/spack/bootstrap/__init__.py index 1dad6597a635e5..1f2f239de3d844 100644 --- a/lib/spack/spack/bootstrap/__init__.py +++ b/lib/spack/spack/bootstrap/__init__.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Function and classes needed to bootstrap Spack itself.""" -from .config import ensure_bootstrap_configuration, is_bootstrapping +from .config import ensure_bootstrap_configuration, is_bootstrapping, store_path from .core import all_core_root_specs, ensure_core_dependencies, ensure_patchelf_in_path_or_raise from .environment import BootstrapEnvironment, ensure_environment_dependencies from .status import status_message @@ -18,4 +18,5 @@ "ensure_environment_dependencies", "BootstrapEnvironment", "status_message", + "store_path", ] diff --git a/lib/spack/spack/bootstrap/_common.py b/lib/spack/spack/bootstrap/_common.py index 555c11d80961d8..69f32d62639d81 100644 --- a/lib/spack/spack/bootstrap/_common.py +++ b/lib/spack/spack/bootstrap/_common.py @@ -50,7 +50,7 @@ def _try_import_from_store( # We have to run as part of this python interpreter query_spec += " ^" + spec_for_current_python() - installed_specs = spack.store.db.query(query_spec, installed=True) + installed_specs = spack.store.STORE.db.query(query_spec, installed=True) for candidate_spec in installed_specs: pkg = candidate_spec["python"].package @@ -183,7 +183,7 @@ def _executables_in_store( executables_str = ", ".join(executables) msg = "[BOOTSTRAP EXECUTABLES {0}] Try installed specs with query '{1}'" tty.debug(msg.format(executables_str, query_spec)) - installed_specs = spack.store.db.query(query_spec, installed=True) + installed_specs = spack.store.STORE.db.query(query_spec, installed=True) if installed_specs: for concrete_spec in installed_specs: bin_dir = concrete_spec.prefix.bin diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py index 1753810a934ff6..6786bc0d3ead00 100644 --- a/lib/spack/spack/bootstrap/config.py +++ b/lib/spack/spack/bootstrap/config.py @@ -124,9 +124,9 @@ def _read_and_sanitize_configuration() -> Dict[str, Any]: def _bootstrap_config_scopes() -> Sequence["spack.config.ConfigScope"]: tty.debug("[BOOTSTRAP CONFIG SCOPE] name=_builtin") config_scopes: MutableSequence["spack.config.ConfigScope"] = [ - spack.config.InternalConfigScope("_builtin", spack.config.config_defaults) + spack.config.InternalConfigScope("_builtin", spack.config.CONFIG_DEFAULTS) ] - configuration_paths = (spack.config.configuration_defaults_path, ("bootstrap", _config_path())) + configuration_paths = (spack.config.CONFIGURATION_DEFAULTS_PATH, ("bootstrap", _config_path())) for name, path in configuration_paths: platform = spack.platforms.host().name platform_scope = spack.config.ConfigScope( @@ -143,25 +143,28 @@ def _bootstrap_config_scopes() -> Sequence["spack.config.ConfigScope"]: def _add_compilers_if_missing() -> None: arch = spack.spec.ArchSpec.frontend_arch() if not spack.compilers.compilers_for_arch(arch): - new_compilers = spack.compilers.find_new_compilers() + new_compilers = spack.compilers.find_new_compilers( + mixed_toolchain=sys.platform == "darwin" + ) if new_compilers: spack.compilers.add_compilers_to_config(new_compilers, init_config=False) @contextlib.contextmanager def _ensure_bootstrap_configuration() -> Generator: + spack.store.ensure_singleton_created() bootstrap_store_path = store_path() user_configuration = _read_and_sanitize_configuration() with spack.environment.no_active_environment(): with spack.platforms.prevent_cray_detection(), spack.platforms.use_platform( spack.platforms.real_host() - ), spack.repo.use_repositories(spack.paths.packages_path), spack.store.use_store( - bootstrap_store_path - ): + ), spack.repo.use_repositories(spack.paths.packages_path): # Default configuration scopes excluding command line # and builtin but accounting for platform specific scopes config_scopes = _bootstrap_config_scopes() - with spack.config.use_configuration(*config_scopes): + with spack.config.use_configuration(*config_scopes), spack.store.use_store( + bootstrap_store_path, extra_data={"padded_length": 0} + ): # We may need to compile code from sources, so ensure we # have compilers for the current platform _add_compilers_if_missing() diff --git a/lib/spack/spack/bootstrap/core.py b/lib/spack/spack/bootstrap/core.py index e545ea5854fc35..5f73c7bfaf49c8 100644 --- a/lib/spack/spack/bootstrap/core.py +++ b/lib/spack/spack/bootstrap/core.py @@ -214,7 +214,7 @@ def _install_and_test( with spack.config.override(self.mirror_scope): # This index is currently needed to get the compiler used to build some # specs that we know by dag hash. - spack.binary_distribution.binary_index.regenerate_spec_cache() + spack.binary_distribution.BINARY_INDEX.regenerate_spec_cache() index = spack.binary_distribution.update_cache_and_get_specs() if not index: @@ -228,7 +228,7 @@ def _install_and_test( if not abstract_spec.intersects(candidate_spec): continue - if python_spec is not None and python_spec not in abstract_spec: + if python_spec is not None and not abstract_spec.intersects(f"^{python_spec}"): continue for _, pkg_hash, pkg_sha256 in item["binaries"]: @@ -291,6 +291,10 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool: with spack_python_interpreter(): # Add hint to use frontend operating system on Cray concrete_spec = spack.spec.Spec(abstract_spec_str + " ^" + spec_for_current_python()) + # This is needed to help the old concretizer taking the `setuptools` dependency + # only when bootstrapping from sources on Python 3.12 + if spec_for_current_python() == "python@3.12": + concrete_spec.constrain("+force_setuptools") if module == "clingo": # TODO: remove when the old concretizer is deprecated # pylint: disable=fixme @@ -446,16 +450,11 @@ def ensure_executables_in_path_or_raise( current_bootstrapper.last_search["spec"], current_bootstrapper.last_search["command"], ) - env_mods = spack.util.environment.EnvironmentModifications() - for dep in concrete_spec.traverse( - root=True, order="post", deptype=("link", "run") - ): - env_mods.extend( - spack.user_environment.environment_modifications_for_spec( - dep, set_package_py_globals=False - ) + cmd.add_default_envmod( + spack.user_environment.environment_modifications_for_specs( + concrete_spec, set_package_py_globals=False ) - cmd.add_default_envmod(env_mods) + ) return cmd assert exception_handler, ( @@ -476,15 +475,22 @@ def ensure_executables_in_path_or_raise( def _add_externals_if_missing() -> None: search_list = [ # clingo - spack.repo.path.get_pkg_class("cmake"), - spack.repo.path.get_pkg_class("bison"), + "cmake", + "bison", # GnuPG - spack.repo.path.get_pkg_class("gawk"), + "gawk", + # develop deps + "git", ] if IS_WINDOWS: - search_list.append(spack.repo.path.get_pkg_class("winbison")) - detected_packages = spack.detection.by_executable(search_list) - spack.detection.update_configuration(detected_packages, scope="bootstrap") + search_list.append("winbison") + externals = spack.detection.by_path(search_list) + # System git is typically deprecated, so mark as non-buildable to force it as external + non_buildable_externals = {k: externals.pop(k) for k in ("git",) if k in externals} + spack.detection.update_configuration(externals, scope="bootstrap", buildable=True) + spack.detection.update_configuration( + non_buildable_externals, scope="bootstrap", buildable=False + ) def clingo_root_spec() -> str: diff --git a/lib/spack/spack/bootstrap/environment.py b/lib/spack/spack/bootstrap/environment.py index 158b7ecae35d8a..2a2fc37b454afa 100644 --- a/lib/spack/spack/bootstrap/environment.py +++ b/lib/spack/spack/bootstrap/environment.py @@ -15,14 +15,15 @@ from llnl.util import tty -import spack.build_environment import spack.environment import spack.tengine +import spack.util.cpus import spack.util.executable from spack.environment import depfile from ._common import _root_spec from .config import root_path, spec_for_current_python, store_path +from .core import _add_externals_if_missing class BootstrapEnvironment(spack.environment.Environment): @@ -136,7 +137,7 @@ def _install_with_depfile(self) -> None: "-C", str(self.environment_root()), "-j", - str(spack.build_environment.determine_number_of_jobs(parallel=True)), + str(spack.util.cpus.determine_number_of_jobs(parallel=True)), **kwargs, ) @@ -185,6 +186,7 @@ def pytest_root_spec() -> str: def ensure_environment_dependencies() -> None: """Ensure Spack dependencies from the bootstrap environment are installed and ready to use""" + _add_externals_if_missing() with BootstrapEnvironment() as env: env.update_installations() env.update_syspath_and_environ() diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index ddcbc9acc2a506..36574259d0394d 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -40,11 +40,15 @@ import sys import traceback import types +from collections import defaultdict +from enum import Flag, auto +from itertools import chain from typing import List, Tuple import llnl.util.tty as tty +from llnl.string import plural from llnl.util.filesystem import join_path -from llnl.util.lang import dedupe +from llnl.util.lang import dedupe, stable_partition from llnl.util.symlink import symlink from llnl.util.tty.color import cescape, colorize from llnl.util.tty.log import MultiProcessFd @@ -54,35 +58,37 @@ import spack.build_systems.python import spack.builder import spack.config +import spack.deptypes as dt import spack.main import spack.package_base import spack.paths import spack.platforms import spack.repo import spack.schema.environment +import spack.spec import spack.store import spack.subprocess_context import spack.user_environment import spack.util.path import spack.util.pattern +from spack import traverse +from spack.context import Context from spack.error import NoHeadersError, NoLibrariesError from spack.install_test import spack_install_test_log from spack.installer import InstallError -from spack.util.cpus import cpus_available +from spack.util.cpus import determine_number_of_jobs from spack.util.environment import ( SYSTEM_DIRS, EnvironmentModifications, env_flag, filter_system_paths, get_path, - inspect_path, is_system_path, validate, ) from spack.util.executable import Executable from spack.util.log_parse import make_log_context, parse_log_events from spack.util.module_cmd import load_module, module, path_from_modules -from spack.util.string import plural # # This can be set by the user to globally disable parallel builds. @@ -109,7 +115,6 @@ SPACK_CCACHE_BINARY = "SPACK_CCACHE_BINARY" SPACK_SYSTEM_DIRS = "SPACK_SYSTEM_DIRS" - # Platform-specific library suffix. if sys.platform == "darwin": dso_suffix = "dylib" @@ -406,19 +411,13 @@ def set_compiler_environment_variables(pkg, env): def set_wrapper_variables(pkg, env): - """Set environment variables used by the Spack compiler wrapper - (which have the prefix `SPACK_`) and also add the compiler wrappers - to PATH. - - This determines the injected -L/-I/-rpath options; each - of these specifies a search order and this function computes these - options in a manner that is intended to match the DAG traversal order - in `modifications_from_dependencies`: that method uses a post-order - traversal so that `PrependPath` actions from dependencies take lower - precedence; we use a post-order traversal here to match the visitation - order of `modifications_from_dependencies` (so we are visiting the - lowest priority packages first). - """ + """Set environment variables used by the Spack compiler wrapper (which have the prefix + `SPACK_`) and also add the compiler wrappers to PATH. + + This determines the injected -L/-I/-rpath options; each of these specifies a search order and + this function computes these options in a manner that is intended to match the DAG traversal + order in `SetupContext`. TODO: this is not the case yet, we're using post order, SetupContext + is using topo order.""" # Set environment variables if specified for # the given compiler compiler = pkg.compiler @@ -537,78 +536,42 @@ def update_compiler_args_for_dep(dep): env.set(SPACK_RPATH_DIRS, ":".join(rpath_dirs)) -def determine_number_of_jobs( - parallel=False, command_line=None, config_default=None, max_cpus=None -): - """ - Packages that require sequential builds need 1 job. Otherwise we use the - number of jobs set on the command line. If not set, then we use the config - defaults (which is usually set through the builtin config scope), but we - cap to the number of CPUs available to avoid oversubscription. - - Parameters: - parallel (bool or None): true when package supports parallel builds - command_line (int or None): command line override - config_default (int or None): config default number of jobs - max_cpus (int or None): maximum number of CPUs available. When None, this - value is automatically determined. - """ - if not parallel: - return 1 - - if command_line is None and "command_line" in spack.config.scopes(): - command_line = spack.config.get("config:build_jobs", scope="command_line") - - if command_line is not None: - return command_line - - max_cpus = max_cpus or cpus_available() - - # in some rare cases _builtin config may not be set, so default to max 16 - config_default = config_default or spack.config.get("config:build_jobs", 16) - - return min(max_cpus, config_default) - - -def set_module_variables_for_package(pkg): +def set_package_py_globals(pkg, context: Context = Context.BUILD): """Populate the Python module of a package with some useful global names. This makes things easier for package writers. """ - # Put a marker on this module so that it won't execute the body of this - # function again, since it is not needed - marker = "_set_run_already_called" - if getattr(pkg.module, marker, False): - return - module = ModuleChangePropagator(pkg) - jobs = determine_number_of_jobs(parallel=pkg.parallel) - m = module - m.make_jobs = jobs - - # TODO: make these build deps that can be installed if not found. - m.make = MakeExecutable("make", jobs) - m.ninja = MakeExecutable("ninja", jobs, supports_jobserver=False) - # TODO: johnwparent: add package or builder support to define these build tools - # for now there is no entrypoint for builders to define these on their - # own - if sys.platform == "win32": - m.nmake = Executable("nmake") - m.msbuild = Executable("msbuild") - # analog to configure for win32 - m.cscript = Executable("cscript") - - # Find the configure script in the archive path - # Don't use which for this; we want to find it in the current dir. - m.configure = Executable("./configure") - - # Standard CMake arguments - m.std_cmake_args = spack.build_systems.cmake.CMakeBuilder.std_args(pkg) - m.std_meson_args = spack.build_systems.meson.MesonBuilder.std_args(pkg) - m.std_pip_args = spack.build_systems.python.PythonPipBuilder.std_args(pkg) - - # Put spack compiler paths in module scope. + + if context == Context.BUILD: + jobs = determine_number_of_jobs(parallel=pkg.parallel) + m.make_jobs = jobs + + # TODO: make these build deps that can be installed if not found. + m.make = MakeExecutable("make", jobs) + m.gmake = MakeExecutable("gmake", jobs) + m.ninja = MakeExecutable("ninja", jobs, supports_jobserver=False) + # TODO: johnwparent: add package or builder support to define these build tools + # for now there is no entrypoint for builders to define these on their + # own + if sys.platform == "win32": + m.nmake = Executable("nmake") + m.msbuild = Executable("msbuild") + # analog to configure for win32 + m.cscript = Executable("cscript") + + # Find the configure script in the archive path + # Don't use which for this; we want to find it in the current dir. + m.configure = Executable("./configure") + + # Standard CMake arguments + m.std_cmake_args = spack.build_systems.cmake.CMakeBuilder.std_args(pkg) + m.std_meson_args = spack.build_systems.meson.MesonBuilder.std_args(pkg) + m.std_pip_args = spack.build_systems.python.PythonPipBuilder.std_args(pkg) + + # Put spack compiler paths in module scope. (Some packages use it + # in setup_run_environment etc, so don't put it context == build) link_dir = spack.paths.build_env_path m.spack_cc = os.path.join(link_dir, pkg.compiler.link_paths["cc"]) m.spack_cxx = os.path.join(link_dir, pkg.compiler.link_paths["cxx"]) @@ -632,9 +595,6 @@ def static_to_shared_library(static_lib, shared_lib=None, **kwargs): m.static_to_shared_library = static_to_shared_library - # Put a marker on this module so that it won't execute the body of this - # function again, since it is not needed - setattr(m, marker, True) module.propagate_changes_to_mro() @@ -760,12 +720,15 @@ def load_external_modules(pkg): load_module(external_module) -def setup_package(pkg, dirty, context="build"): +def setup_package(pkg, dirty, context: Context = Context.BUILD): """Execute all environment setup routines.""" - if context not in ["build", "test"]: - raise ValueError("'context' must be one of ['build', 'test'] - got: {0}".format(context)) + if context not in (Context.BUILD, Context.TEST): + raise ValueError(f"'context' must be Context.BUILD or Context.TEST - got {context}") - set_module_variables_for_package(pkg) + # First populate the package.py's module with the relevant globals that could be used in any + # of the setup_* functions. + setup_context = SetupContext(pkg.spec, context=context) + setup_context.set_all_package_py_globals() # Keep track of env changes from packages separately, since we want to # issue warnings when packages make "suspicious" modifications. @@ -773,42 +736,30 @@ def setup_package(pkg, dirty, context="build"): env_mods = EnvironmentModifications() # setup compilers for build contexts - need_compiler = context == "build" or (context == "test" and pkg.test_requires_compiler) + need_compiler = context == Context.BUILD or ( + context == Context.TEST and pkg.test_requires_compiler + ) if need_compiler: set_compiler_environment_variables(pkg, env_mods) set_wrapper_variables(pkg, env_mods) - tty.debug("setup_package: grabbing modifications from dependencies") - env_mods.extend(modifications_from_dependencies(pkg.spec, context, custom_mods_only=False)) - tty.debug("setup_package: collected all modifications from dependencies") - - # architecture specific setup + # Platform specific setup goes before package specific setup. This is for setting + # defaults like MACOSX_DEPLOYMENT_TARGET on macOS. platform = spack.platforms.by_name(pkg.spec.architecture.platform) target = platform.target(pkg.spec.architecture.target) platform.setup_platform_environment(pkg, env_mods) - if context == "build": - tty.debug("setup_package: setup build environment for root") - builder = spack.builder.create(pkg) - builder.setup_build_environment(env_mods) + tty.debug("setup_package: grabbing modifications from dependencies") + env_mods.extend(setup_context.get_env_modifications()) + tty.debug("setup_package: collected all modifications from dependencies") - if (not dirty) and (not env_mods.is_unset("CPATH")): - tty.debug( - "A dependency has updated CPATH, this may lead pkg-" - "config to assume that the package is part of the system" - " includes and omit it when invoked with '--cflags'." - ) - elif context == "test": - tty.debug("setup_package: setup test environment for root") - env_mods.extend( - inspect_path( - pkg.spec.prefix, - spack.user_environment.prefix_inspections(pkg.spec.platform), - exclude=is_system_path, - ) - ) - pkg.setup_run_environment(env_mods) + if context == Context.TEST: env_mods.prepend_path("PATH", ".") + elif context == Context.BUILD and not dirty and not env_mods.is_unset("CPATH"): + tty.debug( + "A dependency has updated CPATH, this may lead pkg-config to assume that the package " + "is part of the system includes and omit it when invoked with '--cflags'." + ) # First apply the clean environment changes env_base.apply_modifications() @@ -846,158 +797,257 @@ def setup_package(pkg, dirty, context="build"): return env_base -def _make_runnable(pkg, env): - # Helper method which prepends a Package's bin/ prefix to the PATH - # environment variable - prefix = pkg.prefix +class EnvironmentVisitor: + def __init__(self, *roots: spack.spec.Spec, context: Context): + # For the roots (well, marked specs) we follow different edges + # than for their deps, depending on the context. + self.root_hashes = set(s.dag_hash() for s in roots) + + if context == Context.BUILD: + # Drop direct run deps in build context + # We don't really distinguish between install and build time test deps, + # so we include them here as build-time test deps. + self.root_depflag = dt.BUILD | dt.TEST | dt.LINK + elif context == Context.TEST: + # This is more of an extended run environment + self.root_depflag = dt.TEST | dt.RUN | dt.LINK + elif context == Context.RUN: + self.root_depflag = dt.RUN | dt.LINK + + def neighbors(self, item): + spec = item.edge.spec + if spec.dag_hash() in self.root_hashes: + depflag = self.root_depflag + else: + depflag = dt.LINK | dt.RUN + return traverse.sort_edges(spec.edges_to_dependencies(depflag=depflag)) - for dirname in ["bin", "bin64"]: - bin_dir = os.path.join(prefix, dirname) - if os.path.isdir(bin_dir): - env.prepend_path("PATH", bin_dir) +class UseMode(Flag): + #: Entrypoint spec (a spec to be built; an env root, etc) + ROOT = auto() -def modifications_from_dependencies( - spec, context, custom_mods_only=True, set_package_py_globals=True -): - """Returns the environment modifications that are required by - the dependencies of a spec and also applies modifications - to this spec's package at module scope, if need be. + #: A spec used at runtime, but no executables in PATH + RUNTIME = auto() - Environment modifications include: + #: A spec used at runtime, with executables in PATH + RUNTIME_EXECUTABLE = auto() - - Updating PATH so that executables can be found - - Updating CMAKE_PREFIX_PATH and PKG_CONFIG_PATH so that their respective - tools can find Spack-built dependencies - - Running custom package environment modifications + #: A spec that's a direct build or test dep + BUILDTIME_DIRECT = auto() - Custom package modifications can conflict with the default PATH changes - we make (specifically for the PATH, CMAKE_PREFIX_PATH, and PKG_CONFIG_PATH - environment variables), so this applies changes in a fixed order: + #: A spec that should be visible in search paths in a build env. + BUILDTIME = auto() - - All modifications (custom and default) from external deps first - - All modifications from non-external deps afterwards + #: Flag is set when the (node, mode) is finalized + ADDED = auto() - With that order, `PrependPath` actions from non-external default - environment modifications will take precedence over custom modifications - from external packages. - A secondary constraint is that custom and default modifications are - grouped on a per-package basis: combined with the post-order traversal this - means that default modifications of dependents can override custom - modifications of dependencies (again, this would only occur for PATH, - CMAKE_PREFIX_PATH, or PKG_CONFIG_PATH). - - Args: - spec (spack.spec.Spec): spec for which we want the modifications - context (str): either 'build' for build-time modifications or 'run' - for run-time modifications - custom_mods_only (bool): if True returns only custom modifications, if False - returns custom and default modifications - set_package_py_globals (bool): whether or not to set the global variables in the - package.py files (this may be problematic when using buildcaches that have - been built on a different but compatible OS) - """ - if context not in ["build", "run", "test"]: - raise ValueError( - "Expecting context to be one of ['build', 'run', 'test'], " "got: {0}".format(context) +def effective_deptypes( + *specs: spack.spec.Spec, context: Context = Context.BUILD +) -> List[Tuple[spack.spec.Spec, UseMode]]: + """Given a list of input specs and a context, return a list of tuples of + all specs that contribute to (environment) modifications, together with + a flag specifying in what way they do so. The list is ordered topologically + from root to leaf, meaning that environment modifications should be applied + in reverse so that dependents override dependencies, not the other way around.""" + visitor = traverse.TopoVisitor( + EnvironmentVisitor(*specs, context=context), + key=lambda x: x.dag_hash(), + root=True, + all_edges=True, + ) + traverse.traverse_depth_first_with_visitor(traverse.with_artificial_edges(specs), visitor) + + # Dictionary with "no mode" as default value, so it's easy to write modes[x] |= flag. + use_modes = defaultdict(lambda: UseMode(0)) + nodes_with_type = [] + + for edge in visitor.edges: + parent, child, depflag = edge.parent, edge.spec, edge.depflag + + # Mark the starting point + if parent is None: + use_modes[child] = UseMode.ROOT + continue + + parent_mode = use_modes[parent] + + # Nothing to propagate. + if not parent_mode: + continue + + # Dependending on the context, include particular deps from the root. + if UseMode.ROOT & parent_mode: + if context == Context.BUILD: + if (dt.BUILD | dt.TEST) & depflag: + use_modes[child] |= UseMode.BUILDTIME_DIRECT + if dt.LINK & depflag: + use_modes[child] |= UseMode.BUILDTIME + + elif context == Context.TEST: + if (dt.RUN | dt.TEST) & depflag: + use_modes[child] |= UseMode.RUNTIME_EXECUTABLE + elif dt.LINK & depflag: + use_modes[child] |= UseMode.RUNTIME + + elif context == Context.RUN: + if dt.RUN & depflag: + use_modes[child] |= UseMode.RUNTIME_EXECUTABLE + elif dt.LINK & depflag: + use_modes[child] |= UseMode.RUNTIME + + # Propagate RUNTIME and RUNTIME_EXECUTABLE through link and run deps. + if (UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE | UseMode.BUILDTIME_DIRECT) & parent_mode: + if dt.LINK & depflag: + use_modes[child] |= UseMode.RUNTIME + if dt.RUN & depflag: + use_modes[child] |= UseMode.RUNTIME_EXECUTABLE + + # Propagate BUILDTIME through link deps. + if UseMode.BUILDTIME & parent_mode: + if dt.LINK & depflag: + use_modes[child] |= UseMode.BUILDTIME + + # Finalize the spec; the invariant is that all in-edges are processed + # before out-edges, meaning that parent is done. + if not (UseMode.ADDED & parent_mode): + use_modes[parent] |= UseMode.ADDED + nodes_with_type.append((parent, parent_mode)) + + # Attach the leaf nodes, since we only added nodes with out-edges. + for spec, parent_mode in use_modes.items(): + if parent_mode and not (UseMode.ADDED & parent_mode): + nodes_with_type.append((spec, parent_mode)) + + return nodes_with_type + + +class SetupContext: + """This class encapsulates the logic to determine environment modifications, and is used as + well to set globals in modules of package.py.""" + + def __init__(self, *specs: spack.spec.Spec, context: Context) -> None: + """Construct a ModificationsFromDag object. + Args: + specs: single root spec for build/test context, possibly more for run context + context: build, run, or test""" + if (context == Context.BUILD or context == Context.TEST) and not len(specs) == 1: + raise ValueError("Cannot setup build environment for multiple specs") + specs_with_type = effective_deptypes(*specs, context=context) + + self.specs = specs + self.context = context + self.external: List[Tuple[spack.spec.Spec, UseMode]] + self.nonexternal: List[Tuple[spack.spec.Spec, UseMode]] + # Reverse so we go from leaf to root + self.nodes_in_subdag = set(id(s) for s, _ in specs_with_type) + + # Split into non-external and external, maintaining topo order per group. + self.external, self.nonexternal = stable_partition( + reversed(specs_with_type), lambda t: t[0].external ) + self.should_be_runnable = UseMode.BUILDTIME_DIRECT | UseMode.RUNTIME_EXECUTABLE + self.should_setup_run_env = ( + UseMode.BUILDTIME_DIRECT | UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE + ) + self.should_setup_dependent_build_env = UseMode.BUILDTIME | UseMode.BUILDTIME_DIRECT + self.should_setup_build_env = UseMode.ROOT if context == Context.BUILD else UseMode(0) - env = EnvironmentModifications() - - # Note: see computation of 'custom_mod_deps' and 'exe_deps' later in this - # function; these sets form the building blocks of those collections. - build_deps = set(spec.dependencies(deptype=("build", "test"))) - link_deps = set(spec.traverse(root=False, deptype="link")) - build_link_deps = build_deps | link_deps - build_and_supporting_deps = set() - for build_dep in build_deps: - build_and_supporting_deps.update(build_dep.traverse(deptype="run")) - run_and_supporting_deps = set(spec.traverse(root=False, deptype=("run", "link"))) - test_and_supporting_deps = set() - for test_dep in set(spec.dependencies(deptype="test")): - test_and_supporting_deps.update(test_dep.traverse(deptype="run")) - - # All dependencies that might have environment modifications to apply - custom_mod_deps = set() - if context == "build": - custom_mod_deps.update(build_and_supporting_deps) - # Tests may be performed after build - custom_mod_deps.update(test_and_supporting_deps) - else: - # test/run context - custom_mod_deps.update(run_and_supporting_deps) - if context == "test": - custom_mod_deps.update(test_and_supporting_deps) - custom_mod_deps.update(link_deps) - - # Determine 'exe_deps': the set of packages with binaries we want to use - if context == "build": - exe_deps = build_and_supporting_deps | test_and_supporting_deps - elif context == "run": - exe_deps = set(spec.traverse(deptype="run")) - elif context == "test": - exe_deps = test_and_supporting_deps - - def default_modifications_for_dep(dep): - if dep in build_link_deps and not is_system_path(dep.prefix) and context == "build": - prefix = dep.prefix - - env.prepend_path("CMAKE_PREFIX_PATH", prefix) - - for directory in ("lib", "lib64", "share"): - pcdir = os.path.join(prefix, directory, "pkgconfig") - if os.path.isdir(pcdir): - env.prepend_path("PKG_CONFIG_PATH", pcdir) - - if dep in exe_deps and not is_system_path(dep.prefix): - _make_runnable(dep, env) - - def add_modifications_for_dep(dep): - tty.debug("Adding env modifications for {0}".format(dep.name)) - # Some callers of this function only want the custom modifications. - # For callers that want both custom and default modifications, we want - # to perform the default modifications here (this groups custom - # and default modifications together on a per-package basis). - if not custom_mods_only: - default_modifications_for_dep(dep) - - # Perform custom modifications here (PrependPath actions performed in - # the custom method override the default environment modifications - # we do to help the build, namely for PATH, CMAKE_PREFIX_PATH, and - # PKG_CONFIG_PATH) - if dep in custom_mod_deps: - dpkg = dep.package - if set_package_py_globals: - set_module_variables_for_package(dpkg) - - current_module = ModuleChangePropagator(spec.package) - dpkg.setup_dependent_package(current_module, spec) - current_module.propagate_changes_to_mro() - - if context == "build": - builder = spack.builder.create(dpkg) - builder.setup_dependent_build_environment(env, spec) - else: - dpkg.setup_dependent_run_environment(env, spec) - tty.debug("Added env modifications for {0}".format(dep.name)) - - # Note that we want to perform environment modifications in a fixed order. - # The Spec.traverse method provides this: i.e. in addition to - # the post-order semantics, it also guarantees a fixed traversal order - # among dependencies which are not constrained by post-order semantics. - for dspec in spec.traverse(root=False, order="post"): - if dspec.external: - add_modifications_for_dep(dspec) - - for dspec in spec.traverse(root=False, order="post"): - # Default env modifications for non-external packages can override - # custom modifications of external packages (this can only occur - # for modifications to PATH, CMAKE_PREFIX_PATH, and PKG_CONFIG_PATH) - if not dspec.external: - add_modifications_for_dep(dspec) + if context == Context.RUN or context == Context.TEST: + self.should_be_runnable |= UseMode.ROOT + self.should_setup_run_env |= UseMode.ROOT - return env + # Everything that calls setup_run_environment and setup_dependent_* needs globals set. + self.should_set_package_py_globals = ( + self.should_setup_dependent_build_env | self.should_setup_run_env | UseMode.ROOT + ) + # In a build context, the root and direct build deps need build-specific globals set. + self.needs_build_context = UseMode.ROOT | UseMode.BUILDTIME_DIRECT + + def set_all_package_py_globals(self): + """Set the globals in modules of package.py files.""" + for dspec, flag in chain(self.external, self.nonexternal): + pkg = dspec.package + + if self.should_set_package_py_globals & flag: + if self.context == Context.BUILD and self.needs_build_context & flag: + set_package_py_globals(pkg, context=Context.BUILD) + else: + # This includes runtime dependencies, also runtime deps of direct build deps. + set_package_py_globals(pkg, context=Context.RUN) + + for spec in dspec.dependents(): + # Note: some specs have dependents that are unreachable from the root, so avoid + # setting globals for those. + if id(spec) not in self.nodes_in_subdag: + continue + dependent_module = ModuleChangePropagator(spec.package) + pkg.setup_dependent_package(dependent_module, spec) + dependent_module.propagate_changes_to_mro() + + def get_env_modifications(self) -> EnvironmentModifications: + """Returns the environment variable modifications for the given input specs and context. + Environment modifications include: + - Updating PATH for packages that are required at runtime + - Updating CMAKE_PREFIX_PATH and PKG_CONFIG_PATH so that their respective + tools can find Spack-built dependencies (when context=build) + - Running custom package environment modifications: setup_run_environment, + setup_dependent_run_environment, setup_build_environment, + setup_dependent_build_environment. + + The (partial) order imposed on the specs is externals first, then topological + from leaf to root. That way externals cannot contribute search paths that would shadow + Spack's prefixes, and dependents override variables set by dependencies.""" + env = EnvironmentModifications() + for dspec, flag in chain(self.external, self.nonexternal): + tty.debug(f"Adding env modifications for {dspec.name}") + pkg = dspec.package + + if self.should_setup_dependent_build_env & flag: + self._make_buildtime_detectable(dspec, env) + + for root in self.specs: # there is only one root in build context + spack.builder.create(pkg).setup_dependent_build_environment(env, root) + + if self.should_setup_build_env & flag: + spack.builder.create(pkg).setup_build_environment(env) + + if self.should_be_runnable & flag: + self._make_runnable(dspec, env) + + if self.should_setup_run_env & flag: + run_env_mods = EnvironmentModifications() + for spec in dspec.dependents(deptype=dt.LINK | dt.RUN): + if id(spec) in self.nodes_in_subdag: + pkg.setup_dependent_run_environment(run_env_mods, spec) + pkg.setup_run_environment(run_env_mods) + if self.context == Context.BUILD: + # Don't let the runtime environment of comiler like dependencies leak into the + # build env + run_env_mods.drop("CC", "CXX", "F77", "FC") + env.extend(run_env_mods) + + return env + + def _make_buildtime_detectable(self, dep: spack.spec.Spec, env: EnvironmentModifications): + if is_system_path(dep.prefix): + return + + env.prepend_path("CMAKE_PREFIX_PATH", dep.prefix) + for d in ("lib", "lib64", "share"): + pcdir = os.path.join(dep.prefix, d, "pkgconfig") + if os.path.isdir(pcdir): + env.prepend_path("PKG_CONFIG_PATH", pcdir) + + def _make_runnable(self, dep: spack.spec.Spec, env: EnvironmentModifications): + if is_system_path(dep.prefix): + return + + for d in ("bin", "bin64"): + bin_dir = os.path.join(dep.prefix, d) + if os.path.isdir(bin_dir): + env.prepend_path("PATH", bin_dir) def get_cmake_prefix_path(pkg): @@ -1027,9 +1077,9 @@ def get_cmake_prefix_path(pkg): def _setup_pkg_and_run( - serialized_pkg, function, kwargs, child_pipe, input_multiprocess_fd, jsfd1, jsfd2 + serialized_pkg, function, kwargs, write_pipe, input_multiprocess_fd, jsfd1, jsfd2 ): - context = kwargs.get("context", "build") + context: str = kwargs.get("context", "build") try: # We are in the child process. Python sets sys.stdin to @@ -1045,15 +1095,15 @@ def _setup_pkg_and_run( if not kwargs.get("fake", False): kwargs["unmodified_env"] = os.environ.copy() kwargs["env_modifications"] = setup_package( - pkg, dirty=kwargs.get("dirty", False), context=context + pkg, dirty=kwargs.get("dirty", False), context=Context.from_string(context) ) return_value = function(pkg, kwargs) - child_pipe.send(return_value) + write_pipe.send(return_value) except StopPhase as e: # Do not create a full ChildError from this, it's not an error # it's a control statement. - child_pipe.send(e) + write_pipe.send(e) except BaseException: # catch ANYTHING that goes wrong in the child process exc_type, exc, tb = sys.exc_info() @@ -1102,10 +1152,10 @@ def _setup_pkg_and_run( context, package_context, ) - child_pipe.send(ce) + write_pipe.send(ce) finally: - child_pipe.close() + write_pipe.close() if input_multiprocess_fd is not None: input_multiprocess_fd.close() @@ -1149,7 +1199,7 @@ def child_fun(): For more information on `multiprocessing` child process creation mechanisms, see https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods """ - parent_pipe, child_pipe = multiprocessing.Pipe() + read_pipe, write_pipe = multiprocessing.Pipe(duplex=False) input_multiprocess_fd = None jobserver_fd1 = None jobserver_fd2 = None @@ -1174,7 +1224,7 @@ def child_fun(): serialized_pkg, function, kwargs, - child_pipe, + write_pipe, input_multiprocess_fd, jobserver_fd1, jobserver_fd2, @@ -1183,6 +1233,12 @@ def child_fun(): p.start() + # We close the writable end of the pipe now to be sure that p is the + # only process which owns a handle for it. This ensures that when p + # closes its handle for the writable end, read_pipe.recv() will + # promptly report the readable end as being ready. + write_pipe.close() + except InstallError as e: e.pkg = pkg raise @@ -1192,7 +1248,16 @@ def child_fun(): if input_multiprocess_fd is not None: input_multiprocess_fd.close() - child_result = parent_pipe.recv() + def exitcode_msg(p): + typ = "exit" if p.exitcode >= 0 else "signal" + return f"{typ} {abs(p.exitcode)}" + + try: + child_result = read_pipe.recv() + except EOFError: + p.join() + raise InstallError(f"The process has stopped unexpectedly ({exitcode_msg(p)})") + p.join() # If returns a StopPhase, raise it @@ -1212,6 +1277,10 @@ def child_fun(): child_result.print_context() raise child_result + # Fallback. Usually caught beforehand in EOFError above. + if p.exitcode != 0: + raise InstallError(f"The process failed unexpectedly ({exitcode_msg(p)})") + return child_result @@ -1256,9 +1325,8 @@ def make_stack(tb, stack=None): func = getattr(obj, tb.tb_frame.f_code.co_name, "") if func: typename, *_ = func.__qualname__.partition(".") - - if isinstance(obj, CONTEXT_BASES) and typename not in basenames: - break + if isinstance(obj, CONTEXT_BASES) and typename not in basenames: + break else: return None diff --git a/lib/spack/spack/build_systems/_checks.py b/lib/spack/spack/build_systems/_checks.py index 94c59aaa055560..38fada308f5aed 100644 --- a/lib/spack/spack/build_systems/_checks.py +++ b/lib/spack/spack/build_systems/_checks.py @@ -39,7 +39,7 @@ def check_paths(path_list, filetype, predicate): check_paths(pkg.sanity_check_is_file, "file", os.path.isfile) check_paths(pkg.sanity_check_is_dir, "directory", os.path.isdir) - ignore_file = llnl.util.lang.match_predicate(spack.store.layout.hidden_file_regexes) + ignore_file = llnl.util.lang.match_predicate(spack.store.STORE.layout.hidden_file_regexes) if all(map(ignore_file, os.listdir(pkg.prefix))): msg = "Install failed for {0}. Nothing was installed!" raise spack.installer.InstallError(msg.format(pkg.name)) diff --git a/lib/spack/spack/build_systems/autotools.py b/lib/spack/spack/build_systems/autotools.py index a2594203ff3e33..760faf307bbc65 100644 --- a/lib/spack/spack/build_systems/autotools.py +++ b/lib/spack/spack/build_systems/autotools.py @@ -46,6 +46,7 @@ class AutotoolsPackage(spack.package_base.PackageBase): depends_on("gnuconfig", type="build", when="target=ppc64le:") depends_on("gnuconfig", type="build", when="target=aarch64:") depends_on("gnuconfig", type="build", when="target=riscv64:") + depends_on("gmake", type="build") conflicts("platform=windows") def flags_to_build_system_args(self, flags): @@ -55,7 +56,8 @@ def flags_to_build_system_args(self, flags): setattr(self, "configure_flag_args", []) for flag, values in flags.items(): if values: - values_str = "{0}={1}".format(flag.upper(), " ".join(values)) + var_name = "LIBS" if flag == "ldlibs" else flag.upper() + values_str = "{0}={1}".format(var_name, " ".join(values)) self.configure_flag_args.append(values_str) # Spack's fflags are meant for both F77 and FC, therefore we # additionaly set FCFLAGS if required. diff --git a/lib/spack/spack/build_systems/cached_cmake.py b/lib/spack/spack/build_systems/cached_cmake.py index 457a722e1b4d87..d85c2b7e199352 100644 --- a/lib/spack/spack/build_systems/cached_cmake.py +++ b/lib/spack/spack/build_systems/cached_cmake.py @@ -162,17 +162,6 @@ def initconfig_compiler_entries(self): libs_string = libs_format_string.format(lang) entries.append(cmake_cache_string(libs_string, libs_flags)) - # Set the generator in the cached config - if self.spec.satisfies("generator=make"): - entries.append(cmake_cache_string("CMAKE_GENERATOR", "Unix Makefiles")) - if self.spec.satisfies("generator=ninja"): - entries.append(cmake_cache_string("CMAKE_GENERATOR", "Ninja")) - entries.append( - cmake_cache_string( - "CMAKE_MAKE_PROGRAM", "{0}/ninja".format(spec["ninja"].prefix.bin) - ) - ) - return entries def initconfig_mpi_entries(self): diff --git a/lib/spack/spack/build_systems/cmake.py b/lib/spack/spack/build_systems/cmake.py index 8e819f39581652..1859e40fe51d20 100644 --- a/lib/spack/spack/build_systems/cmake.py +++ b/lib/spack/spack/build_systems/cmake.py @@ -142,10 +142,10 @@ def flags_to_build_system_args(self, flags): # We specify for each of them. if flags["ldflags"]: ldflags = " ".join(flags["ldflags"]) - ld_string = "-DCMAKE_{0}_LINKER_FLAGS={1}" # cmake has separate linker arguments for types of builds. - for type in ["EXE", "MODULE", "SHARED", "STATIC"]: - self.cmake_flag_args.append(ld_string.format(type, ldflags)) + self.cmake_flag_args.append(f"-DCMAKE_EXE_LINKER_FLAGS={ldflags}") + self.cmake_flag_args.append(f"-DCMAKE_MODULE_LINKER_FLAGS={ldflags}") + self.cmake_flag_args.append(f"-DCMAKE_SHARED_LINKER_FLAGS={ldflags}") # CMake has libs options separated by language. Apply ours to each. if flags["ldlibs"]: @@ -248,7 +248,8 @@ def std_cmake_args(self): @staticmethod def std_args(pkg, generator=None): """Computes the standard cmake arguments for a generic package""" - generator = generator or "Unix Makefiles" + default_generator = "Ninja" if sys.platform == "win32" else "Unix Makefiles" + generator = generator or default_generator valid_primary_generators = ["Unix Makefiles", "Ninja"] primary_generator = _extract_primary_generator(generator) if primary_generator not in valid_primary_generators: @@ -273,7 +274,6 @@ def std_args(pkg, generator=None): generator, define("CMAKE_INSTALL_PREFIX", pathlib.Path(pkg.prefix).as_posix()), define("CMAKE_BUILD_TYPE", build_type), - define("BUILD_TESTING", pkg.run_tests), ] # CMAKE_INTERPROCEDURAL_OPTIMIZATION only exists for CMake >= 3.9 @@ -296,8 +296,46 @@ def std_args(pkg, generator=None): define("CMAKE_PREFIX_PATH", spack.build_environment.get_cmake_prefix_path(pkg)), ] ) + return args + @staticmethod + def define_cuda_architectures(pkg): + """Returns the str ``-DCMAKE_CUDA_ARCHITECTURES:STRING=(expanded cuda_arch)``. + + ``cuda_arch`` is variant composed of a list of target CUDA architectures and + it is declared in the cuda package. + + This method is no-op for cmake<3.18 and when ``cuda_arch`` variant is not set. + + """ + cmake_flag = str() + if "cuda_arch" in pkg.spec.variants and pkg.spec.satisfies("^cmake@3.18:"): + cmake_flag = CMakeBuilder.define( + "CMAKE_CUDA_ARCHITECTURES", pkg.spec.variants["cuda_arch"].value + ) + + return cmake_flag + + @staticmethod + def define_hip_architectures(pkg): + """Returns the str ``-DCMAKE_HIP_ARCHITECTURES:STRING=(expanded amdgpu_target)``. + + ``amdgpu_target`` is variant composed of a list of the target HIP + architectures and it is declared in the rocm package. + + This method is no-op for cmake<3.18 and when ``amdgpu_target`` variant is + not set. + + """ + cmake_flag = str() + if "amdgpu_target" in pkg.spec.variants and pkg.spec.satisfies("^cmake@3.21:"): + cmake_flag = CMakeBuilder.define( + "CMAKE_HIP_ARCHITECTURES", pkg.spec.variants["amdgpu_target"].value + ) + + return cmake_flag + @staticmethod def define(cmake_var, value): """Return a CMake command line argument that defines a variable. @@ -412,7 +450,6 @@ def cmake_args(self): * CMAKE_INSTALL_PREFIX * CMAKE_BUILD_TYPE - * BUILD_TESTING which will be set automatically. """ diff --git a/lib/spack/spack/build_systems/cuda.py b/lib/spack/spack/build_systems/cuda.py index fd7e2a0df3d5c1..4a07c8b2c4cf44 100644 --- a/lib/spack/spack/build_systems/cuda.py +++ b/lib/spack/spack/build_systems/cuda.py @@ -154,7 +154,7 @@ def cuda_flags(arch_list): conflicts("%pgi@:15.3,15.5:", when="+cuda ^cuda@7.5 target=x86_64:") conflicts("%pgi@:16.2,16.0:16.3", when="+cuda ^cuda@8 target=x86_64:") conflicts("%pgi@:15,18:", when="+cuda ^cuda@9.0:9.1 target=x86_64:") - conflicts("%pgi@:16,19:", when="+cuda ^cuda@9.2.88:10 target=x86_64:") + conflicts("%pgi@:16,19:", when="+cuda ^cuda@9.2.88:10.0 target=x86_64:") conflicts("%pgi@:17,20:", when="+cuda ^cuda@10.1.105:10.2.89 target=x86_64:") conflicts("%pgi@:17,21:", when="+cuda ^cuda@11.0.2:11.1.0 target=x86_64:") conflicts("%clang@:3.4", when="+cuda ^cuda@:7.5 target=x86_64:") diff --git a/lib/spack/spack/build_systems/makefile.py b/lib/spack/spack/build_systems/makefile.py index feb6d37f24325b..25eec07095b176 100644 --- a/lib/spack/spack/build_systems/makefile.py +++ b/lib/spack/spack/build_systems/makefile.py @@ -9,7 +9,8 @@ import spack.builder import spack.package_base -from spack.directives import build_system, conflicts +from spack.directives import build_system, conflicts, depends_on +from spack.multimethod import when from ._checks import ( BaseBuilder, @@ -29,7 +30,10 @@ class MakefilePackage(spack.package_base.PackageBase): legacy_buildsystem = "makefile" build_system("makefile") - conflicts("platform=windows", when="build_system=makefile") + + with when("build_system=makefile"): + conflicts("platform=windows") + depends_on("gmake", type="build") @spack.builder.builder("makefile") diff --git a/lib/spack/spack/build_systems/meson.py b/lib/spack/spack/build_systems/meson.py index be02685e54c324..38939dc7adc323 100644 --- a/lib/spack/spack/build_systems/meson.py +++ b/lib/spack/spack/build_systems/meson.py @@ -10,7 +10,7 @@ import spack.builder import spack.package_base -from spack.directives import build_system, depends_on, variant +from spack.directives import build_system, conflicts, depends_on, variant from spack.multimethod import when from ._checks import BaseBuilder, execute_build_time_tests @@ -47,6 +47,13 @@ class MesonPackage(spack.package_base.PackageBase): variant("strip", default=False, description="Strip targets on install") depends_on("meson", type="build") depends_on("ninja", type="build") + # Python detection in meson requires distutils to be importable, but distutils no longer + # exists in Python 3.12. In Spack, we can't use setuptools as distutils replacement, + # because the distutils-precedence.pth startup file that setuptools ships with is not run + # when setuptools is in PYTHONPATH; it has to be in system site-packages. In a future meson + # release, the distutils requirement will be dropped, so this conflict can be relaxed. + # We have patches to make it work with meson 1.1 and above. + conflicts("^python@3.12:", when="^meson@:1.0") def flags_to_build_system_args(self, flags): """Produces a list of all command line arguments to pass the specified @@ -209,5 +216,5 @@ def install(self, pkg, spec, prefix): def check(self): """Search Meson-generated files for the target ``test`` and run it if found.""" with fs.working_dir(self.build_directory): - self._if_ninja_target_execute("test") - self._if_ninja_target_execute("check") + self.pkg._if_ninja_target_execute("test") + self.pkg._if_ninja_target_execute("check") diff --git a/lib/spack/spack/build_systems/nmake.py b/lib/spack/spack/build_systems/nmake.py index 7b0f7f18baf7ce..3349ff892966f7 100644 --- a/lib/spack/spack/build_systems/nmake.py +++ b/lib/spack/spack/build_systems/nmake.py @@ -95,7 +95,7 @@ def makefile_root(self): return self.stage.source_path @property - def nmakefile_name(self): + def makefile_name(self): """Name of the current makefile. This is currently an empty value. If a project defines this value, it will be used with the /f argument to provide nmake an explicit makefile. This is usefule in scenarios where @@ -126,8 +126,8 @@ def build(self, pkg, spec, prefix): """Run "nmake" on the build targets specified by the builder.""" opts = self.std_nmake_args opts += self.nmake_args() - if self.nmakefile_name: - opts.append("/f {}".format(self.nmakefile_name)) + if self.makefile_name: + opts.append("/F{}".format(self.makefile_name)) with fs.working_dir(self.build_directory): inspect.getmodule(self.pkg).nmake( *opts, *self.build_targets, ignore_quotes=self.ignore_quotes @@ -139,8 +139,8 @@ def install(self, pkg, spec, prefix): opts = self.std_nmake_args opts += self.nmake_args() opts += self.nmake_install_args() - if self.nmakefile_name: - opts.append("/f {}".format(self.nmakefile_name)) + if self.makefile_name: + opts.append("/F{}".format(self.makefile_name)) opts.append(self.define("PREFIX", prefix)) with fs.working_dir(self.build_directory): inspect.getmodule(self.pkg).nmake( diff --git a/lib/spack/spack/build_systems/oneapi.py b/lib/spack/spack/build_systems/oneapi.py index fddbd3410c73d4..4c432c0cace6d6 100644 --- a/lib/spack/spack/build_systems/oneapi.py +++ b/lib/spack/spack/build_systems/oneapi.py @@ -9,11 +9,10 @@ import shutil from os.path import basename, dirname, isdir -from llnl.util.filesystem import find_headers, find_libraries, join_path +from llnl.util.filesystem import find_headers, find_libraries, join_path, mkdirp from llnl.util.link_tree import LinkTree from spack.directives import conflicts, variant -from spack.package import mkdirp from spack.util.environment import EnvironmentModifications from spack.util.executable import Executable @@ -61,6 +60,11 @@ def component_prefix(self): """Path to component //.""" return self.prefix.join(join_path(self.component_dir, self.spec.version)) + @property + def env_script_args(self): + """Additional arguments to pass to vars.sh script.""" + return () + def install(self, spec, prefix): self.install_component(basename(self.url_for_version(spec.version))) @@ -124,7 +128,7 @@ def setup_run_environment(self, env): if "~envmods" not in self.spec: env.extend( EnvironmentModifications.from_sourcing_file( - join_path(self.component_prefix, "env", "vars.sh") + join_path(self.component_prefix, "env", "vars.sh"), *self.env_script_args ) ) @@ -207,3 +211,7 @@ def link_flags(self): @property def ld_flags(self): return "{0} {1}".format(self.search_flags, self.link_flags) + + +#: Tuple of Intel math libraries, exported to packages +INTEL_MATH_LIBRARIES = ("intel-mkl", "intel-oneapi-mkl", "intel-parallel-studio") diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py index 7a355490f7a0d9..521994b1ec95f7 100644 --- a/lib/spack/spack/build_systems/python.py +++ b/lib/spack/spack/build_systems/python.py @@ -16,21 +16,38 @@ import spack.builder import spack.config +import spack.deptypes as dt import spack.detection import spack.multimethod import spack.package_base import spack.spec import spack.store from spack.directives import build_system, depends_on, extends, maintainers -from spack.error import NoHeadersError, NoLibrariesError, SpecError +from spack.error import NoHeadersError, NoLibrariesError from spack.install_test import test_part -from spack.version import Version from ._checks import BaseBuilder, execute_install_time_tests +def _flatten_dict(dictionary): + """Iterable that yields KEY=VALUE paths through a dictionary. + Args: + dictionary: Possibly nested dictionary of arbitrary keys and values. + Yields: + A single path through the dictionary. + """ + for key, item in dictionary.items(): + if isinstance(item, dict): + # Recursive case + for value in _flatten_dict(item): + yield f"{key}={value}" + else: + # Base case + yield f"{key}={item}" + + class PythonExtension(spack.package_base.PackageBase): - maintainers("adamjstewart", "pradyunsg") + maintainers("adamjstewart") @property def import_modules(self): @@ -173,7 +190,7 @@ def test_imports(self): # Make sure we are importing the installed modules, # not the ones in the source directory - python = inspect.getmodule(self).python.path + python = inspect.getmodule(self).python for module in self.import_modules: with test_part( self, @@ -201,7 +218,7 @@ def update_external_dependencies(self, extendee_spec=None): else: python = self.get_external_python_for_prefix() if not python.concrete: - repo = spack.repo.path.repo_for_pkg(python) + repo = spack.repo.PATH.repo_for_pkg(python) python.namespace = repo.namespace # Ensure architecture information is present @@ -226,7 +243,48 @@ def update_external_dependencies(self, extendee_spec=None): python.external_path = self.spec.external_path python._mark_concrete() - self.spec.add_dependency_edge(python, deptypes=("build", "link", "run"), virtuals=()) + self.spec.add_dependency_edge(python, depflag=dt.BUILD | dt.LINK | dt.RUN, virtuals=()) + + def get_external_python_for_prefix(self): + """ + For an external package that extends python, find the most likely spec for the python + it depends on. + + First search: an "installed" external that shares a prefix with this package + Second search: a configured external that shares a prefix with this package + Third search: search this prefix for a python package + + Returns: + spack.spec.Spec: The external Spec for python most likely to be compatible with self.spec + """ + python_externals_installed = [ + s for s in spack.store.STORE.db.query("python") if s.prefix == self.spec.external_path + ] + if python_externals_installed: + return python_externals_installed[0] + + python_external_config = spack.config.get("packages:python:externals", []) + python_externals_configured = [ + spack.spec.parse_with_version_concrete(item["spec"]) + for item in python_external_config + if item["prefix"] == self.spec.external_path + ] + if python_externals_configured: + return python_externals_configured[0] + + python_externals_detection = spack.detection.by_path( + ["python"], path_hints=[self.spec.external_path] + ) + + python_externals_detected = [ + d.spec + for d in python_externals_detection.get("python", []) + if d.prefix == self.spec.external_path + ] + if python_externals_detected: + return python_externals_detected[0] + + raise StopIteration("No external python could be detected for %s to depend on" % self.spec) class PythonPackage(PythonExtension): @@ -273,54 +331,16 @@ def list_url(cls): name = cls.pypi.split("/")[0] return "https://pypi.org/simple/" + name + "/" - def get_external_python_for_prefix(self): - """ - For an external package that extends python, find the most likely spec for the python - it depends on. - - First search: an "installed" external that shares a prefix with this package - Second search: a configured external that shares a prefix with this package - Third search: search this prefix for a python package - - Returns: - spack.spec.Spec: The external Spec for python most likely to be compatible with self.spec - """ - python_externals_installed = [ - s for s in spack.store.db.query("python") if s.prefix == self.spec.external_path - ] - if python_externals_installed: - return python_externals_installed[0] - - python_external_config = spack.config.get("packages:python:externals", []) - python_externals_configured = [ - spack.spec.parse_with_version_concrete(item["spec"]) - for item in python_external_config - if item["prefix"] == self.spec.external_path - ] - if python_externals_configured: - return python_externals_configured[0] - - python_externals_detection = spack.detection.by_executable( - [spack.repo.path.get_pkg_class("python")], path_hints=[self.spec.external_path] - ) - - python_externals_detected = [ - d.spec - for d in python_externals_detection.get("python", []) - if d.prefix == self.spec.external_path - ] - if python_externals_detected: - return python_externals_detected[0] - - raise StopIteration("No external python could be detected for %s to depend on" % self.spec) - @property def headers(self): """Discover header files in platlib.""" + # Remove py- prefix in package name + name = self.spec.name[3:] + # Headers may be in either location - include = self.prefix.join(self.spec["python"].package.include) - platlib = self.prefix.join(self.spec["python"].package.platlib) + include = self.prefix.join(self.spec["python"].package.include).join(name) + platlib = self.prefix.join(self.spec["python"].package.platlib).join(name) headers = fs.find_all_headers(include) + fs.find_all_headers(platlib) if headers: @@ -334,13 +354,14 @@ def libs(self): """Discover libraries in platlib.""" # Remove py- prefix in package name - library = "lib" + self.spec.name[3:].replace("-", "?") - root = self.prefix.join(self.spec["python"].package.platlib) + name = self.spec.name[3:] - for shared in [True, False]: - libs = fs.find_libraries(library, root, shared=shared, recursive=True) - if libs: - return libs + root = self.prefix.join(self.spec["python"].package.platlib).join(name) + + libs = fs.find_all_libraries(root, recursive=True) + + if libs: + return libs msg = "Unable to recursively locate {} libraries in {}" raise NoLibrariesError(msg.format(self.spec.name, root)) @@ -357,7 +378,7 @@ class PythonPipBuilder(BaseBuilder): legacy_long_methods = ("install_options", "global_options", "config_settings") #: Names associated with package attributes in the old build-system format - legacy_attributes = ("build_directory", "install_time_test_callbacks") + legacy_attributes = ("archive_files", "build_directory", "install_time_test_callbacks") #: Callback names for install-time test install_time_test_callbacks = ["test"] @@ -401,20 +422,24 @@ def build_directory(self): def config_settings(self, spec, prefix): """Configuration settings to be passed to the PEP 517 build backend. - Requires pip 22.1+, which requires Python 3.7+. + + Requires pip 22.1 or newer for keys that appear only a single time, + or pip 23.1 or newer if the same key appears multiple times. Args: spec (spack.spec.Spec): build spec prefix (spack.util.prefix.Prefix): installation prefix Returns: - dict: dictionary of KEY, VALUE settings + dict: Possibly nested dictionary of KEY, VALUE settings """ return {} def install_options(self, spec, prefix): """Extra arguments to be supplied to the setup.py install command. + Requires pip 23.0 or older. + Args: spec (spack.spec.Spec): build spec prefix (spack.util.prefix.Prefix): installation prefix @@ -428,6 +453,8 @@ def global_options(self, spec, prefix): """Extra global options to be supplied to the setup.py call before the install or bdist_wheel command. + Deprecated in pip 23.1. + Args: spec (spack.spec.Spec): build spec prefix (spack.util.prefix.Prefix): installation prefix @@ -440,29 +467,28 @@ def global_options(self, spec, prefix): def install(self, pkg, spec, prefix): """Install everything from build directory.""" - args = PythonPipBuilder.std_args(pkg) + ["--prefix=" + prefix] - - for key, value in self.config_settings(spec, prefix).items(): - if spec["py-pip"].version < Version("22.1"): - raise SpecError( - "'{}' package uses 'config_settings' which is only supported by " - "pip 22.1+. Add the following line to the package to fix this:\n\n" - ' depends_on("py-pip@22.1:", type="build")'.format(spec.name) - ) - - args.append("--config-settings={}={}".format(key, value)) + args = PythonPipBuilder.std_args(pkg) + [f"--prefix={prefix}"] + for setting in _flatten_dict(self.config_settings(spec, prefix)): + args.append(f"--config-settings={setting}") for option in self.install_options(spec, prefix): - args.append("--install-option=" + option) + args.append(f"--install-option={option}") for option in self.global_options(spec, prefix): - args.append("--global-option=" + option) + args.append(f"--global-option={option}") if pkg.stage.archive_file and pkg.stage.archive_file.endswith(".whl"): args.append(pkg.stage.archive_file) else: args.append(".") - pip = inspect.getmodule(pkg).pip + pip = spec["python"].command + # Hide user packages, since we don't have build isolation. This is + # necessary because pip / setuptools may run hooks from arbitrary + # packages during the build. There is no equivalent variable to hide + # system packages, so this is not reliable for external Python. + pip.add_default_env("PYTHONNOUSERSITE", "1") + pip.add_default_arg("-m") + pip.add_default_arg("pip") with fs.working_dir(self.build_directory): pip(*args) diff --git a/lib/spack/spack/build_systems/qmake.py b/lib/spack/spack/build_systems/qmake.py index cd8fe0904bccfd..538054f3b05643 100644 --- a/lib/spack/spack/build_systems/qmake.py +++ b/lib/spack/spack/build_systems/qmake.py @@ -28,7 +28,7 @@ class QMakePackage(spack.package_base.PackageBase): build_system("qmake") - depends_on("qt", type="build", when="build_system=qmake") + depends_on("qmake", type="build", when="build_system=qmake") @spack.builder.builder("qmake") diff --git a/lib/spack/spack/build_systems/racket.py b/lib/spack/spack/build_systems/racket.py index 8a07773a292e14..50c4944a5d7ccb 100644 --- a/lib/spack/spack/build_systems/racket.py +++ b/lib/spack/spack/build_systems/racket.py @@ -10,9 +10,10 @@ import llnl.util.tty as tty import spack.builder -from spack.build_environment import SPACK_NO_PARALLEL_MAKE, determine_number_of_jobs +from spack.build_environment import SPACK_NO_PARALLEL_MAKE from spack.directives import build_system, extends, maintainers from spack.package_base import PackageBase +from spack.util.cpus import determine_number_of_jobs from spack.util.environment import env_flag from spack.util.executable import Executable, ProcessError @@ -63,7 +64,7 @@ class RacketBuilder(spack.builder.Builder): @property def subdirectory(self): - if self.racket_name: + if self.pkg.racket_name: return "pkgs/{0}".format(self.pkg.racket_name) return None @@ -92,7 +93,7 @@ def install(self, pkg, spec, prefix): "--copy", "-i", "-j", - str(determine_number_of_jobs(parallel)), + str(determine_number_of_jobs(parallel=parallel)), "--", os.getcwd(), ] diff --git a/lib/spack/spack/build_systems/rocm.py b/lib/spack/spack/build_systems/rocm.py index e219ee1f8917c3..a1c6d661f50549 100644 --- a/lib/spack/spack/build_systems/rocm.py +++ b/lib/spack/spack/build_systems/rocm.py @@ -140,8 +140,6 @@ class ROCmPackage(PackageBase): depends_on("hsa-rocr-dev", when="+rocm") depends_on("hip +rocm", when="+rocm") - conflicts("^blt@:0.3.6", when="+rocm") - # need amd gpu type for rocm builds conflicts("amdgpu_target=none", when="+rocm") diff --git a/lib/spack/spack/build_systems/sip.py b/lib/spack/spack/build_systems/sip.py index 3ec43f5f822879..884c740d561d4f 100644 --- a/lib/spack/spack/build_systems/sip.py +++ b/lib/spack/spack/build_systems/sip.py @@ -7,13 +7,14 @@ import re import llnl.util.tty as tty -from llnl.util.filesystem import find, join_path, working_dir +from llnl.util.filesystem import find, working_dir import spack.builder import spack.install_test import spack.package_base from spack.directives import build_system, depends_on, extends from spack.multimethod import when +from spack.util.executable import Executable from ._checks import BaseBuilder, execute_install_time_tests @@ -39,9 +40,8 @@ class SIPPackage(spack.package_base.PackageBase): build_system("sip") with when("build_system=sip"): - extends("python") - depends_on("qt") - depends_on("py-sip") + extends("python", type=("build", "link", "run")) + depends_on("py-sip", type="build") @property def import_modules(self): @@ -113,13 +113,13 @@ class SIPBuilder(BaseBuilder): * install The configure phase already adds a set of default flags. To see more - options, run ``python configure.py --help``. + options, run ``sip-build --help``. """ phases = ("configure", "build", "install") #: Names associated with package methods in the old build-system format - legacy_methods = ("configure_file", "configure_args", "build_args", "install_args") + legacy_methods = ("configure_args", "build_args", "install_args") #: Names associated with package attributes in the old build-system format legacy_attributes = ( @@ -130,34 +130,17 @@ class SIPBuilder(BaseBuilder): "build_directory", ) - def configure_file(self): - """Returns the name of the configure file to use.""" - return "configure.py" + build_directory = "build" def configure(self, pkg, spec, prefix): """Configure the package.""" - configure = self.configure_file() - - args = self.configure_args() - - args.extend( - [ - "--verbose", - "--confirm-license", - "--qmake", - spec["qt"].prefix.bin.qmake, - "--sip", - spec["py-sip"].prefix.bin.sip, - "--sip-incdir", - join_path(spec["py-sip"].prefix, spec["python"].package.include), - "--bindir", - prefix.bin, - "--destdir", - inspect.getmodule(self.pkg).python_platlib, - ] - ) - - self.pkg.python(configure, *args) + + # https://www.riverbankcomputing.com/static/Docs/sip/command_line_tools.html + args = ["--verbose", "--target-dir", inspect.getmodule(self.pkg).python_platlib] + args.extend(self.configure_args()) + + sip_build = Executable(spec["py-sip"].prefix.bin.join("sip-build")) + sip_build(*args) def configure_args(self): """Arguments to pass to configure.""" @@ -167,7 +150,8 @@ def build(self, pkg, spec, prefix): """Build the package.""" args = self.build_args() - inspect.getmodule(self.pkg).make(*args) + with working_dir(self.build_directory): + inspect.getmodule(self.pkg).make(*args) def build_args(self): """Arguments to pass to build.""" @@ -177,21 +161,11 @@ def install(self, pkg, spec, prefix): """Install the package.""" args = self.install_args() - inspect.getmodule(self.pkg).make("install", parallel=False, *args) + with working_dir(self.build_directory): + inspect.getmodule(self.pkg).make("install", *args) def install_args(self): """Arguments to pass to install.""" return [] spack.builder.run_after("install")(execute_install_time_tests) - - @spack.builder.run_after("install") - def extend_path_setup(self): - # See github issue #14121 and PR #15297 - module = self.pkg.spec["py-sip"].variants["module"].value - if module != "sip": - module = module.split(".")[0] - with working_dir(inspect.getmodule(self.pkg).python_platlib): - with open(os.path.join(module, "__init__.py"), "a") as f: - f.write("from pkgutil import extend_path\n") - f.write("__path__ = extend_path(__path__, __name__)\n") diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index 87ecd30ed4e525..14c7571f4832db 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -20,9 +20,9 @@ def misc_cache_location(): - """The ``misc_cache`` is Spack's cache for small data. + """The ``MISC_CACHE`` is Spack's cache for small data. - Currently the ``misc_cache`` stores indexes for virtual dependency + Currently the ``MISC_CACHE`` stores indexes for virtual dependency providers and for which packages provide which tags. """ path = spack.config.get("config:misc_cache", spack.paths.default_misc_cache_path) @@ -35,7 +35,7 @@ def _misc_cache(): #: Spack's cache for small data -misc_cache: Union[ +MISC_CACHE: Union[ spack.util.file_cache.FileCache, llnl.util.lang.Singleton ] = llnl.util.lang.Singleton(_misc_cache) @@ -91,6 +91,6 @@ def symlink(self, mirror_ref): #: Spack's local cache for downloaded source archives -fetch_cache: Union[ +FETCH_CACHE: Union[ spack.fetch_strategy.FsCache, llnl.util.lang.Singleton ] = llnl.util.lang.Singleton(_fetch_cache) diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index cfe87e52141df9..afad3b7a45197e 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -25,6 +25,7 @@ import llnl.util.filesystem as fs import llnl.util.tty as tty from llnl.util.lang import memoized +from llnl.util.tty.color import cescape, colorize import spack import spack.binary_distribution as bindist @@ -49,7 +50,11 @@ TEMP_STORAGE_MIRROR_NAME = "ci_temporary_mirror" SPACK_RESERVED_TAGS = ["public", "protected", "notary"] +# TODO: Remove this in Spack 0.23 SHARED_PR_MIRROR_URL = "s3://spack-binaries-prs/shared_pr_mirror" +JOB_NAME_FORMAT = ( + "{name}{@version} {/hash:7} {%compiler.name}{@compiler.version}{arch=architecture}" +) spack_gpg = spack.main.SpackCommand("gpg") spack_compiler = spack.main.SpackCommand("compiler") @@ -69,48 +74,23 @@ def __exit__(self, exc_type, exc_value, exc_traceback): return False -def get_job_name(spec, osarch, build_group): - """Given the necessary parts, format the gitlab job name +def get_job_name(spec: spack.spec.Spec, build_group: str = ""): + """Given a spec and possibly a build group, return the job name. If the + resulting name is longer than 255 characters, it will be truncated. Arguments: spec (spack.spec.Spec): Spec job will build - osarch: Architecture TODO: (this is a spack.spec.ArchSpec, - but sphinx doesn't recognize the type and fails). build_group (str): Name of build group this job belongs to (a CDash notion) Returns: The job name """ - item_idx = 0 - format_str = "" - format_args = [] - - format_str += "{{{0}}}".format(item_idx) - format_args.append(spec.name) - item_idx += 1 - - format_str += "/{{{0}}}".format(item_idx) - format_args.append(spec.dag_hash(7)) - item_idx += 1 - - format_str += " {{{0}}}".format(item_idx) - format_args.append(spec.version) - item_idx += 1 - - format_str += " {{{0}}}".format(item_idx) - format_args.append(spec.compiler) - item_idx += 1 - - format_str += " {{{0}}}".format(item_idx) - format_args.append(osarch) - item_idx += 1 + job_name = spec.format(JOB_NAME_FORMAT) if build_group: - format_str += " {{{0}}}".format(item_idx) - format_args.append(build_group) - item_idx += 1 + job_name = "{0} {1}".format(job_name, build_group) - return format_str.format(*format_args) + return job_name[:255] def _remove_reserved_tags(tags): @@ -118,15 +98,6 @@ def _remove_reserved_tags(tags): return [tag for tag in tags if tag not in SPACK_RESERVED_TAGS] -def _get_spec_string(spec): - format_elements = ["{name}{@version}", "{%compiler}"] - - if spec.architecture: - format_elements.append(" {arch=architecture}") - - return spec.format("".join(format_elements)) - - def _spec_deps_key(s): return "{0}/{1}".format(s.name, s.dag_hash(7)) @@ -224,29 +195,29 @@ def _print_staging_summary(spec_labels, stages, mirrors_to_check, rebuild_decisi if not stages: return - mirrors = spack.mirror.MirrorCollection(mirrors=mirrors_to_check) + mirrors = spack.mirror.MirrorCollection(mirrors=mirrors_to_check, binary=True) tty.msg("Checked the following mirrors for binaries:") for m in mirrors.values(): tty.msg(" {0}".format(m.fetch_url)) tty.msg("Staging summary ([x] means a job needs rebuilding):") for stage_index, stage in enumerate(stages): - tty.msg(" stage {0} ({1} jobs):".format(stage_index, len(stage))) + tty.msg(f" stage {stage_index} ({len(stage)} jobs):") - for job in sorted(stage): + for job in sorted(stage, key=lambda j: (not rebuild_decisions[j].rebuild, j)): s = spec_labels[job] - rebuild = rebuild_decisions[job].rebuild reason = rebuild_decisions[job].reason - reason_msg = " ({0})".format(reason) if reason else "" - tty.msg( - " [{1}] {0} -> {2}{3}".format( - job, "x" if rebuild else " ", _get_spec_string(s), reason_msg - ) - ) - if rebuild_decisions[job].mirrors: - tty.msg(" found on the following mirrors:") - for murl in rebuild_decisions[job].mirrors: - tty.msg(" {0}".format(murl)) + reason_msg = f" ({reason})" if reason else "" + spec_fmt = "{name}{@version}{%compiler}{/hash:7}" + if rebuild_decisions[job].rebuild: + status = colorize("@*g{[x]} ") + msg = f" {status}{s.cformat(spec_fmt)}{reason_msg}" + else: + msg = f"{s.format(spec_fmt)}{reason_msg}" + if rebuild_decisions[job].mirrors: + msg += f" [{', '.join(rebuild_decisions[job].mirrors)}]" + msg = colorize(f" @K - {cescape(msg)}@.") + tty.msg(msg) def _compute_spec_deps(spec_list): @@ -308,7 +279,7 @@ def append_dep(s, d): dependencies.append({"spec": s, "depends": d}) for spec in spec_list: - for s in spec.traverse(deptype=all): + for s in spec.traverse(deptype="all"): if s.external: tty.msg("Will not stage external pkg: {0}".format(s)) continue @@ -316,7 +287,7 @@ def append_dep(s, d): skey = _spec_deps_key(s) spec_labels[skey] = s - for d in s.dependencies(deptype=all): + for d in s.dependencies(deptype="all"): dkey = _spec_deps_key(d) if d.external: tty.msg("Will not stage external dep: {0}".format(d)) @@ -337,7 +308,7 @@ def _spec_matches(spec, match_string): def _format_job_needs( - dep_jobs, osname, build_group, prune_dag, rebuild_decisions, enable_artifacts_buildcache + dep_jobs, build_group, prune_dag, rebuild_decisions, enable_artifacts_buildcache ): needs_list = [] for dep_job in dep_jobs: @@ -347,7 +318,7 @@ def _format_job_needs( if not prune_dag or rebuild: needs_list.append( { - "job": get_job_name(dep_job, dep_job.architecture, build_group), + "job": get_job_name(dep_job, build_group), "artifacts": enable_artifacts_buildcache, } ) @@ -535,7 +506,7 @@ def __job_name(name, suffix=""): """Compute the name of a named job with appropriate suffix. Valid suffixes are either '-remove' or empty string or None """ - assert type(name) == str + assert isinstance(name, str) jname = name if suffix: @@ -700,7 +671,7 @@ def generate_gitlab_ci_yaml( remote_mirror_override (str): Typically only needed when one spack.yaml is used to populate several mirrors with binaries, based on some criteria. Spack protected pipelines populate different mirrors based - on branch name, facilitated by this option. + on branch name, facilitated by this option. DEPRECATED """ with spack.concretize.disable_compiler_existence_check(): with env.write_transaction(): @@ -797,17 +768,39 @@ def generate_gitlab_ci_yaml( "instead.", ) - if "mirrors" not in yaml_root or len(yaml_root["mirrors"].values()) < 1: - tty.die("spack ci generate requires an env containing a mirror") + pipeline_mirrors = spack.mirror.MirrorCollection(binary=True) + deprecated_mirror_config = False + buildcache_destination = None + if "buildcache-destination" in pipeline_mirrors: + if remote_mirror_override: + tty.die( + "Using the deprecated --buildcache-destination cli option and " + "having a mirror named 'buildcache-destination' at the same time " + "is not allowed" + ) + buildcache_destination = pipeline_mirrors["buildcache-destination"] + else: + deprecated_mirror_config = True + # TODO: This will be an error in Spack 0.23 + + # TODO: Remove this block in spack 0.23 + remote_mirror_url = None + if deprecated_mirror_config: + if "mirrors" not in yaml_root or len(yaml_root["mirrors"].values()) < 1: + tty.die("spack ci generate requires an env containing a mirror") - ci_mirrors = yaml_root["mirrors"] - mirror_urls = [url for url in ci_mirrors.values()] - remote_mirror_url = mirror_urls[0] + ci_mirrors = yaml_root["mirrors"] + mirror_urls = [url for url in ci_mirrors.values()] + remote_mirror_url = mirror_urls[0] spack_buildcache_copy = os.environ.get("SPACK_COPY_BUILDCACHE", None) if spack_buildcache_copy: buildcache_copies = {} - buildcache_copy_src_prefix = remote_mirror_override or remote_mirror_url + buildcache_copy_src_prefix = ( + buildcache_destination.fetch_url + if buildcache_destination + else remote_mirror_override or remote_mirror_url + ) buildcache_copy_dest_prefix = spack_buildcache_copy # Check for a list of "known broken" specs that we should not bother @@ -819,6 +812,7 @@ def generate_gitlab_ci_yaml( enable_artifacts_buildcache = False if "enable-artifacts-buildcache" in ci_config: + tty.warn("Support for enable-artifacts-buildcache will be removed in Spack 0.23") enable_artifacts_buildcache = ci_config["enable-artifacts-buildcache"] rebuild_index_enabled = True @@ -827,13 +821,15 @@ def generate_gitlab_ci_yaml( temp_storage_url_prefix = None if "temporary-storage-url-prefix" in ci_config: + tty.warn("Support for temporary-storage-url-prefix will be removed in Spack 0.23") temp_storage_url_prefix = ci_config["temporary-storage-url-prefix"] # If a remote mirror override (alternate buildcache destination) was # specified, add it here in case it has already built hashes we might # generate. + # TODO: Remove this block in Spack 0.23 mirrors_to_check = None - if remote_mirror_override: + if deprecated_mirror_config and remote_mirror_override: if spack_pipeline_type == "spack_protected_branch": # Overriding the main mirror in this case might result # in skipping jobs on a release pipeline because specs are @@ -853,8 +849,9 @@ def generate_gitlab_ci_yaml( cfg.default_modify_scope(), ) + # TODO: Remove this block in Spack 0.23 shared_pr_mirror = None - if spack_pipeline_type == "spack_pull_request": + if deprecated_mirror_config and spack_pipeline_type == "spack_pull_request": stack_name = os.environ.get("SPACK_CI_STACK_NAME", "") shared_pr_mirror = url_util.join(SHARED_PR_MIRROR_URL, stack_name) spack.mirror.add( @@ -885,7 +882,7 @@ def generate_gitlab_ci_yaml( cli_scopes = [ os.path.relpath(s.path, concrete_env_dir) for s in cfg.scopes().values() - if type(s) == cfg.ImmutableConfigScope + if isinstance(s, cfg.ImmutableConfigScope) and s.path not in env_includes and os.path.exists(s.path) ] @@ -906,6 +903,7 @@ def generate_gitlab_ci_yaml( job_log_dir = os.path.join(pipeline_artifacts_dir, "logs") job_repro_dir = os.path.join(pipeline_artifacts_dir, "reproduction") job_test_dir = os.path.join(pipeline_artifacts_dir, "tests") + # TODO: Remove this line in Spack 0.23 local_mirror_dir = os.path.join(pipeline_artifacts_dir, "mirror") user_artifacts_dir = os.path.join(pipeline_artifacts_dir, "user_data") @@ -920,13 +918,13 @@ def generate_gitlab_ci_yaml( rel_job_log_dir = os.path.relpath(job_log_dir, ci_project_dir) rel_job_repro_dir = os.path.relpath(job_repro_dir, ci_project_dir) rel_job_test_dir = os.path.relpath(job_test_dir, ci_project_dir) + # TODO: Remove this line in Spack 0.23 rel_local_mirror_dir = os.path.join(local_mirror_dir, ci_project_dir) rel_user_artifacts_dir = os.path.relpath(user_artifacts_dir, ci_project_dir) # Speed up staging by first fetching binary indices from all mirrors - # (including the override mirror we may have just added above). try: - bindist.binary_index.update() + bindist.BINARY_INDEX.update() except bindist.FetchCacheError as e: tty.warn(e) @@ -1023,19 +1021,23 @@ def main_script_replacements(cmd): if "after_script" in job_object: job_object["after_script"] = _unpack_script(job_object["after_script"]) - osname = str(release_spec.architecture) - job_name = get_job_name(release_spec, osname, build_group) + job_name = get_job_name(release_spec, build_group) job_vars = job_object.setdefault("variables", {}) job_vars["SPACK_JOB_SPEC_DAG_HASH"] = release_spec_dag_hash job_vars["SPACK_JOB_SPEC_PKG_NAME"] = release_spec.name + job_vars["SPACK_JOB_SPEC_PKG_VERSION"] = release_spec.format("{version}") + job_vars["SPACK_JOB_SPEC_COMPILER_NAME"] = release_spec.format("{compiler.name}") + job_vars["SPACK_JOB_SPEC_COMPILER_VERSION"] = release_spec.format("{compiler.version}") + job_vars["SPACK_JOB_SPEC_ARCH"] = release_spec.format("{architecture}") + job_vars["SPACK_JOB_SPEC_VARIANTS"] = release_spec.format("{variants}") job_object["needs"] = [] if spec_label in dependencies: if enable_artifacts_buildcache: # Get dependencies transitively, so they're all # available in the artifacts buildcache. - dep_jobs = [d for d in release_spec.traverse(deptype=all, root=False)] + dep_jobs = [d for d in release_spec.traverse(deptype="all", root=False)] else: # In this case, "needs" is only used for scheduling # purposes, so we only get the direct dependencies. @@ -1046,7 +1048,6 @@ def main_script_replacements(cmd): job_object["needs"].extend( _format_job_needs( dep_jobs, - osname, build_group, prune_dag, rebuild_decisions, @@ -1132,6 +1133,7 @@ def main_script_replacements(cmd): }, ) + # TODO: Remove this block in Spack 0.23 if enable_artifacts_buildcache: bc_root = os.path.join(local_mirror_dir, "build_cache") job_object["artifacts"]["paths"].extend( @@ -1161,10 +1163,12 @@ def main_script_replacements(cmd): _print_staging_summary(spec_labels, stages, mirrors_to_check, rebuild_decisions) # Clean up remote mirror override if enabled - if remote_mirror_override: - spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope()) - if spack_pipeline_type == "spack_pull_request": - spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope()) + # TODO: Remove this block in Spack 0.23 + if deprecated_mirror_config: + if remote_mirror_override: + spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope()) + if spack_pipeline_type == "spack_pull_request": + spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope()) tty.debug("{0} build jobs generated in {1} stages".format(job_id, stage_id)) @@ -1195,10 +1199,28 @@ def main_script_replacements(cmd): sync_job["needs"] = [ {"job": generate_job_name, "pipeline": "{0}".format(parent_pipeline_id)} ] + + if "variables" not in sync_job: + sync_job["variables"] = {} + + sync_job["variables"]["SPACK_COPY_ONLY_DESTINATION"] = ( + buildcache_destination.fetch_url + if buildcache_destination + else remote_mirror_override or remote_mirror_url + ) + + if "buildcache-source" in pipeline_mirrors: + buildcache_source = pipeline_mirrors["buildcache-source"].fetch_url + else: + # TODO: Remove this condition in Spack 0.23 + buildcache_source = os.environ.get("SPACK_SOURCE_MIRROR", None) + sync_job["variables"]["SPACK_BUILDCACHE_SOURCE"] = buildcache_source + output_object["copy"] = sync_job job_id += 1 if job_id > 0: + # TODO: Remove this block in Spack 0.23 if temp_storage_url_prefix: # There were some rebuild jobs scheduled, so we will need to # schedule a job to clean up the temporary storage location @@ -1232,6 +1254,13 @@ def main_script_replacements(cmd): signing_job["when"] = "always" signing_job["retry"] = {"max": 2, "when": ["always"]} signing_job["interruptible"] = True + if "variables" not in signing_job: + signing_job["variables"] = {} + signing_job["variables"]["SPACK_BUILDCACHE_DESTINATION"] = ( + buildcache_destination.push_url # need the s3 url for aws s3 sync + if buildcache_destination + else remote_mirror_override or remote_mirror_url + ) output_object["sign-pkgs"] = signing_job @@ -1240,13 +1269,13 @@ def main_script_replacements(cmd): stage_names.append("stage-rebuild-index") final_job = spack_ci_ir["jobs"]["reindex"]["attributes"] - index_target_mirror = mirror_urls[0] - if remote_mirror_override: - index_target_mirror = remote_mirror_override final_job["stage"] = "stage-rebuild-index" + target_mirror = remote_mirror_override or remote_mirror_url + if buildcache_destination: + target_mirror = buildcache_destination.push_url final_job["script"] = _unpack_script( final_job["script"], - op=lambda cmd: cmd.replace("{index_target_mirror}", index_target_mirror), + op=lambda cmd: cmd.replace("{index_target_mirror}", target_mirror), ) final_job["when"] = "always" @@ -1268,28 +1297,30 @@ def main_script_replacements(cmd): "SPACK_CONCRETE_ENV_DIR": rel_concrete_env_dir, "SPACK_VERSION": spack_version, "SPACK_CHECKOUT_VERSION": version_to_clone, + # TODO: Remove this line in Spack 0.23 "SPACK_REMOTE_MIRROR_URL": remote_mirror_url, "SPACK_JOB_LOG_DIR": rel_job_log_dir, "SPACK_JOB_REPRO_DIR": rel_job_repro_dir, "SPACK_JOB_TEST_DIR": rel_job_test_dir, + # TODO: Remove this line in Spack 0.23 "SPACK_LOCAL_MIRROR_DIR": rel_local_mirror_dir, "SPACK_PIPELINE_TYPE": str(spack_pipeline_type), "SPACK_CI_STACK_NAME": os.environ.get("SPACK_CI_STACK_NAME", "None"), + # TODO: Remove this line in Spack 0.23 "SPACK_CI_SHARED_PR_MIRROR_URL": shared_pr_mirror or "None", "SPACK_REBUILD_CHECK_UP_TO_DATE": str(prune_dag), "SPACK_REBUILD_EVERYTHING": str(rebuild_everything), + "SPACK_REQUIRE_SIGNING": os.environ.get("SPACK_REQUIRE_SIGNING", "False"), } - if remote_mirror_override: + # TODO: Remove this block in Spack 0.23 + if deprecated_mirror_config and remote_mirror_override: (output_object["variables"]["SPACK_REMOTE_MIRROR_OVERRIDE"]) = remote_mirror_override spack_stack_name = os.environ.get("SPACK_CI_STACK_NAME", None) if spack_stack_name: output_object["variables"]["SPACK_CI_STACK_NAME"] = spack_stack_name - # Ensure the child pipeline always runs - output_object["workflow"] = {"rules": [{"when": "always"}]} - if spack_buildcache_copy: # Write out the file describing specs that should be copied copy_specs_dir = os.path.join(pipeline_artifacts_dir, "specs_to_copy") @@ -1305,21 +1336,17 @@ def main_script_replacements(cmd): with open(copy_specs_file, "w") as fd: fd.write(json.dumps(buildcache_copies)) - sorted_output = {} - for output_key, output_value in sorted(output_object.items()): - sorted_output[output_key] = output_value - # TODO(opadron): remove this or refactor if run_optimizer: import spack.ci_optimization as ci_opt - sorted_output = ci_opt.optimizer(sorted_output) + output_object = ci_opt.optimizer(output_object) # TODO(opadron): remove this or refactor if use_dependencies: import spack.ci_needs_workaround as cinw - sorted_output = cinw.needs_to_dependencies(sorted_output) + output_object = cinw.needs_to_dependencies(output_object) else: # No jobs were generated noop_job = spack_ci_ir["jobs"]["noop"]["attributes"] @@ -1330,10 +1357,17 @@ def main_script_replacements(cmd): noop_job["script"] = [ 'echo "copy-only pipelines are not supported with deprecated ci configs"' ] - sorted_output = {"unsupported-copy": noop_job} + output_object = {"unsupported-copy": noop_job} else: tty.debug("No specs to rebuild, generating no-op job") - sorted_output = {"no-specs-to-rebuild": noop_job} + output_object = {"no-specs-to-rebuild": noop_job} + + # Ensure the child pipeline always runs + output_object["workflow"] = {"rules": [{"when": "always"}]} + + sorted_output = {} + for output_key, output_value in sorted(output_object.items()): + sorted_output[output_key] = output_value if known_broken_specs_encountered: tty.error("This pipeline generated hashes known to be broken on develop:") @@ -1419,9 +1453,7 @@ def _push_mirror_contents(input_spec, sign_binaries, mirror_url): unsigned = not sign_binaries tty.debug("Creating buildcache ({0})".format("unsigned" if unsigned else "signed")) push_url = spack.mirror.Mirror.from_url(mirror_url).push_url - return bindist.push( - input_spec, push_url, bindist.PushOptions(force=True, allow_root=True, unsigned=unsigned) - ) + return bindist.push(input_spec, push_url, bindist.PushOptions(force=True, unsigned=unsigned)) def push_mirror_contents(input_spec: spack.spec.Spec, mirror_url, sign_binaries): @@ -1505,7 +1537,7 @@ def copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) -> return try: - pkg_cls = spack.repo.path.get_pkg_class(job_spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(job_spec.name) job_pkg = pkg_cls(job_spec) tty.debug("job package: {0}".format(job_pkg)) except AssertionError: @@ -1691,7 +1723,7 @@ def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None): return True -def reproduce_ci_job(url, work_dir): +def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime): """Given a url to gitlab artifacts.zip from a failed 'spack ci rebuild' job, attempt to setup an environment in which the failure can be reproduced locally. This entails the following: @@ -1707,6 +1739,11 @@ def reproduce_ci_job(url, work_dir): work_dir = os.path.realpath(work_dir) download_and_extract_artifacts(url, work_dir) + gpg_path = None + if gpg_url: + gpg_path = web_util.fetch_url_text(gpg_url, dest_dir=os.path.join(work_dir, "_pgp")) + rel_gpg_path = gpg_path.replace(work_dir, "").lstrip(os.path.sep) + lock_file = fs.find(work_dir, "spack.lock")[0] repro_lock_dir = os.path.dirname(lock_file) @@ -1799,60 +1836,63 @@ def reproduce_ci_job(url, work_dir): # more faithful reproducer if everything appears to run in the same # absolute path used during the CI build. mount_as_dir = "/work" + mounted_workdir = "/reproducer" if repro_details: mount_as_dir = repro_details["ci_project_dir"] mounted_repro_dir = os.path.join(mount_as_dir, rel_repro_dir) mounted_env_dir = os.path.join(mount_as_dir, relative_concrete_env_dir) - - # We will also try to clone spack from your local checkout and - # reproduce the state present during the CI build, and put that into - # the bind-mounted reproducer directory. - - # Regular expressions for parsing that HEAD commit. If the pipeline - # was on the gitlab spack mirror, it will have been a merge commit made by - # gitub and pushed by the sync script. If the pipeline was run on some - # environment repo, then the tested spack commit will likely have been - # a regular commit. - commit_1 = None - commit_2 = None - commit_regex = re.compile(r"commit\s+([^\s]+)") - merge_commit_regex = re.compile(r"Merge\s+([^\s]+)\s+into\s+([^\s]+)") - - # Try the more specific merge commit regex first - m = merge_commit_regex.search(spack_info) + if gpg_path: + mounted_gpg_path = os.path.join(mounted_workdir, rel_gpg_path) + + # We will also try to clone spack from your local checkout and + # reproduce the state present during the CI build, and put that into + # the bind-mounted reproducer directory. + + # Regular expressions for parsing that HEAD commit. If the pipeline + # was on the gitlab spack mirror, it will have been a merge commit made by + # gitub and pushed by the sync script. If the pipeline was run on some + # environment repo, then the tested spack commit will likely have been + # a regular commit. + commit_1 = None + commit_2 = None + commit_regex = re.compile(r"commit\s+([^\s]+)") + merge_commit_regex = re.compile(r"Merge\s+([^\s]+)\s+into\s+([^\s]+)") + + # Try the more specific merge commit regex first + m = merge_commit_regex.search(spack_info) + if m: + # This was a merge commit and we captured the parents + commit_1 = m.group(1) + commit_2 = m.group(2) + else: + # Not a merge commit, just get the commit sha + m = commit_regex.search(spack_info) if m: - # This was a merge commit and we captured the parents commit_1 = m.group(1) - commit_2 = m.group(2) + + setup_result = False + if commit_1: + if commit_2: + setup_result = setup_spack_repro_version(work_dir, commit_2, merge_commit=commit_1) else: - # Not a merge commit, just get the commit sha - m = commit_regex.search(spack_info) - if m: - commit_1 = m.group(1) - - setup_result = False - if commit_1: - if commit_2: - setup_result = setup_spack_repro_version(work_dir, commit_2, merge_commit=commit_1) - else: - setup_result = setup_spack_repro_version(work_dir, commit_1) - - if not setup_result: - setup_msg = """ - This can happen if the spack you are using to run this command is not a git - repo, or if it is a git repo, but it does not have the commits needed to - recreate the tested merge commit. If you are trying to reproduce a spack - PR pipeline job failure, try fetching the latest develop commits from - mainline spack and make sure you have the most recent commit of the PR - branch in your local spack repo. Then run this command again. - Alternatively, you can also manually clone spack if you know the version - you want to test. - """ - tty.error( - "Failed to automatically setup the tested version of spack " - "in your local reproduction directory." - ) - print(setup_msg) + setup_result = setup_spack_repro_version(work_dir, commit_1) + + if not setup_result: + setup_msg = """ + This can happen if the spack you are using to run this command is not a git + repo, or if it is a git repo, but it does not have the commits needed to + recreate the tested merge commit. If you are trying to reproduce a spack + PR pipeline job failure, try fetching the latest develop commits from + mainline spack and make sure you have the most recent commit of the PR + branch in your local spack repo. Then run this command again. + Alternatively, you can also manually clone spack if you know the version + you want to test. + """ + tty.error( + "Failed to automatically setup the tested version of spack " + "in your local reproduction directory." + ) + print(setup_msg) # In cases where CI build was run on a shell runner, it might be useful # to see what tags were applied to the job so the user knows what shell @@ -1863,45 +1903,92 @@ def reproduce_ci_job(url, work_dir): job_tags = job_yaml["tags"] tty.msg("Job ran with the following tags: {0}".format(job_tags)) - inst_list = [] + entrypoint_script = [ + ["git", "config", "--global", "--add", "safe.directory", mount_as_dir], + [".", os.path.join(mount_as_dir if job_image else work_dir, "share/spack/setup-env.sh")], + ["spack", "gpg", "trust", mounted_gpg_path if job_image else gpg_path] if gpg_path else [], + ["spack", "env", "activate", mounted_env_dir if job_image else repro_dir], + [os.path.join(mounted_repro_dir, "install.sh") if job_image else install_script], + ] + inst_list = [] # Finally, print out some instructions to reproduce the build if job_image: - inst_list.append("\nRun the following command:\n\n") - inst_list.append( - " $ docker run --rm --name spack_reproducer -v {0}:{1}:Z -ti {2}\n".format( - work_dir, mount_as_dir, job_image - ) + # Allow interactive + entrypoint_script.extend( + [ + [ + "echo", + "Re-run install script using:\n\t{0}".format( + os.path.join(mounted_repro_dir, "install.sh") + if job_image + else install_script + ), + ], + # Allow interactive + ["exec", "$@"], + ] + ) + process_command( + "entrypoint", entrypoint_script, work_dir, run=False, exit_on_failure=False ) - inst_list.append("\nOnce inside the container:\n\n") - else: - inst_list.append("\nOnce on the tagged runner:\n\n") - if not setup_result: - inst_list.append(" - Clone spack and acquire tested commit\n") - inst_list.append("{0}".format(spack_info)) - spack_root = "" + docker_command = [ + [ + runtime, + "run", + "-i", + "-t", + "--rm", + "--name", + "spack_reproducer", + "-v", + ":".join([work_dir, mounted_workdir, "Z"]), + "-v", + ":".join( + [ + os.path.join(work_dir, "jobs_scratch_dir"), + os.path.join(mount_as_dir, "jobs_scratch_dir"), + "Z", + ] + ), + "-v", + ":".join([os.path.join(work_dir, "spack"), mount_as_dir, "Z"]), + "--entrypoint", + os.path.join(mounted_workdir, "entrypoint.sh"), + job_image, + "bash", + ] + ] + autostart = autostart and setup_result + process_command("start", docker_command, work_dir, run=autostart) + + if not autostart: + inst_list.append("\nTo run the docker reproducer:\n\n") + inst_list.extend( + [ + " - Start the docker container install", + " $ {0}/start.sh".format(work_dir), + ] + ) else: - spack_root = "{0}/spack".format(mount_as_dir) + process_command("reproducer", entrypoint_script, work_dir, run=False) - inst_list.append(" - Activate the environment\n\n") - inst_list.append(" $ source {0}/share/spack/setup-env.sh\n".format(spack_root)) - inst_list.append( - " $ spack env activate --without-view {0}\n\n".format( - mounted_env_dir if job_image else repro_dir - ) - ) - inst_list.append(" - Run the install script\n\n") - inst_list.append( - " $ {0}\n".format( - os.path.join(mounted_repro_dir, "install.sh") if job_image else install_script + inst_list.append("\nOnce on the tagged runner:\n\n") + inst_list.extent( + [" - Run the reproducer script", " $ {0}/reproducer.sh".format(work_dir)] ) - ) - print("".join(inst_list)) + if not setup_result: + inst_list.append("\n - Clone spack and acquire tested commit") + inst_list.append("\n {0}\n".format(spack_info)) + inst_list.append("\n") + inst_list.append("\n Path to clone spack: {0}/spack\n\n".format(work_dir)) + + tty.msg("".join(inst_list)) -def process_command(name, commands, repro_dir): +def process_command(name, commands, repro_dir, run=True, exit_on_failure=True): """ Create a script for and run the command. Copy the script to the reproducibility directory. @@ -1911,6 +1998,7 @@ def process_command(name, commands, repro_dir): commands (list): list of arguments for single command or list of lists of arguments for multiple commands. No shell escape is performed. repro_dir (str): Job reproducibility directory + run (bool): Run the script and return the exit code if True Returns: the exit code from processing the command """ @@ -1929,7 +2017,8 @@ def process_command(name, commands, repro_dir): with open(script, "w") as fd: fd.write("#!/bin/sh\n\n") fd.write("\n# spack {0} command\n".format(name)) - fd.write("set -e\n") + if exit_on_failure: + fd.write("set -e\n") if os.environ.get("SPACK_VERBOSE_SCRIPT"): fd.write("set -x\n") fd.write(full_command) @@ -1940,62 +2029,48 @@ def process_command(name, commands, repro_dir): copy_path = os.path.join(repro_dir, script) shutil.copyfile(script, copy_path) + st = os.stat(copy_path) + os.chmod(copy_path, st.st_mode | stat.S_IEXEC) # Run the generated install.sh shell script as if it were being run in # a login shell. - try: - cmd_process = subprocess.Popen(["/bin/sh", "./{0}".format(script)]) - cmd_process.wait() - exit_code = cmd_process.returncode - except (ValueError, subprocess.CalledProcessError, OSError) as err: - tty.error("Encountered error running {0} script".format(name)) - tty.error(err) - exit_code = 1 - - tty.debug("spack {0} exited {1}".format(name, exit_code)) + exit_code = None + if run: + try: + cmd_process = subprocess.Popen(["/bin/sh", "./{0}".format(script)]) + cmd_process.wait() + exit_code = cmd_process.returncode + except (ValueError, subprocess.CalledProcessError, OSError) as err: + tty.error("Encountered error running {0} script".format(name)) + tty.error(err) + exit_code = 1 + + tty.debug("spack {0} exited {1}".format(name, exit_code)) + else: + # Delete the script, it is copied to the destination dir + os.remove(script) + return exit_code def create_buildcache( - input_spec: spack.spec.Spec, - *, - pr_pipeline: bool, - pipeline_mirror_url: Optional[str] = None, - buildcache_mirror_url: Optional[str] = None, + input_spec: spack.spec.Spec, *, destination_mirror_urls: List[str], sign_binaries: bool = False ) -> List[PushResult]: """Create the buildcache at the provided mirror(s). Arguments: input_spec: Installed spec to package and push - buildcache_mirror_url: URL for the buildcache mirror - pipeline_mirror_url: URL for the pipeline mirror - pr_pipeline: True if the CI job is for a PR + destination_mirror_urls: List of urls to push to + sign_binaries: Whether or not to sign buildcache entry Returns: A list of PushResults, indicating success or failure. """ - sign_binaries = pr_pipeline is False and can_sign_binaries() - results = [] - # Create buildcache in either the main remote mirror, or in the - # per-PR mirror, if this is a PR pipeline - if buildcache_mirror_url: - results.append( - PushResult( - success=push_mirror_contents(input_spec, buildcache_mirror_url, sign_binaries), - url=buildcache_mirror_url, - ) - ) - - # Create another copy of that buildcache in the per-pipeline - # temporary storage mirror (this is only done if either - # artifacts buildcache is enabled or a temporary storage url - # prefix is set) - if pipeline_mirror_url: + for mirror_url in destination_mirror_urls: results.append( PushResult( - success=push_mirror_contents(input_spec, pipeline_mirror_url, sign_binaries), - url=pipeline_mirror_url, + success=push_mirror_contents(input_spec, mirror_url, sign_binaries), url=mirror_url ) ) @@ -2175,13 +2250,13 @@ def build_name(self): spec.architecture, self.build_group, ) - tty.verbose( + tty.debug( "Generated CDash build name ({0}) from the {1}".format(build_name, spec.name) ) return build_name build_name = os.environ.get("SPACK_CDASH_BUILD_NAME") - tty.verbose("Using CDash build name ({0}) from the environment".format(build_name)) + tty.debug("Using CDash build name ({0}) from the environment".format(build_name)) return build_name @property # type: ignore @@ -2195,11 +2270,11 @@ def build_stamp(self): Returns: (str) current CDash build stamp""" build_stamp = os.environ.get("SPACK_CDASH_BUILD_STAMP") if build_stamp: - tty.verbose("Using build stamp ({0}) from the environment".format(build_stamp)) + tty.debug("Using build stamp ({0}) from the environment".format(build_stamp)) return build_stamp build_stamp = cdash_build_stamp(self.build_group, time.time()) - tty.verbose("Generated new build stamp ({0})".format(build_stamp)) + tty.debug("Generated new build stamp ({0})".format(build_stamp)) return build_stamp @property # type: ignore diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 74daeb6243a961..c26ab181c10230 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -11,6 +11,7 @@ from textwrap import dedent from typing import List, Match, Tuple +import llnl.string import llnl.util.tty as tty from llnl.util.filesystem import join_path from llnl.util.lang import attr_setdefault, index_by @@ -29,7 +30,6 @@ import spack.user_environment as uenv import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml -import spack.util.string # cmd has a submodule called "list" so preserve the python list module python_list = list @@ -273,9 +273,9 @@ def disambiguate_spec_from_hashes(spec, hashes, local=False, installed=True, fir See ``spack.database.Database._query`` for details. """ if local: - matching_specs = spack.store.db.query_local(spec, hashes=hashes, installed=installed) + matching_specs = spack.store.STORE.db.query_local(spec, hashes=hashes, installed=installed) else: - matching_specs = spack.store.db.query(spec, hashes=hashes, installed=installed) + matching_specs = spack.store.STORE.db.query(spec, hashes=hashes, installed=installed) if not matching_specs: tty.die("Spec '%s' matches no installed packages." % spec) @@ -291,7 +291,7 @@ def ensure_single_spec_or_die(spec, matching_specs): if len(matching_specs) <= 1: return - format_string = "{name}{@version}{%compiler}{arch=architecture}" + format_string = "{name}{@version}{%compiler.name}{@compiler.version}{arch=architecture}" args = ["%s matches multiple packages." % spec, "Matching packages:"] args += [ colorize(" @K{%s} " % s.dag_hash(7)) + s.cformat(format_string) for s in matching_specs @@ -342,9 +342,9 @@ def iter_groups(specs, indent, all_headers): print() header = "%s{%s} / %s{%s}" % ( - spack.spec.architecture_color, + spack.spec.ARCHITECTURE_COLOR, architecture if architecture else "no arch", - spack.spec.compiler_color, + spack.spec.COMPILER_COLOR, f"{compiler.display_str}" if compiler else "no compiler", ) @@ -383,7 +383,7 @@ def display_specs(specs, args=None, **kwargs): deps (bool): Display dependencies with specs long (bool): Display short hashes with specs very_long (bool): Display full hashes with specs (supersedes ``long``) - namespace (bool): Print namespaces along with names + namespaces (bool): Print namespaces along with names show_flags (bool): Show compiler flags with specs variants (bool): Show variants with specs indent (int): indent each line this much @@ -407,7 +407,7 @@ def get_arg(name, default=None): paths = get_arg("paths", False) deps = get_arg("deps", False) hashes = get_arg("long", False) - namespace = get_arg("namespace", False) + namespaces = get_arg("namespaces", False) flags = get_arg("show_flags", False) full_compiler = get_arg("show_full_compiler", False) variants = get_arg("variants", False) @@ -428,7 +428,7 @@ def get_arg(name, default=None): format_string = get_arg("format", None) if format_string is None: - nfmt = "{fullname}" if namespace else "{name}" + nfmt = "{fullname}" if namespaces else "{name}" ffmt = "" if full_compiler or flags: ffmt += "{%compiler.name}" @@ -473,7 +473,7 @@ def format_list(specs): out = "" # getting lots of prefixes requires DB lookups. Ensure # all spec.prefix calls are in one transaction. - with spack.store.db.read_transaction(): + with spack.store.STORE.db.read_transaction(): for string, spec in formatted: if not string: # print newline from above @@ -516,7 +516,7 @@ def print_how_many_pkgs(specs, pkg_type=""): category, e.g. if pkg_type is "installed" then the message would be "3 installed packages" """ - tty.msg("%s" % spack.util.string.plural(len(specs), pkg_type + " package")) + tty.msg("%s" % llnl.string.plural(len(specs), pkg_type + " package")) def spack_is_git_repo(): @@ -584,14 +584,14 @@ def require_active_env(cmd_name): if env: return env - else: - tty.die( - "`spack %s` requires an environment" % cmd_name, - "activate an environment first:", - " spack env activate ENV", - "or use:", - " spack -e ENV %s ..." % cmd_name, - ) + + tty.die( + "`spack %s` requires an environment" % cmd_name, + "activate an environment first:", + " spack env activate ENV", + "or use:", + " spack -e ENV %s ..." % cmd_name, + ) def find_environment(args): diff --git a/lib/spack/spack/cmd/audit.py b/lib/spack/spack/cmd/audit.py index cb71dbbcb2248d..86eea9f7bc8b9a 100644 --- a/lib/spack/spack/cmd/audit.py +++ b/lib/spack/spack/cmd/audit.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import llnl.util.tty as tty +import llnl.util.tty.colify import llnl.util.tty.color as cl import spack.audit @@ -20,6 +21,15 @@ def setup_parser(subparser): # Audit configuration files sp.add_parser("configs", help="audit configuration files") + # Audit package recipes + external_parser = sp.add_parser("externals", help="check external detection in packages") + external_parser.add_argument( + "--list", + action="store_true", + dest="list_externals", + help="if passed, list which packages have detection tests", + ) + # Https and other linting https_parser = sp.add_parser("packages-https", help="check https in packages") https_parser.add_argument( @@ -29,7 +39,7 @@ def setup_parser(subparser): # Audit package recipes pkg_parser = sp.add_parser("packages", help="audit package recipes") - for group in [pkg_parser, https_parser]: + for group in [pkg_parser, https_parser, external_parser]: group.add_argument( "name", metavar="PKG", @@ -47,7 +57,7 @@ def configs(parser, args): def packages(parser, args): - pkgs = args.name or spack.repo.path.all_package_names() + pkgs = args.name or spack.repo.PATH.all_package_names() reports = spack.audit.run_group(args.subcommand, pkgs=pkgs) _process_reports(reports) @@ -57,7 +67,19 @@ def packages_https(parser, args): if not args.check_all and not args.name: tty.die("Please specify one or more packages to audit, or --all.") - pkgs = args.name or spack.repo.path.all_package_names() + pkgs = args.name or spack.repo.PATH.all_package_names() + reports = spack.audit.run_group(args.subcommand, pkgs=pkgs) + _process_reports(reports) + + +def externals(parser, args): + if args.list_externals: + msg = "@*{The following packages have detection tests:}" + tty.msg(cl.colorize(msg)) + llnl.util.tty.colify.colify(spack.audit.packages_with_detection_tests(), indent=2) + return + + pkgs = args.name or spack.repo.PATH.all_package_names() reports = spack.audit.run_group(args.subcommand, pkgs=pkgs) _process_reports(reports) @@ -78,6 +100,7 @@ def list(parser, args): def audit(parser, args): subcommands = { "configs": configs, + "externals": externals, "packages": packages, "packages-https": packages_https, "list": list, diff --git a/lib/spack/spack/cmd/blame.py b/lib/spack/spack/cmd/blame.py index cc1f58ff38bf55..c7e34904858dd1 100644 --- a/lib/spack/spack/cmd/blame.py +++ b/lib/spack/spack/cmd/blame.py @@ -59,7 +59,7 @@ def setup_parser(subparser): subparser.add_argument( "package_or_file", - help="name of package to show contributions for, " "or path to a file in the spack repo", + help="name of package to show contributions for, or path to a file in the spack repo", ) @@ -126,7 +126,7 @@ def blame(parser, args): blame_file = path if not blame_file: - pkg_cls = spack.repo.path.get_pkg_class(args.package_or_file) + pkg_cls = spack.repo.PATH.get_pkg_class(args.package_or_file) blame_file = pkg_cls.module.__file__.rstrip("c") # .pyc -> .py # get git blame for the package diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index be2e64dbce5f0a..71c3d3a04526e7 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os.path import shutil +import sys import tempfile import llnl.util.filesystem @@ -68,11 +69,10 @@ def _add_scope_option(parser): scopes = spack.config.scopes() - scopes_metavar = spack.config.scopes_metavar parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, help="configuration scope to read/modify", ) @@ -169,7 +169,7 @@ def _reset(args): if not ok_to_continue: raise RuntimeError("Aborting") - for scope in spack.config.config.file_scopes: + for scope in spack.config.CONFIG.file_scopes: # The default scope should stay untouched if scope.name == "defaults": continue @@ -186,7 +186,7 @@ def _reset(args): if os.path.exists(bootstrap_yaml): shutil.move(bootstrap_yaml, backup_file) - spack.config.config.clear_caches() + spack.config.CONFIG.clear_caches() def _root(args): @@ -326,6 +326,7 @@ def _status(args): if missing: print(llnl.util.tty.color.colorize(legend)) print() + sys.exit(1) def _add(args): diff --git a/lib/spack/spack/cmd/build_env.py b/lib/spack/spack/cmd/build_env.py index 7da9213c5b0e7b..f5efca6e230484 100644 --- a/lib/spack/spack/cmd/build_env.py +++ b/lib/spack/spack/cmd/build_env.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spack.cmd.common.env_utility as env_utility +from spack.context import Context description = ( "run a command in a spec's install environment, or dump its environment to screen or file" @@ -14,4 +15,4 @@ def build_env(parser, args): - env_utility.emulate_env_utility("build-env", "build", args) + env_utility.emulate_env_utility("build-env", Context.BUILD, args) diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index e1b917cf18831f..94cce16030be32 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -2,15 +2,21 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import argparse +import copy import glob +import hashlib import json +import multiprocessing.pool import os import shutil import sys import tempfile +import urllib.request +from typing import Dict, List, Optional, Tuple import llnl.util.tty as tty -import llnl.util.tty.color as clr +from llnl.string import plural from llnl.util.lang import elide_list import spack.binary_distribution as bindist @@ -18,77 +24,103 @@ import spack.cmd.common.arguments as arguments import spack.config import spack.environment as ev +import spack.error import spack.hash_types as ht import spack.mirror +import spack.oci.oci +import spack.oci.opener import spack.relocate import spack.repo import spack.spec +import spack.stage import spack.store +import spack.user_environment import spack.util.crypto import spack.util.url as url_util import spack.util.web as web_util +from spack.build_environment import determine_number_of_jobs from spack.cmd import display_specs -from spack.error import SpecError +from spack.oci.image import ( + Digest, + ImageReference, + default_config, + default_index_tag, + default_manifest, + default_tag, + tag_is_spec, +) +from spack.oci.oci import ( + copy_missing_layers_with_retry, + get_manifest_and_config_with_retry, + upload_blob_with_retry, + upload_manifest_with_retry, +) from spack.spec import Spec, save_dependency_specfiles -from spack.stage import Stage -from spack.util.string import plural description = "create, download and install binary packages" section = "packaging" level = "long" -def setup_parser(subparser): - setup_parser.parser = subparser +def setup_parser(subparser: argparse.ArgumentParser): + setattr(setup_parser, "parser", subparser) subparsers = subparser.add_subparsers(help="buildcache sub-commands") push = subparsers.add_parser("push", aliases=["create"], help=push_fn.__doc__) - push.add_argument("-f", "--force", action="store_true", help="overwrite tarball if it exists.") + push.add_argument("-f", "--force", action="store_true", help="overwrite tarball if it exists") push.add_argument( - "-u", "--unsigned", action="store_true", help="push unsigned buildcache tarballs" - ) - push.add_argument( - "-a", "--allow-root", + "-a", action="store_true", help="allow install root string in binary files after RPATH substitution", ) + push_sign = push.add_mutually_exclusive_group(required=False) + push_sign.add_argument( + "--unsigned", "-u", action="store_true", help="push unsigned buildcache tarballs" + ) + push_sign.add_argument( + "--key", "-k", metavar="key", type=str, default=None, help="key for signing" + ) push.add_argument( - "-k", "--key", metavar="key", type=str, default=None, help="Key for signing." + "mirror", type=arguments.mirror_name_or_url, help="mirror name, path, or URL" ) - push.add_argument("mirror", type=str, help="Mirror name, path, or URL.") push.add_argument( "--update-index", "--rebuild-index", action="store_true", default=False, - help="Regenerate buildcache index after building package(s)", + help="regenerate buildcache index after building package(s)", ) push.add_argument( - "--spec-file", default=None, help="Create buildcache entry for spec from json or yaml file" + "--spec-file", default=None, help="create buildcache entry for spec from json or yaml file" ) push.add_argument( "--only", default="package,dependencies", dest="things_to_install", choices=["package", "dependencies"], - help=( - "Select the buildcache mode. the default is to" - " build a cache for the package along with all" - " its dependencies. Alternatively, one can" - " decide to build a cache for only the package" - " or only the dependencies" - ), - ) - arguments.add_common_arguments(push, ["specs"]) + help="select the buildcache mode. " + "The default is to build a cache for the package along with all its dependencies. " + "Alternatively, one can decide to build a cache for only the package or only the " + "dependencies", + ) + push.add_argument( + "--fail-fast", + action="store_true", + help="stop pushing on first failure (default is best effort)", + ) + push.add_argument( + "--base-image", default=None, help="specify the base image for the buildcache. " + ) + arguments.add_common_arguments(push, ["specs", "jobs"]) push.set_defaults(func=push_fn) install = subparsers.add_parser("install", help=install_fn.__doc__) install.add_argument( - "-f", "--force", action="store_true", help="overwrite install directory if it exists." + "-f", "--force", action="store_true", help="overwrite install directory if it exists" ) install.add_argument( - "-m", "--multiple", action="store_true", help="allow all matching packages " + "-m", "--multiple", action="store_true", help="allow all matching packages" ) install.add_argument( "-u", @@ -107,7 +139,7 @@ def setup_parser(subparser): install.set_defaults(func=install_fn) listcache = subparsers.add_parser("list", help=list_fn.__doc__) - arguments.add_common_arguments(listcache, ["long", "very_long"]) + arguments.add_common_arguments(listcache, ["long", "very_long", "namespaces"]) listcache.add_argument( "-v", "--variants", @@ -142,49 +174,49 @@ def setup_parser(subparser): "-m", "--mirror-url", default=None, - help="Override any configured mirrors with this mirror URL", + help="override any configured mirrors with this mirror URL", ) check.add_argument( - "-o", "--output-file", default=None, help="File where rebuild info should be written" + "-o", "--output-file", default=None, help="file where rebuild info should be written" ) # used to construct scope arguments below scopes = spack.config.scopes() - scopes_metavar = spack.config.scopes_metavar check.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_modify_scope(), help="configuration scope containing mirrors to check", ) - - check.add_argument( - "-s", "--spec", default=None, help="Check single spec instead of release specs file" + check_spec_or_specfile = check.add_mutually_exclusive_group(required=True) + check_spec_or_specfile.add_argument( + "-s", "--spec", help="check single spec instead of release specs file" ) - - check.add_argument( + check_spec_or_specfile.add_argument( "--spec-file", - default=None, - help=("Check single spec from json or yaml file instead of release specs file"), + help="check single spec from json or yaml file instead of release specs file", ) check.set_defaults(func=check_fn) # Download tarball and specfile download = subparsers.add_parser("download", help=download_fn.__doc__) - download.add_argument( - "-s", "--spec", default=None, help="Download built tarball for spec from mirror" + download_spec_or_specfile = download.add_mutually_exclusive_group(required=True) + download_spec_or_specfile.add_argument( + "-s", "--spec", help="download built tarball for spec from mirror" ) - download.add_argument( - "--spec-file", - default=None, - help=("Download built tarball for spec (from json or yaml file) from mirror"), + download_spec_or_specfile.add_argument( + "--spec-file", help="download built tarball for spec (from json or yaml file) from mirror" ) download.add_argument( - "-p", "--path", default=None, help="Path to directory where tarball should be downloaded" + "-p", + "--path", + required=True, + default=None, + help="path to directory where tarball should be downloaded", ) download.set_defaults(func=download_fn) @@ -192,53 +224,53 @@ def setup_parser(subparser): getbuildcachename = subparsers.add_parser( "get-buildcache-name", help=get_buildcache_name_fn.__doc__ ) - getbuildcachename.add_argument( - "-s", "--spec", default=None, help="Spec string for which buildcache name is desired" + getbuildcachename_spec_or_specfile = getbuildcachename.add_mutually_exclusive_group( + required=True ) - getbuildcachename.add_argument( - "--spec-file", - default=None, - help=("Path to spec json or yaml file for which buildcache name is desired"), + getbuildcachename_spec_or_specfile.add_argument( + "-s", "--spec", help="spec string for which buildcache name is desired" + ) + getbuildcachename_spec_or_specfile.add_argument( + "--spec-file", help="path to spec json or yaml file for which buildcache name is desired" ) getbuildcachename.set_defaults(func=get_buildcache_name_fn) # Given the root spec, save the yaml of the dependent spec to a file savespecfile = subparsers.add_parser("save-specfile", help=save_specfile_fn.__doc__) - savespecfile.add_argument("--root-spec", default=None, help="Root spec of dependent spec") - savespecfile.add_argument( - "--root-specfile", - default=None, - help="Path to json or yaml file containing root spec of dependent spec", + savespecfile_spec_or_specfile = savespecfile.add_mutually_exclusive_group(required=True) + savespecfile_spec_or_specfile.add_argument("--root-spec", help="root spec of dependent spec") + savespecfile_spec_or_specfile.add_argument( + "--root-specfile", help="path to json or yaml file containing root spec of dependent spec" ) savespecfile.add_argument( "-s", "--specs", - default=None, - help="List of dependent specs for which saved yaml is desired", + required=True, + help="list of dependent specs for which saved yaml is desired", ) savespecfile.add_argument( - "--specfile-dir", default=None, help="Path to directory where spec yamls should be saved" + "--specfile-dir", required=True, help="path to directory where spec yamls should be saved" ) savespecfile.set_defaults(func=save_specfile_fn) # Sync buildcache entries from one mirror to another sync = subparsers.add_parser("sync", help=sync_fn.__doc__) sync.add_argument( - "--manifest-glob", help="A quoted glob pattern identifying copy manifest files" + "--manifest-glob", help="a quoted glob pattern identifying copy manifest files" ) sync.add_argument( "src_mirror", metavar="source mirror", type=arguments.mirror_name_or_url, nargs="?", - help="Source mirror name, path, or URL", + help="source mirror name, path, or URL", ) sync.add_argument( "dest_mirror", metavar="destination mirror", type=arguments.mirror_name_or_url, nargs="?", - help="Destination mirror name, path, or URL", + help="destination mirror name, path, or URL", ) sync.set_defaults(func=sync_fn) @@ -247,118 +279,131 @@ def setup_parser(subparser): "update-index", aliases=["rebuild-index"], help=update_index_fn.__doc__ ) update_index.add_argument( - "mirror", type=arguments.mirror_name_or_url, help="Destination mirror name, path, or URL" + "mirror", type=arguments.mirror_name_or_url, help="destination mirror name, path, or URL" ) update_index.add_argument( "-k", "--keys", default=False, action="store_true", - help="If provided, key index will be updated as well as package index", + help="if provided, key index will be updated as well as package index", ) update_index.set_defaults(func=update_index_fn) -def _matching_specs(specs, spec_file): - """Return a list of matching specs read from either a spec file (JSON or YAML), - a query over the store or a query over the active environment. - """ - env = ev.active_environment() - hashes = env.all_hashes() if env else None - if spec_file: - return spack.store.specfile_matches(spec_file, hashes=hashes) - - if specs: - constraints = spack.cmd.parse_specs(specs) - return spack.store.find(constraints, hashes=hashes) - - if env: - return [concrete for _, concrete in env.concretized_specs()] +def _matching_specs(specs: List[Spec]) -> List[Spec]: + """Disambiguate specs and return a list of matching specs""" + return [spack.cmd.disambiguate_spec(s, ev.active_environment(), installed=any) for s in specs] - tty.die( - "build cache file creation requires at least one" - " installed package spec, an active environment," - " or else a path to a json or yaml file containing a spec" - " to install" - ) +def _format_spec(spec: Spec) -> str: + return spec.cformat("{name}{@version}{/hash:7}") -def _concrete_spec_from_args(args): - spec_str, specfile_path = args.spec, args.spec_file - - if not spec_str and not specfile_path: - tty.error("must provide either spec string or path to YAML or JSON specfile") - sys.exit(1) - if spec_str: - try: - constraints = spack.cmd.parse_specs(spec_str) - spec = spack.store.find(constraints)[0] - spec.concretize() - except SpecError as spec_error: - tty.error("Unable to concretize spec {0}".format(spec_str)) - tty.debug(spec_error) - sys.exit(1) +def _progress(i: int, total: int): + if total > 1: + digits = len(str(total)) + return f"[{i+1:{digits}}/{total}] " + return "" - return spec - return Spec.from_specfile(specfile_path) +def _make_pool(): + return multiprocessing.pool.Pool(determine_number_of_jobs(parallel=True)) def push_fn(args): """create a binary package and push it to a mirror""" - mirror = arguments.mirror_name_or_url(args.mirror) + if args.spec_file: + tty.warn( + "The flag `--spec-file` is deprecated and will be removed in Spack 0.22. " + "Use positional arguments instead." + ) - url = mirror.push_url + if args.specs or args.spec_file: + specs = _matching_specs(spack.cmd.parse_specs(args.specs or args.spec_file)) + else: + specs = spack.cmd.require_active_env("buildcache push").all_specs() + + if args.allow_root: + tty.warn( + "The flag `--allow-root` is the default in Spack 0.21, will be removed in Spack 0.22" + ) + # Check if this is an OCI image. + try: + image_ref = spack.oci.oci.image_from_mirror(args.mirror) + except ValueError: + image_ref = None + + # For OCI images, we require dependencies to be pushed for now. + if image_ref: + if "dependencies" not in args.things_to_install: + tty.die("Dependencies must be pushed for OCI images.") + if not args.unsigned: + tty.warn( + "Code signing is currently not supported for OCI images. " + "Use --unsigned to silence this warning." + ) + + # This is a list of installed, non-external specs. specs = bindist.specs_to_be_packaged( - _matching_specs(args.specs, args.spec_file), + specs, root="package" in args.things_to_install, dependencies="dependencies" in args.things_to_install, ) + url = args.mirror.push_url + # When pushing multiple specs, print the url once ahead of time, as well as how # many specs are being pushed. if len(specs) > 1: tty.info(f"Selected {len(specs)} specs to push to {url}") - skipped = [] - - # tty printing - color = clr.get_color_when() - format_spec = lambda s: s.format("{name}{@version}{/hash:7}", color=color) - total_specs = len(specs) - digits = len(str(total_specs)) + failed = [] - for i, spec in enumerate(specs): - try: - bindist.push_or_raise( - spec, - url, - bindist.PushOptions( - force=args.force, - unsigned=args.unsigned, - allow_root=args.allow_root, - key=args.key, - regenerate_index=args.update_index, - ), - ) + # TODO: unify this logic in the future. + if image_ref: + with tempfile.TemporaryDirectory( + dir=spack.stage.get_stage_root() + ) as tmpdir, _make_pool() as pool: + skipped = _push_oci(args, image_ref, specs, tmpdir, pool) + else: + skipped = [] + + for i, spec in enumerate(specs): + try: + bindist.push_or_raise( + spec, + url, + bindist.PushOptions( + force=args.force, + unsigned=args.unsigned, + key=args.key, + regenerate_index=args.update_index, + ), + ) - if total_specs > 1: - msg = f"[{i+1:{digits}}/{total_specs}] Pushed {format_spec(spec)}" - else: - msg = f"Pushed {format_spec(spec)} to {url}" + msg = f"{_progress(i, len(specs))}Pushed {_format_spec(spec)}" + if len(specs) == 1: + msg += f" to {url}" + tty.info(msg) - tty.info(msg) + except bindist.NoOverwriteException: + skipped.append(_format_spec(spec)) - except bindist.NoOverwriteException: - skipped.append(format_spec(spec)) + # Catch any other exception unless the fail fast option is set + except Exception as e: + if args.fail_fast or isinstance( + e, (bindist.PickKeyException, bindist.NoKeyException) + ): + raise + failed.append((_format_spec(spec), e)) if skipped: if len(specs) == 1: tty.info("The spec is already in the buildcache. Use --force to overwrite it.") elif len(skipped) == len(specs): - tty.info("All specs are already in the buildcache. Use --force to overwite them.") + tty.info("All specs are already in the buildcache. Use --force to overwrite them.") else: tty.info( "The following {} specs were skipped as they already exist in the buildcache:\n" @@ -368,6 +413,352 @@ def push_fn(args): ) ) + if failed: + if len(failed) == 1: + raise failed[0][1] + + raise spack.error.SpackError( + f"The following {len(failed)} errors occurred while pushing specs to the buildcache", + "\n".join( + elide_list([f" {spec}: {e.__class__.__name__}: {e}" for spec, e in failed], 5) + ), + ) + + # Update the index if requested + # TODO: remove update index logic out of bindist; should be once after all specs are pushed + # not once per spec. + if image_ref and len(skipped) < len(specs) and args.update_index: + with tempfile.TemporaryDirectory( + dir=spack.stage.get_stage_root() + ) as tmpdir, _make_pool() as pool: + _update_index_oci(image_ref, tmpdir, pool) + + +def _get_spack_binary_blob(image_ref: ImageReference) -> Optional[spack.oci.oci.Blob]: + """Get the spack tarball layer digests and size if it exists""" + try: + manifest, config = get_manifest_and_config_with_retry(image_ref) + + return spack.oci.oci.Blob( + compressed_digest=Digest.from_string(manifest["layers"][-1]["digest"]), + uncompressed_digest=Digest.from_string(config["rootfs"]["diff_ids"][-1]), + size=manifest["layers"][-1]["size"], + ) + except Exception: + return None + + +def _push_single_spack_binary_blob(image_ref: ImageReference, spec: spack.spec.Spec, tmpdir: str): + filename = os.path.join(tmpdir, f"{spec.dag_hash()}.tar.gz") + + # Create an oci.image.layer aka tarball of the package + compressed_tarfile_checksum, tarfile_checksum = spack.oci.oci.create_tarball(spec, filename) + + blob = spack.oci.oci.Blob( + Digest.from_sha256(compressed_tarfile_checksum), + Digest.from_sha256(tarfile_checksum), + os.path.getsize(filename), + ) + + # Upload the blob + upload_blob_with_retry(image_ref, file=filename, digest=blob.compressed_digest) + + # delete the file + os.unlink(filename) + + return blob + + +def _retrieve_env_dict_from_config(config: dict) -> dict: + """Retrieve the environment variables from the image config file. + Sets a default value for PATH if it is not present. + + Args: + config (dict): The image config file. + + Returns: + dict: The environment variables. + """ + env = {"PATH": "/bin:/usr/bin"} + + if "Env" in config.get("config", {}): + for entry in config["config"]["Env"]: + key, value = entry.split("=", 1) + env[key] = value + return env + + +def _archspec_to_gooarch(spec: spack.spec.Spec) -> str: + name = spec.target.family.name + name_map = {"aarch64": "arm64", "x86_64": "amd64"} + return name_map.get(name, name) + + +def _put_manifest( + base_images: Dict[str, Tuple[dict, dict]], + checksums: Dict[str, spack.oci.oci.Blob], + spec: spack.spec.Spec, + image_ref: ImageReference, + tmpdir: str, +): + architecture = _archspec_to_gooarch(spec) + + dependencies = list( + reversed( + list( + s + for s in spec.traverse(order="topo", deptype=("link", "run"), root=True) + if not s.external + ) + ) + ) + + base_manifest, base_config = base_images[architecture] + env = _retrieve_env_dict_from_config(base_config) + + spack.user_environment.environment_modifications_for_specs(spec).apply_modifications(env) + + # Create an oci.image.config file + config = copy.deepcopy(base_config) + + # Add the diff ids of the dependencies + for s in dependencies: + config["rootfs"]["diff_ids"].append(str(checksums[s.dag_hash()].uncompressed_digest)) + + # Set the environment variables + config["config"]["Env"] = [f"{k}={v}" for k, v in env.items()] + + # From the OCI v1.0 spec: + # > Any extra fields in the Image JSON struct are considered implementation + # > specific and MUST be ignored by any implementations which are unable to + # > interpret them. + # We use this to store the Spack spec, so we can use it to create an index. + spec_dict = spec.to_dict(hash=ht.dag_hash) + spec_dict["buildcache_layout_version"] = 1 + spec_dict["binary_cache_checksum"] = { + "hash_algorithm": "sha256", + "hash": checksums[spec.dag_hash()].compressed_digest.digest, + } + config.update(spec_dict) + + config_file = os.path.join(tmpdir, f"{spec.dag_hash()}.config.json") + + with open(config_file, "w") as f: + json.dump(config, f, separators=(",", ":")) + + config_file_checksum = Digest.from_sha256( + spack.util.crypto.checksum(hashlib.sha256, config_file) + ) + + # Upload the config file + upload_blob_with_retry(image_ref, file=config_file, digest=config_file_checksum) + + oci_manifest = { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "schemaVersion": 2, + "config": { + "mediaType": base_manifest["config"]["mediaType"], + "digest": str(config_file_checksum), + "size": os.path.getsize(config_file), + }, + "layers": [ + *(layer for layer in base_manifest["layers"]), + *( + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": str(checksums[s.dag_hash()].compressed_digest), + "size": checksums[s.dag_hash()].size, + } + for s in dependencies + ), + ], + "annotations": {"org.opencontainers.image.description": spec.format()}, + } + + image_ref_for_spec = image_ref.with_tag(default_tag(spec)) + + # Finally upload the manifest + upload_manifest_with_retry(image_ref_for_spec, oci_manifest=oci_manifest) + + # delete the config file + os.unlink(config_file) + + return image_ref_for_spec + + +def _push_oci( + args, + image_ref: ImageReference, + installed_specs_with_deps: List[Spec], + tmpdir: str, + pool: multiprocessing.pool.Pool, +) -> List[str]: + """Push specs to an OCI registry + + Args: + args: The command line arguments. + image_ref: The image reference. + installed_specs_with_deps: The installed specs to push, excluding externals, + including deps, ordered from roots to leaves. + + Returns: + List[str]: The list of skipped specs (already in the buildcache). + """ + + # Reverse the order + installed_specs_with_deps = list(reversed(installed_specs_with_deps)) + + # The base image to use for the package. When not set, we use + # the OCI registry only for storage, and do not use any base image. + base_image_ref: Optional[ImageReference] = ( + ImageReference.from_string(args.base_image) if args.base_image else None + ) + + # Spec dag hash -> blob + checksums: Dict[str, spack.oci.oci.Blob] = {} + + # arch -> (manifest, config) + base_images: Dict[str, Tuple[dict, dict]] = {} + + # Specs not uploaded because they already exist + skipped = [] + + if not args.force: + tty.info("Checking for existing specs in the buildcache") + to_be_uploaded = [] + + tags_to_check = (image_ref.with_tag(default_tag(s)) for s in installed_specs_with_deps) + available_blobs = pool.map(_get_spack_binary_blob, tags_to_check) + + for spec, maybe_blob in zip(installed_specs_with_deps, available_blobs): + if maybe_blob is not None: + checksums[spec.dag_hash()] = maybe_blob + skipped.append(_format_spec(spec)) + else: + to_be_uploaded.append(spec) + else: + to_be_uploaded = installed_specs_with_deps + + if not to_be_uploaded: + return skipped + + tty.info( + f"{len(to_be_uploaded)} specs need to be pushed to {image_ref.domain}/{image_ref.name}" + ) + + # Upload blobs + new_blobs = pool.starmap( + _push_single_spack_binary_blob, ((image_ref, spec, tmpdir) for spec in to_be_uploaded) + ) + + # And update the spec to blob mapping + for spec, blob in zip(to_be_uploaded, new_blobs): + checksums[spec.dag_hash()] = blob + + # Copy base image layers, probably fine to do sequentially. + for spec in to_be_uploaded: + architecture = _archspec_to_gooarch(spec) + # Get base image details, if we don't have them yet + if architecture in base_images: + continue + if base_image_ref is None: + base_images[architecture] = (default_manifest(), default_config(architecture, "linux")) + else: + base_images[architecture] = copy_missing_layers_with_retry( + base_image_ref, image_ref, architecture + ) + + # Upload manifests + tty.info("Uploading manifests") + pushed_image_ref = pool.starmap( + _put_manifest, + ((base_images, checksums, spec, image_ref, tmpdir) for spec in to_be_uploaded), + ) + + # Print the image names of the top-level specs + for spec, ref in zip(to_be_uploaded, pushed_image_ref): + tty.info(f"Pushed {_format_spec(spec)} to {ref}") + + return skipped + + +def _config_from_tag(image_ref: ImageReference, tag: str) -> Optional[dict]: + # Don't allow recursion here, since Spack itself always uploads + # vnd.oci.image.manifest.v1+json, not vnd.oci.image.index.v1+json + _, config = get_manifest_and_config_with_retry(image_ref.with_tag(tag), tag, recurse=0) + + # Do very basic validation: if "spec" is a key in the config, it + # must be a Spec object too. + return config if "spec" in config else None + + +def _update_index_oci( + image_ref: ImageReference, tmpdir: str, pool: multiprocessing.pool.Pool +) -> None: + response = spack.oci.opener.urlopen(urllib.request.Request(url=image_ref.tags_url())) + spack.oci.opener.ensure_status(response, 200) + tags = json.load(response)["tags"] + + # Fetch all image config files in parallel + spec_dicts = pool.starmap( + _config_from_tag, ((image_ref, tag) for tag in tags if tag_is_spec(tag)) + ) + + # Populate the database + db_root_dir = os.path.join(tmpdir, "db_root") + db = bindist.BuildCacheDatabase(db_root_dir) + + for spec_dict in spec_dicts: + spec = Spec.from_dict(spec_dict) + db.add(spec, directory_layout=None) + db.mark(spec, "in_buildcache", True) + + # Create the index.json file + index_json_path = os.path.join(tmpdir, "index.json") + with open(index_json_path, "w") as f: + db._write_to_file(f) + + # Create an empty config.json file + empty_config_json_path = os.path.join(tmpdir, "config.json") + with open(empty_config_json_path, "wb") as f: + f.write(b"{}") + + # Upload the index.json file + index_shasum = Digest.from_sha256(spack.util.crypto.checksum(hashlib.sha256, index_json_path)) + upload_blob_with_retry(image_ref, file=index_json_path, digest=index_shasum) + + # Upload the config.json file + empty_config_digest = Digest.from_sha256( + spack.util.crypto.checksum(hashlib.sha256, empty_config_json_path) + ) + upload_blob_with_retry(image_ref, file=empty_config_json_path, digest=empty_config_digest) + + # Push a manifest file that references the index.json file as a layer + # Notice that we push this as if it is an image, which it of course is not. + # When the ORAS spec becomes official, we can use that instead of a fake image. + # For now we just use the OCI image spec, so that we don't run into issues with + # automatic garbage collection of blobs that are not referenced by any image manifest. + oci_manifest = { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "schemaVersion": 2, + # Config is just an empty {} file for now, and irrelevant + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": str(empty_config_digest), + "size": os.path.getsize(empty_config_json_path), + }, + # The buildcache index is the only layer, and is not a tarball, we lie here. + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": str(index_shasum), + "size": os.path.getsize(index_json_path), + } + ], + } + + upload_manifest_with_retry(image_ref.with_tag(default_index_tag), oci_manifest) + def install_fn(args): """install from a binary package""" @@ -411,32 +802,31 @@ def keys_fn(args): def preview_fn(args): - """analyze an installed spec and reports whether executables - and libraries are relocatable - """ - constraints = spack.cmd.parse_specs(args.specs) - specs = spack.store.find(constraints, multiple=True) + """analyze an installed spec and reports whether executables and libraries are relocatable""" + tty.warn( + "`spack buildcache preview` is deprecated since `spack buildcache push --allow-root` is " + "now the default. This command will be removed in Spack 0.22" + ) - # Cycle over the specs that match - for spec in specs: - print("Relocatable nodes") - print("--------------------------------") - print(spec.tree(status_fn=spack.relocate.is_relocatable)) +def check_fn(args: argparse.Namespace): + """check specs against remote binary mirror(s) to see if any need to be rebuilt -def check_fn(args): - """Check specs (either a single spec from --spec, or else the full set - of release specs) against remote binary mirror(s) to see if any need - to be rebuilt. This command uses the process exit code to indicate - its result, specifically, if the exit code is non-zero, then at least - one of the indicated specs needs to be rebuilt. + this command uses the process exit code to indicate its result, specifically, if the + exit code is non-zero, then at least one of the indicated specs needs to be rebuilt """ - if args.spec or args.spec_file: - specs = [_concrete_spec_from_args(args)] + if args.spec_file: + tty.warn( + "The flag `--spec-file` is deprecated and will be removed in Spack 0.22. " + "Use --spec instead." + ) + + specs = spack.cmd.parse_specs(args.spec or args.spec_file) + + if specs: + specs = _matching_specs(specs) else: - env = spack.cmd.require_active_env(cmd_name="buildcache") - env.concretize() - specs = env.all_specs() + specs = spack.cmd.require_active_env("buildcache check").all_specs() if not specs: tty.msg("No specs provided, exiting.") @@ -460,63 +850,61 @@ def check_fn(args): def download_fn(args): - """Download buildcache entry from a remote mirror to local folder. This - command uses the process exit code to indicate its result, specifically, - a non-zero exit code indicates that the command failed to download at - least one of the required buildcache components.""" - if not args.spec and not args.spec_file: - tty.msg("No specs provided, exiting.") - return + """download buildcache entry from a remote mirror to local folder - if not args.path: - tty.msg("No download path provided, exiting") - return + this command uses the process exit code to indicate its result, specifically, a non-zero exit + code indicates that the command failed to download at least one of the required buildcache + components + """ + if args.spec_file: + tty.warn( + "The flag `--spec-file` is deprecated and will be removed in Spack 0.22. " + "Use --spec instead." + ) - spec = _concrete_spec_from_args(args) - result = bindist.download_single_spec(spec, args.path) + specs = _matching_specs(spack.cmd.parse_specs(args.spec or args.spec_file)) - if not result: + if len(specs) != 1: + tty.die("a single spec argument is required to download from a buildcache") + + if not bindist.download_single_spec(specs[0], args.path): sys.exit(1) def get_buildcache_name_fn(args): - """Get name (prefix) of buildcache entries for this spec""" - spec = _concrete_spec_from_args(args) - buildcache_name = bindist.tarball_name(spec, "") - print("{0}".format(buildcache_name)) + """get name (prefix) of buildcache entries for this spec""" + tty.warn("This command is deprecated and will be removed in Spack 0.22.") + specs = _matching_specs(spack.cmd.parse_specs(args.spec or args.spec_file)) + if len(specs) != 1: + tty.die("a single spec argument is required to get buildcache name") + print(bindist.tarball_name(specs[0], "")) def save_specfile_fn(args): - """Get full spec for dependencies, relative to root spec, and write them - to files in the specified output directory. Uses exit code to signal - success or failure. An exit code of zero means the command was likely - successful. If any errors or exceptions are encountered, or if expected - command-line arguments are not provided, then the exit code will be - non-zero. + """get full spec for dependencies and write them to files in the specified output directory + + uses exit code to signal success or failure. an exit code of zero means the command was likely + successful. if any errors or exceptions are encountered, or if expected command-line arguments + are not provided, then the exit code will be non-zero """ - if not args.root_spec and not args.root_specfile: - tty.msg("No root spec provided, exiting.") - sys.exit(1) + if args.root_specfile: + tty.warn( + "The flag `--root-specfile` is deprecated and will be removed in Spack 0.22. " + "Use --root-spec instead." + ) - if not args.specs: - tty.msg("No dependent specs provided, exiting.") - sys.exit(1) + specs = spack.cmd.parse_specs(args.root_spec or args.root_specfile) - if not args.specfile_dir: - tty.msg("No yaml directory provided, exiting.") - sys.exit(1) + if len(specs) != 1: + tty.die("a single spec argument is required to save specfile") + + root = specs[0] + + if not root.concrete: + root.concretize() - if args.root_specfile: - with open(args.root_specfile) as fd: - root_spec_as_json = fd.read() - spec_format = "yaml" if args.root_specfile.endswith("yaml") else "json" - else: - root_spec = Spec(args.root_spec) - root_spec.concretize() - root_spec_as_json = root_spec.to_json(hash=ht.dag_hash) - spec_format = "json" save_dependency_specfiles( - root_spec_as_json, args.specfile_dir, args.specs.split(), spec_format + root, args.specfile_dir, dependencies=spack.cmd.parse_specs(args.specs) ) @@ -529,12 +917,12 @@ def copy_buildcache_file(src_url, dest_url, local_path=None): local_path = os.path.join(tmpdir, os.path.basename(src_url)) try: - temp_stage = Stage(src_url, path=os.path.dirname(local_path)) + temp_stage = spack.stage.Stage(src_url, path=os.path.dirname(local_path)) try: temp_stage.create() temp_stage.fetch() web_util.push_to_url(local_path, dest_url, keep_original=True) - except web_util.FetchError as e: + except spack.error.FetchError as e: # Expected, since we have to try all the possible extensions tty.debug("no such file: {0}".format(src_url)) tty.debug(e) @@ -546,12 +934,9 @@ def copy_buildcache_file(src_url, dest_url, local_path=None): def sync_fn(args): - """Syncs binaries (and associated metadata) from one mirror to another. - Requires an active environment in order to know which specs to sync. + """sync binaries (and associated metadata) from one mirror to another - Args: - src (str): Source mirror URL - dest (str): Destination mirror URL + requires an active environment in order to know which specs to sync """ if args.manifest_glob: manifest_copy(glob.glob(args.manifest_glob)) @@ -626,6 +1011,20 @@ def manifest_copy(manifest_file_list): def update_index(mirror: spack.mirror.Mirror, update_keys=False): + # Special case OCI images for now. + try: + image_ref = spack.oci.oci.image_from_mirror(mirror) + except ValueError: + image_ref = None + + if image_ref: + with tempfile.TemporaryDirectory( + dir=spack.stage.get_stage_root() + ) as tmpdir, _make_pool() as pool: + _update_index_oci(image_ref, tmpdir, pool) + return + + # Otherwise, assume a normal mirror. url = mirror.push_url bindist.generate_package_index(url_util.join(url, bindist.build_cache_relative_path())) @@ -639,7 +1038,7 @@ def update_index(mirror: spack.mirror.Mirror, update_keys=False): def update_index_fn(args): - """Update a buildcache index.""" + """update a buildcache index""" update_index(args.mirror, update_keys=args.keys) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index c6b39132c85d22..f927d2d922a26d 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -3,20 +3,23 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse +import re import sys -import llnl.util.tty as tty +import llnl.string +import llnl.util.lang +from llnl.util import tty import spack.cmd -import spack.cmd.common.arguments as arguments import spack.repo import spack.spec import spack.stage import spack.util.crypto -from spack.package_base import deprecated_version, preferred_version +import spack.util.web as web_util +from spack.cmd.common import arguments +from spack.package_base import PackageBase, deprecated_version, preferred_version from spack.util.editor import editor -from spack.util.naming import valid_fully_qualified_module_name +from spack.util.format import get_version_lines from spack.version import Version description = "checksum available versions of a package" @@ -31,135 +34,245 @@ def setup_parser(subparser): default=False, help="don't clean up staging area when command completes", ) - sp = subparser.add_mutually_exclusive_group() - sp.add_argument( - "-b", + subparser.add_argument( "--batch", + "-b", action="store_true", default=False, help="don't ask which versions to checksum", ) - sp.add_argument( - "-l", + subparser.add_argument( "--latest", + "-l", action="store_true", default=False, - help="checksum the latest available version only", + help="checksum the latest available version", ) - sp.add_argument( - "-p", + subparser.add_argument( "--preferred", + "-p", action="store_true", default=False, - help="checksum the preferred version only", + help="checksum the known Spack preferred version", ) - subparser.add_argument( - "-a", + modes_parser = subparser.add_mutually_exclusive_group() + modes_parser.add_argument( "--add-to-package", + "-a", action="store_true", default=False, help="add new versions to package", ) - arguments.add_common_arguments(subparser, ["package"]) + modes_parser.add_argument( + "--verify", action="store_true", default=False, help="verify known package checksums" + ) + subparser.add_argument("package", help="name or spec (e.g. `cmake` or `cmake@3.18`)") subparser.add_argument( - "versions", nargs=argparse.REMAINDER, help="versions to generate checksums for" + "versions", + nargs="*", + help="checksum these specific versions (if omitted, Spack searches for remote versions)", + ) + arguments.add_common_arguments(subparser, ["jobs"]) + subparser.epilog = ( + "examples:\n" + " `spack checksum zlib@1.2` autodetects versions 1.2.0 to 1.2.13 from the remote\n" + " `spack checksum zlib 1.2.13` checksums exact version 1.2.13 directly without search\n" ) def checksum(parser, args): - # Did the user pass 'package@version' string? - if len(args.versions) == 0 and "@" in args.package: - args.versions = [args.package.split("@")[1]] - args.package = args.package.split("@")[0] - - # Make sure the user provided a package and not a URL - if not valid_fully_qualified_module_name(args.package): - tty.die("`spack checksum` accepts package names, not URLs.") + spec = spack.spec.Spec(args.package) # Get the package we're going to generate checksums for - pkg_cls = spack.repo.path.get_pkg_class(args.package) - pkg = pkg_cls(spack.spec.Spec(args.package)) + pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec) + + versions = [Version(v) for v in args.versions] + + # Define placeholder for remote versions. + # This'll help reduce redundant work if we need to check for the existance + # of remote versions more than once. + remote_versions = None + + # Add latest version if requested + if args.latest: + remote_versions = pkg.fetch_remote_versions(args.jobs) + if len(remote_versions) > 0: + latest_version = sorted(remote_versions.keys(), reverse=True)[0] + versions.append(latest_version) + + # Add preferred version if requested + if args.preferred: + versions.append(preferred_version(pkg)) + # Store a dict of the form version -> URL url_dict = {} - if not args.versions and args.preferred: - versions = [preferred_version(pkg)] - else: - versions = [Version(v) for v in args.versions] - - if versions: - remote_versions = None - for version in versions: - if deprecated_version(pkg, version): - tty.warn("Version {0} is deprecated".format(version)) - - url = pkg.find_valid_url_for_version(version) - if url is not None: - url_dict[version] = url - continue - # if we get here, it's because no valid url was provided by the package - # do expensive fallback to try to recover - if remote_versions is None: - remote_versions = pkg.fetch_remote_versions() - if version in remote_versions: - url_dict[version] = remote_versions[version] - else: - url_dict = pkg.fetch_remote_versions() + + for version in versions: + if deprecated_version(pkg, version): + tty.warn(f"Version {version} is deprecated") + + url = pkg.find_valid_url_for_version(version) + if url is not None: + url_dict[version] = url + continue + # if we get here, it's because no valid url was provided by the package + # do expensive fallback to try to recover + if remote_versions is None: + remote_versions = pkg.fetch_remote_versions(args.jobs) + if version in remote_versions: + url_dict[version] = remote_versions[version] + + if len(versions) <= 0: + if remote_versions is None: + remote_versions = pkg.fetch_remote_versions(args.jobs) + url_dict = remote_versions + + # A spidered URL can differ from the package.py *computed* URL, pointing to different tarballs. + # For example, GitHub release pages sometimes have multiple tarballs with different shasum: + # - releases/download/1.0/-1.0.tar.gz (uploaded tarball) + # - archive/refs/tags/1.0.tar.gz (generated tarball) + # We wanna ensure that `spack checksum` and `spack install` ultimately use the same URL, so + # here we check whether the crawled and computed URLs disagree, and if so, prioritize the + # former if that URL exists (just sending a HEAD request that is). + url_changed_for_version = set() + for version, url in url_dict.items(): + possible_urls = pkg.all_urls_for_version(version) + if url not in possible_urls: + for possible_url in possible_urls: + if web_util.url_exists(possible_url): + url_dict[version] = possible_url + break + else: + url_changed_for_version.add(version) if not url_dict: - tty.die("Could not find any remote versions for {0}".format(pkg.name)) - - version_lines = spack.stage.get_checksums_for_versions( - url_dict, - pkg.name, - keep_stage=args.keep_stage, - batch=(args.batch or len(args.versions) > 0 or len(url_dict) == 1), - latest=args.latest, - fetch_options=pkg.fetch_options, + tty.die(f"Could not find any remote versions for {pkg.name}") + elif len(url_dict) > 1 and not args.batch and sys.stdin.isatty(): + filtered_url_dict = spack.stage.interactive_version_filter( + url_dict, + pkg.versions, + url_changes=url_changed_for_version, + initial_verion_filter=spec.versions, + ) + if not filtered_url_dict: + exit(0) + url_dict = filtered_url_dict + else: + tty.info(f"Found {llnl.string.plural(len(url_dict), 'version')} of {pkg.name}") + + version_hashes = spack.stage.get_checksums_for_versions( + url_dict, pkg.name, keep_stage=args.keep_stage, fetch_options=pkg.fetch_options ) + if args.verify: + print_checksum_status(pkg, version_hashes) + sys.exit(0) + + # convert dict into package.py version statements + version_lines = get_version_lines(version_hashes, url_dict) print() print(version_lines) print() if args.add_to_package: - filename = spack.repo.path.filename_for_package_name(pkg.name) - # Make sure we also have a newline after the last version - versions = [v + "\n" for v in version_lines.splitlines()] - versions.append("\n") - # We need to insert the versions in reversed order - versions.reverse() - versions.append(" # FIXME: Added by `spack checksum`\n") - version_line = None - - with open(filename, "r") as f: - lines = f.readlines() - for i in range(len(lines)): - # Black is drunk, so this is what it looks like for now - # See https://github.com/psf/black/issues/2156 for more information - if lines[i].startswith(" # FIXME: Added by `spack checksum`") or lines[ - i - ].startswith(" version("): - version_line = i - break + add_versions_to_package(pkg, version_lines) + - if version_line is not None: - for v in versions: - lines.insert(version_line, v) +def print_checksum_status(pkg: PackageBase, version_hashes: dict): + """ + Verify checksums present in version_hashes against those present + in the package's instructions. - with open(filename, "w") as f: - f.writelines(lines) + Args: + pkg (spack.package_base.PackageBase): A package class for a given package in Spack. + version_hashes (dict): A dictionary of the form: version -> checksum. - msg = "opening editor to verify" + """ + results = [] + num_verified = 0 + failed = False - if not sys.stdout.isatty(): - msg = "please verify" + max_len = max(len(str(v)) for v in version_hashes) + num_total = len(version_hashes) - tty.info( - "Added {0} new versions to {1}, " - "{2}.".format(len(versions) - 2, args.package, msg) - ) + for version, sha in version_hashes.items(): + if version not in pkg.versions: + msg = "No previous checksum" + status = "-" + + elif sha == pkg.versions[version]["sha256"]: + msg = "Correct" + status = "=" + num_verified += 1 - if sys.stdout.isatty(): - editor(filename) else: - tty.warn("Could not add new versions to {0}.".format(args.package)) + msg = sha + status = "x" + failed = True + + results.append("{0:{1}} {2} {3}".format(str(version), max_len, f"[{status}]", msg)) + + # Display table of checksum results. + tty.msg(f"Verified {num_verified} of {num_total}", "", *llnl.util.lang.elide_list(results), "") + + # Terminate at the end of function to prevent additional output. + if failed: + print() + tty.die("Invalid checksums found.") + + +def add_versions_to_package(pkg: PackageBase, version_lines: str): + """ + Add checksumed versions to a package's instructions and open a user's + editor so they may double check the work of the function. + + Args: + pkg (spack.package_base.PackageBase): A package class for a given package in Spack. + version_lines (str): A string of rendered version lines. + + """ + # Get filename and path for package + filename = spack.repo.PATH.filename_for_package_name(pkg.name) + num_versions_added = 0 + + version_statement_re = re.compile(r"([\t ]+version\([^\)]*\))") + version_re = re.compile(r'[\t ]+version\(\s*"([^"]+)"[^\)]*\)') + + # Split rendered version lines into tuple of (version, version_line) + # We reverse sort here to make sure the versions match the version_lines + new_versions = [] + for ver_line in version_lines.split("\n"): + match = version_re.match(ver_line) + if match: + new_versions.append((Version(match.group(1)), ver_line)) + + with open(filename, "r+") as f: + contents = f.read() + split_contents = version_statement_re.split(contents) + + for i, subsection in enumerate(split_contents): + # If there are no more versions to add we should exit + if len(new_versions) <= 0: + break + + # Check if the section contains a version + contents_version = version_re.match(subsection) + if contents_version is not None: + parsed_version = Version(contents_version.group(1)) + + if parsed_version < new_versions[0][0]: + split_contents[i:i] = [new_versions.pop(0)[1], " # FIXME", "\n"] + num_versions_added += 1 + + elif parsed_version == new_versions[0][0]: + new_versions.pop(0) + + # Seek back to the start of the file so we can rewrite the file contents. + f.seek(0) + f.writelines("".join(split_contents)) + + tty.msg(f"Added {num_versions_added} new versions to {pkg.name}") + tty.msg(f"Open {filename} to review the additions.") + + if sys.stdout.isatty(): + editor(filename) diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index f0facf712e504e..6c573193026fbf 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -18,6 +18,8 @@ import spack.environment as ev import spack.hash_types as ht import spack.mirror +import spack.util.gpg as gpg_util +import spack.util.timer as timer import spack.util.url as url_util import spack.util.web as web_util @@ -47,40 +49,36 @@ def setup_parser(subparser): generate.add_argument( "--output-file", default=None, - help="""pathname for the generated gitlab ci yaml file - Path to the file where generated jobs file should -be written. Default is .gitlab-ci.yml in the root of -the repository.""", + help="pathname for the generated gitlab ci yaml file\n\n" + "path to the file where generated jobs file should be written. " + "default is .gitlab-ci.yml in the root of the repository", ) generate.add_argument( "--copy-to", default=None, - help="""path to additional directory for job files - This option provides an absolute path to a directory -where the generated jobs yaml file should be copied. -Default is not to copy.""", + help="path to additional directory for job files\n\n" + "this option provides an absolute path to a directory where the generated " + "jobs yaml file should be copied. default is not to copy", ) generate.add_argument( "--optimize", action="store_true", default=False, - help="""(Experimental) optimize the gitlab yaml file for size - Run the generated document through a series of -optimization passes designed to reduce the size -of the generated file.""", + help="(experimental) optimize the gitlab yaml file for size\n\n" + "run the generated document through a series of optimization passes " + "designed to reduce the size of the generated file", ) generate.add_argument( "--dependencies", action="store_true", default=False, - help="(Experimental) disable DAG scheduling; use " ' "plain" dependencies.', + help="(experimental) disable DAG scheduling (use 'plain' dependencies)", ) generate.add_argument( "--buildcache-destination", default=None, - help="Override the mirror configured in the environment (spack.yaml) " - + "in order to push binaries from the generated pipeline to a " - + "different location.", + help="override the mirror configured in the environment\n\n" + "allows for pushing binaries from the generated pipeline to a different location", ) prune_group = generate.add_mutually_exclusive_group() prune_group.add_argument( @@ -88,45 +86,37 @@ def setup_parser(subparser): action="store_true", dest="prune_dag", default=True, - help="""skip up-to-date specs - Do not generate jobs for specs that are up-to-date -on the mirror.""", + help="skip up-to-date specs\n\n" + "do not generate jobs for specs that are up-to-date on the mirror", ) prune_group.add_argument( "--no-prune-dag", action="store_false", dest="prune_dag", default=True, - help="""process up-to-date specs - Generate jobs for specs even when they are up-to-date -on the mirror.""", + help="process up-to-date specs\n\n" + "generate jobs for specs even when they are up-to-date on the mirror", ) generate.add_argument( "--check-index-only", action="store_true", dest="index_only", default=False, - help="""only check spec state from buildcache indices - Spack always checks specs against configured binary -mirrors, regardless of the DAG pruning option. - If enabled, Spack will assume all remote buildcache -indices are up-to-date when assessing whether the spec -on the mirror, if present, is up-to-date. This has the -benefit of reducing pipeline generation time but at the -potential cost of needlessly rebuilding specs when the -indices are outdated. - If not enabled, Spack will fetch remote spec files -directly to assess whether the spec on the mirror is -up-to-date.""", + help="only check spec state from buildcache indices\n\n" + "Spack always checks specs against configured binary mirrors, regardless of the DAG " + "pruning option. if enabled, Spack will assume all remote buildcache indices are " + "up-to-date when assessing whether the spec on the mirror, if present, is up-to-date. " + "this has the benefit of reducing pipeline generation time but at the potential cost of " + "needlessly rebuilding specs when the indices are outdated. if not enabled, Spack will " + "fetch remote spec files directly to assess whether the spec on the mirror is up-to-date", ) generate.add_argument( "--artifacts-root", default=None, - help="""path to the root of the artifacts directory - If provided, concrete environment files (spack.yaml, -spack.lock) will be generated under this directory. -Their location will be passed to generated child jobs -through the SPACK_CONCRETE_ENVIRONMENT_PATH variable.""", + help="path to the root of the artifacts directory\n\n" + "if provided, concrete environment files (spack.yaml, spack.lock) will be generated under " + "this directory. their location will be passed to generated child jobs through the " + "SPACK_CONCRETE_ENVIRONMENT_PATH variable", ) generate.set_defaults(func=ci_generate) @@ -150,13 +140,13 @@ def setup_parser(subparser): "--tests", action="store_true", default=False, - help="""run stand-alone tests after the build""", + help="run stand-alone tests after the build", ) rebuild.add_argument( "--fail-fast", action="store_true", default=False, - help="""stop stand-alone tests after the first failure""", + help="stop stand-alone tests after the first failure", ) rebuild.set_defaults(func=ci_rebuild) @@ -166,25 +156,49 @@ def setup_parser(subparser): description=deindent(ci_reproduce.__doc__), help=spack.cmd.first_line(ci_reproduce.__doc__), ) - reproduce.add_argument("job_url", help="Url of job artifacts bundle") + reproduce.add_argument("job_url", help="URL of job artifacts bundle") + reproduce.add_argument( + "--runtime", + help="Container runtime to use.", + default="docker", + choices=["docker", "podman"], + ) reproduce.add_argument( "--working-dir", - help="Where to unpack artifacts", + help="where to unpack artifacts", default=os.path.join(os.getcwd(), "ci_reproduction"), ) + reproduce.add_argument( + "-s", "--autostart", help="Run docker reproducer automatically", action="store_true" + ) + gpg_group = reproduce.add_mutually_exclusive_group(required=False) + gpg_group.add_argument( + "--gpg-file", help="Path to public GPG key for validating binary cache installs" + ) + gpg_group.add_argument( + "--gpg-url", help="URL to public GPG key for validating binary cache installs" + ) reproduce.set_defaults(func=ci_reproduce) def ci_generate(args): - """Generate jobs file from a CI-aware spack file. + """generate jobs file from a CI-aware spack file - If you want to report the results on CDash, you will need to set - the SPACK_CDASH_AUTH_TOKEN before invoking this command. The - value must be the CDash authorization token needed to create a - build group and register all generated jobs under it.""" + if you want to report the results on CDash, you will need to set the SPACK_CDASH_AUTH_TOKEN + before invoking this command. the value must be the CDash authorization token needed to create + a build group and register all generated jobs under it + """ env = spack.cmd.require_active_env(cmd_name="ci generate") + if args.copy_to: + tty.warn("The flag --copy-to is deprecated and will be removed in Spack 0.23") + + if args.buildcache_destination: + tty.warn( + "The flag --buildcache-destination is deprecated and will be removed in Spack 0.23" + ) + output_file = args.output_file copy_yaml_to = args.copy_to run_optimizer = args.optimize @@ -223,10 +237,11 @@ def ci_generate(args): def ci_reindex(args): - """Rebuild the buildcache index for the remote mirror. + """rebuild the buildcache index for the remote mirror - Use the active, gitlab-enabled environment to rebuild the buildcache - index for the associated mirror.""" + use the active, gitlab-enabled environment to rebuild the buildcache index for the associated + mirror + """ env = spack.cmd.require_active_env(cmd_name="ci rebuild-index") yaml_root = env.manifest[ev.TOP_LEVEL_KEY] @@ -242,10 +257,13 @@ def ci_reindex(args): def ci_rebuild(args): - """Rebuild a spec if it is not on the remote mirror. + """rebuild a spec if it is not on the remote mirror + + check a single spec against the remote mirror, and rebuild it from source if the mirror does + not contain the hash + """ + rebuild_timer = timer.Timer() - Check a single spec against the remote mirror, and rebuild it from - source if the mirror does not contain the hash.""" env = spack.cmd.require_active_env(cmd_name="ci rebuild") # Make sure the environment is "gitlab-enabled", or else there's nothing @@ -254,12 +272,6 @@ def ci_rebuild(args): if not ci_config: tty.die("spack ci rebuild requires an env containing ci cfg") - tty.msg( - "SPACK_BUILDCACHE_DESTINATION={0}".format( - os.environ.get("SPACK_BUILDCACHE_DESTINATION", None) - ) - ) - # Grab the environment variables we need. These either come from the # pipeline generation step ("spack ci generate"), where they were written # out as variables, or else provided by GitLab itself. @@ -267,6 +279,7 @@ def ci_rebuild(args): job_log_dir = os.environ.get("SPACK_JOB_LOG_DIR") job_test_dir = os.environ.get("SPACK_JOB_TEST_DIR") repro_dir = os.environ.get("SPACK_JOB_REPRO_DIR") + # TODO: Remove this in Spack 0.23 local_mirror_dir = os.environ.get("SPACK_LOCAL_MIRROR_DIR") concrete_env_dir = os.environ.get("SPACK_CONCRETE_ENV_DIR") ci_pipeline_id = os.environ.get("CI_PIPELINE_ID") @@ -275,11 +288,25 @@ def ci_rebuild(args): job_spec_pkg_name = os.environ.get("SPACK_JOB_SPEC_PKG_NAME") job_spec_dag_hash = os.environ.get("SPACK_JOB_SPEC_DAG_HASH") spack_pipeline_type = os.environ.get("SPACK_PIPELINE_TYPE") + # TODO: Remove this in Spack 0.23 remote_mirror_override = os.environ.get("SPACK_REMOTE_MIRROR_OVERRIDE") + # TODO: Remove this in Spack 0.23 remote_mirror_url = os.environ.get("SPACK_REMOTE_MIRROR_URL") spack_ci_stack_name = os.environ.get("SPACK_CI_STACK_NAME") + # TODO: Remove this in Spack 0.23 shared_pr_mirror_url = os.environ.get("SPACK_CI_SHARED_PR_MIRROR_URL") rebuild_everything = os.environ.get("SPACK_REBUILD_EVERYTHING") + require_signing = os.environ.get("SPACK_REQUIRE_SIGNING") + + # If signing key was provided via "SPACK_SIGNING_KEY", then try to import it. + if signing_key: + spack_ci.import_signing_key(signing_key) + + # Fail early if signing is required but we don't have a signing key + sign_binaries = require_signing is not None and require_signing.lower() == "true" + if sign_binaries and not spack_ci.can_sign_binaries(): + gpg_util.list(False, True) + tty.die("SPACK_REQUIRE_SIGNING=True => spack must have exactly one signing key") # Construct absolute paths relative to current $CI_PROJECT_DIR ci_project_dir = os.environ.get("CI_PROJECT_DIR") @@ -323,21 +350,36 @@ def ci_rebuild(args): full_rebuild = True if rebuild_everything and rebuild_everything.lower() == "true" else False + pipeline_mirrors = spack.mirror.MirrorCollection(binary=True) + deprecated_mirror_config = False + buildcache_destination = None + if "buildcache-destination" in pipeline_mirrors: + buildcache_destination = pipeline_mirrors["buildcache-destination"] + else: + deprecated_mirror_config = True + # TODO: This will be an error in Spack 0.23 + # If no override url exists, then just push binary package to the # normal remote mirror url. + # TODO: Remove in Spack 0.23 buildcache_mirror_url = remote_mirror_override or remote_mirror_url + if buildcache_destination: + buildcache_mirror_url = buildcache_destination.push_url # Figure out what is our temporary storage mirror: Is it artifacts # buildcache? Or temporary-storage-url-prefix? In some cases we need to # force something or pipelines might not have a way to propagate build # artifacts from upstream to downstream jobs. + # TODO: Remove this in Spack 0.23 pipeline_mirror_url = None + # TODO: Remove this in Spack 0.23 temp_storage_url_prefix = None if "temporary-storage-url-prefix" in ci_config: temp_storage_url_prefix = ci_config["temporary-storage-url-prefix"] pipeline_mirror_url = url_util.join(temp_storage_url_prefix, ci_pipeline_id) + # TODO: Remove this in Spack 0.23 enable_artifacts_mirror = False if "enable-artifacts-buildcache" in ci_config: enable_artifacts_mirror = ci_config["enable-artifacts-buildcache"] @@ -404,11 +446,6 @@ def ci_rebuild(args): dst_file = os.path.join(repro_dir, file_name) shutil.copyfile(src_file, dst_file) - # If signing key was provided via "SPACK_SIGNING_KEY", then try to - # import it. - if signing_key: - spack_ci.import_signing_key(signing_key) - # Write this job's spec json into the reproduction directory, and it will # also be used in the generated "spack install" command to install the spec tty.debug("job concrete spec path: {0}".format(job_spec_json_path)) @@ -438,12 +475,14 @@ def ci_rebuild(args): # If we decided there should be a temporary storage mechanism, add that # mirror now so it's used when we check for a hash match already # built for this spec. + # TODO: Remove this block in Spack 0.23 if pipeline_mirror_url: mirror = spack.mirror.Mirror(pipeline_mirror_url, name=spack_ci.TEMP_STORAGE_MIRROR_NAME) spack.mirror.add(mirror, cfg.default_modify_scope()) pipeline_mirrors.append(pipeline_mirror_url) # Check configured mirrors for a built spec with a matching hash + # TODO: Remove this block in Spack 0.23 mirrors_to_check = None if remote_mirror_override: if spack_pipeline_type == "spack_protected_branch": @@ -461,7 +500,8 @@ def ci_rebuild(args): ) pipeline_mirrors.append(remote_mirror_override) - if spack_pipeline_type == "spack_pull_request": + # TODO: Remove this in Spack 0.23 + if deprecated_mirror_config and spack_pipeline_type == "spack_pull_request": if shared_pr_mirror_url != "None": pipeline_mirrors.append(shared_pr_mirror_url) @@ -483,6 +523,7 @@ def ci_rebuild(args): tty.msg("No need to rebuild {0}, found hash match at: ".format(job_spec_pkg_name)) for match in matches: tty.msg(" {0}".format(match["mirror_url"])) + # TODO: Remove this block in Spack 0.23 if enable_artifacts_mirror: matching_mirror = matches[0]["mirror_url"] build_cache_dir = os.path.join(local_mirror_dir, "build_cache") @@ -497,7 +538,8 @@ def ci_rebuild(args): # only want to keep the mirror being used by the current pipeline as it's binary # package destination. This ensures that the when we rebuild everything, we only # consume binary dependencies built in this pipeline. - if full_rebuild: + # TODO: Remove this in Spack 0.23 + if deprecated_mirror_config and full_rebuild: spack_ci.remove_other_mirrors(pipeline_mirrors, cfg.default_modify_scope()) # No hash match anywhere means we need to rebuild spec @@ -563,7 +605,9 @@ def ci_rebuild(args): "SPACK_COLOR=always", "SPACK_INSTALL_FLAGS={}".format(args_to_string(deps_install_args)), "-j$(nproc)", - "install-deps/{}".format(job_spec.format("{name}-{version}-{hash}")), + "install-deps/{}".format( + ev.depfile.MakefileSpec(job_spec).safe_format("{name}-{version}-{hash}") + ), ], spack_cmd + ["install"] + root_install_args, ] @@ -606,7 +650,7 @@ def ci_rebuild(args): ) reports_dir = fs.join_path(os.getcwd(), "cdash_report") if args.tests and broken_tests: - tty.warn("Unable to run stand-alone tests since listed in " "ci's 'broken-tests-packages'") + tty.warn("Unable to run stand-alone tests since listed in ci's 'broken-tests-packages'") if cdash_handler: msg = "Package is listed in ci's broken-tests-packages" cdash_handler.report_skipped(job_spec, reports_dir, reason=msg) @@ -649,7 +693,7 @@ def ci_rebuild(args): tty.warn("No recognized test results reporting option") else: - tty.warn("Unable to run stand-alone tests due to unsuccessful " "installation") + tty.warn("Unable to run stand-alone tests due to unsuccessful installation") if cdash_handler: msg = "Failed to install the package" cdash_handler.report_skipped(job_spec, reports_dir, reason=msg) @@ -660,21 +704,25 @@ def ci_rebuild(args): # print out some instructions on how to reproduce this build failure # outside of the pipeline environment. if install_exit_code == 0: - if buildcache_mirror_url or pipeline_mirror_url: - for result in spack_ci.create_buildcache( - input_spec=job_spec, - buildcache_mirror_url=buildcache_mirror_url, - pipeline_mirror_url=pipeline_mirror_url, - pr_pipeline=spack_is_pr_pipeline, - ): - msg = tty.msg if result.success else tty.warn - msg( - "{} {} to {}".format( - "Pushed" if result.success else "Failed to push", - job_spec.format("{name}{@version}{/hash:7}", color=clr.get_color_when()), - result.url, - ) + mirror_urls = [buildcache_mirror_url] + + # TODO: Remove this block in Spack 0.23 + if pipeline_mirror_url: + mirror_urls.append(pipeline_mirror_url) + + for result in spack_ci.create_buildcache( + input_spec=job_spec, + destination_mirror_urls=mirror_urls, + sign_binaries=spack_ci.can_sign_binaries(), + ): + msg = tty.msg if result.success else tty.warn + msg( + "{} {} to {}".format( + "Pushed" if result.success else "Failed to push", + job_spec.format("{name}{@version}{/hash:7}", color=clr.get_color_when()), + result.url, ) + ) # If this is a develop pipeline, check if the spec that we just built is # on the broken-specs list. If so, remove it. @@ -709,7 +757,7 @@ def ci_rebuild(args): \033[34mTo reproduce this build locally, run: - spack ci reproduce-build {0} [--working-dir ] + spack ci reproduce-build {0} [--working-dir ] [--autostart] If this project does not have public pipelines, you will need to first: @@ -723,19 +771,38 @@ def ci_rebuild(args): print(reproduce_msg) + rebuild_timer.stop() + try: + with open("install_timers.json", "w") as timelog: + extra_attributes = {"name": ".ci-rebuild"} + rebuild_timer.write_json(timelog, extra_attributes=extra_attributes) + except Exception as e: + tty.debug(str(e)) + # Tie job success/failure to the success/failure of building the spec return install_exit_code def ci_reproduce(args): - """Generate instructions for reproducing the spec rebuild job. + """generate instructions for reproducing the spec rebuild job - Artifacts of the provided gitlab pipeline rebuild job's URL will be - used to derive instructions for reproducing the build locally.""" + artifacts of the provided gitlab pipeline rebuild job's URL will be used to derive + instructions for reproducing the build locally + """ job_url = args.job_url work_dir = args.working_dir + autostart = args.autostart + runtime = args.runtime + + # Allow passing GPG key for reprocuding protected CI jobs + if args.gpg_file: + gpg_key_url = url_util.path_to_file_url(args.gpg_file) + elif args.gpg_url: + gpg_key_url = args.gpg_url + else: + gpg_key_url = None - return spack_ci.reproduce_ci_job(job_url, work_dir) + return spack_ci.reproduce_ci_job(job_url, work_dir, autostart, gpg_key_url, runtime) def ci(parser, args): diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index eeb37f909d7d96..5a1831cda5a43d 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -17,6 +17,7 @@ import spack.config import spack.repo import spack.stage +import spack.store import spack.util.path from spack.paths import lib_path, var_path @@ -114,22 +115,18 @@ def clean(parser, args): if args.stage: tty.msg("Removing all temporary build stages") spack.stage.purge() - # Temp directory where buildcaches are extracted - extract_tmp = os.path.join(spack.store.layout.root, ".tmp") - if os.path.exists(extract_tmp): - tty.debug("Removing {0}".format(extract_tmp)) - shutil.rmtree(extract_tmp) + if args.downloads: tty.msg("Removing cached downloads") - spack.caches.fetch_cache.destroy() + spack.caches.FETCH_CACHE.destroy() if args.failures: tty.msg("Removing install failure marks") - spack.installer.clear_failures() + spack.store.STORE.failure_tracker.clear_all() if args.misc_cache: tty.msg("Removing cached information on repositories") - spack.caches.misc_cache.destroy() + spack.caches.MISC_CACHE.destroy() if args.python_cache: tty.msg("Removing python cache files") diff --git a/lib/spack/spack/cmd/clone.py b/lib/spack/spack/cmd/clone.py index 836e9653da1f2f..ade3c2a7390658 100644 --- a/lib/spack/spack/cmd/clone.py +++ b/lib/spack/spack/cmd/clone.py @@ -48,7 +48,7 @@ def get_origin_info(remote): ) except ProcessError: origin_url = _SPACK_UPSTREAM - tty.warn("No git repository found; " "using default upstream URL: %s" % origin_url) + tty.warn("No git repository found; using default upstream URL: %s" % origin_url) return (origin_url.strip(), branch.strip()) @@ -69,7 +69,7 @@ def clone(parser, args): files_in_the_way = os.listdir(prefix) if files_in_the_way: tty.die( - "There are already files there! " "Delete these files before boostrapping spack.", + "There are already files there! Delete these files before boostrapping spack.", *files_in_the_way, ) diff --git a/lib/spack/spack/cmd/commands.py b/lib/spack/spack/cmd/commands.py index 6af7bb54e89ad3..25e1a24d0077d0 100644 --- a/lib/spack/spack/cmd/commands.py +++ b/lib/spack/spack/cmd/commands.py @@ -9,16 +9,11 @@ import re import sys from argparse import ArgumentParser, Namespace -from typing import IO, Any, Callable, Dict, Sequence, Set +from typing import IO, Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union import llnl.util.filesystem as fs import llnl.util.tty as tty -from llnl.util.argparsewriter import ( - ArgparseCompletionWriter, - ArgparseRstWriter, - ArgparseWriter, - Command, -) +from llnl.util.argparsewriter import ArgparseRstWriter, ArgparseWriter, Command from llnl.util.tty.colify import colify import spack.cmd @@ -41,9 +36,15 @@ "bash": { "aliases": True, "format": "bash", - "header": os.path.join(spack.paths.share_path, "bash", "spack-completion.in"), + "header": os.path.join(spack.paths.share_path, "bash", "spack-completion.bash"), "update": os.path.join(spack.paths.share_path, "spack-completion.bash"), - } + }, + "fish": { + "aliases": True, + "format": "fish", + "header": os.path.join(spack.paths.share_path, "fish", "spack-completion.fish"), + "update": os.path.join(spack.paths.share_path, "spack-completion.fish"), + }, } @@ -178,9 +179,63 @@ def format(self, cmd: Command) -> str: } -class BashCompletionWriter(ArgparseCompletionWriter): +class BashCompletionWriter(ArgparseWriter): """Write argparse output as bash programmable tab completion.""" + def format(self, cmd: Command) -> str: + """Return the string representation of a single node in the parser tree. + + Args: + cmd: Parsed information about a command or subcommand. + + Returns: + String representation of this subcommand. + """ + + assert cmd.optionals # we should always at least have -h, --help + assert not (cmd.positionals and cmd.subcommands) # one or the other + + # We only care about the arguments/flags, not the help messages + positionals: Tuple[str, ...] = () + if cmd.positionals: + positionals, _, _, _ = zip(*cmd.positionals) + optionals, _, _, _, _ = zip(*cmd.optionals) + subcommands: Tuple[str, ...] = () + if cmd.subcommands: + _, subcommands, _ = zip(*cmd.subcommands) + + # Flatten lists of lists + optionals = [x for xx in optionals for x in xx] + + return ( + self.start_function(cmd.prog) + + self.body(positionals, optionals, subcommands) + + self.end_function(cmd.prog) + ) + + def start_function(self, prog: str) -> str: + """Return the syntax needed to begin a function definition. + + Args: + prog: Program name. + + Returns: + Function definition beginning. + """ + name = prog.replace("-", "_").replace(" ", "_") + return "\n_{0}() {{".format(name) + + def end_function(self, prog: str) -> str: + """Return the syntax needed to end a function definition. + + Args: + prog: Program name + + Returns: + Function definition ending. + """ + return "}\n" + def body( self, positionals: Sequence[str], optionals: Sequence[str], subcommands: Sequence[str] ) -> str: @@ -264,6 +319,396 @@ def subcommands(self, subcommands: Sequence[str]) -> str: return 'SPACK_COMPREPLY="{0}"'.format(" ".join(subcommands)) +# Map argument destination names to their complete commands +# Earlier items in the list have higher precedence +_dest_to_fish_complete = { + ("activate", "view"): "-f -a '(__fish_complete_directories)'", + ("bootstrap root", "path"): "-f -a '(__fish_complete_directories)'", + ("mirror add", "mirror"): "-f", + ("repo add", "path"): "-f -a '(__fish_complete_directories)'", + ("test find", "filter"): "-f -a '(__fish_spack_tests)'", + ("bootstrap", "name"): "-f -a '(__fish_spack_bootstrap_names)'", + ("buildcache create", "key"): "-f -a '(__fish_spack_gpg_keys)'", + ("build-env", r"spec \[--\].*"): "-f -a '(__fish_spack_build_env_spec)'", + ("checksum", "package"): "-f -a '(__fish_spack_packages)'", + ( + "checksum", + "versions", + ): "-f -a '(__fish_spack_package_versions $__fish_spack_argparse_argv[1])'", + ("config", "path"): "-f -a '(__fish_spack_colon_path)'", + ("config", "section"): "-f -a '(__fish_spack_config_sections)'", + ("develop", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", + ("diff", "specs?"): "-f -a '(__fish_spack_installed_specs)'", + ("gpg sign", "output"): "-f -a '(__fish_complete_directories)'", + ("gpg", "keys?"): "-f -a '(__fish_spack_gpg_keys)'", + ("graph", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", + ("help", "help_command"): "-f -a '(__fish_spack_commands)'", + ("list", "filter"): "-f -a '(__fish_spack_packages)'", + ("mirror", "mirror"): "-f -a '(__fish_spack_mirrors)'", + ("pkg", "package"): "-f -a '(__fish_spack_pkg_packages)'", + ("remove", "specs?"): "-f -a '(__fish_spack_installed_specs)'", + ("repo", "namespace_or_path"): "$__fish_spack_force_files -a '(__fish_spack_repos)'", + ("restage", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", + ("rm", "specs?"): "-f -a '(__fish_spack_installed_specs)'", + ("solve", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", + ("spec", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", + ("stage", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", + ("test-env", r"spec \[--\].*"): "-f -a '(__fish_spack_build_env_spec)'", + ("test", r"\[?name.*"): "-f -a '(__fish_spack_tests)'", + ("undevelop", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", + ("verify", "specs_or_files"): "$__fish_spack_force_files -a '(__fish_spack_installed_specs)'", + ("view", "path"): "-f -a '(__fish_complete_directories)'", + ("", "comment"): "-f", + ("", "compiler_spec"): "-f -a '(__fish_spack_installed_compilers)'", + ("", "config_scopes"): "-f -a '(__fish_complete_directories)'", + ("", "extendable"): "-f -a '(__fish_spack_extensions)'", + ("", "installed_specs?"): "-f -a '(__fish_spack_installed_specs)'", + ("", "job_url"): "-f", + ("", "location_env"): "-f -a '(__fish_complete_directories)'", + ("", "pytest_args"): "-f -a '(__fish_spack_unit_tests)'", + ("", "package_or_file"): "$__fish_spack_force_files -a '(__fish_spack_packages)'", + ("", "package_or_user"): "-f -a '(__fish_spack_packages)'", + ("", "package"): "-f -a '(__fish_spack_packages)'", + ("", "PKG"): "-f -a '(__fish_spack_packages)'", + ("", "prefix"): "-f -a '(__fish_complete_directories)'", + ("", r"rev\d?"): "-f -a '(__fish_spack_git_rev)'", + ("", "specs?"): "-f -k -a '(__fish_spack_specs)'", + ("", "tags?"): "-f -a '(__fish_spack_tags)'", + ("", "virtual_package"): "-f -a '(__fish_spack_providers)'", + ("", "working_dir"): "-f -a '(__fish_complete_directories)'", + ("", r"(\w*_)?env"): "-f -a '(__fish_spack_environments)'", + ("", r"(\w*_)?dir(ectory)?"): "-f -a '(__fish_spack_environments)'", + ("", r"(\w*_)?mirror_name"): "-f -a '(__fish_spack_mirrors)'", +} + + +def _fish_dest_get_complete(prog: str, dest: str) -> Optional[str]: + """Map from subcommand to autocompletion argument. + + Args: + prog: Program name. + dest: Destination. + + Returns: + Autocompletion argument. + """ + s = prog.split(None, 1) + subcmd = s[1] if len(s) == 2 else "" + + for (prog_key, pos_key), value in _dest_to_fish_complete.items(): + if subcmd.startswith(prog_key) and re.match("^" + pos_key + "$", dest): + return value + return None + + +class FishCompletionWriter(ArgparseWriter): + """Write argparse output as bash programmable tab completion.""" + + def format(self, cmd: Command) -> str: + """Return the string representation of a single node in the parser tree. + + Args: + cmd: Parsed information about a command or subcommand. + + Returns: + String representation of a node. + """ + assert cmd.optionals # we should always at least have -h, --help + assert not (cmd.positionals and cmd.subcommands) # one or the other + + # We also need help messages and how arguments are used + # So we pass everything to completion writer + positionals = cmd.positionals + optionals = cmd.optionals + subcommands = cmd.subcommands + + return ( + self.prog_comment(cmd.prog) + + self.optspecs(cmd.prog, optionals) + + self.complete(cmd.prog, positionals, optionals, subcommands) + ) + + def _quote(self, string: str) -> str: + """Quote string and escape special characters if necessary. + + Args: + string: Input string. + + Returns: + Quoted string. + """ + # Goal here is to match fish_indent behavior + + # Strings without spaces (or other special characters) do not need to be escaped + if not any([sub in string for sub in [" ", "'", '"']]): + return string + + string = string.replace("'", r"\'") + return f"'{string}'" + + def optspecs( + self, + prog: str, + optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]], + ) -> str: + """Read the optionals and return the command to set optspec. + + Args: + prog: Program name. + optionals: List of optional arguments. + + Returns: + Command to set optspec variable. + """ + # Variables of optspecs + optspec_var = "__fish_spack_optspecs_" + prog.replace(" ", "_").replace("-", "_") + + if optionals is None: + return "set -g %s\n" % optspec_var + + # Build optspec by iterating over options + args = [] + + for flags, dest, _, nargs, _ in optionals: + if len(flags) == 0: + continue + + required = "" + + # Because nargs '?' is treated differently in fish, we treat it as required. + # Because multi-argument options are not supported, we treat it like one argument. + required = "=" + if nargs == 0: + required = "" + + # Pair short options with long options + + # We need to do this because fish doesn't support multiple short + # or long options. + # However, since we are paring options only, this is fine + + short = [f[1:] for f in flags if f.startswith("-") and len(f) == 2] + long = [f[2:] for f in flags if f.startswith("--")] + + while len(short) > 0 and len(long) > 0: + arg = "%s/%s%s" % (short.pop(), long.pop(), required) + while len(short) > 0: + arg = "%s/%s" % (short.pop(), required) + while len(long) > 0: + arg = "%s%s" % (long.pop(), required) + + args.append(arg) + + # Even if there is no option, we still set variable. + # In fish such variable is an empty array, we use it to + # indicate that such subcommand exists. + args = " ".join(args) + + return "set -g %s %s\n" % (optspec_var, args) + + @staticmethod + def complete_head( + prog: str, index: Optional[int] = None, nargs: Optional[Union[int, str]] = None + ) -> str: + """Return the head of the completion command. + + Args: + prog: Program name. + index: Index of positional argument. + nargs: Number of arguments. + + Returns: + Head of the completion command. + """ + # Split command and subcommand + s = prog.split(None, 1) + subcmd = s[1] if len(s) == 2 else "" + + if index is None: + return "complete -c %s -n '__fish_spack_using_command %s'" % (s[0], subcmd) + elif nargs in [argparse.ZERO_OR_MORE, argparse.ONE_OR_MORE, argparse.REMAINDER]: + head = "complete -c %s -n '__fish_spack_using_command_pos_remainder %d %s'" + else: + head = "complete -c %s -n '__fish_spack_using_command_pos %d %s'" + return head % (s[0], index, subcmd) + + def complete( + self, + prog: str, + positionals: List[Tuple[str, Optional[Iterable[Any]], Union[int, str, None], str]], + optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]], + subcommands: List[Tuple[ArgumentParser, str, str]], + ) -> str: + """Return all the completion commands. + + Args: + prog: Program name. + positionals: List of positional arguments. + optionals: List of optional arguments. + subcommands: List of subcommand parsers. + + Returns: + Completion command. + """ + commands = [] + + if positionals: + commands.append(self.positionals(prog, positionals)) + + if subcommands: + commands.append(self.subcommands(prog, subcommands)) + + if optionals: + commands.append(self.optionals(prog, optionals)) + + return "".join(commands) + + def positionals( + self, + prog: str, + positionals: List[Tuple[str, Optional[Iterable[Any]], Union[int, str, None], str]], + ) -> str: + """Return the completion for positional arguments. + + Args: + prog: Program name. + positionals: List of positional arguments. + + Returns: + Completion command. + """ + commands = [] + + for idx, (args, choices, nargs, help) in enumerate(positionals): + # Make sure we always get same order of output + if isinstance(choices, dict): + choices = sorted(choices.keys()) + elif isinstance(choices, (set, frozenset)): + choices = sorted(choices) + + # Remove platform-specific choices to avoid hard-coding the platform. + if choices is not None: + valid_choices = [] + for choice in choices: + if spack.platforms.host().name not in choice: + valid_choices.append(choice) + choices = valid_choices + + head = self.complete_head(prog, idx, nargs) + + if choices is not None: + # If there are choices, we provide a completion for all possible values. + commands.append(head + " -f -a %s" % self._quote(" ".join(choices))) + else: + # Otherwise, we try to find a predefined completion for it + value = _fish_dest_get_complete(prog, args) + if value is not None: + commands.append(head + " " + value) + + return "\n".join(commands) + "\n" + + def prog_comment(self, prog: str) -> str: + """Return a comment line for the command. + + Args: + prog: Program name. + + Returns: + Comment line. + """ + return "\n# %s\n" % prog + + def optionals( + self, + prog: str, + optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]], + ) -> str: + """Return the completion for optional arguments. + + Args: + prog: Program name. + optionals: List of optional arguments. + + Returns: + Completion command. + """ + commands = [] + head = self.complete_head(prog) + + for flags, dest, _, nargs, help in optionals: + # Make sure we always get same order of output + if isinstance(dest, dict): + dest = sorted(dest.keys()) + elif isinstance(dest, (set, frozenset)): + dest = sorted(dest) + + # Remove platform-specific choices to avoid hard-coding the platform. + if dest is not None: + valid_choices = [] + for choice in dest: + if spack.platforms.host().name not in choice: + valid_choices.append(choice) + dest = valid_choices + + # To provide description for optionals, and also possible values, + # we need to use two split completion command. + # Otherwise, each option will have same description. + prefix = head + + # Add all flags to the completion + for f in flags: + if f.startswith("--"): + long = f[2:] + prefix += " -l %s" % long + elif f.startswith("-"): + short = f[1:] + assert len(short) == 1 + prefix += " -s %s" % short + + # Check if option require argument. + # Currently multi-argument options are not supported, so we treat it like one argument. + if nargs != 0: + prefix += " -r" + + if dest is not None: + # If there are choices, we provide a completion for all possible values. + commands.append(prefix + " -f -a %s" % self._quote(" ".join(dest))) + else: + # Otherwise, we try to find a predefined completion for it + value = _fish_dest_get_complete(prog, dest) + if value is not None: + commands.append(prefix + " " + value) + + if help: + commands.append(prefix + " -d %s" % self._quote(help)) + + return "\n".join(commands) + "\n" + + def subcommands(self, prog: str, subcommands: List[Tuple[ArgumentParser, str, str]]) -> str: + """Return the completion for subcommands. + + Args: + prog: Program name. + subcommands: List of subcommand parsers. + + Returns: + Completion command. + """ + commands = [] + head = self.complete_head(prog, 0) + + for _, subcommand, help in subcommands: + command = head + " -f -a %s" % self._quote(subcommand) + + if help is not None and len(help) > 0: + help = help.split("\n")[0] + command += " -d %s" % self._quote(help) + + commands.append(command) + + return "\n".join(commands) + "\n" + + @formatter def subcommands(args: Namespace, out: IO) -> None: """Hierarchical tree of subcommands. @@ -351,7 +796,9 @@ def names(args: Namespace, out: IO) -> None: commands = copy.copy(spack.cmd.all_commands()) if args.aliases: - commands.extend(spack.main.aliases.keys()) + aliases = spack.config.get("config:aliases") + if aliases: + commands.extend(aliases.keys()) colify(commands, output=out) @@ -367,10 +814,24 @@ def bash(args: Namespace, out: IO) -> None: parser = spack.main.make_argument_parser() spack.main.add_all_commands(parser) + aliases_config = spack.config.get("config:aliases") + if aliases_config: + aliases = ";".join(f"{key}:{val}" for key, val in aliases_config.items()) + out.write(f'SPACK_ALIASES="{aliases}"\n\n') + writer = BashCompletionWriter(parser.prog, out, args.aliases) writer.write(parser) +@formatter +def fish(args, out): + parser = spack.main.make_argument_parser() + spack.main.add_all_commands(parser) + + writer = FishCompletionWriter(parser.prog, out, args.aliases) + writer.write(parser) + + def prepend_header(args: Namespace, out: IO) -> None: """Prepend header text at the beginning of a file. diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index 09e3c32ec00d1c..9aa3edac479f50 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -12,7 +12,7 @@ import spack.cmd import spack.config -import spack.dependency as dep +import spack.deptypes as dt import spack.environment as ev import spack.mirror import spack.modules @@ -82,12 +82,12 @@ def _specs(self, **kwargs): # return everything for an empty query. if not qspecs: - return spack.store.db.query(**kwargs) + return spack.store.STORE.db.query(**kwargs) # Return only matching stuff otherwise. specs = {} for spec in qspecs: - for s in spack.store.db.query(spec, **kwargs): + for s in spack.store.STORE.db.query(spec, **kwargs): # This is fast for already-concrete specs specs[s.dag_hash()] = s @@ -114,16 +114,13 @@ def __call__(self, parser, namespace, jobs, option_string): class DeptypeAction(argparse.Action): - """Creates a tuple of valid dependency types from a deptype argument.""" + """Creates a flag of valid dependency types from a deptype argument.""" def __call__(self, parser, namespace, values, option_string=None): - deptype = dep.all_deptypes - if values: - deptype = tuple(x.strip() for x in values.split(",")) - if deptype == ("all",): - deptype = "all" - deptype = dep.canonical_deptype(deptype) - + if not values or values == "all": + deptype = dt.ALL + else: + deptype = dt.canonicalize(values.split(",")) setattr(namespace, self.dest, deptype) @@ -265,7 +262,7 @@ def recurse_dependents(): "--dependents", action="store_true", dest="dependents", - help="also uninstall any packages that depend on the ones given " "via command line", + help="also uninstall any packages that depend on the ones given via command line", ) @@ -285,9 +282,8 @@ def deptype(): return Args( "--deptype", action=DeptypeAction, - default=dep.all_deptypes, - help="comma-separated list of deptypes to traverse\ndefault=%s" - % ",".join(dep.all_deptypes), + default=dt.ALL, + help="comma-separated list of deptypes to traverse (default=%s)" % ",".join(dt.ALL_TYPES), ) @@ -331,6 +327,17 @@ def tags(): ) +@arg +def namespaces(): + return Args( + "-N", + "--namespaces", + action="store_true", + default=False, + help="show fully qualified package names", + ) + + @arg def jobs(): return Args( @@ -350,9 +357,9 @@ def install_status(): "--install-status", action="store_true", default=True, - help="show install status of packages. packages can be: " + help="show install status of packages\n\npackages can be: " "installed [+], missing and needed by an installed package [-], " - "installed in and upstream instance [^], " + "installed in an upstream instance [^], " "or not installed (no annotation)", ) @@ -393,24 +400,23 @@ def add_cdash_args(subparser, add_help): cdash_help = {} if add_help: cdash_help["upload-url"] = "CDash URL where reports will be uploaded" - cdash_help[ - "build" - ] = """The name of the build that will be reported to CDash. -Defaults to spec of the package to operate on.""" - cdash_help[ - "site" - ] = """The site name that will be reported to CDash. -Defaults to current system hostname.""" - cdash_help[ - "track" - ] = """Results will be reported to this group on CDash. -Defaults to Experimental.""" - cdash_help[ - "buildstamp" - ] = """Instead of letting the CDash reporter prepare the -buildstamp which, when combined with build name, site and project, -uniquely identifies the build, provide this argument to identify -the build yourself. Format: %%Y%%m%%d-%%H%%M-[cdash-track]""" + cdash_help["build"] = ( + "name of the build that will be reported to CDash\n\n" + "defaults to spec of the package to operate on" + ) + cdash_help["site"] = ( + "site name that will be reported to CDash\n\n" "defaults to current system hostname" + ) + cdash_help["track"] = ( + "results will be reported to this group on CDash\n\n" "defaults to Experimental" + ) + cdash_help["buildstamp"] = ( + "use custom buildstamp\n\n" + "instead of letting the CDash reporter prepare the " + "buildstamp which, when combined with build name, site and project, " + "uniquely identifies the build, provide this argument to identify " + "the build yourself. format: %%Y%%m%%d-%%H%%M-[cdash-track]" + ) else: cdash_help["upload-url"] = argparse.SUPPRESS cdash_help["build"] = argparse.SUPPRESS @@ -537,22 +543,24 @@ def add_concretizer_args(subparser): ) -def add_s3_connection_args(subparser, add_help): +def add_connection_args(subparser, add_help): subparser.add_argument( "--s3-access-key-id", help="ID string to use to connect to this S3 mirror" ) subparser.add_argument( - "--s3-access-key-secret", help="Secret string to use to connect to this S3 mirror" + "--s3-access-key-secret", help="secret string to use to connect to this S3 mirror" ) subparser.add_argument( - "--s3-access-token", help="Access Token to use to connect to this S3 mirror" + "--s3-access-token", help="access token to use to connect to this S3 mirror" ) subparser.add_argument( "--s3-profile", help="S3 profile name to use to connect to this S3 mirror", default=None ) subparser.add_argument( - "--s3-endpoint-url", help="Endpoint URL to use to connect to this S3 mirror" + "--s3-endpoint-url", help="endpoint URL to use to connect to this S3 mirror" ) + subparser.add_argument("--oci-username", help="username to use to connect to this OCI mirror") + subparser.add_argument("--oci-password", help="password to use to connect to this OCI mirror") def use_buildcache(cli_arg_value): diff --git a/lib/spack/spack/cmd/common/confirmation.py b/lib/spack/spack/cmd/common/confirmation.py new file mode 100644 index 00000000000000..8a5cd2592b44e9 --- /dev/null +++ b/lib/spack/spack/cmd/common/confirmation.py @@ -0,0 +1,30 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import sys +from typing import List + +import llnl.util.tty as tty + +import spack.cmd + +display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4} + + +def confirm_action(specs: List[spack.spec.Spec], participle: str, noun: str): + """Display the list of specs to be acted on and ask for confirmation. + + Args: + specs: specs to be removed + participle: action expressed as a participle, e.g. "uninstalled" + noun: action expressed as a noun, e.g. "uninstallation" + """ + tty.msg(f"The following {len(specs)} packages will be {participle}:\n") + spack.cmd.display_specs(specs, **display_args) + print("") + answer = tty.get_yes_or_no("Do you want to proceed?", default=False) + if not answer: + tty.msg(f"Aborting {noun}") + sys.exit(0) diff --git a/lib/spack/spack/cmd/common/env_utility.py b/lib/spack/spack/cmd/common/env_utility.py index a616b79d8f1062..b8a6338d924f83 100644 --- a/lib/spack/spack/cmd/common/env_utility.py +++ b/lib/spack/spack/cmd/common/env_utility.py @@ -7,14 +7,15 @@ import llnl.util.tty as tty -import spack.build_environment as build_environment import spack.cmd import spack.cmd.common.arguments as arguments +import spack.deptypes as dt import spack.error import spack.paths import spack.spec import spack.store -from spack import traverse +from spack import build_environment, traverse +from spack.context import Context from spack.util.environment import dump_environment, pickle_environment @@ -41,14 +42,14 @@ def setup_parser(subparser): class AreDepsInstalledVisitor: - def __init__(self, context="build"): - if context not in ("build", "test"): - raise ValueError("context can only be build or test") - - if context == "build": - self.direct_deps = ("build", "link", "run") + def __init__(self, context: Context = Context.BUILD): + if context == Context.BUILD: + # TODO: run deps shouldn't be required for build env. + self.direct_deps = dt.BUILD | dt.LINK | dt.RUN + elif context == Context.TEST: + self.direct_deps = dt.BUILD | dt.TEST | dt.LINK | dt.RUN else: - self.direct_deps = ("build", "test", "link", "run") + raise ValueError("context can only be Context.BUILD or Context.TEST") self.has_uninstalled_deps = False @@ -71,11 +72,11 @@ def accept(self, item): def neighbors(self, item): # Direct deps: follow build & test edges. # Transitive deps: follow link / run. - deptypes = self.direct_deps if item.depth == 0 else ("link", "run") - return item.edge.spec.edges_to_dependencies(deptype=deptypes) + depflag = self.direct_deps if item.depth == 0 else dt.LINK | dt.RUN + return item.edge.spec.edges_to_dependencies(depflag=depflag) -def emulate_env_utility(cmd_name, context, args): +def emulate_env_utility(cmd_name, context: Context, args): if not args.spec: tty.die("spack %s requires a spec." % cmd_name) @@ -106,7 +107,7 @@ def emulate_env_utility(cmd_name, context, args): visitor = AreDepsInstalledVisitor(context=context) # Mass install check needs read transaction. - with spack.store.db.read_transaction(): + with spack.store.STORE.db.read_transaction(): traverse.traverse_breadth_first_with_visitor([spec], traverse.CoverNodesVisitor(visitor)) if visitor.has_uninstalled_deps: @@ -119,7 +120,7 @@ def emulate_env_utility(cmd_name, context, args): hashes=True, # This shows more than necessary, but we cannot dynamically change deptypes # in Spec.tree(...). - deptypes="all" if context == "build" else ("build", "test", "link", "run"), + deptypes="all" if context == Context.BUILD else ("build", "test", "link", "run"), ), ) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index e16ab4d7f4c7aa..76eb8d31508a67 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -24,7 +24,6 @@ def setup_parser(subparser): sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="compiler_command") scopes = spack.config.scopes() - scopes_metavar = spack.config.scopes_metavar # Find find_parser = sp.add_parser( @@ -32,11 +31,24 @@ def setup_parser(subparser): aliases=["add"], help="search the system for compilers to add to Spack configuration", ) + mixed_toolchain_group = find_parser.add_mutually_exclusive_group() + mixed_toolchain_group.add_argument( + "--mixed-toolchain", + action="store_true", + default=sys.platform == "darwin", + help="Allow mixed toolchains (for example: clang, clang++, gfortran)", + ) + mixed_toolchain_group.add_argument( + "--no-mixed-toolchain", + action="store_false", + dest="mixed_toolchain", + help="Do not allow mixed toolchains (for example: clang, clang++, gfortran)", + ) find_parser.add_argument("add_paths", nargs=argparse.REMAINDER) find_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_modify_scope("compilers"), help="configuration scope to modify", ) @@ -50,7 +62,7 @@ def setup_parser(subparser): remove_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=None, help="configuration scope to modify", ) @@ -60,7 +72,7 @@ def setup_parser(subparser): list_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_list_scope(), help="configuration scope to read from", ) @@ -71,7 +83,7 @@ def setup_parser(subparser): info_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_list_scope(), help="configuration scope to read from", ) @@ -87,13 +99,15 @@ def compiler_find(args): # Below scope=None because we want new compilers that don't appear # in any other configuration. - new_compilers = spack.compilers.find_new_compilers(paths, scope=None) + new_compilers = spack.compilers.find_new_compilers( + paths, scope=None, mixed_toolchain=args.mixed_toolchain + ) if new_compilers: spack.compilers.add_compilers_to_config(new_compilers, scope=args.scope, init_config=False) n = len(new_compilers) s = "s" if n > 1 else "" - config = spack.config.config + config = spack.config.CONFIG filename = config.get_config_filename(args.scope, "compilers") tty.msg("Added %d new compiler%s to %s" % (n, s, filename)) colify(reversed(sorted(c.spec.display_str for c in new_compilers)), indent=4) @@ -186,7 +200,7 @@ def compiler_list(args): os_str = os if target: os_str += "-%s" % target - cname = "%s{%s} %s" % (spack.spec.compiler_color, name, os_str) + cname = "%s{%s} %s" % (spack.spec.COMPILER_COLOR, name, os_str) tty.hline(colorize(cname), char="-") colify(reversed(sorted(c.spec.display_str for c in compilers))) diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index 81f222aff1f0af..6d0ff43ea2e88d 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -13,12 +13,11 @@ def setup_parser(subparser): scopes = spack.config.scopes() - scopes_metavar = spack.config.scopes_metavar subparser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, help="configuration scope to read/modify", ) diff --git a/lib/spack/spack/cmd/concretize.py b/lib/spack/spack/cmd/concretize.py index 1b3ad66f93626f..47cccef9ba2bbf 100644 --- a/lib/spack/spack/cmd/concretize.py +++ b/lib/spack/spack/cmd/concretize.py @@ -14,18 +14,16 @@ def setup_parser(subparser): subparser.add_argument( - "-f", "--force", action="store_true", help="Re-concretize even if already concretized." + "-f", "--force", action="store_true", help="re-concretize even if already concretized" ) subparser.add_argument( "--test", default=None, choices=["root", "all"], - help="""Concretize with test dependencies. When 'root' is chosen, test -dependencies are only added for the environment's root specs. When 'all' is -chosen, test dependencies are enabled for all packages in the environment.""", + help="concretize with test dependencies of only root packages or all packages", ) subparser.add_argument( - "-q", "--quiet", action="store_true", help="Don't print concretized specs" + "-q", "--quiet", action="store_true", help="don't print concretized specs" ) spack.cmd.common.arguments.add_concretizer_args(subparser) diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 0d60970d3dc4c1..14514400a86ceb 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -27,13 +27,12 @@ def setup_parser(subparser): scopes = spack.config.scopes() - scopes_metavar = spack.config.scopes_metavar # User can only choose one subparser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, help="configuration scope to read/modify", ) @@ -42,10 +41,10 @@ def setup_parser(subparser): get_parser = sp.add_parser("get", help="print configuration values") get_parser.add_argument( "section", - help="configuration section to print. " "options: %(choices)s", + help="configuration section to print\n\noptions: %(choices)s", nargs="?", metavar="section", - choices=spack.config.section_schemas, + choices=spack.config.SECTION_SCHEMAS, ) blame_parser = sp.add_parser( @@ -53,18 +52,18 @@ def setup_parser(subparser): ) blame_parser.add_argument( "section", - help="configuration section to print. " "options: %(choices)s", + help="configuration section to print\n\noptions: %(choices)s", metavar="section", - choices=spack.config.section_schemas, + choices=spack.config.SECTION_SCHEMAS, ) edit_parser = sp.add_parser("edit", help="edit configuration file") edit_parser.add_argument( "section", - help="configuration section to edit. " "options: %(choices)s", + help="configuration section to edit\n\noptions: %(choices)s", metavar="section", nargs="?", - choices=spack.config.section_schemas, + choices=spack.config.SECTION_SCHEMAS, ) edit_parser.add_argument( "--print-file", action="store_true", help="print the file name that would be edited" @@ -76,7 +75,7 @@ def setup_parser(subparser): add_parser.add_argument( "path", nargs="?", - help="colon-separated path to config that should be added," " e.g. 'config:default:true'", + help="colon-separated path to config that should be added, e.g. 'config:default:true'", ) add_parser.add_argument("-f", "--file", help="file from which to set all config values") @@ -88,7 +87,7 @@ def setup_parser(subparser): "--local", action="store_true", default=False, - help="Set packages preferences based on local installs, rather " "than upstream.", + help="set packages preferences based on local installs, rather than upstream", ) remove_parser = sp.add_parser("remove", aliases=["rm"], help="remove configuration parameters") @@ -146,10 +145,10 @@ def config_get(args): scope, section = _get_scope_and_section(args) if section is not None: - spack.config.config.print_section(section) + spack.config.CONFIG.print_section(section) elif scope and scope.startswith("env:"): - config_file = spack.config.config.get_config_filename(scope, section) + config_file = spack.config.CONFIG.get_config_filename(scope, section) if os.path.exists(config_file): with open(config_file) as f: print(f.read()) @@ -157,12 +156,12 @@ def config_get(args): tty.die("environment has no %s file" % ev.manifest_name) else: - tty.die("`spack config get` requires a section argument " "or an active environment.") + tty.die("`spack config get` requires a section argument or an active environment.") def config_blame(args): """Print out line-by-line blame of merged YAML.""" - spack.config.config.print_section(args.section, blame=True) + spack.config.CONFIG.print_section(args.section, blame=True) def config_edit(args): @@ -180,8 +179,8 @@ def config_edit(args): # If we aren't editing a spack.yaml file, get config path from scope. scope, section = _get_scope_and_section(args) if not scope and not section: - tty.die("`spack config edit` requires a section argument " "or an active environment.") - config_file = spack.config.config.get_config_filename(scope, section) + tty.die("`spack config edit` requires a section argument or an active environment.") + config_file = spack.config.CONFIG.get_config_filename(scope, section) if args.print_file: print(config_file) @@ -194,7 +193,7 @@ def config_list(args): Used primarily for shell tab completion scripts. """ - print(" ".join(list(spack.config.section_schemas))) + print(" ".join(list(spack.config.SECTION_SCHEMAS))) def config_add(args): @@ -251,19 +250,19 @@ def _can_update_config_file(scope: spack.config.ConfigScope, cfg_file): def config_update(args): # Read the configuration files - spack.config.config.get_config(args.section, scope=args.scope) + spack.config.CONFIG.get_config(args.section, scope=args.scope) updates: List[spack.config.ConfigScope] = list( filter( lambda s: not isinstance( s, (spack.config.InternalConfigScope, spack.config.ImmutableConfigScope) ), - spack.config.config.format_updates[args.section], + spack.config.CONFIG.format_updates[args.section], ) ) cannot_overwrite, skip_system_scope = [], False for scope in updates: - cfg_file = spack.config.config.get_config_filename(scope.name, args.section) + cfg_file = spack.config.CONFIG.get_config_filename(scope.name, args.section) can_be_updated = _can_update_config_file(scope, cfg_file) if not can_be_updated: if scope.name == "system": @@ -302,7 +301,7 @@ def config_update(args): " the latest schema format:\n\n" ) for scope in updates: - cfg_file = spack.config.config.get_config_filename(scope.name, args.section) + cfg_file = spack.config.CONFIG.get_config_filename(scope.name, args.section) msg += "\t[scope={0}, file={1}]\n".format(scope.name, cfg_file) msg += ( "\nIf the configuration files are updated, versions of Spack " @@ -325,7 +324,7 @@ def config_update(args): # Make a backup copy and rewrite the file bkp_file = cfg_file + ".bkp" shutil.copy(cfg_file, bkp_file) - spack.config.config.update_config(args.section, data, scope=scope.name, force=True) + spack.config.CONFIG.update_config(args.section, data, scope=scope.name, force=True) tty.msg(f'File "{cfg_file}" update [backup={bkp_file}]') @@ -337,13 +336,13 @@ def _can_revert_update(scope_dir, cfg_file, bkp_file): def config_revert(args): - scopes = [args.scope] if args.scope else [x.name for x in spack.config.config.file_scopes] + scopes = [args.scope] if args.scope else [x.name for x in spack.config.CONFIG.file_scopes] # Search for backup files in the configuration scopes Entry = collections.namedtuple("Entry", ["scope", "cfg", "bkp"]) to_be_restored, cannot_overwrite = [], [] for scope in scopes: - cfg_file = spack.config.config.get_config_filename(scope, args.section) + cfg_file = spack.config.CONFIG.get_config_filename(scope, args.section) bkp_file = cfg_file + ".bkp" # If the backup files doesn't exist move to the next scope @@ -374,7 +373,7 @@ def config_revert(args): proceed = True if not args.yes_to_all: - msg = "The following scopes will be restored from the corresponding" " backup files:\n" + msg = "The following scopes will be restored from the corresponding backup files:\n" for entry in to_be_restored: msg += "\t[scope={0.scope}, bkp={0.bkp}]\n".format(entry) msg += "This operation cannot be undone." @@ -399,8 +398,8 @@ def config_prefer_upstream(args): if scope is None: scope = spack.config.default_modify_scope("packages") - all_specs = set(spack.store.db.query(installed=True)) - local_specs = set(spack.store.db.query_local(installed=True)) + all_specs = set(spack.store.STORE.db.query(installed=True)) + local_specs = set(spack.store.STORE.db.query_local(installed=True)) pref_specs = local_specs if args.local else all_specs - local_specs conflicting_variants = set() @@ -408,7 +407,9 @@ def config_prefer_upstream(args): pkgs = {} for spec in pref_specs: # Collect all the upstream compilers and versions for this package. - pkg = pkgs.get(spec.name, {"version": [], "compiler": []}) + pkg = pkgs.get(spec.name, {"version": []}) + all = pkgs.get("all", {"compiler": []}) + pkgs["all"] = all pkgs[spec.name] = pkg # We have no existing variant if this is our first added version. @@ -419,8 +420,8 @@ def config_prefer_upstream(args): pkg["version"].append(version) compiler = str(spec.compiler) - if compiler not in pkg["compiler"]: - pkg["compiler"].append(compiler) + if compiler not in all["compiler"]: + all["compiler"].append(compiler) # Get and list all the variants that differ from the default. variants = [] @@ -457,7 +458,7 @@ def config_prefer_upstream(args): existing = spack.config.get("packages", scope=scope) new = spack.config.merge_yaml(existing, pkgs) spack.config.set("packages", new, scope) - config_file = spack.config.config.get_config_filename(scope, section) + config_file = spack.config.CONFIG.get_config_filename(scope, section) tty.msg("Updated config at {0}".format(config_file)) diff --git a/lib/spack/spack/cmd/containerize.py b/lib/spack/spack/cmd/containerize.py index 8445e4cb6a9fb5..b34b7783b68829 100644 --- a/lib/spack/spack/cmd/containerize.py +++ b/lib/spack/spack/cmd/containerize.py @@ -10,7 +10,7 @@ import spack.container import spack.container.images -description = "creates recipes to build images for different" " container runtimes" +description = "creates recipes to build images for different container runtimes" section = "container" level = "long" diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 2012257239723c..946e9bc8b960d3 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -5,6 +5,7 @@ import os import re +import sys import urllib.parse import llnl.util.tty as tty @@ -17,6 +18,7 @@ from spack.url import UndetectableNameError, UndetectableVersionError, parse_name, parse_version from spack.util.editor import editor from spack.util.executable import ProcessError, which +from spack.util.format import get_version_lines from spack.util.naming import mod_to_class, simplify_name, valid_fully_qualified_module_name description = "create a new package file" @@ -61,6 +63,10 @@ class {class_name}({base_class_name}): # notify when the package is updated. # maintainers("github_user1", "github_user2") + # FIXME: Add the SPDX identifier of the project's license below. + # See https://spdx.org/licenses/ for a list. + license("UNKNOWN") + {versions} {dependencies} @@ -325,6 +331,7 @@ class PythonPackageTemplate(PackageTemplate): # FIXME: Add a build backend, usually defined in pyproject.toml. If no such file # exists, use setuptools. # depends_on("py-setuptools", type="build") + # depends_on("py-hatchling", type="build") # depends_on("py-flit-core", type="build") # depends_on("py-poetry-core", type="build") @@ -332,17 +339,11 @@ class PythonPackageTemplate(PackageTemplate): # depends_on("py-foo", type=("build", "run"))""" body_def = """\ - def global_options(self, spec, prefix): - # FIXME: Add options to pass to setup.py - # FIXME: If not needed, delete this function - options = [] - return options - - def install_options(self, spec, prefix): - # FIXME: Add options to pass to setup.py install + def config_settings(self, spec, prefix): + # FIXME: Add configuration settings to be passed to the build backend # FIXME: If not needed, delete this function - options = [] - return options""" + settings = {} + return settings""" def __init__(self, name, url, *args, **kwargs): # If the user provided `--name py-numpy`, don't rename it py-py-numpy @@ -612,7 +613,7 @@ def setup_parser(subparser): "--template", metavar="TEMPLATE", choices=sorted(templates.keys()), - help="build system template to use. options: %(choices)s", + help="build system template to use\n\noptions: %(choices)s", ) subparser.add_argument( "-r", "--repo", help="path to a repository where the package should be created" @@ -620,7 +621,7 @@ def setup_parser(subparser): subparser.add_argument( "-N", "--namespace", - help="specify a namespace for the package. must be the namespace of " + help="specify a namespace for the package\n\nmust be the namespace of " "a repository registered with Spack", ) subparser.add_argument( @@ -826,7 +827,12 @@ def get_versions(args, name): if args.url is not None and args.template != "bundle" and valid_url: # Find available versions try: - url_dict = spack.util.web.find_versions_of_archive(args.url) + url_dict = spack.url.find_versions_of_archive(args.url) + if len(url_dict) > 1 and not args.batch and sys.stdin.isatty(): + url_dict_filtered = spack.stage.interactive_version_filter(url_dict) + if url_dict_filtered is None: + exit(0) + url_dict = url_dict_filtered except UndetectableVersionError: # Use fake versions tty.warn("Couldn't detect version in: {0}".format(args.url)) @@ -837,13 +843,11 @@ def get_versions(args, name): version = parse_version(args.url) url_dict = {version: args.url} - versions = spack.stage.get_checksums_for_versions( - url_dict, - name, - first_stage_function=guesser, - keep_stage=args.keep_stage, - batch=(args.batch or len(url_dict) == 1), + version_hashes = spack.stage.get_checksums_for_versions( + url_dict, name, first_stage_function=guesser, keep_stage=args.keep_stage ) + + versions = get_version_lines(version_hashes, url_dict) else: versions = unhashed_versions @@ -878,7 +882,7 @@ def get_build_system(template, url, guesser): # Use whatever build system the guesser detected selected_template = guesser.build_system if selected_template == "generic": - tty.warn("Unable to detect a build system. " "Using a generic package template.") + tty.warn("Unable to detect a build system. Using a generic package template.") else: msg = "This package looks like it uses the {0} build system" tty.msg(msg.format(selected_template)) @@ -917,11 +921,11 @@ def get_repository(args, name): ) else: if spec.namespace: - repo = spack.repo.path.get_repo(spec.namespace, None) + repo = spack.repo.PATH.get_repo(spec.namespace, None) if not repo: tty.die("Unknown namespace: '{0}'".format(spec.namespace)) else: - repo = spack.repo.path.first_repo() + repo = spack.repo.PATH.first_repo() # Set the namespace on the spec if it's not there already if not spec.namespace: diff --git a/lib/spack/spack/cmd/debug.py b/lib/spack/spack/cmd/debug.py index b3224f0b1b9f5b..7f729ad83eb114 100644 --- a/lib/spack/spack/cmd/debug.py +++ b/lib/spack/spack/cmd/debug.py @@ -60,16 +60,16 @@ def create_db_tarball(args): tarball_name = "spack-db.%s.tar.gz" % _debug_tarball_suffix() tarball_path = os.path.abspath(tarball_name) - base = os.path.basename(str(spack.store.root)) + base = os.path.basename(str(spack.store.STORE.root)) transform_args = [] if "GNU" in tar("--version", output=str): transform_args = ["--transform", "s/^%s/%s/" % (base, tarball_name)] else: transform_args = ["-s", "/^%s/%s/" % (base, tarball_name)] - wd = os.path.dirname(str(spack.store.root)) + wd = os.path.dirname(str(spack.store.STORE.root)) with working_dir(wd): - files = [spack.store.db._index_path] + files = [spack.store.STORE.db._index_path] files += glob("%s/*/*/*/.spack/spec.json" % base) files += glob("%s/*/*/*/.spack/spec.yaml" % base) files = [os.path.relpath(f) for f in files] diff --git a/lib/spack/spack/cmd/deconcretize.py b/lib/spack/spack/cmd/deconcretize.py new file mode 100644 index 00000000000000..dbcf72ea8b3a29 --- /dev/null +++ b/lib/spack/spack/cmd/deconcretize.py @@ -0,0 +1,103 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import argparse +import sys +from typing import List + +import llnl.util.tty as tty + +import spack.cmd +import spack.cmd.common.arguments as arguments +import spack.cmd.common.confirmation as confirmation +import spack.environment as ev +import spack.spec + +description = "remove specs from the concretized lockfile of an environment" +section = "environments" +level = "long" + +# Arguments for display_specs when we find ambiguity +display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4} + + +def setup_parser(subparser): + subparser.add_argument( + "--root", action="store_true", help="deconcretize only specific environment roots" + ) + arguments.add_common_arguments(subparser, ["yes_to_all", "specs"]) + subparser.add_argument( + "-a", + "--all", + action="store_true", + dest="all", + help="deconcretize ALL specs that match each supplied spec", + ) + + +def get_deconcretize_list( + args: argparse.Namespace, specs: List[spack.spec.Spec], env: ev.Environment +) -> List[spack.spec.Spec]: + """ + Get list of environment roots to deconcretize + """ + env_specs = [s for _, s in env.concretized_specs()] + to_deconcretize = [] + errors = [] + + for s in specs: + if args.root: + # find all roots matching given spec + to_deconc = [e for e in env_specs if e.satisfies(s)] + else: + # find all roots matching or depending on a matching spec + to_deconc = [e for e in env_specs if any(d.satisfies(s) for d in e.traverse())] + + if len(to_deconc) < 1: + tty.warn(f"No matching specs to deconcretize for {s}") + + elif len(to_deconc) > 1 and not args.all: + errors.append((s, to_deconc)) + + to_deconcretize.extend(to_deconc) + + if errors: + for spec, matching in errors: + tty.error(f"{spec} matches multiple concrete specs:") + sys.stderr.write("\n") + spack.cmd.display_specs(matching, output=sys.stderr, **display_args) + sys.stderr.write("\n") + sys.stderr.flush() + tty.die("Use '--all' to deconcretize all matching specs, or be more specific") + + return to_deconcretize + + +def deconcretize_specs(args, specs): + env = spack.cmd.require_active_env(cmd_name="deconcretize") + + if args.specs: + deconcretize_list = get_deconcretize_list(args, specs, env) + else: + deconcretize_list = [s for _, s in env.concretized_specs()] + + if not args.yes_to_all: + confirmation.confirm_action(deconcretize_list, "deconcretized", "deconcretization") + + with env.write_transaction(): + for spec in deconcretize_list: + env.deconcretize(spec) + env.write() + + +def deconcretize(parser, args): + if not args.specs and not args.all: + tty.die( + "deconcretize requires at least one spec argument.", + " Use `spack deconcretize --all` to deconcretize ALL specs.", + ) + + specs = spack.cmd.parse_specs(args.specs) if args.specs else [any] + deconcretize_specs(args, specs) diff --git a/lib/spack/spack/cmd/dependencies.py b/lib/spack/spack/cmd/dependencies.py index 355bb2918cae83..ed85d47d2266d0 100644 --- a/lib/spack/spack/cmd/dependencies.py +++ b/lib/spack/spack/cmd/dependencies.py @@ -26,8 +26,8 @@ def setup_parser(subparser): "--installed", action="store_true", default=False, - help="List installed dependencies of an installed spec, " - "instead of possible dependencies of a package.", + help="list installed dependencies of an installed spec " + "instead of possible dependencies of a package", ) subparser.add_argument( "-t", @@ -60,7 +60,7 @@ def dependencies(parser, args): format_string = "{name}{@version}{%compiler}{/hash:7}" if sys.stdout.isatty(): tty.msg("Dependencies of %s" % spec.format(format_string, color=True)) - deps = spack.store.db.installed_relatives( + deps = spack.store.STORE.db.installed_relatives( spec, "children", args.transitive, deptype=args.deptype ) if deps: @@ -74,7 +74,7 @@ def dependencies(parser, args): spec, transitive=args.transitive, expand_virtuals=args.expand_virtuals, - deptype=args.deptype, + depflag=args.deptype, ) if spec.name in dependencies: diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py index fa951be903a13a..733feaf71559b9 100644 --- a/lib/spack/spack/cmd/dependents.py +++ b/lib/spack/spack/cmd/dependents.py @@ -25,15 +25,15 @@ def setup_parser(subparser): "--installed", action="store_true", default=False, - help="List installed dependents of an installed spec, " - "instead of possible dependents of a package.", + help="list installed dependents of an installed spec " + "instead of possible dependents of a package", ) subparser.add_argument( "-t", "--transitive", action="store_true", default=False, - help="Show all transitive dependents.", + help="show all transitive dependents", ) arguments.add_common_arguments(subparser, ["spec"]) @@ -47,14 +47,14 @@ def inverted_dependencies(): actual dependents. """ dag = {} - for pkg_cls in spack.repo.path.all_package_classes(): + for pkg_cls in spack.repo.PATH.all_package_classes(): dag.setdefault(pkg_cls.name, set()) for dep in pkg_cls.dependencies: deps = [dep] # expand virtuals if necessary - if spack.repo.path.is_virtual(dep): - deps += [s.name for s in spack.repo.path.providers_for(dep)] + if spack.repo.PATH.is_virtual(dep): + deps += [s.name for s in spack.repo.PATH.providers_for(dep)] for d in deps: dag.setdefault(d, set()).add(pkg_cls.name) @@ -96,7 +96,7 @@ def dependents(parser, args): format_string = "{name}{@version}{%compiler}{/hash:7}" if sys.stdout.isatty(): tty.msg("Dependents of %s" % spec.cformat(format_string)) - deps = spack.store.db.installed_relatives(spec, "parents", args.transitive) + deps = spack.store.STORE.db.installed_relatives(spec, "parents", args.transitive) if deps: spack.cmd.display_specs(deps, long=True) else: diff --git a/lib/spack/spack/cmd/deprecate.py b/lib/spack/spack/cmd/deprecate.py index faecd910e74d0d..dca9b49756b501 100644 --- a/lib/spack/spack/cmd/deprecate.py +++ b/lib/spack/spack/cmd/deprecate.py @@ -26,7 +26,7 @@ from spack.database import InstallStatuses from spack.error import SpackError -description = "Replace one package with another via symlinks" +description = "replace one package with another via symlinks" section = "admin" level = "long" @@ -46,7 +46,7 @@ def setup_parser(sp): action="store_true", default=True, dest="dependencies", - help="Deprecate dependencies (default)", + help="deprecate dependencies (default)", ) deps.add_argument( "-D", @@ -54,7 +54,7 @@ def setup_parser(sp): action="store_false", default=True, dest="dependencies", - help="Do not deprecate dependencies", + help="do not deprecate dependencies", ) install = sp.add_mutually_exclusive_group() @@ -64,7 +64,7 @@ def setup_parser(sp): action="store_true", default=False, dest="install", - help="Concretize and install deprecator spec", + help="concretize and install deprecator spec", ) install.add_argument( "-I", @@ -72,7 +72,7 @@ def setup_parser(sp): action="store_false", default=False, dest="install", - help="Deprecator spec must already be installed (default)", + help="deprecator spec must already be installed (default)", ) sp.add_argument( @@ -81,7 +81,7 @@ def setup_parser(sp): type=str, default="soft", choices=["soft", "hard"], - help="Type of filesystem link to use for deprecation (default soft)", + help="type of filesystem link to use for deprecation (default soft)", ) sp.add_argument( @@ -130,7 +130,7 @@ def deprecate(parser, args): already_deprecated = [] already_deprecated_for = [] for spec in all_deprecate: - deprecated_for = spack.store.db.deprecator(spec) + deprecated_for = spack.store.STORE.db.deprecator(spec) if deprecated_for: already_deprecated.append(spec) already_deprecated_for.append(deprecated_for) diff --git a/lib/spack/spack/cmd/dev_build.py b/lib/spack/spack/cmd/dev_build.py index c837b58c626cfa..90008c8b3ef601 100644 --- a/lib/spack/spack/cmd/dev_build.py +++ b/lib/spack/spack/cmd/dev_build.py @@ -25,14 +25,14 @@ def setup_parser(subparser): "--source-path", dest="source_path", default=None, - help="path to source directory. defaults to the current directory", + help="path to source directory (defaults to the current directory)", ) subparser.add_argument( "-i", "--ignore-dependencies", action="store_true", dest="ignore_deps", - help="don't try to install dependencies of requested packages", + help="do not try to install dependencies of requested packages", ) arguments.add_common_arguments(subparser, ["no_checksum", "deprecated"]) subparser.add_argument( @@ -55,16 +55,13 @@ def setup_parser(subparser): type=str, dest="shell", default=None, - help="drop into a build environment in a new shell, e.g. bash, zsh", + help="drop into a build environment in a new shell, e.g., bash", ) subparser.add_argument( "--test", default=None, choices=["root", "all"], - help="""If 'root' is chosen, run package tests during -installation for top-level packages (but skip tests for dependencies). -if 'all' is chosen, run package tests during installation for all -packages. If neither are chosen, don't run tests for any packages.""", + help="run tests on only root packages or all packages", ) arguments.add_common_arguments(subparser, ["spec"]) @@ -101,11 +98,8 @@ def dev_build(self, args): tty.die("spack dev-build only takes one spec.") spec = specs[0] - if not spack.repo.path.exists(spec.name): - tty.die( - "No package for '{0}' was found.".format(spec.name), - " Use `spack create` to create a new package", - ) + if not spack.repo.PATH.exists(spec.name): + raise spack.repo.UnknownPackageError(spec.name) if not spec.versions.concrete_range_as_version: tty.die( diff --git a/lib/spack/spack/cmd/develop.py b/lib/spack/spack/cmd/develop.py index 4746f9c96e3766..ac7e89596fa851 100644 --- a/lib/spack/spack/cmd/develop.py +++ b/lib/spack/spack/cmd/develop.py @@ -20,7 +20,7 @@ def setup_parser(subparser): - subparser.add_argument("-p", "--path", help="Source location of package") + subparser.add_argument("-p", "--path", help="source location of package") clone_group = subparser.add_mutually_exclusive_group() clone_group.add_argument( @@ -28,18 +28,18 @@ def setup_parser(subparser): action="store_false", dest="clone", default=None, - help="Do not clone. The package already exists at the source path", + help="do not clone, the package already exists at the source path", ) clone_group.add_argument( "--clone", action="store_true", dest="clone", default=None, - help="Clone the package even if the path already exists", + help="clone the package even if the path already exists", ) subparser.add_argument( - "-f", "--force", help="Remove any files or directories that block cloning source code" + "-f", "--force", help="remove any files or directories that block cloning source code" ) arguments.add_common_arguments(subparser, ["spec"]) @@ -66,8 +66,7 @@ def develop(parser, args): # Both old syntax `spack develop pkg@x` and new syntax `spack develop pkg@=x` # are currently supported. spec = spack.spec.parse_with_version_concrete(entry["spec"]) - pkg_cls = spack.repo.path.get_pkg_class(spec.name) - pkg_cls(spec).stage.steal_source(abspath) + env.develop(spec=spec, path=path, clone=True) if not env.dev_specs: tty.warn("No develop specs to download") diff --git a/lib/spack/spack/cmd/diff.py b/lib/spack/spack/cmd/diff.py index 13c27aec2c592b..c654f6a5b8c6db 100644 --- a/lib/spack/spack/cmd/diff.py +++ b/lib/spack/spack/cmd/diff.py @@ -29,7 +29,7 @@ def setup_parser(subparser): action="store_true", default=False, dest="dump_json", - help="Dump json output instead of pretty printing.", + help="dump json output instead of pretty printing", ) subparser.add_argument( "--first", diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index 621bd114930d65..79f441a67adf57 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -31,9 +31,9 @@ def edit_package(name, repo_path, namespace): if repo_path: repo = spack.repo.Repo(repo_path) elif namespace: - repo = spack.repo.path.get_repo(namespace) + repo = spack.repo.PATH.get_repo(namespace) else: - repo = spack.repo.path + repo = spack.repo.PATH path = repo.filename_for_package_name(name) spec = Spec(name) @@ -43,10 +43,7 @@ def edit_package(name, repo_path, namespace): if not os.access(path, os.R_OK): tty.die("Insufficient permissions on '%s'!" % path) else: - tty.die( - "No package for '{0}' was found.".format(spec.name), - " Use `spack create` to create a new package", - ) + raise spack.repo.UnknownPackageError(spec.name) editor(path) @@ -62,7 +59,7 @@ def setup_parser(subparser): dest="path", action="store_const", const=spack.paths.build_systems_path, - help="Edit the build system with the supplied name.", + help="edit the build system with the supplied name", ) excl_args.add_argument( "-c", diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index afcf61ab08d017..bb1ad13ec2b985 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -5,10 +5,13 @@ import argparse import os +import shlex import shutil import sys import tempfile +from typing import Optional +import llnl.string as string import llnl.util.filesystem as fs import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -28,7 +31,6 @@ import spack.schema.env import spack.spec import spack.tengine -import spack.util.string as string from spack.util.environment import EnvironmentModifications description = "manage virtual environments" @@ -96,22 +98,16 @@ def env_activate_setup_parser(subparser): view_options = subparser.add_mutually_exclusive_group() view_options.add_argument( - "-v", "--with-view", - action="store_const", - dest="with_view", - const=True, - default=True, - help="update PATH etc. with associated view", + "-v", + metavar="name", + help="set runtime environment variables for specific view", ) view_options.add_argument( - "-V", "--without-view", - action="store_const", - dest="with_view", - const=False, - default=True, - help="do not update PATH etc. with associated view", + "-V", + action="store_true", + help="do not set runtime environment variables for any view", ) subparser.add_argument( @@ -149,10 +145,13 @@ def create_temp_env_directory(): return tempfile.mkdtemp(prefix="spack-") -def env_activate(args): - if not args.activate_env and not args.dir and not args.temp: - tty.die("spack env activate requires an environment name, directory, or --temp") +def _tty_info(msg): + """tty.info like function that prints the equivalent printf statement for eval.""" + decorated = f'{colorize("@*b{==>}")} {msg}\n' + print(f"printf {shlex.quote(decorated)};") + +def env_activate(args): if not args.shell: spack.cmd.common.shell_init_instructions( "spack env activate", " eval `spack env activate {sh_arg} [...]`" @@ -161,16 +160,29 @@ def env_activate(args): # Error out when -e, -E, -D flags are given, cause they are ambiguous. if args.env or args.no_env or args.env_dir: - tty.die("Calling spack env activate with --env, --env-dir and --no-env " "is ambiguous") + tty.die("Calling spack env activate with --env, --env-dir and --no-env is ambiguous") env_name_or_dir = args.activate_env or args.dir + # When executing `spack env activate` without further arguments, activate + # the default environment. It's created when it doesn't exist yet. + if not env_name_or_dir and not args.temp: + short_name = "default" + if not ev.exists(short_name): + ev.create(short_name) + action = "Created and activated" + else: + action = "Activated" + env_path = ev.root(short_name) + _tty_info(f"{action} default environment in {env_path}") + # Temporary environment - if args.temp: + elif args.temp: env = create_temp_env_directory() env_path = os.path.abspath(env) short_name = os.path.basename(env_path) ev.create_in_dir(env).write(regenerate=False) + _tty_info(f"Created and activated temporary environment in {env_path}") # Managed environment elif ev.exists(env_name_or_dir) and not args.dir: @@ -197,10 +209,20 @@ def env_activate(args): # Activate new environment active_env = ev.Environment(env_path) + + # Check if runtime environment variables are requested, and if so, for what view. + view: Optional[str] = None + if args.with_view: + view = args.with_view + if not active_env.has_view(view): + tty.die(f"The environment does not have a view named '{view}'") + elif not args.without_view and active_env.has_view(ev.default_view_name): + view = ev.default_view_name + cmds += spack.environment.shell.activate_header( - env=active_env, shell=args.shell, prompt=env_prompt if args.prompt else None + env=active_env, shell=args.shell, prompt=env_prompt if args.prompt else None, view=view ) - env_mods.extend(spack.environment.shell.activate(env=active_env, add_view=args.with_view)) + env_mods.extend(spack.environment.shell.activate(env=active_env, view=view)) cmds += env_mods.shell_modifications(args.shell) sys.stdout.write(cmds) @@ -239,6 +261,13 @@ def env_deactivate_setup_parser(subparser): const="bat", help="print bat commands to activate the environment", ) + shells.add_argument( + "--pwsh", + action="store_const", + dest="shell", + const="pwsh", + help="print pwsh commands to activate the environment", + ) def env_deactivate(args): @@ -250,7 +279,7 @@ def env_deactivate(args): # Error out when -e, -E, -D flags are given, cause they are ambiguous. if args.env or args.no_env or args.env_dir: - tty.die("Calling spack env deactivate with --env, --env-dir and --no-env " "is ambiguous") + tty.die("Calling spack env deactivate with --env, --env-dir and --no-env is ambiguous") if ev.active_environment() is None: tty.die("No environment is currently active.") @@ -290,7 +319,7 @@ def env_create_setup_parser(subparser): "envfile", nargs="?", default=None, - help="either a lockfile (must end with '.json' or '.lock') or a manifest file.", + help="either a lockfile (must end with '.json' or '.lock') or a manifest file", ) @@ -368,28 +397,33 @@ def env_remove(args): and manifests embedded in repositories should be removed manually. """ read_envs = [] + bad_envs = [] for env_name in args.rm_env: - env = ev.read(env_name) - read_envs.append(env) + try: + env = ev.read(env_name) + read_envs.append(env) + except (spack.config.ConfigFormatError, ev.SpackEnvironmentConfigError): + bad_envs.append(env_name) if not args.yes_to_all: - answer = tty.get_yes_or_no( - "Really remove %s %s?" - % ( - string.plural(len(args.rm_env), "environment", show_n=False), - string.comma_and(args.rm_env), - ), - default=False, - ) + environments = string.plural(len(args.rm_env), "environment", show_n=False) + envs = string.comma_and(args.rm_env) + answer = tty.get_yes_or_no(f"Really remove {environments} {envs}?", default=False) if not answer: tty.die("Will not remove any environments") for env in read_envs: + name = env.name if env.active: - tty.die("Environment %s can't be removed while activated." % env.name) - + tty.die(f"Environment {name} can't be removed while activated.") env.destroy() - tty.msg("Successfully removed environment '%s'" % env.name) + tty.msg(f"Successfully removed environment '{name}'") + + for bad_env_name in bad_envs: + shutil.rmtree( + spack.environment.environment.environment_dir_from_name(bad_env_name, exists_ok=True) + ) + tty.msg(f"Successfully removed environment '{bad_env_name}'") # @@ -536,8 +570,8 @@ def env_update_setup_parser(subparser): def env_update(args): manifest_file = ev.manifest_file(args.update_env) backup_file = manifest_file + ".bkp" - needs_update = not ev.is_latest_format(manifest_file) + needs_update = not ev.is_latest_format(manifest_file) if not needs_update: tty.msg('No update needed for the environment "{0}"'.format(args.update_env)) return @@ -608,16 +642,16 @@ def env_depfile_setup_parser(subparser): "--make-target-prefix", default=None, metavar="TARGET", - help="prefix Makefile targets (and variables) with /. By default " + help="prefix Makefile targets (and variables) with /\n\nby default " "the absolute path to the directory makedeps under the environment metadata dir is " - "used. Can be set to an empty string --make-prefix ''.", + "used. can be set to an empty string --make-prefix ''", ) subparser.add_argument( "--make-disable-jobserver", default=True, action="store_false", dest="jobserver", - help="disable POSIX jobserver support.", + help="disable POSIX jobserver support", ) subparser.add_argument( "--use-buildcache", @@ -625,8 +659,8 @@ def env_depfile_setup_parser(subparser): type=arguments.use_buildcache, default="package:auto,dependencies:auto", metavar="[{auto,only,never},][package:{auto,only,never},][dependencies:{auto,only,never}]", - help="When using `only`, redundant build dependencies are pruned from the DAG. " - "This flag is passed on to the generated spack install commands.", + help="when using `only`, redundant build dependencies are pruned from the DAG\n\n" + "this flag is passed on to the generated spack install commands", ) subparser.add_argument( "-o", @@ -640,7 +674,7 @@ def env_depfile_setup_parser(subparser): "--generator", default="make", choices=("make",), - help="specify the depfile type. Currently only make is supported.", + help="specify the depfile type\n\ncurrently only make is supported", ) subparser.add_argument( metavar="specs", @@ -655,18 +689,31 @@ def env_depfile(args): # Currently only make is supported. spack.cmd.require_active_env(cmd_name="env depfile") + env = ev.active_environment() + # What things do we build when running make? By default, we build the # root specs. If specific specs are provided as input, we build those. filter_specs = spack.cmd.parse_specs(args.specs) if args.specs else None template = spack.tengine.make_environment().get_template(os.path.join("depfile", "Makefile")) model = depfile.MakefileModel.from_env( - ev.active_environment(), + env, filter_specs=filter_specs, pkg_buildcache=depfile.UseBuildCache.from_string(args.use_buildcache[0]), dep_buildcache=depfile.UseBuildCache.from_string(args.use_buildcache[1]), make_prefix=args.make_prefix, jobserver=args.jobserver, ) + + # Warn in case we're generating a depfile for an empty environment. We don't automatically + # concretize; the user should do that explicitly. Could be changed in the future if requested. + if model.empty: + if not env.user_specs: + tty.warn("no specs in the environment") + elif filter_specs is not None: + tty.warn("no concrete matching specs found in environment") + else: + tty.warn("environment is not concretized. Run `spack concretize` first") + makefile = template.render(model.to_dict()) # Finally write to stdout/file. diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 05a5fe48ac1d34..1bf5c9ecffc4d9 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -22,7 +22,7 @@ def setup_parser(subparser): subparser.epilog = ( - "If called without argument returns " "the list of all valid extendable packages" + "If called without argument returns the list of all valid extendable packages" ) arguments.add_common_arguments(subparser, ["long", "very_long"]) subparser.add_argument( @@ -58,7 +58,7 @@ def extensions(parser, args): extendable_pkgs = [] for name in spack.repo.all_package_names(): - pkg_cls = spack.repo.path.get_pkg_class(name) + pkg_cls = spack.repo.PATH.get_pkg_class(name) if pkg_cls.extendable: extendable_pkgs.append(name) @@ -81,7 +81,7 @@ def extensions(parser, args): if args.show in ("packages", "all"): # List package names of extensions - extensions = spack.repo.path.extensions_for(spec) + extensions = spack.repo.PATH.extensions_for(spec) if not extensions: tty.msg("%s has no extensions." % spec.cshort_spec) else: @@ -91,7 +91,7 @@ def extensions(parser, args): if args.show in ("installed", "all"): # List specs of installed extensions. - installed = [s.spec for s in spack.store.db.installed_extensions_for(spec)] + installed = [s.spec for s in spack.store.STORE.db.installed_extensions_for(spec)] if args.show == "all": print diff --git a/lib/spack/spack/cmd/external.py b/lib/spack/spack/cmd/external.py index 00e52b3b6e1a30..081ec8039438bd 100644 --- a/lib/spack/spack/cmd/external.py +++ b/lib/spack/spack/cmd/external.py @@ -5,7 +5,9 @@ import argparse import errno import os +import re import sys +from typing import List, Optional import llnl.util.tty as tty import llnl.util.tty.colify as colify @@ -13,6 +15,7 @@ import spack import spack.cmd import spack.cmd.common.arguments +import spack.config import spack.cray_manifest as cray_manifest import spack.detection import spack.error @@ -27,7 +30,6 @@ def setup_parser(subparser): sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="external_command") scopes = spack.config.scopes() - scopes_metavar = spack.config.scopes_metavar find_parser = sp.add_parser("find", help="add external packages to packages.yaml") find_parser.add_argument( @@ -42,19 +44,19 @@ def setup_parser(subparser): "--path", default=None, action="append", - help="Alternative search paths for finding externals. May be repeated", + help="one or more alternative search paths for finding externals", ) find_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_modify_scope("packages"), help="configuration scope to modify", ) find_parser.add_argument( "--all", action="store_true", help="search for all packages that Spack knows about" ) - spack.cmd.common.arguments.add_common_arguments(find_parser, ["tags"]) + spack.cmd.common.arguments.add_common_arguments(find_parser, ["tags", "jobs"]) find_parser.add_argument("packages", nargs=argparse.REMAINDER) find_parser.epilog = ( 'The search is by default on packages tagged with the "build-tools" or ' @@ -66,10 +68,8 @@ def setup_parser(subparser): read_cray_manifest = sp.add_parser( "read-cray-manifest", - help=( - "consume a Spack-compatible description of externally-installed " - "packages, including dependency relationships" - ), + help="consume a Spack-compatible description of externally-installed packages, including " + "dependency relationships", ) read_cray_manifest.add_argument( "--file", default=None, help="specify a location other than the default" @@ -92,7 +92,7 @@ def setup_parser(subparser): read_cray_manifest.add_argument( "--fail-on-error", action="store_true", - help=("if a manifest file cannot be parsed, fail and report the " "full stack trace"), + help="if a manifest file cannot be parsed, fail and report the full stack trace", ) @@ -111,70 +111,69 @@ def external_find(args): # For most exceptions, just print a warning and continue. # Note that KeyboardInterrupt does not subclass Exception # (so CTRL-C will terminate the program as expected). - skip_msg = "Skipping manifest and continuing with other external " "checks" + skip_msg = "Skipping manifest and continuing with other external checks" if (isinstance(e, IOError) or isinstance(e, OSError)) and e.errno in [ errno.EPERM, errno.EACCES, ]: # The manifest file does not have sufficient permissions enabled: # print a warning and keep going - tty.warn("Unable to read manifest due to insufficient " "permissions.", skip_msg) + tty.warn("Unable to read manifest due to insufficient permissions.", skip_msg) else: tty.warn("Unable to read manifest, unexpected error: {0}".format(str(e)), skip_msg) - # If the user didn't specify anything, search for build tools by default - if not args.tags and not args.all and not args.packages: - args.tags = ["core-packages", "build-tools"] + # Outside the Cray manifest, the search is done by tag for performance reasons, + # since tags are cached. # If the user specified both --all and --tag, then --all has precedence - if args.all and args.tags: - args.tags = [] - - # Construct the list of possible packages to be detected - pkg_cls_to_check = [] - - # Add the packages that have been required explicitly - if args.packages: - pkg_cls_to_check = [spack.repo.path.get_pkg_class(pkg) for pkg in args.packages] - if args.tags: - allowed = set(spack.repo.path.packages_with_tags(*args.tags)) - pkg_cls_to_check = [x for x in pkg_cls_to_check if x.name in allowed] - - if args.tags and not pkg_cls_to_check: - # If we arrived here we didn't have any explicit package passed - # as argument, which means to search all packages. - # Since tags are cached it's much faster to construct what we need - # to search directly, rather than filtering after the fact - pkg_cls_to_check = [ - spack.repo.path.get_pkg_class(pkg_name) - for tag in args.tags - for pkg_name in spack.repo.path.packages_with_tags(tag) - ] - pkg_cls_to_check = list(set(pkg_cls_to_check)) - - # If the list of packages is empty, search for every possible package - if not args.tags and not pkg_cls_to_check: - pkg_cls_to_check = list(spack.repo.path.all_package_classes()) - - # If the user specified any packages to exclude from external find, add them here - if args.exclude: - pkg_cls_to_check = [pkg for pkg in pkg_cls_to_check if pkg.name not in args.exclude] - - detected_packages = spack.detection.by_executable(pkg_cls_to_check, path_hints=args.path) - detected_packages.update(spack.detection.by_library(pkg_cls_to_check, path_hints=args.path)) + if args.all or args.packages: + # Each detectable package has at least the detectable tag + args.tags = ["detectable"] + elif not args.tags: + # If the user didn't specify anything, search for build tools by default + args.tags = ["core-packages", "build-tools"] + + candidate_packages = packages_to_search_for( + names=args.packages, tags=args.tags, exclude=args.exclude + ) + detected_packages = spack.detection.by_path( + candidate_packages, path_hints=args.path, max_workers=args.jobs + ) new_entries = spack.detection.update_configuration( detected_packages, scope=args.scope, buildable=not args.not_buildable ) if new_entries: - path = spack.config.config.get_config_filename(args.scope, "packages") - msg = "The following specs have been detected on this system " "and added to {0}" + path = spack.config.CONFIG.get_config_filename(args.scope, "packages") + msg = "The following specs have been detected on this system and added to {0}" tty.msg(msg.format(path)) spack.cmd.display_specs(new_entries) else: tty.msg("No new external packages detected") +def packages_to_search_for( + *, names: Optional[List[str]], tags: List[str], exclude: Optional[List[str]] +): + result = [] + for current_tag in tags: + result.extend(spack.repo.PATH.packages_with_tags(current_tag, full=True)) + + if names: + # Match both fully qualified and unqualified + parts = [rf"(^{x}$|[.]{x}$)" for x in names] + select_re = re.compile("|".join(parts)) + result = [x for x in result if select_re.search(x)] + + if exclude: + # Match both fully qualified and unqualified + parts = [rf"(^{x}$|[.]{x}$)" for x in exclude] + select_re = re.compile("|".join(parts)) + result = [x for x in result if not select_re.search(x)] + + return result + + def external_read_cray_manifest(args): _collect_and_consume_cray_manifest_files( manifest_file=args.file, @@ -236,12 +235,12 @@ def _collect_and_consume_cray_manifest_files( if fail_on_error: raise else: - tty.warn("Failure reading manifest file: {0}" "\n\t{1}".format(path, str(e))) + tty.warn("Failure reading manifest file: {0}\n\t{1}".format(path, str(e))) def external_list(args): # Trigger a read of all packages, might take a long time. - list(spack.repo.path.all_package_classes()) + list(spack.repo.PATH.all_package_classes()) # Print all the detectable packages tty.msg("Detectable packages per repository") for namespace, pkgs in sorted(spack.package_base.detectable_packages.items()): diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 663c57c4cba755..227f7e89732a27 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -10,6 +10,7 @@ import spack.config import spack.environment as ev import spack.repo +import spack.traverse description = "fetch archives for packages" section = "build" @@ -36,6 +37,12 @@ def setup_parser(subparser): def fetch(parser, args): + if args.no_checksum: + spack.config.set("config:checksum", False, scope="command_line") + + if args.deprecated: + spack.config.set("config:deprecated", True, scope="command_line") + if args.specs: specs = spack.cmd.parse_specs(args.specs, concretize=True) else: @@ -51,24 +58,21 @@ def fetch(parser, args): else: specs = env.all_specs() if specs == []: - tty.die( - "No uninstalled specs in environment. Did you " "run `spack concretize` yet?" - ) + tty.die("No uninstalled specs in environment. Did you run `spack concretize` yet?") else: tty.die("fetch requires at least one spec argument") - if args.no_checksum: - spack.config.set("config:checksum", False, scope="command_line") + if args.dependencies or args.missing: + to_be_fetched = spack.traverse.traverse_nodes(specs, key=spack.traverse.by_dag_hash) + else: + to_be_fetched = specs - if args.deprecated: - spack.config.set("config:deprecated", True, scope="command_line") + for spec in to_be_fetched: + if args.missing and spec.installed: + continue - for spec in specs: - if args.missing or args.dependencies: - for s in spec.traverse(root=False): - # Skip already-installed packages with --missing - if args.missing and s.installed: - continue + pkg = spec.package - s.package.do_fetch() - spec.package.do_fetch() + pkg.stage.keep = True + with pkg.stage: + pkg.do_fetch() diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 15ea5c77097ed1..a0570038456018 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -67,7 +67,7 @@ def setup_parser(subparser): help="do not group specs by arch/compiler", ) - arguments.add_common_arguments(subparser, ["long", "very_long", "tags"]) + arguments.add_common_arguments(subparser, ["long", "very_long", "tags", "namespaces"]) subparser.add_argument( "-c", @@ -140,9 +140,6 @@ def setup_parser(subparser): subparser.add_argument( "--only-deprecated", action="store_true", help="show only deprecated packages" ) - subparser.add_argument( - "-N", "--namespace", action="store_true", help="show fully qualified package names" - ) subparser.add_argument("--start-date", help="earliest date of installation [YYYY-MM-DD]") subparser.add_argument("--end-date", help="latest date of installation [YYYY-MM-DD]") @@ -230,7 +227,7 @@ def display_env(env, args, decorator, results): env.user_specs, root_args, decorator=lambda s, f: color.colorize("@*{%s}" % f), - namespace=True, + namespaces=True, show_flags=True, show_full_compiler=True, variants=True, @@ -271,7 +268,7 @@ def find(parser, args): # If tags have been specified on the command line, filter by tags if args.tags: - packages_with_tags = spack.repo.path.packages_with_tags(*args.tags) + packages_with_tags = spack.repo.PATH.packages_with_tags(*args.tags) results = [x for x in results if x.name in packages_with_tags] if args.loaded: diff --git a/lib/spack/spack/cmd/gc.py b/lib/spack/spack/cmd/gc.py index bbeb8882ba4522..9918bf7479fd05 100644 --- a/lib/spack/spack/cmd/gc.py +++ b/lib/spack/spack/cmd/gc.py @@ -6,6 +6,7 @@ import llnl.util.tty as tty import spack.cmd.common.arguments +import spack.cmd.common.confirmation import spack.cmd.uninstall import spack.environment as ev import spack.store @@ -20,7 +21,7 @@ def setup_parser(subparser): def gc(parser, args): - specs = spack.store.db.unused_specs + specs = spack.store.STORE.db.unused_specs # Restrict garbage collection to the active environment # speculating over roots that are yet to be installed @@ -41,6 +42,6 @@ def gc(parser, args): return if not args.yes_to_all: - spack.cmd.uninstall.confirm_removal(specs) + spack.cmd.common.confirmation.confirm_action(specs, "uninstalled", "uninstallation") spack.cmd.uninstall.do_uninstall(specs, force=False) diff --git a/lib/spack/spack/cmd/gpg.py b/lib/spack/spack/cmd/gpg.py index 80957dd6e9dde0..0bce5f473664c5 100644 --- a/lib/spack/spack/cmd/gpg.py +++ b/lib/spack/spack/cmd/gpg.py @@ -68,7 +68,7 @@ def setup_parser(subparser): metavar="DEST", type=str, dest="secret", - help="export the private key to a file.", + help="export the private key to a file", ) create.set_defaults(func=gpg_create) @@ -86,7 +86,7 @@ def setup_parser(subparser): export = subparsers.add_parser("export", help=gpg_export.__doc__) export.add_argument("location", type=str, help="where to export keys") export.add_argument( - "keys", nargs="*", help="the keys to export; " "all public keys if unspecified" + "keys", nargs="*", help="the keys to export (all public keys if unspecified)" ) export.add_argument("--secret", action="store_true", help="export secret keys") export.set_defaults(func=gpg_export) @@ -99,29 +99,29 @@ def setup_parser(subparser): "--directory", metavar="directory", type=str, - help="local directory where keys will be published.", + help="local directory where keys will be published", ) output.add_argument( "-m", "--mirror-name", metavar="mirror-name", type=str, - help="name of the mirror where " + "keys will be published.", + help="name of the mirror where keys will be published", ) output.add_argument( "--mirror-url", metavar="mirror-url", type=str, - help="URL of the mirror where " + "keys will be published.", + help="URL of the mirror where keys will be published", ) publish.add_argument( "--rebuild-index", action="store_true", default=False, - help=("Regenerate buildcache key index " "after publishing key(s)"), + help="regenerate buildcache key index after publishing key(s)", ) publish.add_argument( - "keys", nargs="*", help="the keys to publish; " "all public keys if unspecified" + "keys", nargs="*", help="keys to publish (all public keys if unspecified)" ) publish.set_defaults(func=gpg_publish) @@ -146,7 +146,7 @@ def gpg_create(args): def gpg_export(args): - """export a gpg key, optionally including secret key.""" + """export a gpg key, optionally including secret key""" keys = args.keys if not keys: keys = spack.util.gpg.signing_keys() @@ -168,7 +168,7 @@ def gpg_sign(args): elif not keys: raise RuntimeError("no signing keys are available") else: - raise RuntimeError("multiple signing keys are available; " "please choose one") + raise RuntimeError("multiple signing keys are available; please choose one") output = args.output if not output: output = args.spec[0] + ".asc" @@ -216,7 +216,7 @@ def gpg_publish(args): url = spack.util.url.path_to_file_url(args.directory) mirror = spack.mirror.Mirror(url, url) elif args.mirror_name: - mirror = spack.mirror.MirrorCollection().lookup(args.mirror_name) + mirror = spack.mirror.MirrorCollection(binary=True).lookup(args.mirror_name) elif args.mirror_url: mirror = spack.mirror.Mirror(args.mirror_url, args.mirror_url) diff --git a/lib/spack/spack/cmd/graph.py b/lib/spack/spack/cmd/graph.py index a0fd16f19fee87..eeced40720e07f 100644 --- a/lib/spack/spack/cmd/graph.py +++ b/lib/spack/spack/cmd/graph.py @@ -63,7 +63,7 @@ def graph(parser, args): if env: specs = env.all_specs() else: - specs = spack.store.db.query() + specs = spack.store.STORE.db.query() else: specs = spack.cmd.parse_specs(args.specs, concretize=not args.static) @@ -74,19 +74,19 @@ def graph(parser, args): if args.static: args.dot = True - static_graph_dot(specs, deptype=args.deptype) + static_graph_dot(specs, depflag=args.deptype) return if args.dot: builder = SimpleDAG() if args.color: builder = DAGWithDependencyTypes() - graph_dot(specs, builder=builder, deptype=args.deptype) + graph_dot(specs, builder=builder, depflag=args.deptype) return # ascii is default: user doesn't need to provide it explicitly debug = spack.config.get("config:debug") - graph_ascii(specs[0], debug=debug, deptype=args.deptype) + graph_ascii(specs[0], debug=debug, depflag=args.deptype) for spec in specs[1:]: print() # extra line bt/w independent graphs graph_ascii(spec, debug=debug) diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 68b95083a61b58..dd56c25451083a 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import sys import textwrap from itertools import zip_longest @@ -11,10 +12,12 @@ from llnl.util.tty.colify import colify import spack.cmd.common.arguments as arguments +import spack.deptypes as dt import spack.fetch_strategy as fs import spack.install_test import spack.repo import spack.spec +import spack.version from spack.package_base import preferred_version description = "get detailed information on a particular package" @@ -52,6 +55,7 @@ def setup_parser(subparser): ("--tags", print_tags.__doc__), ("--tests", print_tests.__doc__), ("--virtuals", print_virtuals.__doc__), + ("--variants-by-name", "list variants in strict name order; don't group by condition"), ] for opt, help_comment in options: subparser.add_argument(opt, action="store_true", help=help_comment) @@ -64,42 +68,21 @@ def section_title(s): def version(s): - return spack.spec.version_color + s + plain_format + return spack.spec.VERSION_COLOR + s + plain_format def variant(s): - return spack.spec.enabled_variant_color + s + plain_format + return spack.spec.ENABLED_VARIANT_COLOR + s + plain_format -class VariantFormatter: - def __init__(self, variants): - self.variants = variants - self.headers = ("Name [Default]", "When", "Allowed values", "Description") +def license(s): + return spack.spec.VERSION_COLOR + s + plain_format - # Formats - fmt_name = "{0} [{1}]" - - # Initialize column widths with the length of the - # corresponding headers, as they cannot be shorter - # than that - self.column_widths = [len(x) for x in self.headers] - - # Expand columns based on max line lengths - for k, e in variants.items(): - v, w = e - candidate_max_widths = ( - len(fmt_name.format(k, self.default(v))), # Name [Default] - len(str(w)), - len(v.allowed_values), # Allowed values - len(v.description), # Description - ) - self.column_widths = ( - max(self.column_widths[0], candidate_max_widths[0]), - max(self.column_widths[1], candidate_max_widths[1]), - max(self.column_widths[2], candidate_max_widths[2]), - max(self.column_widths[3], candidate_max_widths[3]), - ) +class VariantFormatter: + def __init__(self, pkg): + self.variants = pkg.variants + self.headers = ("Name [Default]", "When", "Allowed values", "Description") # Don't let name or possible values be less than max widths _, cols = tty.terminal_size() @@ -132,6 +115,8 @@ def default(self, v): def lines(self): if not self.variants: yield " None" + return + else: yield " " + self.fmt % self.headers underline = tuple([w * "=" for w in self.column_widths]) @@ -160,7 +145,7 @@ def print_dependencies(pkg): for deptype in ("build", "link", "run"): color.cprint("") color.cprint(section_title("%s Dependencies:" % deptype.capitalize())) - deps = sorted(pkg.dependencies_of_type(deptype)) + deps = sorted(pkg.dependencies_of_type(dt.flag_from_string(deptype))) if deps: colify(deps, indent=4) else: @@ -266,15 +251,165 @@ def print_tests(pkg): color.cprint(" None") -def print_variants(pkg): +def _fmt_value(v): + if v is None or isinstance(v, bool): + return str(v).lower() + else: + return str(v) + + +def _fmt_name_and_default(variant): + """Print colorized name [default] for a variant.""" + return color.colorize(f"@c{{{variant.name}}} @C{{[{_fmt_value(variant.default)}]}}") + + +def _fmt_when(when, indent): + return color.colorize(f"{indent * ' '}@B{{when}} {color.cescape(when)}") + + +def _fmt_variant_description(variant, width, indent): + """Format a variant's description, preserving explicit line breaks.""" + return "\n".join( + textwrap.fill( + line, width=width, initial_indent=indent * " ", subsequent_indent=indent * " " + ) + for line in variant.description.split("\n") + ) + + +def _fmt_variant(variant, max_name_default_len, indent, when=None, out=None): + out = out or sys.stdout + + _, cols = tty.terminal_size() + + name_and_default = _fmt_name_and_default(variant) + name_default_len = color.clen(name_and_default) + + values = variant.values + if not isinstance(variant.values, (tuple, list, spack.variant.DisjointSetsOfValues)): + values = [variant.values] + + # put 'none' first, sort the rest by value + sorted_values = sorted(values, key=lambda v: (v != "none", v)) + + pad = 4 # min padding between 'name [default]' and values + value_indent = (indent + max_name_default_len + pad) * " " # left edge of values + + # This preserves any formatting (i.e., newlines) from how the description was + # written in package.py, but still wraps long lines for small terminals. + # This allows some packages to provide detailed help on their variants (see, e.g., gasnet). + formatted_values = "\n".join( + textwrap.wrap( + f"{', '.join(_fmt_value(v) for v in sorted_values)}", + width=cols - 2, + initial_indent=value_indent, + subsequent_indent=value_indent, + ) + ) + formatted_values = formatted_values[indent + name_default_len + pad :] + + # name [default] value1, value2, value3, ... + padding = pad * " " + color.cprint(f"{indent * ' '}{name_and_default}{padding}@c{{{formatted_values}}}", stream=out) + + # when + description_indent = indent + 4 + if when is not None and when != spack.spec.Spec(): + out.write(_fmt_when(when, description_indent - 2)) + out.write("\n") + + # description, preserving explicit line breaks from the way it's written in the package file + out.write(_fmt_variant_description(variant, cols - 2, description_indent)) + out.write("\n") + + +def _variants_by_name_when(pkg): + """Adaptor to get variants keyed by { name: { when: { [Variant...] } }.""" + # TODO: replace with pkg.variants_by_name(when=True) when unified directive dicts are merged. + variants = {} + for name, (variant, whens) in pkg.variants.items(): + for when in whens: + variants.setdefault(name, {}).setdefault(when, []).append(variant) + return variants + + +def _variants_by_when_name(pkg): + """Adaptor to get variants keyed by { when: { name: Variant } }""" + # TODO: replace with pkg.variants when unified directive dicts are merged. + variants = {} + for name, (variant, whens) in pkg.variants.items(): + for when in whens: + variants.setdefault(when, {})[name] = variant + return variants + + +def _print_variants_header(pkg): """output variants""" + if not pkg.variants: + print(" None") + return + color.cprint("") color.cprint(section_title("Variants:")) - formatter = VariantFormatter(pkg.variants) - for line in formatter.lines: - color.cprint(color.cescape(line)) + variants_by_name = _variants_by_name_when(pkg) + + # Calculate the max length of the "name [default]" part of the variant display + # This lets us know where to print variant values. + max_name_default_len = max( + color.clen(_fmt_name_and_default(variant)) + for name, when_variants in variants_by_name.items() + for variants in when_variants.values() + for variant in variants + ) + + return max_name_default_len, variants_by_name + + +def _unconstrained_ver_first(item): + """sort key that puts specs with open version ranges first""" + spec, _ = item + return (spack.version.any_version not in spec.versions, spec) + + +def print_variants_grouped_by_when(pkg): + max_name_default_len, _ = _print_variants_header(pkg) + + indent = 4 + variants = _variants_by_when_name(pkg) + for when, variants_by_name in sorted(variants.items(), key=_unconstrained_ver_first): + padded_values = max_name_default_len + 4 + start_indent = indent + + if when != spack.spec.Spec(): + sys.stdout.write("\n") + sys.stdout.write(_fmt_when(when, indent)) + sys.stdout.write("\n") + + # indent names slightly inside 'when', but line up values + padded_values -= 2 + start_indent += 2 + + for name, variant in sorted(variants_by_name.items()): + _fmt_variant(variant, padded_values, start_indent, None, out=sys.stdout) + + +def print_variants_by_name(pkg): + max_name_default_len, variants_by_name = _print_variants_header(pkg) + max_name_default_len += 4 + + indent = 4 + for name, when_variants in variants_by_name.items(): + for when, variants in sorted(when_variants.items(), key=_unconstrained_ver_first): + for variant in variants: + _fmt_variant(variant, max_name_default_len, indent, when, out=sys.stdout) + sys.stdout.write("\n") + + +def print_variants(pkg): + """output variants""" + print_variants_grouped_by_when(pkg) def print_versions(pkg): @@ -295,18 +430,24 @@ def print_versions(pkg): pad = padder(pkg.versions, 4) preferred = preferred_version(pkg) - url = "" - if pkg.has_code: - url = fs.for_package_version(pkg, preferred) + def get_url(version): + try: + return fs.for_package_version(pkg, version) + except spack.fetch_strategy.InvalidArgsError: + return "No URL" + + url = get_url(preferred) if pkg.has_code else "" line = version(" {0}".format(pad(preferred))) + color.cescape(url) - color.cprint(line) + color.cwrite(line) + + print() safe = [] deprecated = [] for v in reversed(sorted(pkg.versions)): if pkg.has_code: - url = fs.for_package_version(pkg, v) + url = get_url(v) if pkg.versions[v].get("deprecated", False): deprecated.append((v, url)) else: @@ -347,9 +488,25 @@ def print_virtuals(pkg): color.cprint(" None") +def print_licenses(pkg): + """Output the licenses of the project.""" + + color.cprint("") + color.cprint(section_title("Licenses: ")) + + if len(pkg.licenses) == 0: + color.cprint(" None") + else: + pad = padder(pkg.licenses, 4) + for when_spec in pkg.licenses: + license_identifier = pkg.licenses[when_spec] + line = license(" {0}".format(pad(license_identifier))) + color.cescape(when_spec) + color.cprint(line) + + def info(parser, args): spec = spack.spec.Spec(args.package) - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) pkg = pkg_cls(spec) # Output core package information @@ -363,7 +520,12 @@ def info(parser, args): else: color.cprint(" None") - color.cprint(section_title("Homepage: ") + pkg.homepage) + if getattr(pkg, "homepage"): + color.cprint(section_title("Homepage: ") + pkg.homepage) + + _print_variants = ( + print_variants_by_name if args.variants_by_name else print_variants_grouped_by_when + ) # Now output optional information in expected order sections = [ @@ -371,11 +533,12 @@ def info(parser, args): (args.all or args.detectable, print_detectable), (args.all or args.tags, print_tags), (args.all or not args.no_versions, print_versions), - (args.all or not args.no_variants, print_variants), + (args.all or not args.no_variants, _print_variants), (args.all or args.phases, print_phases), (args.all or not args.no_dependencies, print_dependencies), (args.all or args.virtuals, print_virtuals), (args.all or args.tests, print_tests), + (args.all or True, print_licenses), ] for print_it, func in sections: if print_it: diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 8fe952943bfb43..b74f982755115b 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -75,10 +75,9 @@ def setup_parser(subparser): default="package,dependencies", dest="things_to_install", choices=["package", "dependencies"], - help="""select the mode of installation. -the default is to install the package along with all its dependencies. -alternatively one can decide to install only the package or only -the dependencies""", + help="select the mode of installation\n\n" + "default is to install the package along with all its dependencies. " + "alternatively, one can decide to install only the package or only the dependencies", ) subparser.add_argument( "-u", @@ -143,12 +142,11 @@ def setup_parser(subparser): type=arguments.use_buildcache, default="package:auto,dependencies:auto", metavar="[{auto,only,never},][package:{auto,only,never},][dependencies:{auto,only,never}]", - help="""select the mode of buildcache for the 'package' and 'dependencies'. -Default: package:auto,dependencies:auto -- `auto` behaves like --use-cache -- `only` behaves like --cache-only -- `never` behaves like --no-cache -""", + help="select the mode of buildcache for the 'package' and 'dependencies'\n\n" + "default: package:auto,dependencies:auto\n\n" + "- `auto` behaves like --use-cache\n" + "- `only` behaves like --cache-only\n" + "- `never` behaves like --no-cache", ) subparser.add_argument( @@ -156,8 +154,8 @@ def setup_parser(subparser): action="store_true", dest="include_build_deps", default=False, - help="""include build deps when installing from cache, -which is useful for CI pipeline troubleshooting""", + help="include build deps when installing from cache, " + "useful for CI pipeline troubleshooting", ) subparser.add_argument( @@ -186,7 +184,7 @@ def setup_parser(subparser): dest="install_verbose", help="display verbose build output while installing", ) - subparser.add_argument("--fake", action="store_true", help="fake install for debug purposes.") + subparser.add_argument("--fake", action="store_true", help="fake install for debug purposes") subparser.add_argument( "--only-concrete", action="store_true", @@ -199,14 +197,13 @@ def setup_parser(subparser): "--add", action="store_true", default=False, - help="""(with environment) add spec to the environment as a root.""", + help="(with environment) add spec to the environment as a root", ) updateenv_group.add_argument( "--no-add", action="store_false", dest="add", - help="""(with environment) do not add spec to the environment as a -root (the default behavior).""", + help="(with environment) do not add spec to the environment as a root", ) subparser.add_argument( @@ -216,7 +213,7 @@ def setup_parser(subparser): default=[], dest="specfiles", metavar="SPEC_YAML_FILE", - help="install from file. Read specs to install from .yaml files", + help="read specs to install from .yaml files", ) cd_group = subparser.add_mutually_exclusive_group() @@ -227,19 +224,12 @@ def setup_parser(subparser): "--test", default=None, choices=["root", "all"], - help="""If 'root' is chosen, run package tests during -installation for top-level packages (but skip tests for dependencies). -if 'all' is chosen, run package tests during installation for all -packages. If neither are chosen, don't run tests for any packages.""", + help="run tests on only root packages or all packages", ) arguments.add_common_arguments(subparser, ["log_format"]) + subparser.add_argument("--log-file", default=None, help="filename for the log file") subparser.add_argument( - "--log-file", - default=None, - help="filename for the log file. if not passed a default will be used", - ) - subparser.add_argument( - "--help-cdash", action="store_true", help="Show usage instructions for CDash reporting" + "--help-cdash", action="store_true", help="show usage instructions for CDash reporting" ) arguments.add_cdash_args(subparser, False) arguments.add_common_arguments(subparser, ["yes_to_all", "spec"]) @@ -250,8 +240,7 @@ def default_log_file(spec): """Computes the default filename for the log file and creates the corresponding directory if not present """ - fmt = "test-{x.name}-{x.version}-{hash}.xml" - basename = fmt.format(x=spec, hash=spec.dag_hash()) + basename = spec.format_path("test-{name}-{version}-{hash}.xml") dirname = fs.os.path.join(spack.paths.reports_path, "junit") fs.mkdirp(dirname) return fs.os.path.join(dirname, basename) @@ -276,11 +265,11 @@ def require_user_confirmation_for_overwrite(concrete_specs, args): if args.yes_to_all: return - installed = list(filter(lambda x: x, map(spack.store.db.query_one, concrete_specs))) + installed = list(filter(lambda x: x, map(spack.store.STORE.db.query_one, concrete_specs))) display_args = {"long": True, "show_flags": True, "variants": True} if installed: - tty.msg("The following package specs will be " "reinstalled:\n") + tty.msg("The following package specs will be reinstalled:\n") spack.cmd.display_specs(installed, **display_args) not_installed = list(filter(lambda x: x not in installed, concrete_specs)) diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 544bdda5ad2ab2..a46d7fa5e079c6 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -16,7 +16,7 @@ from llnl.util.tty.colify import colify import spack.cmd.common.arguments as arguments -import spack.dependency +import spack.deptypes as dt import spack.repo from spack.version import VersionList @@ -107,7 +107,7 @@ def match(p, f): if f.match(p): return True - pkg_cls = spack.repo.path.get_pkg_class(p) + pkg_cls = spack.repo.PATH.get_pkg_class(p) if pkg_cls.__doc__: return f.match(pkg_cls.__doc__) return False @@ -149,8 +149,8 @@ def rows_for_ncols(elts, ncols): def get_dependencies(pkg): all_deps = {} - for deptype in spack.dependency.all_deptypes: - deps = pkg.dependencies_of_type(deptype) + for deptype in dt.ALL_TYPES: + deps = pkg.dependencies_of_type(dt.flag_from_string(deptype)) all_deps[deptype] = [d for d in deps] return all_deps @@ -159,7 +159,7 @@ def get_dependencies(pkg): @formatter def version_json(pkg_names, out): """Print all packages with their latest versions.""" - pkg_classes = [spack.repo.path.get_pkg_class(name) for name in pkg_names] + pkg_classes = [spack.repo.PATH.get_pkg_class(name) for name in pkg_names] out.write("[\n") @@ -201,7 +201,7 @@ def html(pkg_names, out): """ # Read in all packages - pkg_classes = [spack.repo.path.get_pkg_class(name) for name in pkg_names] + pkg_classes = [spack.repo.PATH.get_pkg_class(name) for name in pkg_names] # Start at 2 because the title of the page from Sphinx is id1. span_id = 2 @@ -275,8 +275,8 @@ def head(n, span_id, title, anchor=None): out.write("\n") out.write("\n") - for deptype in spack.dependency.all_deptypes: - deps = pkg_cls.dependencies_of_type(deptype) + for deptype in dt.ALL_TYPES: + deps = pkg_cls.dependencies_of_type(dt.flag_from_string(deptype)) if deps: out.write("
%s Dependencies:
\n" % deptype.capitalize()) out.write("
\n") @@ -313,13 +313,13 @@ def list(parser, args): # If tags have been specified on the command line, filter by tags if args.tags: - packages_with_tags = spack.repo.path.packages_with_tags(*args.tags) + packages_with_tags = spack.repo.PATH.packages_with_tags(*args.tags) sorted_packages = [p for p in sorted_packages if p in packages_with_tags] if args.update: # change output stream if user asked for update if os.path.exists(args.update): - if os.path.getmtime(args.update) > spack.repo.path.last_mtime(): + if os.path.getmtime(args.update) > spack.repo.PATH.last_mtime(): tty.msg("File is up to date: %s" % args.update) return diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index ee9ee939e8fa57..5cdd2909c7ae40 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -5,6 +5,8 @@ import sys +import llnl.util.tty as tty + import spack.cmd import spack.cmd.common.arguments as arguments import spack.cmd.find @@ -52,6 +54,13 @@ def setup_parser(subparser): const="bat", help="print bat commands to load the package", ) + shells.add_argument( + "--pwsh", + action="store_const", + dest="shell", + const="pwsh", + help="print pwsh commands to load the package", + ) subparser.add_argument( "--first", @@ -66,10 +75,9 @@ def setup_parser(subparser): default="package,dependencies", dest="things_to_load", choices=["package", "dependencies"], - help="""select whether to load the package and its dependencies -the default is to load the package and all dependencies -alternatively one can decide to load only the package or only -the dependencies""", + help="select whether to load the package and its dependencies\n\n" + "the default is to load the package and all dependencies. alternatively, " + "one can decide to load only the package or only the dependencies", ) subparser.add_argument( @@ -102,16 +110,14 @@ def load(parser, args): ) return 1 - with spack.store.db.read_transaction(): - if "dependencies" in args.things_to_load: - include_roots = "package" in args.things_to_load - specs = [ - dep for spec in specs for dep in spec.traverse(root=include_roots, order="post") - ] + if args.things_to_load != "package,dependencies": + tty.warn( + "The `--only` flag in spack load is deprecated and will be removed in Spack v0.22" + ) - env_mod = spack.util.environment.EnvironmentModifications() + with spack.store.STORE.db.read_transaction(): + env_mod = uenv.environment_modifications_for_specs(*specs) for spec in specs: - env_mod.extend(uenv.environment_modifications_for_spec(spec)) env_mod.prepend_path(uenv.spack_loaded_hashes_var, spec.dag_hash()) cmds = env_mod.shell_modifications(args.shell) diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 48c0fd5d297db0..b4bbaf54b82a8a 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -55,13 +55,13 @@ def setup_parser(subparser): directories.add_argument( "--source-dir", action="store_true", - help="source directory for a spec " "(requires it to be staged first)", + help="source directory for a spec (requires it to be staged first)", ) directories.add_argument( "-b", "--build-dir", action="store_true", - help="build directory for a spec " "(requires it to be staged first)", + help="build directory for a spec (requires it to be staged first)", ) directories.add_argument( "-e", @@ -109,7 +109,7 @@ def location(parser, args): return if args.packages: - print(spack.repo.path.first_repo().root) + print(spack.repo.PATH.first_repo().root) return if args.stages: @@ -135,7 +135,7 @@ def location(parser, args): # Package dir just needs the spec name if args.package_dir: - print(spack.repo.path.dirname_for_package_name(spec.name)) + print(spack.repo.PATH.dirname_for_package_name(spec.name)) return # Either concretize or filter from already concretized environment @@ -162,7 +162,7 @@ def location(parser, args): # source dir remains, which requires the spec to be staged if not pkg.stage.expanded: tty.die( - "Source directory does not exist yet. " "Run this to create it:", + "Source directory does not exist yet. Run this to create it:", "spack stage " + " ".join(args.spec), ) diff --git a/lib/spack/spack/cmd/maintainers.py b/lib/spack/spack/cmd/maintainers.py index 1fd61ef3824f52..80140f6291f014 100644 --- a/lib/spack/spack/cmd/maintainers.py +++ b/lib/spack/spack/cmd/maintainers.py @@ -54,11 +54,11 @@ def setup_parser(subparser): def packages_to_maintainers(package_names=None): if not package_names: - package_names = spack.repo.path.all_package_names() + package_names = spack.repo.PATH.all_package_names() pkg_to_users = defaultdict(lambda: set()) for name in package_names: - cls = spack.repo.path.get_pkg_class(name) + cls = spack.repo.PATH.get_pkg_class(name) for user in cls.maintainers: pkg_to_users[name].add(user) @@ -67,8 +67,8 @@ def packages_to_maintainers(package_names=None): def maintainers_to_packages(users=None): user_to_pkgs = defaultdict(lambda: []) - for name in spack.repo.path.all_package_names(): - cls = spack.repo.path.get_pkg_class(name) + for name in spack.repo.PATH.all_package_names(): + cls = spack.repo.PATH.get_pkg_class(name) for user in cls.maintainers: lower_users = [u.lower() for u in users] if not users or user.lower() in lower_users: @@ -80,8 +80,8 @@ def maintainers_to_packages(users=None): def maintained_packages(): maintained = [] unmaintained = [] - for name in spack.repo.path.all_package_names(): - cls = spack.repo.path.get_pkg_class(name) + for name in spack.repo.PATH.all_package_names(): + cls = spack.repo.PATH.get_pkg_class(name) if cls.maintainers: maintained.append(name) else: diff --git a/lib/spack/spack/cmd/make_installer.py b/lib/spack/spack/cmd/make_installer.py index eedc9db1656564..986631d39f778d 100644 --- a/lib/spack/spack/cmd/make_installer.py +++ b/lib/spack/spack/cmd/make_installer.py @@ -6,10 +6,11 @@ import posixpath import sys +from llnl.path import convert_to_posix_path + import spack.paths import spack.util.executable from spack.spec import Spec -from spack.util.path import convert_to_posix_path description = "generate Windows installer" section = "admin" @@ -39,7 +40,7 @@ def line_to_rtf(str): def setup_parser(subparser): spack_source_group = subparser.add_mutually_exclusive_group(required=True) spack_source_group.add_argument( - "-v", "--spack-version", default="", help="download given spack version e.g. 0.16.0" + "-v", "--spack-version", default="", help="download given spack version" ) spack_source_group.add_argument( "-s", "--spack-source", default="", help="full path to spack source" @@ -50,7 +51,7 @@ def setup_parser(subparser): "--git-installer-verbosity", default="", choices=["SILENT", "VERYSILENT"], - help="Level of verbosity provided by bundled Git Installer. Default is fully verbose", + help="level of verbosity provided by bundled git installer (default is fully verbose)", required=False, action="store", dest="git_verbosity", diff --git a/lib/spack/spack/cmd/mark.py b/lib/spack/spack/cmd/mark.py index ca4f48096d5136..61939c9458322c 100644 --- a/lib/spack/spack/cmd/mark.py +++ b/lib/spack/spack/cmd/mark.py @@ -35,10 +35,7 @@ def setup_parser(subparser): "--all", action="store_true", dest="all", - help="Mark ALL installed packages that match each " - "supplied spec. If you `mark --all libelf`," - " ALL versions of `libelf` are marked. If no spec is " - "supplied, all installed packages will be marked.", + help="mark ALL installed packages that match each supplied spec", ) exim = subparser.add_mutually_exclusive_group(required=True) exim.add_argument( @@ -46,14 +43,14 @@ def setup_parser(subparser): "--explicit", action="store_true", dest="explicit", - help="Mark packages as explicitly installed.", + help="mark packages as explicitly installed", ) exim.add_argument( "-i", "--implicit", action="store_true", dest="implicit", - help="Mark packages as implicitly installed.", + help="mark packages as implicitly installed", ) @@ -74,7 +71,7 @@ def find_matching_specs(specs, allow_multiple_matches=False): for spec in specs: install_query = [InstallStatuses.INSTALLED] - matching = spack.store.db.query_local(spec, installed=install_query) + matching = spack.store.STORE.db.query_local(spec, installed=install_query) # For each spec provided, make sure it refers to only one package. # Fail and ask user to be unambiguous if it doesn't if not allow_multiple_matches and len(matching) > 1: @@ -105,7 +102,7 @@ def do_mark(specs, explicit): explicit (bool): whether to mark specs as explicitly installed """ for spec in specs: - spack.store.db.update_explicit(spec, explicit) + spack.store.STORE.db.update_explicit(spec, explicit) def mark_specs(args, specs): diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 582627cbe26a92..1036dcbe917e35 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -21,7 +21,6 @@ import spack.util.path import spack.util.web as web_util from spack.error import SpackError -from spack.util.spack_yaml import syaml_dict description = "manage mirrors (source and binary)" section = "config" @@ -55,13 +54,13 @@ def setup_parser(subparser): ) create_parser.add_argument( "--exclude-specs", - help="specs which Spack should not try to add to a mirror" " (specified on command line)", + help="specs which Spack should not try to add to a mirror (specified on command line)", ) create_parser.add_argument( "--skip-unstable-versions", action="store_true", - help="don't cache versions unless they identify a stable (unchanging)" " source code", + help="don't cache versions unless they identify a stable (unchanging) source code", ) create_parser.add_argument( "-D", "--dependencies", action="store_true", help="also fetch all dependencies" @@ -91,7 +90,6 @@ def setup_parser(subparser): # used to construct scope arguments below scopes = spack.config.scopes() - scopes_metavar = spack.config.scopes_metavar # Add add_parser = sp.add_parser("add", help=mirror_add.__doc__) @@ -100,18 +98,27 @@ def setup_parser(subparser): add_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_modify_scope(), help="configuration scope to modify", ) - arguments.add_s3_connection_args(add_parser, False) + add_parser.add_argument( + "--type", + action="append", + choices=("binary", "source"), + help=( + "specify the mirror type: for both binary " + "and source use `--type binary --type source` (default)" + ), + ) + arguments.add_connection_args(add_parser, False) # Remove remove_parser = sp.add_parser("remove", aliases=["rm"], help=mirror_remove.__doc__) remove_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror") remove_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_modify_scope(), help="configuration scope to modify", ) @@ -120,129 +127,148 @@ def setup_parser(subparser): set_url_parser = sp.add_parser("set-url", help=mirror_set_url.__doc__) set_url_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror") set_url_parser.add_argument("url", help="url of mirror directory from 'spack mirror create'") - set_url_parser.add_argument( - "--push", action="store_true", help="set only the URL used for uploading new packages" + set_url_push_or_fetch = set_url_parser.add_mutually_exclusive_group(required=False) + set_url_push_or_fetch.add_argument( + "--push", action="store_true", help="set only the URL used for uploading" + ) + set_url_push_or_fetch.add_argument( + "--fetch", action="store_true", help="set only the URL used for downloading" ) set_url_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, + default=spack.config.default_modify_scope(), + help="configuration scope to modify", + ) + arguments.add_connection_args(set_url_parser, False) + + # Set + set_parser = sp.add_parser("set", help=mirror_set.__doc__) + set_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror") + set_parser_push_or_fetch = set_parser.add_mutually_exclusive_group(required=False) + set_parser_push_or_fetch.add_argument( + "--push", action="store_true", help="modify just the push connection details" + ) + set_parser_push_or_fetch.add_argument( + "--fetch", action="store_true", help="modify just the fetch connection details" + ) + set_parser.add_argument( + "--type", + action="append", + choices=("binary", "source"), + help=( + "specify the mirror type: for both binary " + "and source use `--type binary --type source`" + ), + ) + set_parser.add_argument("--url", help="url of mirror directory from 'spack mirror create'") + set_parser.add_argument( + "--scope", + choices=scopes, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_modify_scope(), help="configuration scope to modify", ) - arguments.add_s3_connection_args(set_url_parser, False) + arguments.add_connection_args(set_parser, False) # List list_parser = sp.add_parser("list", help=mirror_list.__doc__) list_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_list_scope(), help="configuration scope to read from", ) def mirror_add(args): - """Add a mirror to Spack.""" + """add a mirror to Spack""" if ( args.s3_access_key_id or args.s3_access_key_secret or args.s3_access_token or args.s3_profile or args.s3_endpoint_url + or args.type + or args.oci_username + or args.oci_password ): connection = {"url": args.url} if args.s3_access_key_id and args.s3_access_key_secret: - connection["access_pair"] = (args.s3_access_key_id, args.s3_access_key_secret) + connection["access_pair"] = [args.s3_access_key_id, args.s3_access_key_secret] if args.s3_access_token: connection["access_token"] = args.s3_access_token if args.s3_profile: connection["profile"] = args.s3_profile if args.s3_endpoint_url: connection["endpoint_url"] = args.s3_endpoint_url - mirror = spack.mirror.Mirror(fetch_url=connection, push_url=connection, name=args.name) + if args.oci_username and args.oci_password: + connection["access_pair"] = [args.oci_username, args.oci_password] + if args.type: + connection["binary"] = "binary" in args.type + connection["source"] = "source" in args.type + mirror = spack.mirror.Mirror(connection, name=args.name) else: mirror = spack.mirror.Mirror(args.url, name=args.name) spack.mirror.add(mirror, args.scope) def mirror_remove(args): - """Remove a mirror by name.""" + """remove a mirror by name""" spack.mirror.remove(args.name, args.scope) -def mirror_set_url(args): - """Change the URL of a mirror.""" - url = args.url +def _configure_mirror(args): mirrors = spack.config.get("mirrors", scope=args.scope) - if not mirrors: - mirrors = syaml_dict() if args.name not in mirrors: - tty.die("No mirror found with name %s." % args.name) - - entry = mirrors[args.name] - key_values = ["s3_access_key_id", "s3_access_token", "s3_profile"] - - if any(value for value in key_values if value in args): - incoming_data = { - "url": url, - "access_pair": (args.s3_access_key_id, args.s3_access_key_secret), - "access_token": args.s3_access_token, - "profile": args.s3_profile, - "endpoint_url": args.s3_endpoint_url, - } - try: - fetch_url = entry["fetch"] - push_url = entry["push"] - except TypeError: - fetch_url, push_url = entry, entry - - changes_made = False - - if args.push: - if isinstance(push_url, dict): - changes_made = changes_made or push_url != incoming_data - push_url = incoming_data - else: - changes_made = changes_made or push_url != url - push_url = url + tty.die(f"No mirror found with name {args.name}.") + + entry = spack.mirror.Mirror(mirrors[args.name], args.name) + direction = "fetch" if args.fetch else "push" if args.push else None + changes = {} + if args.url: + changes["url"] = args.url + if args.s3_access_key_id and args.s3_access_key_secret: + changes["access_pair"] = [args.s3_access_key_id, args.s3_access_key_secret] + if args.s3_access_token: + changes["access_token"] = args.s3_access_token + if args.s3_profile: + changes["profile"] = args.s3_profile + if args.s3_endpoint_url: + changes["endpoint_url"] = args.s3_endpoint_url + if args.oci_username and args.oci_password: + changes["access_pair"] = [args.oci_username, args.oci_password] + + # argparse cannot distinguish between --binary and --no-binary when same dest :( + # notice that set-url does not have these args, so getattr + if getattr(args, "type", None): + changes["binary"] = "binary" in args.type + changes["source"] = "source" in args.type + + changed = entry.update(changes, direction) + + if changed: + mirrors[args.name] = entry.to_dict() + spack.config.set("mirrors", mirrors, scope=args.scope) else: - if isinstance(push_url, dict): - changes_made = changes_made or push_url != incoming_data or push_url != incoming_data - fetch_url, push_url = incoming_data, incoming_data - else: - changes_made = changes_made or push_url != url - fetch_url, push_url = url, url - - items = [ - ( - (n, u) - if n != args.name - else ( - (n, {"fetch": fetch_url, "push": push_url}) - if fetch_url != push_url - else (n, {"fetch": fetch_url, "push": fetch_url}) - ) - ) - for n, u in mirrors.items() - ] + tty.msg("No changes made to mirror %s." % args.name) - mirrors = syaml_dict(items) - spack.config.set("mirrors", mirrors, scope=args.scope) - if changes_made: - tty.msg( - "Changed%s url or connection information for mirror %s." - % ((" (push)" if args.push else ""), args.name) - ) - else: - tty.msg("No changes made to mirror %s." % args.name) +def mirror_set(args): + """configure the connection details of a mirror""" + _configure_mirror(args) + + +def mirror_set_url(args): + """change the URL of a mirror""" + _configure_mirror(args) def mirror_list(args): - """Print out available mirrors to the console.""" + """print out available mirrors to the console""" mirrors = spack.mirror.MirrorCollection(scope=args.scope) if not mirrors: @@ -395,9 +421,7 @@ def process_mirror_stats(present, mirrored, error): def mirror_create(args): - """Create a directory to be used as a spack mirror, and fill it with - package archives. - """ + """create a directory to be used as a spack mirror, and fill it with package archives""" if args.specs and args.all: raise SpackError( "cannot specify specs on command line if you chose to mirror all specs with '--all'" @@ -455,7 +479,7 @@ def create_mirror_for_all_specs(path, skip_unstable_versions, selection_fn): path, skip_unstable_versions=skip_unstable_versions ) for candidate in mirror_specs: - pkg_cls = spack.repo.path.get_pkg_class(candidate.name) + pkg_cls = spack.repo.PATH.get_pkg_class(candidate.name) pkg_obj = pkg_cls(spack.spec.Spec(candidate)) mirror_stats.next_spec(pkg_obj.spec) spack.mirror.create_mirror_from_package_object(pkg_obj, mirror_cache, mirror_stats) @@ -470,7 +494,7 @@ def create_mirror_for_all_specs_inside_environment(path, skip_unstable_versions, def mirror_destroy(args): - """Given a url, recursively delete everything under it.""" + """given a url, recursively delete everything under it""" mirror_url = None if args.mirror_name: @@ -490,6 +514,7 @@ def mirror(parser, args): "remove": mirror_remove, "rm": mirror_remove, "set-url": mirror_set_url, + "set": mirror_set, "list": mirror_list, } diff --git a/lib/spack/spack/cmd/modules/__init__.py b/lib/spack/spack/cmd/modules/__init__.py index 60a7475f3acd6b..65b174a06f6628 100644 --- a/lib/spack/spack/cmd/modules/__init__.py +++ b/lib/spack/spack/cmd/modules/__init__.py @@ -11,6 +11,7 @@ import sys from llnl.util import filesystem, tty +from llnl.util.tty import color import spack.cmd import spack.cmd.common.arguments as arguments @@ -31,7 +32,7 @@ def setup_parser(subparser): action="store", dest="module_set_name", default="default", - help="Named module set to use from modules configuration.", + help="named module set to use from modules configuration", ) sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="subparser_name") @@ -308,7 +309,7 @@ def refresh(module_type, specs, args): # Skip unknown packages. writers = [ - cls(spec, args.module_set_name) for spec in specs if spack.repo.path.exists(spec.name) + cls(spec, args.module_set_name) for spec in specs if spack.repo.PATH.exists(spec.name) ] # Filter excluded packages early @@ -320,12 +321,13 @@ def refresh(module_type, specs, args): file2writer[item.layout.filename].append(item) if len(file2writer) != len(writers): + spec_fmt_str = "{name}@={version}%{compiler}/{hash:7} {variants} arch={arch}" message = "Name clashes detected in module files:\n" for filename, writer_list in file2writer.items(): if len(writer_list) > 1: message += "\nfile: {0}\n".format(filename) for x in writer_list: - message += "spec: {0}\n".format(x.spec.format()) + message += "spec: {0}\n".format(x.spec.format(spec_fmt_str)) tty.error(message) tty.error("Operation aborted") raise SystemExit(1) @@ -347,14 +349,20 @@ def refresh(module_type, specs, args): spack.modules.common.generate_module_index( module_type_root, writers, overwrite=args.delete_tree ) + errors = [] for x in writers: try: x.write(overwrite=True) + except spack.error.SpackError as e: + msg = f"{x.layout.filename}: {e.message}" + errors.append(msg) except Exception as e: - tty.debug(e) - msg = "Could not write module file [{0}]" - tty.warn(msg.format(x.layout.filename)) - tty.warn("\t--> {0} <--".format(str(e))) + msg = f"{x.layout.filename}: {str(e)}" + errors.append(msg) + + if errors: + errors.insert(0, color.colorize("@*{some module files could not be written}")) + tty.warn("\n".join(errors)) #: Dictionary populated with the list of sub-commands. @@ -368,7 +376,9 @@ def refresh(module_type, specs, args): def modules_cmd(parser, args, module_type, callbacks=callbacks): # Qualifiers to be used when querying the db for specs - constraint_qualifiers = {"refresh": {"installed": True, "known": True}} + constraint_qualifiers = { + "refresh": {"installed": True, "known": lambda x: not spack.repo.PATH.exists(x)} + } query_args = constraint_qualifiers.get(args.subparser_name, {}) # Get the specs that match the query from the DB diff --git a/lib/spack/spack/cmd/modules/lmod.py b/lib/spack/spack/cmd/modules/lmod.py index 2697206a449bbb..f973e9b51534f8 100644 --- a/lib/spack/spack/cmd/modules/lmod.py +++ b/lib/spack/spack/cmd/modules/lmod.py @@ -30,7 +30,7 @@ def add_command(parser, command_dict): def setdefault(module_type, specs, args): - """Set the default module file, when multiple are present""" + """set the default module file, when multiple are present""" # For details on the underlying mechanism see: # # https://lmod.readthedocs.io/en/latest/060_locating.html#marking-a-version-as-default diff --git a/lib/spack/spack/cmd/modules/tcl.py b/lib/spack/spack/cmd/modules/tcl.py index e85645b7437d1d..09f9e1d6c92d7f 100644 --- a/lib/spack/spack/cmd/modules/tcl.py +++ b/lib/spack/spack/cmd/modules/tcl.py @@ -29,7 +29,7 @@ def add_command(parser, command_dict): def setdefault(module_type, specs, args): - """Set the default module file, when multiple are present""" + """set the default module file, when multiple are present""" # Currently, accepts only a single matching spec spack.cmd.modules.one_spec_or_raise(specs) spec = specs[0] diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index ab51281904d139..2d2596c9c53e57 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -7,7 +7,11 @@ import spack.cmd import spack.cmd.common.arguments as arguments +import spack.config +import spack.environment as ev +import spack.package_base import spack.repo +import spack.traverse description = "patch expanded archive sources in preparation for install" section = "build" @@ -21,7 +25,10 @@ def setup_parser(subparser): def patch(parser, args): if not args.specs: - tty.die("patch requires at least one spec argument") + env = ev.active_environment() + if not env: + tty.die("`spack patch` requires a spec or an active environment") + return _patch_env(env) if args.no_checksum: spack.config.set("config:checksum", False, scope="command_line") @@ -29,6 +36,19 @@ def patch(parser, args): if args.deprecated: spack.config.set("config:deprecated", True, scope="command_line") - specs = spack.cmd.parse_specs(args.specs, concretize=True) + specs = spack.cmd.parse_specs(args.specs, concretize=False) for spec in specs: - spec.package.do_patch() + _patch(spack.cmd.matching_spec_from_env(spec).package) + + +def _patch_env(env: ev.Environment): + tty.msg(f"Patching specs from environment {env.name}") + for spec in spack.traverse.traverse_nodes(env.concrete_roots()): + _patch(spec.package) + + +def _patch(pkg: spack.package_base.PackageBase): + pkg.stage.keep = True + with pkg.stage: + pkg.do_patch() + tty.msg(f"Patched {pkg.name} in {pkg.stage.path}") diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 1f1dcdb181833c..9bbd34ca812ae5 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -58,7 +58,7 @@ def setup_parser(subparser): "--type", action="store", default="C", - help="Types of changes to show (A: added, R: removed, " "C: changed); default is 'C'", + help="types of changes to show (A: added, R: removed, C: changed); default is 'C'", ) rm_parser = sp.add_parser("removed", help=pkg_removed.__doc__) @@ -81,7 +81,7 @@ def setup_parser(subparser): "--canonical", action="store_true", default=False, - help="dump canonical source as used by package hash.", + help="dump canonical source as used by package hash", ) arguments.add_common_arguments(source_parser, ["spec"]) @@ -143,7 +143,7 @@ def pkg_source(args): tty.die("spack pkg source requires exactly one spec") spec = specs[0] - filename = spack.repo.path.filename_for_package_name(spec.name) + filename = spack.repo.PATH.filename_for_package_name(spec.name) # regular source dump -- just get the package and print its contents if args.canonical: @@ -184,7 +184,7 @@ def pkg_grep(args, unknown_args): grouper = lambda e: e[0] // 500 # set up iterator and save the first group to ensure we don't end up with a group of size 1 - groups = itertools.groupby(enumerate(spack.repo.path.all_package_paths()), grouper) + groups = itertools.groupby(enumerate(spack.repo.PATH.all_package_paths()), grouper) if not groups: return 0 # no packages to search diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 8c98167ef3cc71..8efd5c2a66bee6 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -17,16 +17,14 @@ def setup_parser(subparser): - subparser.epilog = ( - "If called without argument returns " "the list of all valid virtual packages" - ) + subparser.epilog = "If called without argument returns the list of all valid virtual packages" subparser.add_argument( "virtual_package", nargs="*", help="find packages that provide this virtual package" ) def providers(parser, args): - valid_virtuals = sorted(spack.repo.path.provider_index.providers.keys()) + valid_virtuals = sorted(spack.repo.PATH.provider_index.providers.keys()) buffer = io.StringIO() isatty = sys.stdout.isatty() @@ -55,5 +53,5 @@ def providers(parser, args): for spec in specs: if sys.stdout.isatty(): print("{0}:".format(spec)) - spack.cmd.display_specs(sorted(spack.repo.path.providers_for(spec))) + spack.cmd.display_specs(sorted(spack.repo.PATH.providers_for(spec))) print("") diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py index d8a8ae74669563..e2fd0d559af3e0 100644 --- a/lib/spack/spack/cmd/reindex.py +++ b/lib/spack/spack/cmd/reindex.py @@ -11,4 +11,4 @@ def reindex(parser, args): - spack.store.store.reindex() + spack.store.STORE.reindex() diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index f9fda6c066aa04..7953f227e51bc4 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -20,14 +20,13 @@ def setup_parser(subparser): sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="repo_command") scopes = spack.config.scopes() - scopes_metavar = spack.config.scopes_metavar # Create create_parser = sp.add_parser("create", help=repo_create.__doc__) create_parser.add_argument("directory", help="directory to create the repo in") create_parser.add_argument( "namespace", - help="namespace to identify packages in the repository. " "defaults to the directory name", + help="namespace to identify packages in the repository (defaults to the directory name)", nargs="?", ) create_parser.add_argument( @@ -36,10 +35,8 @@ def setup_parser(subparser): action="store", dest="subdir", default=spack.repo.packages_dir_name, - help=( - "subdirectory to store packages in the repository." - " Default 'packages'. Use an empty string for no subdirectory." - ), + help="subdirectory to store packages in the repository\n\n" + "default 'packages'. use an empty string for no subdirectory", ) # List @@ -47,7 +44,7 @@ def setup_parser(subparser): list_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_list_scope(), help="configuration scope to read from", ) @@ -58,7 +55,7 @@ def setup_parser(subparser): add_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_modify_scope(), help="configuration scope to modify", ) @@ -71,21 +68,21 @@ def setup_parser(subparser): remove_parser.add_argument( "--scope", choices=scopes, - metavar=scopes_metavar, + metavar=spack.config.SCOPES_METAVAR, default=spack.config.default_modify_scope(), help="configuration scope to modify", ) def repo_create(args): - """Create a new package repository.""" + """create a new package repository""" full_path, namespace = spack.repo.create_repo(args.directory, args.namespace, args.subdir) tty.msg("Created repo with namespace '%s'." % namespace) tty.msg("To register it with spack, run this command:", "spack repo add %s" % full_path) def repo_add(args): - """Add a package source to Spack's configuration.""" + """add a package source to Spack's configuration""" path = args.path # real_path is absolute and handles substitution. @@ -116,7 +113,7 @@ def repo_add(args): def repo_remove(args): - """Remove a repository from Spack's configuration.""" + """remove a repository from Spack's configuration""" repos = spack.config.get("repos", scope=args.scope) namespace_or_path = args.namespace_or_path @@ -146,7 +143,7 @@ def repo_remove(args): def repo_list(args): - """Show registered repositories and their namespaces.""" + """show registered repositories and their namespaces""" roots = spack.config.get("repos", scope=args.scope) repos = [] for r in roots: diff --git a/lib/spack/spack/cmd/resource.py b/lib/spack/spack/cmd/resource.py index 66b0c14f67eee2..7e3bb04706410a 100644 --- a/lib/spack/spack/cmd/resource.py +++ b/lib/spack/spack/cmd/resource.py @@ -29,7 +29,7 @@ def setup_parser(subparser): def _show_patch(sha256): """Show a record from the patch index.""" - patches = spack.repo.path.patch_index.index + patches = spack.repo.PATH.patch_index.index data = patches.get(sha256) if not data: @@ -47,7 +47,7 @@ def _show_patch(sha256): owner = rec["owner"] if "relative_path" in rec: - pkg_dir = spack.repo.path.get_pkg_class(owner).package_dir + pkg_dir = spack.repo.PATH.get_pkg_class(owner).package_dir path = os.path.join(pkg_dir, rec["relative_path"]) print(" path: %s" % path) else: @@ -60,7 +60,7 @@ def _show_patch(sha256): def resource_list(args): """list all resources known to spack (currently just patches)""" - patches = spack.repo.path.patch_index.index + patches = spack.repo.PATH.patch_index.index for sha256 in patches: if args.only_hashes: print(sha256) diff --git a/lib/spack/spack/cmd/solve.py b/lib/spack/spack/cmd/solve.py index b20afd74e195e2..4a30b1694b9ac2 100644 --- a/lib/spack/spack/cmd/solve.py +++ b/lib/spack/spack/cmd/solve.py @@ -33,7 +33,7 @@ def setup_parser(subparser): "--show", action="store", default="opt,solutions", - help="select outputs: comma-separated list of: \n" + help="select outputs\n\ncomma-separated list of:\n" " asp asp program text\n" " opt optimization criteria for best model\n" " output raw clingo output\n" @@ -42,7 +42,7 @@ def setup_parser(subparser): ) # Below are arguments w.r.t. spec display (like spack spec) - arguments.add_common_arguments(subparser, ["long", "very_long"]) + arguments.add_common_arguments(subparser, ["long", "very_long", "namespaces"]) install_status_group = subparser.add_mutually_exclusive_group() arguments.add_common_arguments(install_status_group, ["install_status", "no_install_status"]) @@ -73,13 +73,6 @@ def setup_parser(subparser): choices=["nodes", "edges", "paths"], help="how extensively to traverse the DAG (default: nodes)", ) - subparser.add_argument( - "-N", - "--namespaces", - action="store_true", - default=False, - help="show fully qualified package names", - ) subparser.add_argument( "-t", "--types", action="store_true", default=False, help="show dependency types" ) @@ -144,7 +137,7 @@ def solve(parser, args): # these are the same options as `spack spec` install_status_fn = spack.spec.Spec.install_status - fmt = spack.spec.display_format + fmt = spack.spec.DISPLAY_FORMAT if args.namespaces: fmt = "{namespace}." + fmt @@ -183,17 +176,29 @@ def solve(parser, args): output = sys.stdout if "asp" in show else None setup_only = set(show) == {"asp"} unify = spack.config.get("concretizer:unify") + allow_deprecated = spack.config.get("config:deprecated", False) if unify != "when_possible": # set up solver parameters # Note: reuse and other concretizer prefs are passed as configuration result = solver.solve( - specs, out=output, timers=args.timers, stats=args.stats, setup_only=setup_only + specs, + out=output, + timers=args.timers, + stats=args.stats, + setup_only=setup_only, + allow_deprecated=allow_deprecated, ) if not setup_only: _process_result(result, show, required_format, kwargs) else: for idx, result in enumerate( - solver.solve_in_rounds(specs, out=output, timers=args.timers, stats=args.stats) + solver.solve_in_rounds( + specs, + out=output, + timers=args.timers, + stats=args.stats, + allow_deprecated=allow_deprecated, + ) ): if "solutions" in show: tty.msg("ROUND {0}".format(idx)) diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index eef1888e7fdd55..0abd63ff6a69b6 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -29,7 +29,7 @@ def setup_parser(subparser): for further documentation regarding the spec syntax, see: spack help --spec """ - arguments.add_common_arguments(subparser, ["long", "very_long"]) + arguments.add_common_arguments(subparser, ["long", "very_long", "namespaces"]) install_status_group = subparser.add_mutually_exclusive_group() arguments.add_common_arguments(install_status_group, ["install_status", "no_install_status"]) @@ -67,13 +67,6 @@ def setup_parser(subparser): choices=["nodes", "edges", "paths"], help="how extensively to traverse the DAG (default: nodes)", ) - subparser.add_argument( - "-N", - "--namespaces", - action="store_true", - default=False, - help="show fully qualified package names", - ) subparser.add_argument( "-t", "--types", action="store_true", default=False, help="show dependency types" ) @@ -84,7 +77,7 @@ def setup_parser(subparser): def spec(parser, args): install_status_fn = spack.spec.Spec.install_status - fmt = spack.spec.display_format + fmt = spack.spec.DISPLAY_FORMAT if args.namespaces: fmt = "{namespace}." + fmt @@ -100,7 +93,7 @@ def spec(parser, args): # spec in the DAG. This avoids repeatedly querying the DB. tree_context = lang.nullcontext if args.install_status: - tree_context = spack.store.db.read_transaction + tree_context = spack.store.STORE.db.read_transaction # Use command line specified specs, otherwise try to use environment specs. if args.specs: diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index a8d069e6bf55a0..4405ddedca47c9 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -9,9 +9,12 @@ import spack.cmd import spack.cmd.common.arguments as arguments +import spack.config import spack.environment as ev +import spack.package_base import spack.repo import spack.stage +import spack.traverse description = "expand downloaded archive in preparation for install" section = "build" @@ -27,24 +30,18 @@ def setup_parser(subparser): def stage(parser, args): - if not args.specs: - env = ev.active_environment() - if env: - tty.msg("Staging specs from environment %s" % env.name) - for spec in env.specs_by_hash.values(): - for dep in spec.traverse(): - dep.package.do_stage() - tty.msg("Staged {0} in {1}".format(dep.package.name, dep.package.stage.path)) - return - else: - tty.die("`spack stage` requires a spec or an active environment") - if args.no_checksum: spack.config.set("config:checksum", False, scope="command_line") if args.deprecated: spack.config.set("config:deprecated", True, scope="command_line") + if not args.specs: + env = ev.active_environment() + if not env: + tty.die("`spack stage` requires a spec or an active environment") + return _stage_env(env) + specs = spack.cmd.parse_specs(args.specs, concretize=False) # We temporarily modify the working directory when setting up a stage, so we need to @@ -57,7 +54,24 @@ def stage(parser, args): for spec in specs: spec = spack.cmd.matching_spec_from_env(spec) + pkg = spec.package + if custom_path: - spec.package.path = custom_path - spec.package.do_stage() - tty.msg("Staged {0} in {1}".format(spec.package.name, spec.package.stage.path)) + pkg.path = custom_path + + _stage(pkg) + + +def _stage_env(env: ev.Environment): + tty.msg(f"Staging specs from environment {env.name}") + for spec in spack.traverse.traverse_nodes(env.concrete_roots()): + _stage(spec.package) + + +def _stage(pkg: spack.package_base.PackageBase): + # Use context manager to ensure we don't restage while an installation is in progress + # keep = True ensures that the stage is not removed after exiting the context manager + pkg.stage.keep = True + with pkg.stage: + pkg.do_stage() + tty.msg(f"Staged {pkg.name} in {pkg.stage.path}") diff --git a/lib/spack/spack/cmd/tags.py b/lib/spack/spack/cmd/tags.py index 755dc702e1c261..0fbc91b6f4980d 100644 --- a/lib/spack/spack/cmd/tags.py +++ b/lib/spack/spack/cmd/tags.py @@ -5,6 +5,7 @@ import io import sys +import llnl.string import llnl.util.tty as tty import llnl.util.tty.colify as colify @@ -12,7 +13,7 @@ import spack.store import spack.tag -description = "Show package tags and associated packages" +description = "show package tags and associated packages" section = "basic" level = "long" @@ -24,7 +25,7 @@ def report_tags(category, tags): if isatty: num = len(tags) fmt = "{0} package tag".format(category) - buffer.write("{0}:\n".format(spack.util.string.plural(num, fmt))) + buffer.write("{0}:\n".format(llnl.string.plural(num, fmt))) if tags: colify.colify(tags, output=buffer, tty=isatty, indent=4) @@ -68,7 +69,7 @@ def tags(parser, args): return # unique list of available tags - available_tags = sorted(spack.repo.path.tag_index.keys()) + available_tags = sorted(spack.repo.PATH.tag_index.keys()) if not available_tags: tty.msg("No tagged packages") return diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 732861acb2a34d..99c2e906047fc5 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -35,39 +35,35 @@ def setup_parser(subparser): "run", description=test_run.__doc__, help=spack.cmd.first_line(test_run.__doc__) ) - alias_help_msg = "Provide an alias for this test-suite" - alias_help_msg += " for subsequent access." - run_parser.add_argument("--alias", help=alias_help_msg) + run_parser.add_argument( + "--alias", help="provide an alias for this test-suite for subsequent access" + ) run_parser.add_argument( "--fail-fast", action="store_true", - help="Stop tests for each package after the first failure.", + help="stop tests for each package after the first failure", ) run_parser.add_argument( - "--fail-first", action="store_true", help="Stop after the first failed package." + "--fail-first", action="store_true", help="stop after the first failed package" ) run_parser.add_argument( - "--externals", action="store_true", help="Test packages that are externally installed." + "--externals", action="store_true", help="test packages that are externally installed" ) run_parser.add_argument( "-x", "--explicit", action="store_true", - help="Only test packages that are explicitly installed.", + help="only test packages that are explicitly installed", ) run_parser.add_argument( - "--keep-stage", action="store_true", help="Keep testing directory for debugging" + "--keep-stage", action="store_true", help="keep testing directory for debugging" ) arguments.add_common_arguments(run_parser, ["log_format"]) - run_parser.add_argument( - "--log-file", - default=None, - help="filename for the log file. if not passed a default will be used", - ) + run_parser.add_argument("--log-file", default=None, help="filename for the log file") arguments.add_cdash_args(run_parser, False) run_parser.add_argument( - "--help-cdash", action="store_true", help="Show usage instructions for CDash reporting" + "--help-cdash", action="store_true", help="show usage instructions for CDash reporting" ) cd_group = run_parser.add_mutually_exclusive_group() @@ -96,7 +92,7 @@ def setup_parser(subparser): find_parser.add_argument( "filter", nargs=argparse.REMAINDER, - help="optional case-insensitive glob patterns to filter results.", + help="optional case-insensitive glob patterns to filter results", ) # Status @@ -104,7 +100,7 @@ def setup_parser(subparser): "status", description=test_status.__doc__, help=spack.cmd.first_line(test_status.__doc__) ) status_parser.add_argument( - "names", nargs=argparse.REMAINDER, help="Test suites for which to print status" + "names", nargs=argparse.REMAINDER, help="test suites for which to print status" ) # Results @@ -142,15 +138,15 @@ def setup_parser(subparser): ) arguments.add_common_arguments(remove_parser, ["yes_to_all"]) remove_parser.add_argument( - "names", nargs=argparse.REMAINDER, help="Test suites to remove from test stage" + "names", nargs=argparse.REMAINDER, help="test suites to remove from test stage" ) def test_run(args): - """Run tests for the specified installed packages. + """run tests for the specified installed packages - If no specs are listed, run tests for all packages in the current - environment or all installed packages if there is no active environment. + if no specs are listed, run tests for all packages in the current + environment or all installed packages if there is no active environment """ if args.alias: suites = spack.install_test.get_named_test_suites(args.alias) @@ -178,7 +174,7 @@ def test_run(args): specs = spack.cmd.parse_specs(args.specs) if args.specs else [None] specs_to_test = [] for spec in specs: - matching = spack.store.db.query_local(spec, hashes=hashes, explicit=explicit) + matching = spack.store.STORE.db.query_local(spec, hashes=hashes, explicit=explicit) if spec and not matching: tty.warn("No {0}installed packages match spec {1}".format(explicit_str, spec)) """ @@ -231,8 +227,8 @@ def create_reporter(args, specs_to_test, test_suite): def test_list(args): - """List installed packages with available tests.""" - tagged = set(spack.repo.path.packages_with_tags(*args.tag)) if args.tag else set() + """list installed packages with available tests""" + tagged = set(spack.repo.PATH.packages_with_tags(*args.tag)) if args.tag else set() def has_test_and_tags(pkg_class): tests = spack.install_test.test_functions(pkg_class) @@ -241,7 +237,7 @@ def has_test_and_tags(pkg_class): if args.list_all: report_packages = [ pkg_class.name - for pkg_class in spack.repo.path.all_package_classes() + for pkg_class in spack.repo.PATH.all_package_classes() if has_test_and_tags(pkg_class) ] @@ -256,17 +252,17 @@ def has_test_and_tags(pkg_class): env = ev.active_environment() hashes = env.all_hashes() if env else None - specs = spack.store.db.query(hashes=hashes) + specs = spack.store.STORE.db.query(hashes=hashes) specs = list(filter(lambda s: has_test_and_tags(s.package_class), specs)) spack.cmd.display_specs(specs, long=True) def test_find(args): # TODO: merge with status (noargs) - """Find tests that are running or have available results. + """find tests that are running or have available results - Displays aliases for tests that have them, otherwise test suite content - hashes.""" + displays aliases for tests that have them, otherwise test suite content hashes + """ test_suites = spack.install_test.get_all_test_suites() # Filter tests by filter argument @@ -302,7 +298,7 @@ def match(t, f): def test_status(args): - """Get the current status for the specified Spack test suite(s).""" + """get the current status for the specified Spack test suite(s)""" if args.names: test_suites = [] for name in args.names: @@ -333,7 +329,7 @@ def _report_suite_results(test_suite, args, constraints): qspecs = spack.cmd.parse_specs(constraints) specs = {} for spec in qspecs: - for s in spack.store.db.query(spec, installed=True): + for s in spack.store.STORE.db.query(spec, installed=True): specs[s.dag_hash()] = s specs = sorted(specs.values()) test_specs = dict((test_suite.test_pkg_id(s), s) for s in test_suite.specs if s in specs) @@ -387,7 +383,7 @@ def _report_suite_results(test_suite, args, constraints): def test_results(args): - """Get the results from Spack test suite(s) (default all).""" + """get the results from Spack test suite(s) (default all)""" if args.names: try: sep_index = args.names.index("--") @@ -414,12 +410,13 @@ def test_results(args): def test_remove(args): - """Remove results from Spack test suite(s) (default all). + """remove results from Spack test suite(s) (default all) - If no test suite is listed, remove results for all suites. + if no test suite is listed, remove results for all suites. - Removed tests can no longer be accessed for results or status, and will not - appear in `spack test list` results.""" + removed tests can no longer be accessed for results or status, and will not + appear in `spack test list` results + """ if args.names: test_suites = [] for name in args.names: diff --git a/lib/spack/spack/cmd/test_env.py b/lib/spack/spack/cmd/test_env.py index 049df9d5c0494b..070b766248d8dc 100644 --- a/lib/spack/spack/cmd/test_env.py +++ b/lib/spack/spack/cmd/test_env.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spack.cmd.common.env_utility as env_utility +from spack.context import Context description = ( "run a command in a spec's test environment, or dump its environment to screen or file" @@ -14,4 +15,4 @@ def test_env(parser, args): - env_utility.emulate_env_utility("test-env", "test", args) + env_utility.emulate_env_utility("test-env", Context.TEST, args) diff --git a/lib/spack/spack/cmd/tutorial.py b/lib/spack/spack/cmd/tutorial.py index a07824f922a74d..5759912b66ffc7 100644 --- a/lib/spack/spack/cmd/tutorial.py +++ b/lib/spack/spack/cmd/tutorial.py @@ -23,7 +23,7 @@ # tutorial configuration parameters -tutorial_branch = "releases/v0.20" +tutorial_branch = "releases/v0.21" tutorial_mirror = "file:///mirror" tutorial_key = os.path.join(spack.paths.share_path, "keys", "tutorial.pub") diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 4607d1c5d5d64c..3288404151d230 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -11,10 +11,9 @@ import spack.cmd import spack.cmd.common.arguments as arguments +import spack.cmd.common.confirmation as confirmation import spack.environment as ev -import spack.error import spack.package_base -import spack.repo import spack.spec import spack.store import spack.traverse as traverse @@ -54,7 +53,7 @@ def setup_parser(subparser): "--force", action="store_true", dest="force", - help="remove regardless of whether other packages or environments " "depend on this one", + help="remove regardless of whether other packages or environments depend on this one", ) subparser.add_argument( "--remove", @@ -103,7 +102,7 @@ def find_matching_specs( has_errors = False for spec in specs: install_query = [InstallStatuses.INSTALLED, InstallStatuses.DEPRECATED] - matching = spack.store.db.query_local( + matching = spack.store.STORE.db.query_local( spec, hashes=hashes, installed=install_query, origin=origin ) # For each spec provided, make sure it refers to only one package. @@ -139,7 +138,7 @@ def installed_dependents(specs: List[spack.spec.Spec]) -> List[spack.spec.Spec]: # input; in that case we return an empty list. def is_installed(spec): - record = spack.store.db.query_local_by_spec_hash(spec.dag_hash()) + record = spack.store.STORE.db.query_local_by_spec_hash(spec.dag_hash()) return record and record.installed specs = traverse.traverse_nodes( @@ -278,7 +277,7 @@ def uninstall_specs(args, specs): return if not args.yes_to_all: - confirm_removal(uninstall_list) + confirmation.confirm_action(uninstall_list, "uninstalled", "uninstallation") # Uninstall everything on the list do_uninstall(uninstall_list, args.force) @@ -292,21 +291,6 @@ def uninstall_specs(args, specs): env.regenerate_views() -def confirm_removal(specs: List[spack.spec.Spec]): - """Display the list of specs to be removed and ask for confirmation. - - Args: - specs: specs to be removed - """ - tty.msg("The following {} packages will be uninstalled:\n".format(len(specs))) - spack.cmd.display_specs(specs, **display_args) - print("") - answer = tty.get_yes_or_no("Do you want to proceed?", default=False) - if not answer: - tty.msg("Aborting uninstallation") - sys.exit(0) - - def uninstall(parser, args): if not args.specs and not args.all: tty.die( diff --git a/lib/spack/spack/cmd/unit_test.py b/lib/spack/spack/cmd/unit_test.py index 615f72bd8f24d3..4963ad14c64c28 100644 --- a/lib/spack/spack/cmd/unit_test.py +++ b/lib/spack/spack/cmd/unit_test.py @@ -209,12 +209,11 @@ def unit_test(parser, args, unknown_args): # mock configuration used by unit tests # Note: skip on windows here because for the moment, # clingo is wholly unsupported from bootstrap - if sys.platform != "win32": - with spack.bootstrap.ensure_bootstrap_configuration(): - spack.bootstrap.ensure_core_dependencies() - if pytest is None: - spack.bootstrap.ensure_environment_dependencies() - import pytest + with spack.bootstrap.ensure_bootstrap_configuration(): + spack.bootstrap.ensure_core_dependencies() + if pytest is None: + spack.bootstrap.ensure_environment_dependencies() + import pytest if args.pytest_help: # make the pytest.main help output more accurate diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py index 9b076ffd66f1b8..7fe634c56de91e 100644 --- a/lib/spack/spack/cmd/unload.py +++ b/lib/spack/spack/cmd/unload.py @@ -51,17 +51,24 @@ def setup_parser(subparser): const="bat", help="print bat commands to load the package", ) + shells.add_argument( + "--pwsh", + action="store_const", + dest="shell", + const="pwsh", + help="print pwsh commands to load the package", + ) subparser.add_argument( - "-a", "--all", action="store_true", help="unload all loaded Spack packages." + "-a", "--all", action="store_true", help="unload all loaded Spack packages" ) def unload(parser, args): - """Unload spack packages from the user environment.""" + """unload spack packages from the user environment""" if args.specs and args.all: raise spack.error.SpackError( - "Cannot specify specs on command line" " when unloading all specs with '--all'" + "Cannot specify specs on command line when unloading all specs with '--all'" ) hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(":") @@ -71,7 +78,7 @@ def unload(parser, args): for spec in spack.cmd.parse_specs(args.specs) ] else: - specs = spack.store.db.query(hashes=hashes) + specs = spack.store.STORE.db.query(hashes=hashes) if not args.shell: specs_str = " ".join(args.specs) or "SPECS" @@ -81,9 +88,8 @@ def unload(parser, args): ) return 1 - env_mod = spack.util.environment.EnvironmentModifications() + env_mod = uenv.environment_modifications_for_specs(*specs).reversed() for spec in specs: - env_mod.extend(uenv.environment_modifications_for_spec(spec).reversed()) env_mod.remove_path(uenv.spack_loaded_hashes_var, spec.dag_hash()) cmds = env_mod.shell_modifications(args.shell) diff --git a/lib/spack/spack/cmd/url.py b/lib/spack/spack/cmd/url.py index 49339d420296fe..25f8ad382a6058 100644 --- a/lib/spack/spack/cmd/url.py +++ b/lib/spack/spack/cmd/url.py @@ -12,6 +12,7 @@ import spack.fetch_strategy as fs import spack.repo import spack.spec +import spack.url import spack.util.crypto as crypto from spack.url import ( UndetectableNameError, @@ -26,7 +27,6 @@ substitution_offsets, ) from spack.util.naming import simplify_name -from spack.util.web import find_versions_of_archive description = "debugging tool for url parsing" section = "developer" @@ -139,7 +139,7 @@ def url_parse(args): if args.spider: print() tty.msg("Spidering for versions:") - versions = find_versions_of_archive(url) + versions = spack.url.find_versions_of_archive(url) if not versions: print(" Found no versions for {0}".format(name)) @@ -155,7 +155,7 @@ def url_list(args): urls = set() # Gather set of URLs from all packages - for pkg_cls in spack.repo.path.all_package_classes(): + for pkg_cls in spack.repo.PATH.all_package_classes(): url = getattr(pkg_cls, "url", None) urls = url_list_parsing(args, urls, url, pkg_cls) @@ -192,7 +192,7 @@ def url_summary(args): tty.msg("Generating a summary of URL parsing in Spack...") # Loop through all packages - for pkg_cls in spack.repo.path.all_package_classes(): + for pkg_cls in spack.repo.PATH.all_package_classes(): urls = set() pkg = pkg_cls(spack.spec.Spec(pkg_cls.name)) @@ -336,7 +336,7 @@ def add(self, pkg_name, fetcher): version_stats = UrlStats() resource_stats = UrlStats() - for pkg_cls in spack.repo.path.all_package_classes(): + for pkg_cls in spack.repo.PATH.all_package_classes(): npkgs += 1 for v in pkg_cls.versions: diff --git a/lib/spack/spack/cmd/verify.py b/lib/spack/spack/cmd/verify.py index 026e025d363a4b..88a52b691858f4 100644 --- a/lib/spack/spack/cmd/verify.py +++ b/lib/spack/spack/cmd/verify.py @@ -10,7 +10,7 @@ import spack.store import spack.verify -description = "Check that all spack packages are on disk as installed" +description = "check that all spack packages are on disk as installed" section = "admin" level = "long" @@ -19,14 +19,14 @@ def setup_parser(subparser): setup_parser.parser = subparser subparser.add_argument( - "-l", "--local", action="store_true", help="Verify only locally installed packages" + "-l", "--local", action="store_true", help="verify only locally installed packages" ) subparser.add_argument( - "-j", "--json", action="store_true", help="Ouptut json-formatted errors" + "-j", "--json", action="store_true", help="ouptut json-formatted errors" ) - subparser.add_argument("-a", "--all", action="store_true", help="Verify all packages") + subparser.add_argument("-a", "--all", action="store_true", help="verify all packages") subparser.add_argument( - "specs_or_files", nargs=argparse.REMAINDER, help="Specs or files to verify" + "specs_or_files", nargs=argparse.REMAINDER, help="specs or files to verify" ) type = subparser.add_mutually_exclusive_group() @@ -37,7 +37,7 @@ def setup_parser(subparser): const="specs", dest="type", default="specs", - help="Treat entries as specs (default)", + help="treat entries as specs (default)", ) type.add_argument( "-f", @@ -46,7 +46,7 @@ def setup_parser(subparser): const="files", dest="type", default="specs", - help="Treat entries as absolute filenames. Cannot be used with '-a'", + help="treat entries as absolute filenames\n\ncannot be used with '-a'", ) @@ -71,7 +71,7 @@ def verify(parser, args): spec_args = spack.cmd.parse_specs(args.specs_or_files) if args.all: - query = spack.store.db.query_local if local else spack.store.db.query + query = spack.store.STORE.db.query_local if local else spack.store.STORE.db.query # construct spec list if spec_args: diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py index 601e10b7e5570e..9ac6c9e4da72ba 100644 --- a/lib/spack/spack/cmd/versions.py +++ b/lib/spack/spack/cmd/versions.py @@ -26,7 +26,7 @@ def setup_parser(subparser): output.add_argument( "--safe-only", action="store_true", - help="[deprecated] only list safe versions " "of the package", + help="[deprecated] only list safe versions of the package", ) output.add_argument( "-r", "--remote", action="store_true", help="only list remote versions of the package" @@ -35,17 +35,14 @@ def setup_parser(subparser): "-n", "--new", action="store_true", - help="only list remote versions newer than " "the latest checksummed version", + help="only list remote versions newer than the latest checksummed version", ) - subparser.add_argument( - "-c", "--concurrency", default=32, type=int, help="number of concurrent requests" - ) - arguments.add_common_arguments(subparser, ["package"]) + arguments.add_common_arguments(subparser, ["package", "jobs"]) def versions(parser, args): spec = spack.spec.Spec(args.package) - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) pkg = pkg_cls(spec) safe_versions = pkg.versions @@ -68,7 +65,7 @@ def versions(parser, args): if args.safe: return - fetched_versions = pkg.fetch_remote_versions(args.concurrency) + fetched_versions = pkg.fetch_remote_versions(args.jobs) if args.new: if sys.stdout.isatty(): diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 522df616cd9f89..de847bd9f655fe 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -44,7 +44,7 @@ from spack.filesystem_view import YamlFilesystemView, view_func_parser from spack.util import spack_yaml as s_yaml -description = "project packages to a compact naming scheme on the filesystem." +description = "project packages to a compact naming scheme on the filesystem" section = "environments" level = "short" @@ -70,7 +70,7 @@ def squash(matching_specs): return matching_in_view[0] if matching_in_view else matching_specs[0] # make function always return a list to keep consistency between py2/3 - return list(map(squash, map(spack.store.db.query, specs))) + return list(map(squash, map(spack.store.STORE.db.query, specs))) def setup_parser(sp): @@ -81,7 +81,7 @@ def setup_parser(sp): "--verbose", action="store_true", default=False, - help="If not verbose only warnings/errors will be printed.", + help="if not verbose only warnings/errors will be printed", ) sp.add_argument( "-e", @@ -95,7 +95,7 @@ def setup_parser(sp): "--dependencies", choices=["true", "false", "yes", "no"], default="true", - help="Link/remove/list dependencies.", + help="link/remove/list dependencies", ) ssp = sp.add_subparsers(metavar="ACTION", dest="action") @@ -137,12 +137,11 @@ def setup_parser(sp): if cmd in ("symlink", "hardlink", "copy"): # invalid for remove/statlink, for those commands the view needs to # already know its own projections. - help_msg = "Initialize view using projections from file." act.add_argument( "--projection-file", dest="projection_file", type=spack.cmd.extant_file, - help=help_msg, + help="initialize view using projections from file", ) if cmd == "remove": @@ -150,7 +149,7 @@ def setup_parser(sp): act.add_argument( "--no-remove-dependents", action="store_true", - help="Do not remove dependents of specified specs.", + help="do not remove dependents of specified specs", ) # with all option, spec is an optional argument @@ -201,7 +200,7 @@ def view(parser, args): view = YamlFilesystemView( path, - spack.store.layout, + spack.store.STORE.layout, projections=ordered_projections, ignore_conflicts=getattr(args, "ignore_conflicts", False), link=link_fn, diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 396ec6b2fb3a15..70da3ca2c42712 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -13,6 +13,7 @@ import tempfile from typing import List, Optional, Sequence +import llnl.path import llnl.util.lang import llnl.util.tty as tty from llnl.util.filesystem import path_contains_subdirectory, paths_containing_libs @@ -24,7 +25,6 @@ import spack.util.module_cmd import spack.version from spack.util.environment import filter_system_paths -from spack.util.path import system_path_filter __all__ = ["Compiler"] @@ -39,10 +39,17 @@ def _get_compiler_version_output(compiler_path, version_arg, ignore_errors=()): version_arg (str): the argument used to extract version information """ compiler = spack.util.executable.Executable(compiler_path) + compiler_invocation_args = { + "output": str, + "error": str, + "ignore_errors": ignore_errors, + "timeout": 120, + "fail_on_error": True, + } if version_arg: - output = compiler(version_arg, output=str, error=str, ignore_errors=ignore_errors) + output = compiler(version_arg, **compiler_invocation_args) else: - output = compiler(output=str, error=str, ignore_errors=ignore_errors) + output = compiler(**compiler_invocation_args) return output @@ -153,7 +160,7 @@ def _parse_link_paths(string): return implicit_link_dirs -@system_path_filter +@llnl.path.system_path_filter def _parse_non_system_link_dirs(string: str) -> List[str]: """Parses link paths out of compiler debug output. @@ -229,6 +236,9 @@ class Compiler: # by any compiler _all_compiler_rpath_libraries = ["libc", "libc++", "libstdc++"] + #: Platform matcher for Platform objects supported by compiler + is_supported_on_platform = lambda x: True + # Default flags used by a compiler to set an rpath @property def cc_rpath_arg(self): @@ -594,8 +604,6 @@ def search_regexps(cls, language): compiler_names = getattr(cls, "{0}_names".format(language)) prefixes = [""] + cls.prefixes suffixes = [""] - # Windows compilers generally have an extension of some sort - # as do most files on Windows, handle that case here if sys.platform == "win32": ext = r"\.(?:exe|bat)" cls_suf = [suf + ext for suf in cls.suffixes] diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 29e4cd85503407..6366fc321b3d06 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -10,7 +10,7 @@ import itertools import multiprocessing.pool import os -from typing import Dict +from typing import Dict, List, Optional, Tuple import archspec.cpu @@ -21,6 +21,7 @@ import spack.compiler import spack.config import spack.error +import spack.operating_systems import spack.paths import spack.platforms import spack.spec @@ -135,7 +136,7 @@ def _init_compiler_config(*, scope): def compiler_config_files(): config_files = list() - config = spack.config.config + config = spack.config.CONFIG for scope in config.file_scopes: name = scope.name compiler_config = config.get("compilers", scope=name) @@ -169,7 +170,7 @@ def remove_compiler_from_config(compiler_spec, scope=None): """ candidate_scopes = [scope] if scope is None: - candidate_scopes = spack.config.config.scopes.keys() + candidate_scopes = spack.config.CONFIG.scopes.keys() removal_happened = False for current_scope in candidate_scopes: @@ -223,13 +224,16 @@ def all_compiler_specs(scope=None, init_config=True): ] -def find_compilers(path_hints=None): +def find_compilers( + path_hints: Optional[List[str]] = None, *, mixed_toolchain=False +) -> List["spack.compiler.Compiler"]: """Return the list of compilers found in the paths given as arguments. Args: - path_hints (list or None): list of path hints where to look for. - A sensible default based on the ``PATH`` environment variable - will be used if the value is None + path_hints: list of path hints where to look for. A sensible default based on the ``PATH`` + environment variable will be used if the value is None + mixed_toolchain: allow mixing compilers from different toolchains if otherwise missing for + a certain language """ if path_hints is None: path_hints = get_path("PATH") @@ -250,7 +254,7 @@ def find_compilers(path_hints=None): finally: tp.close() - def valid_version(item): + def valid_version(item: Tuple[Optional[DetectVersionArgs], Optional[str]]) -> bool: value, error = item if error is None: return True @@ -262,25 +266,37 @@ def valid_version(item): pass return False - def remove_errors(item): + def remove_errors( + item: Tuple[Optional[DetectVersionArgs], Optional[str]] + ) -> DetectVersionArgs: value, _ = item + assert value is not None return value - return make_compiler_list(map(remove_errors, filter(valid_version, detected_versions))) + return make_compiler_list( + [remove_errors(detected) for detected in detected_versions if valid_version(detected)], + mixed_toolchain=mixed_toolchain, + ) -def find_new_compilers(path_hints=None, scope=None): +def find_new_compilers( + path_hints: Optional[List[str]] = None, + scope: Optional[str] = None, + *, + mixed_toolchain: bool = False, +): """Same as ``find_compilers`` but return only the compilers that are not already in compilers.yaml. Args: - path_hints (list or None): list of path hints where to look for. - A sensible default based on the ``PATH`` environment variable - will be used if the value is None - scope (str): scope to look for a compiler. If None consider the - merged configuration. + path_hints: list of path hints where to look for. A sensible default based on the ``PATH`` + environment variable will be used if the value is None + scope: scope to look for a compiler. If None consider the merged configuration. + mixed_toolchain: allow mixing compilers from different toolchains if otherwise missing for + a certain language """ - compilers = find_compilers(path_hints) + compilers = find_compilers(path_hints, mixed_toolchain=mixed_toolchain) + return select_new_compilers(compilers, scope) @@ -298,7 +314,7 @@ def select_new_compilers(compilers, scope=None): return compilers_not_in_config -def supported_compilers(): +def supported_compilers() -> List[str]: """Return a set of names of compilers supported by Spack. See available_compilers() to get a list of all the available @@ -306,10 +322,41 @@ def supported_compilers(): """ # Hack to be able to call the compiler `apple-clang` while still # using a valid python name for the module - return sorted( - name if name != "apple_clang" else "apple-clang" - for name in llnl.util.lang.list_modules(spack.paths.compilers_path) - ) + return sorted(all_compiler_names()) + + +def supported_compilers_for_host_platform() -> List[str]: + """Return a set of compiler class objects supported by Spack + that are also supported by the current host platform + """ + host_plat = spack.platforms.real_host() + return supported_compilers_for_platform(host_plat) + + +def supported_compilers_for_platform(platform: spack.platforms.Platform) -> List[str]: + """Return a set of compiler class objects supported by Spack + that are also supported by the provided platform + + Args: + platform (str): string representation of platform + for which compiler compatability should be determined + """ + return [ + name + for name in supported_compilers() + if class_for_compiler_name(name).is_supported_on_platform(platform) + ] + + +def all_compiler_names() -> List[str]: + def replace_apple_clang(name): + return name if name != "apple_clang" else "apple-clang" + + return [replace_apple_clang(name) for name in all_compiler_module_names()] + + +def all_compiler_module_names() -> List[str]: + return [name for name in llnl.util.lang.list_modules(spack.paths.compilers_path)] @_auto_compiler_spec @@ -515,7 +562,7 @@ def compiler_for_spec(compiler_spec, arch_spec): if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, arch_spec.os) if len(compilers) > 1: - msg = "Multiple definitions of compiler %s" % compiler_spec + msg = "Multiple definitions of compiler %s " % compiler_spec msg += "for architecture %s:\n %s" % (arch_spec, compilers) tty.debug(msg) return compilers[0] @@ -523,7 +570,7 @@ def compiler_for_spec(compiler_spec, arch_spec): @_auto_compiler_spec def get_compiler_duplicates(compiler_spec, arch_spec): - config = spack.config.config + config = spack.config.CONFIG scope_to_compilers = {} for scope in config.scopes: @@ -607,7 +654,9 @@ def all_compiler_types(): ) -def arguments_to_detect_version_fn(operating_system, paths): +def arguments_to_detect_version_fn( + operating_system: spack.operating_systems.OperatingSystem, paths: List[str] +) -> List[DetectVersionArgs]: """Returns a list of DetectVersionArgs tuples to be used in a corresponding function to detect compiler versions. @@ -615,8 +664,7 @@ def arguments_to_detect_version_fn(operating_system, paths): function by providing a method called with the same name. Args: - operating_system (spack.operating_systems.OperatingSystem): the operating system - on which we are looking for compilers + operating_system: the operating system on which we are looking for compilers paths: paths to search for compilers Returns: @@ -625,10 +673,10 @@ def arguments_to_detect_version_fn(operating_system, paths): compilers in this OS. """ - def _default(search_paths): - command_arguments = [] + def _default(search_paths: List[str]) -> List[DetectVersionArgs]: + command_arguments: List[DetectVersionArgs] = [] files_to_be_tested = fs.files_in(*search_paths) - for compiler_name in spack.compilers.supported_compilers(): + for compiler_name in supported_compilers_for_host_platform(): compiler_cls = class_for_compiler_name(compiler_name) for language in ("cc", "cxx", "f77", "fc"): @@ -653,7 +701,9 @@ def _default(search_paths): return fn(paths) -def detect_version(detect_version_args): +def detect_version( + detect_version_args: DetectVersionArgs, +) -> Tuple[Optional[DetectVersionArgs], Optional[str]]: """Computes the version of a compiler and adds it to the information passed as input. @@ -662,8 +712,7 @@ def detect_version(detect_version_args): needs to be checked by the code dispatching the calls. Args: - detect_version_args (DetectVersionArgs): information on the - compiler for which we should detect the version. + detect_version_args: information on the compiler for which we should detect the version. Returns: A ``(DetectVersionArgs, error)`` tuple. If ``error`` is ``None`` the @@ -679,7 +728,7 @@ def _default(fn_args): path = fn_args.path # Get compiler names and the callback to detect their versions - callback = getattr(compiler_cls, "{0}_version".format(language)) + callback = getattr(compiler_cls, f"{language}_version") try: version = callback(path) @@ -687,9 +736,11 @@ def _default(fn_args): value = fn_args._replace(id=compiler_id._replace(version=version)) return value, None - error = "Couldn't get version for compiler {0}".format(path) + error = f"Couldn't get version for compiler {path}".format(path) except spack.util.executable.ProcessError as e: - error = "Couldn't get version for compiler {0}\n".format(path) + str(e) + error = f"Couldn't get version for compiler {path}\n" + str(e) + except spack.util.executable.ProcessTimeoutError as e: + error = f"Couldn't get version for compiler {path}\n" + str(e) except Exception as e: # Catching "Exception" here is fine because it just # means something went wrong running a candidate executable. @@ -703,13 +754,15 @@ def _default(fn_args): return fn(detect_version_args) -def make_compiler_list(detected_versions): +def make_compiler_list( + detected_versions: List[DetectVersionArgs], mixed_toolchain: bool = False +) -> List["spack.compiler.Compiler"]: """Process a list of detected versions and turn them into a list of compiler specs. Args: - detected_versions (list): list of DetectVersionArgs containing a - valid version + detected_versions: list of DetectVersionArgs containing a valid version + mixed_toolchain: allow mixing compilers from different toolchains if langauge is missing Returns: list: list of Compiler objects @@ -718,7 +771,7 @@ def make_compiler_list(detected_versions): sorted_compilers = sorted(detected_versions, key=group_fn) # Gather items in a dictionary by the id, name variation and language - compilers_d = {} + compilers_d: Dict[CompilerID, Dict[NameVariation, dict]] = {} for sort_key, group in itertools.groupby(sorted_compilers, key=group_fn): compiler_id, name_variation, language = sort_key by_compiler_id = compilers_d.setdefault(compiler_id, {}) @@ -727,7 +780,7 @@ def make_compiler_list(detected_versions): def _default_make_compilers(cmp_id, paths): operating_system, compiler_name, version = cmp_id - compiler_cls = spack.compilers.class_for_compiler_name(compiler_name) + compiler_cls = class_for_compiler_name(compiler_name) spec = spack.spec.CompilerSpec(compiler_cls.name, f"={version}") paths = [paths.get(x, None) for x in ("cc", "cxx", "f77", "fc")] # TODO: johnwparent - revist the following line as per discussion at: @@ -749,13 +802,14 @@ def _default_make_compilers(cmp_id, paths): getattr(variation, "suffix", None), ) - compilers = [] + # Flatten to a list of compiler id, primary variation and compiler dictionary + flat_compilers: List[Tuple[CompilerID, NameVariation, dict]] = [] for compiler_id, by_compiler_id in compilers_d.items(): ordered = sorted(by_compiler_id, key=sort_fn) selected_variation = ordered[0] selected = by_compiler_id[selected_variation] - # fill any missing parts from subsequent entries + # Fill any missing parts from subsequent entries (without mixing toolchains) for lang in ["cxx", "f77", "fc"]: if lang not in selected: next_lang = next( @@ -764,14 +818,63 @@ def _default_make_compilers(cmp_id, paths): if next_lang: selected[lang] = next_lang - operating_system, _, _ = compiler_id - make_compilers = getattr(operating_system, "make_compilers", _default_make_compilers) + flat_compilers.append((compiler_id, selected_variation, selected)) - compilers.extend(make_compilers(compiler_id, selected)) + # Next, fill out the blanks of missing compilers by creating a mixed toolchain (if requested) + if mixed_toolchain: + make_mixed_toolchain(flat_compilers) + + # Finally, create the compiler list + compilers = [] + for compiler_id, _, compiler in flat_compilers: + make_compilers = getattr(compiler_id.os, "make_compilers", _default_make_compilers) + compilers.extend(make_compilers(compiler_id, compiler)) return compilers +def make_mixed_toolchain(compilers: List[Tuple[CompilerID, NameVariation, dict]]) -> None: + """Add missing compilers across toolchains when they are missing for a particular language. + This currently only adds the most sensible gfortran to (apple)-clang if it doesn't have a + fortran compiler (no flang).""" + + # First collect the clangs that are missing a fortran compiler + clangs_without_flang = [ + (id, variation, compiler) + for id, variation, compiler in compilers + if id.compiler_name in ("clang", "apple-clang") + and "f77" not in compiler + and "fc" not in compiler + ] + if not clangs_without_flang: + return + + # Filter on GCCs with fortran compiler + gccs_with_fortran = [ + (id, variation, compiler) + for id, variation, compiler in compilers + if id.compiler_name == "gcc" and "f77" in compiler and "fc" in compiler + ] + + # Sort these GCCs by "best variation" (no prefix / suffix first) + gccs_with_fortran.sort( + key=lambda x: (getattr(x[1], "prefix", None), getattr(x[1], "suffix", None)) + ) + + # Attach the optimal GCC fortran compiler to the clangs that don't have one + for clang_id, _, clang_compiler in clangs_without_flang: + gcc_compiler = next( + (gcc[2] for gcc in gccs_with_fortran if gcc[0].os == clang_id.os), None + ) + + if not gcc_compiler: + continue + + # Update the fc / f77 entries + clang_compiler["f77"] = gcc_compiler["f77"] + clang_compiler["fc"] = gcc_compiler["fc"] + + def is_mixed_toolchain(compiler): """Returns True if the current compiler is a mixed toolchain, False otherwise. diff --git a/lib/spack/spack/compilers/aocc.py b/lib/spack/spack/compilers/aocc.py index 51f7b02e2bd987..326522c93cfc91 100644 --- a/lib/spack/spack/compilers/aocc.py +++ b/lib/spack/spack/compilers/aocc.py @@ -5,7 +5,6 @@ import os import re -import sys import llnl.util.lang @@ -112,17 +111,7 @@ def extract_version_from_output(cls, output): match = re.search(r"AOCC_(\d+)[._](\d+)[._](\d+)", output) if match: return ".".join(match.groups()) - - @classmethod - def fc_version(cls, fortran_compiler): - if sys.platform == "darwin": - return cls.default_version("clang") - - return cls.default_version(fortran_compiler) - - @classmethod - def f77_version(cls, f77): - return cls.fc_version(f77) + return "unknown" @property def stdcxx_libs(self): diff --git a/lib/spack/spack/compilers/apple_clang.py b/lib/spack/spack/compilers/apple_clang.py index 1373ac0bf2dc8f..e81703b2d2d4f5 100644 --- a/lib/spack/spack/compilers/apple_clang.py +++ b/lib/spack/spack/compilers/apple_clang.py @@ -2,13 +2,9 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import os.path import re -import shutil import llnl.util.lang -import llnl.util.tty as tty -from llnl.util.symlink import symlink import spack.compiler import spack.compilers.clang @@ -119,108 +115,3 @@ def c23_flag(self): self, "the C23 standard", "c23_flag", "< 11.0.3" ) return "-std=c2x" - - def setup_custom_environment(self, pkg, env): - """Set the DEVELOPER_DIR environment for the Xcode toolchain. - - On macOS, not all buildsystems support querying CC and CXX for the - compilers to use and instead query the Xcode toolchain for what - compiler to run. This side-steps the spack wrappers. In order to inject - spack into this setup, we need to copy (a subset of) Xcode.app and - replace the compiler executables with symlinks to the spack wrapper. - Currently, the stage is used to store the Xcode.app copies. We then set - the 'DEVELOPER_DIR' environment variables to cause the xcrun and - related tools to use this Xcode.app. - """ - super().setup_custom_environment(pkg, env) - - if not pkg.use_xcode: - # if we do it for all packages, we get into big troubles with MPI: - # filter_compilers(self) will use mockup XCode compilers on macOS - # with Clang. Those point to Spack's compiler wrappers and - # consequently render MPI non-functional outside of Spack. - return - - # Use special XCode versions of compiler wrappers when using XCode - # Overwrites build_environment's setting of SPACK_CC and SPACK_CXX - xcrun = spack.util.executable.Executable("xcrun") - xcode_clang = xcrun("-f", "clang", output=str).strip() - xcode_clangpp = xcrun("-f", "clang++", output=str).strip() - env.set("SPACK_CC", xcode_clang, force=True) - env.set("SPACK_CXX", xcode_clangpp, force=True) - - xcode_select = spack.util.executable.Executable("xcode-select") - - # Get the path of the active developer directory - real_root = xcode_select("--print-path", output=str).strip() - - # The path name can be used to determine whether the full Xcode suite - # or just the command-line tools are installed - if real_root.endswith("Developer"): - # The full Xcode suite is installed - pass - else: - if real_root.endswith("CommandLineTools"): - # Only the command-line tools are installed - msg = "It appears that you have the Xcode command-line tools " - msg += "but not the full Xcode suite installed.\n" - - else: - # Xcode is not installed - msg = "It appears that you do not have Xcode installed.\n" - - msg += "In order to use Spack to build the requested application, " - msg += "you need the full Xcode suite. It can be installed " - msg += "through the App Store. Make sure you launch the " - msg += "application and accept the license agreement.\n" - - raise OSError(msg) - - real_root = os.path.dirname(os.path.dirname(real_root)) - developer_root = os.path.join( - spack.stage.get_stage_root(), "xcode-select", self.name, str(self.version) - ) - xcode_link = os.path.join(developer_root, "Xcode.app") - - if not os.path.exists(developer_root): - tty.warn( - "Copying Xcode from %s to %s in order to add spack " - "wrappers to it. Please do not interrupt." % (real_root, developer_root) - ) - - # We need to make a new Xcode.app instance, but with symlinks to - # the spack wrappers for the compilers it ships. This is necessary - # because some projects insist on just asking xcrun and related - # tools where the compiler runs. These tools are very hard to trick - # as they do realpath and end up ignoring the symlinks in a - # "softer" tree of nothing but symlinks in the right places. - shutil.copytree( - real_root, - developer_root, - symlinks=True, - ignore=shutil.ignore_patterns( - "AppleTV*.platform", - "Watch*.platform", - "iPhone*.platform", - "Documentation", - "swift*", - ), - ) - - real_dirs = ["Toolchains/XcodeDefault.xctoolchain/usr/bin", "usr/bin"] - - bins = ["c++", "c89", "c99", "cc", "clang", "clang++", "cpp"] - - for real_dir in real_dirs: - dev_dir = os.path.join(developer_root, "Contents", "Developer", real_dir) - for fname in os.listdir(dev_dir): - if fname in bins: - os.unlink(os.path.join(dev_dir, fname)) - symlink( - os.path.join(spack.paths.build_env_path, "cc"), - os.path.join(dev_dir, fname), - ) - - symlink(developer_root, xcode_link) - - env.set("DEVELOPER_DIR", xcode_link) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index a9356227de5d76..71837bfe5e1c81 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -5,7 +5,6 @@ import os import re -import sys import llnl.util.lang @@ -39,10 +38,10 @@ class Clang(Compiler): cxx_names = ["clang++"] # Subclasses use possible names of Fortran 77 compiler - f77_names = ["flang", "gfortran", "xlf_r"] + f77_names = ["flang"] # Subclasses use possible names of Fortran 90 compiler - fc_names = ["flang", "gfortran", "xlf90_r"] + fc_names = ["flang"] version_argument = "--version" @@ -182,16 +181,3 @@ def extract_version_from_output(cls, output): if match: ver = match.group(match.lastindex) return ver - - @classmethod - def fc_version(cls, fc): - # We could map from gcc/gfortran version to clang version, but on macOS - # we normally mix any version of gfortran with any version of clang. - if sys.platform == "darwin": - return cls.default_version("clang") - else: - return cls.default_version(fc) - - @classmethod - def f77_version(cls, f77): - return cls.fc_version(f77) diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index ae8d5aa97efc16..adef8a8277dd2f 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -99,6 +99,28 @@ def cxx17_flag(self): else: return "-std=c++17" + @property + def cxx20_flag(self): + if self.real_version < Version("8.0"): + raise spack.compiler.UnsupportedCompilerFlag( + self, "the C++20 standard", "cxx20_flag", "< 8.0" + ) + elif self.real_version < Version("11.0"): + return "-std=c++2a" + else: + return "-std=c++20" + + @property + def cxx23_flag(self): + if self.real_version < Version("11.0"): + raise spack.compiler.UnsupportedCompilerFlag( + self, "the C++23 standard", "cxx23_flag", "< 11.0" + ) + elif self.real_version < Version("14.0"): + return "-std=c++2b" + else: + return "-std=c++23" + @property def c99_flag(self): if self.real_version < Version("4.5"): diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 4ec29605259739..d4ff7c5ebc9067 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -85,6 +85,14 @@ def cxx14_flag(self): else: return "-std=c++14" + @property + def cxx17_flag(self): + # https://www.intel.com/content/www/us/en/developer/articles/news/c17-features-supported-by-c-compiler.html + if self.real_version < Version("19"): + raise UnsupportedCompilerFlag(self, "the C++17 standard", "cxx17_flag", "< 19") + else: + return "-std=c++17" + @property def c99_flag(self): if self.real_version < Version("12"): diff --git a/lib/spack/spack/compilers/msvc.py b/lib/spack/spack/compilers/msvc.py index c9efa264eea4f1..3699a65481dd60 100644 --- a/lib/spack/spack/compilers/msvc.py +++ b/lib/spack/spack/compilers/msvc.py @@ -7,7 +7,6 @@ import re import subprocess import sys -from distutils.version import StrictVersion from typing import Dict, List, Set import spack.compiler @@ -29,13 +28,97 @@ } +class CmdCall: + """Compose a call to `cmd` for an ordered series of cmd commands/scripts""" + + def __init__(self, *cmds): + if not cmds: + raise RuntimeError( + """Attempting to run commands from CMD without specifying commands. + Please add commands to be run.""" + ) + self._cmds = cmds + + def __call__(self): + out = subprocess.check_output(self.cmd_line, stderr=subprocess.STDOUT) # novermin + return out.decode("utf-16le", errors="replace") # novermin + + @property + def cmd_line(self): + base_call = "cmd /u /c " + commands = " && ".join([x.command_str() for x in self._cmds]) + # If multiple commands are being invoked by a single subshell + # they must be encapsulated by a double quote. Always double + # quote to be sure of proper handling + # cmd will properly resolve nested double quotes as needed + # + # `set`` writes out the active env to the subshell stdout, + # and in this context we are always trying to obtain env + # state so it should always be appended + return base_call + f'"{commands} && set"' + + +class VarsInvocation: + def __init__(self, script): + self._script = script + + def command_str(self): + return f'"{self._script}"' + + @property + def script(self): + return self._script + + +class VCVarsInvocation(VarsInvocation): + def __init__(self, script, arch, msvc_version): + super(VCVarsInvocation, self).__init__(script) + self._arch = arch + self._msvc_version = msvc_version + + @property + def sdk_ver(self): + """Accessor for Windows SDK version property + + Note: This property may not be set by + the calling context and as such this property will + return an empty string + + This property will ONLY be set if the SDK package + is a dependency somewhere in the Spack DAG of the package + for which we are constructing an MSVC compiler env. + Otherwise this property should be unset to allow the VCVARS + script to use its internal heuristics to determine appropriate + SDK version + """ + if getattr(self, "_sdk_ver", None): + return self._sdk_ver + ".0" + return "" + + @sdk_ver.setter + def sdk_ver(self, val): + self._sdk_ver = val + + @property + def arch(self): + return self._arch + + @property + def vcvars_ver(self): + return f"-vcvars_ver={self._msvc_version}" + + def command_str(self): + script = super(VCVarsInvocation, self).command_str() + return f"{script} {self.arch} {self.sdk_ver} {self.vcvars_ver}" + + def get_valid_fortran_pth(comp_ver): cl_ver = str(comp_ver) - sort_fn = lambda fc_ver: StrictVersion(fc_ver) + sort_fn = lambda fc_ver: Version(fc_ver) sort_fc_ver = sorted(list(avail_fc_version), key=sort_fn) for ver in sort_fc_ver: if ver in fortran_mapping: - if StrictVersion(cl_ver) <= StrictVersion(fortran_mapping[ver]): + if Version(cl_ver) <= Version(fortran_mapping[ver]): return fc_path[ver] return None @@ -70,27 +153,58 @@ class Msvc(Compiler): #: Regex used to extract version from compiler's output version_regex = r"([1-9][0-9]*\.[0-9]*\.[0-9]*)" + # The MSVC compiler class overrides this to prevent instances + # of erroneous matching on executable names that cannot be msvc + # compilers + suffixes = [] - # Initialize, deferring to base class but then adding the vcvarsallfile - # file based on compiler executable path. + is_supported_on_platform = lambda x: isinstance(x, spack.platforms.Windows) def __init__(self, *args, **kwargs): - new_pth = [pth if pth else get_valid_fortran_pth(args[0].version) for pth in args[3]] - args[3][:] = new_pth + # This positional argument "paths" is later parsed and process by the base class + # via the call to `super` later in this method + paths = args[3] + # This positional argument "cspec" is also parsed and handled by the base class + # constructor + cspec = args[0] + new_pth = [pth if pth else get_valid_fortran_pth(cspec.version) for pth in paths] + paths[:] = new_pth + # Initialize, deferring to base class but then adding the vcvarsallfile + # file based on compiler executable path. super().__init__(*args, **kwargs) - if os.getenv("ONEAPI_ROOT"): + # To use the MSVC compilers, VCVARS must be invoked + # VCVARS is located at a fixed location, referencable + # idiomatically by the following relative path from the + # compiler. + # Spack first finds the compilers via VSWHERE + # and stores their path, but their respective VCVARS + # file must be invoked before useage. + env_cmds = [] + compiler_root = os.path.join(self.cc, "../../../../../../..") + vcvars_script_path = os.path.join(compiler_root, "Auxiliary", "Build", "vcvars64.bat") + # get current platform architecture and format for vcvars argument + arch = spack.platforms.real_host().default.lower() + arch = arch.replace("-", "_") + self.vcvars_call = VCVarsInvocation(vcvars_script_path, arch, self.msvc_version) + env_cmds.append(self.vcvars_call) + # Below is a check for a valid fortran path + # paths has c, cxx, fc, and f77 paths in that order + # paths[2] refers to the fc path and is a generic check + # for a fortran compiler + if paths[2]: # If this found, it sets all the vars - self.setvarsfile = os.path.join(os.getenv("ONEAPI_ROOT"), "setvars.bat") - else: - # To use the MSVC compilers, VCVARS must be invoked - # VCVARS is located at a fixed location, referencable - # idiomatically by the following relative path from the - # compiler. - # Spack first finds the compilers via VSWHERE - # and stores their path, but their respective VCVARS - # file must be invoked before useage. - self.setvarsfile = os.path.abspath(os.path.join(self.cc, "../../../../../../..")) - self.setvarsfile = os.path.join(self.setvarsfile, "Auxiliary", "Build", "vcvars64.bat") + oneapi_root = os.getenv("ONEAPI_ROOT") + oneapi_root_setvars = os.path.join(oneapi_root, "setvars.bat") + oneapi_version_setvars = os.path.join( + oneapi_root, "compiler", str(self.ifx_version), "env", "vars.bat" + ) + # order matters here, the specific version env must be invoked first, + # otherwise it will be ignored if the root setvars sets up the oneapi + # env first + env_cmds.extend( + [VarsInvocation(oneapi_version_setvars), VarsInvocation(oneapi_root_setvars)] + ) + self.msvc_compiler_environment = CmdCall(*env_cmds) @property def msvc_version(self): @@ -119,16 +233,30 @@ def platform_toolset_ver(self): """ return self.msvc_version[:2].joined.string[:3] - @property - def cl_version(self): - """Cl toolset version""" + def _compiler_version(self, compiler): + """Returns version object for given compiler""" + # ignore_errors below is true here due to ifx's + # non zero return code if it is not provided + # and input file return Version( re.search( Msvc.version_regex, - spack.compiler.get_compiler_version_output(self.cc, version_arg=None), + spack.compiler.get_compiler_version_output( + compiler, version_arg=None, ignore_errors=True + ), ).group(1) ) + @property + def cl_version(self): + """Cl toolset version""" + return self._compiler_version(self.cc) + + @property + def ifx_version(self): + """Ifx compiler version associated with this version of MSVC""" + return self._compiler_version(self.fc) + @property def vs_root(self): # The MSVC install root is located at a fix level above the compiler @@ -146,27 +274,12 @@ def setup_custom_environment(self, pkg, env): # output, sort into dictionary, use that to make the build # environment. - # get current platform architecture and format for vcvars argument - arch = spack.platforms.real_host().default.lower() - arch = arch.replace("-", "_") # vcvars can target specific sdk versions, force it to pick up concretized sdk # version, if needed by spec - sdk_ver = ( - "" - if "win-sdk" not in pkg.spec or pkg.name == "win-sdk" - else pkg.spec["win-sdk"].version.string + ".0" - ) - # provide vcvars with msvc version selected by concretization, - # not whatever it happens to pick up on the system (highest available version) - out = subprocess.check_output( # novermin - 'cmd /u /c "{}" {} {} {} && set'.format( - self.setvarsfile, arch, sdk_ver, "-vcvars_ver=%s" % self.msvc_version - ), - stderr=subprocess.STDOUT, - ) - if sys.version_info[0] >= 3: - out = out.decode("utf-16le", errors="replace") # novermin + if pkg.name != "win-sdk" and "win-sdk" in pkg.spec: + self.vcvars_call.sdk_ver = pkg.spec["win-sdk"].version.string + out = self.msvc_compiler_environment() int_env = dict( (key, value) for key, _, value in (line.partition("=") for line in out.splitlines()) diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index 20207023baa588..18b7ac4db3a77d 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -4,8 +4,11 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os +import re from typing import List +import llnl.util.lang + import spack.compiler @@ -32,7 +35,13 @@ class Nag(spack.compiler.Compiler): } version_argument = "-V" - version_regex = r"NAG Fortran Compiler Release ([0-9.]+)" + + @classmethod + @llnl.util.lang.memoized + def extract_version_from_output(cls, output): + match = re.search(r"NAG Fortran Compiler Release (\d+).(\d+)\(.*\) Build (\d+)", output) + if match: + return ".".join(match.groups()) @property def verbose_flag(self): diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index c961e71189b0fa..6e85d66b154f51 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -28,6 +28,7 @@ import spack.abi import spack.compilers +import spack.config import spack.environment import spack.error import spack.platforms @@ -37,7 +38,6 @@ import spack.tengine import spack.util.path import spack.variant as vt -from spack.config import config from spack.package_prefs import PackagePrefs, is_spec_buildable, spec_externals from spack.version import ClosedOpenRange, VersionList, ver @@ -76,7 +76,7 @@ class Concretizer: def __init__(self, abstract_spec=None): if Concretizer.check_for_compiler_existence is None: - Concretizer.check_for_compiler_existence = not config.get( + Concretizer.check_for_compiler_existence = not spack.config.get( "config:install_missing_compilers", False ) self.abstract_spec = abstract_spec @@ -113,7 +113,7 @@ def _valid_virtuals_and_externals(self, spec): pref_key = lambda spec: 0 # no-op pref key if spec.virtual: - candidates = spack.repo.path.providers_for(spec) + candidates = spack.repo.PATH.providers_for(spec) if not candidates: raise spack.error.UnsatisfiableProviderSpecError(candidates[0], spec) @@ -155,7 +155,7 @@ def _valid_virtuals_and_externals(self, spec): ), ) - def choose_virtual_or_external(self, spec): + def choose_virtual_or_external(self, spec: spack.spec.Spec): """Given a list of candidate virtual and external packages, try to find one that is most ABI compatible. """ @@ -744,8 +744,11 @@ def concretize_specs_together(*abstract_specs, **kwargs): def _concretize_specs_together_new(*abstract_specs, **kwargs): import spack.solver.asp + allow_deprecated = spack.config.get("config:deprecated", False) solver = spack.solver.asp.Solver() - result = solver.solve(abstract_specs, tests=kwargs.get("tests", False)) + result = solver.solve( + abstract_specs, tests=kwargs.get("tests", False), allow_deprecated=allow_deprecated + ) result.raise_if_unsat() return [s.copy() for s in result.specs] diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index d7f4638b7c88e8..cd1be71c9d0a33 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -47,6 +47,8 @@ import spack.platforms import spack.schema import spack.schema.bootstrap +import spack.schema.cdash +import spack.schema.ci import spack.schema.compilers import spack.schema.concretizer import spack.schema.config @@ -64,9 +66,10 @@ from spack.util.cpus import cpus_available #: Dict from section names -> schema for that section -section_schemas = { +SECTION_SCHEMAS = { "compilers": spack.schema.compilers.schema, "concretizer": spack.schema.concretizer.schema, + "definitions": spack.schema.definitions.schema, "mirrors": spack.schema.mirrors.schema, "repos": spack.schema.repos.schema, "packages": spack.schema.packages.schema, @@ -80,16 +83,16 @@ # Same as above, but including keys for environments # this allows us to unify config reading between configs and environments -all_schemas = copy.deepcopy(section_schemas) -all_schemas.update({spack.schema.env.TOP_LEVEL_KEY: spack.schema.env.schema}) +_ALL_SCHEMAS = copy.deepcopy(SECTION_SCHEMAS) +_ALL_SCHEMAS.update({spack.schema.env.TOP_LEVEL_KEY: spack.schema.env.schema}) #: Path to the default configuration -configuration_defaults_path = ("defaults", os.path.join(spack.paths.etc_path, "defaults")) +CONFIGURATION_DEFAULTS_PATH = ("defaults", os.path.join(spack.paths.etc_path, "defaults")) #: Hard-coded default values for some key configuration options. #: This ensures that Spack will still work even if config.yaml in #: the defaults scope is removed. -config_defaults = { +CONFIG_DEFAULTS = { "config": { "debug": False, "connect_timeout": 10, @@ -105,10 +108,10 @@ #: metavar to use for commands that accept scopes #: this is shorter and more readable than listing all choices -scopes_metavar = "{defaults,system,site,user}[/PLATFORM] or env:ENVIRONMENT" +SCOPES_METAVAR = "{defaults,system,site,user}[/PLATFORM] or env:ENVIRONMENT" #: Base name for the (internal) overrides scope. -overrides_base_name = "overrides-" +_OVERRIDES_BASE_NAME = "overrides-" class ConfigScope: @@ -134,7 +137,7 @@ def get_section_filename(self, section): def get_section(self, section): if section not in self.sections: path = self.get_section_filename(section) - schema = section_schemas[section] + schema = SECTION_SCHEMAS[section] data = read_config_file(path, schema) self.sections[section] = data return self.sections[section] @@ -145,7 +148,7 @@ def _write_section(self, section): # We copy data here to avoid adding defaults at write time validate_data = copy.deepcopy(data) - validate(validate_data, section_schemas[section]) + validate(validate_data, SECTION_SCHEMAS[section]) try: mkdirp(self.path) @@ -317,7 +320,7 @@ def __init__(self, name, data=None): data = InternalConfigScope._process_dict_keyname_overrides(data) for section in data: dsec = data[section] - validate({section: dsec}, section_schemas[section]) + validate({section: dsec}, SECTION_SCHEMAS[section]) self.sections[section] = _mark_internal(syaml.syaml_dict({section: dsec}), name) def get_section_filename(self, section): @@ -333,7 +336,7 @@ def _write_section(self, section): """This only validates, as the data is already in memory.""" data = self.get_section(section) if data is not None: - validate(data, section_schemas[section]) + validate(data, SECTION_SCHEMAS[section]) self.sections[section] = _mark_internal(data, self.name) def __repr__(self): @@ -430,7 +433,7 @@ def file_scopes(self) -> List[ConfigScope]: return [ s for s in self.scopes.values() - if (type(s) == ConfigScope or type(s) == SingleFileScope) + if (type(s) is ConfigScope or type(s) is SingleFileScope) ] def highest_precedence_scope(self) -> ConfigScope: @@ -711,11 +714,11 @@ def override(path_or_scope, value=None): """ if isinstance(path_or_scope, ConfigScope): overrides = path_or_scope - config.push_scope(path_or_scope) + CONFIG.push_scope(path_or_scope) else: - base_name = overrides_base_name + base_name = _OVERRIDES_BASE_NAME # Ensure the new override gets a unique scope name - current_overrides = [s.name for s in config.matching_scopes(r"^{0}".format(base_name))] + current_overrides = [s.name for s in CONFIG.matching_scopes(r"^{0}".format(base_name))] num_overrides = len(current_overrides) while True: scope_name = "{0}{1}".format(base_name, num_overrides) @@ -725,19 +728,19 @@ def override(path_or_scope, value=None): break overrides = InternalConfigScope(scope_name) - config.push_scope(overrides) - config.set(path_or_scope, value, scope=scope_name) + CONFIG.push_scope(overrides) + CONFIG.set(path_or_scope, value, scope=scope_name) try: - yield config + yield CONFIG finally: - scope = config.remove_scope(overrides.name) + scope = CONFIG.remove_scope(overrides.name) assert scope is overrides #: configuration scopes added on the command line #: set by ``spack.main.main()``. -command_line_scopes: List[str] = [] +COMMAND_LINE_SCOPES: List[str] = [] def _add_platform_scope(cfg, scope_type, name, path): @@ -767,7 +770,7 @@ def _add_command_line_scopes(cfg, command_line_scopes): _add_platform_scope(cfg, ImmutableConfigScope, name, path) -def _config(): +def create(): """Singleton Configuration instance. This constructs one instance associated with this module and returns @@ -781,14 +784,14 @@ def _config(): cfg = Configuration() # first do the builtin, hardcoded defaults - builtin = InternalConfigScope("_builtin", config_defaults) + builtin = InternalConfigScope("_builtin", CONFIG_DEFAULTS) cfg.push_scope(builtin) # Builtin paths to configuration files in Spack configuration_paths = [ # Default configuration scope is the lowest-level scope. These are # versioned with Spack and can be overridden by systems, sites or users - configuration_defaults_path + CONFIGURATION_DEFAULTS_PATH ] disable_local_config = "SPACK_DISABLE_LOCAL_CONFIG" in os.environ @@ -815,7 +818,7 @@ def _config(): _add_platform_scope(cfg, ConfigScope, name, path) # add command-line scopes - _add_command_line_scopes(cfg, command_line_scopes) + _add_command_line_scopes(cfg, COMMAND_LINE_SCOPES) # we make a special scope for spack commands so that they can # override configuration options. @@ -825,7 +828,7 @@ def _config(): #: This is the singleton configuration instance for Spack. -config: Union[Configuration, llnl.util.lang.Singleton] = llnl.util.lang.Singleton(_config) +CONFIG: Union[Configuration, llnl.util.lang.Singleton] = llnl.util.lang.Singleton(create) def add_from_file(filename, scope=None): @@ -838,7 +841,7 @@ def add_from_file(filename, scope=None): # update all sections from config dict # We have to iterate on keys to keep overrides from the file for section in data.keys(): - if section in section_schemas.keys(): + if section in SECTION_SCHEMAS.keys(): # Special handling for compiler scope difference # Has to be handled after we choose a section if scope is None: @@ -849,18 +852,18 @@ def add_from_file(filename, scope=None): new = merge_yaml(existing, value) # We cannot call config.set directly (set is a type) - config.set(section, new, scope) + CONFIG.set(section, new, scope) def add(fullpath, scope=None): """Add the given configuration to the specified config scope. Add accepts a path. If you want to add from a filename, use add_from_file""" - components = process_config_path(fullpath) has_existing_value = True path = "" override = False + value = syaml.load_config(components[-1]) for idx, name in enumerate(components[:-1]): # First handle double colons in constructing path colon = "::" if override else ":" if path else "" @@ -881,14 +884,14 @@ def add(fullpath, scope=None): existing = get_valid_type(path) # construct value from this point down - value = syaml.load_config(components[-1]) for component in reversed(components[idx + 1 : -1]): value = {component: value} break + if override: + path += "::" + if has_existing_value: - path, _, value = fullpath.rpartition(":") - value = syaml.load_config(value) existing = get(path, scope=scope) # append values to lists @@ -897,12 +900,12 @@ def add(fullpath, scope=None): # merge value into existing new = merge_yaml(existing, value) - config.set(path, new, scope) + CONFIG.set(path, new, scope) def get(path, default=None, scope=None): """Module-level wrapper for ``Configuration.get()``.""" - return config.get(path, default, scope) + return CONFIG.get(path, default, scope) def set(path, value, scope=None): @@ -910,26 +913,26 @@ def set(path, value, scope=None): Accepts the path syntax described in ``get()``. """ - return config.set(path, value, scope) + return CONFIG.set(path, value, scope) def add_default_platform_scope(platform): plat_name = os.path.join("defaults", platform) - plat_path = os.path.join(configuration_defaults_path[1], platform) - config.push_scope(ConfigScope(plat_name, plat_path)) + plat_path = os.path.join(CONFIGURATION_DEFAULTS_PATH[1], platform) + CONFIG.push_scope(ConfigScope(plat_name, plat_path)) def scopes(): """Convenience function to get list of configuration scopes.""" - return config.scopes + return CONFIG.scopes def _validate_section_name(section): """Exit if the section is not a valid section.""" - if section not in section_schemas: + if section not in SECTION_SCHEMAS: raise ConfigSectionError( "Invalid config section: '%s'. Options are: %s" - % (section, " ".join(section_schemas.keys())) + % (section, " ".join(SECTION_SCHEMAS.keys())) ) @@ -990,8 +993,9 @@ def read_config_file(filename, schema=None): if data: if not schema: key = next(iter(data)) - schema = all_schemas[key] + schema = _ALL_SCHEMAS[key] validate(data, schema) + return data except StopIteration: @@ -1089,7 +1093,7 @@ def get_valid_type(path): test_data = {component: test_data} try: - validate(test_data, section_schemas[section]) + validate(test_data, SECTION_SCHEMAS[section]) except (ConfigFormatError, AttributeError) as e: jsonschema_error = e.validation_error if jsonschema_error.validator == "type": @@ -1229,11 +1233,17 @@ def they_are(t): return copy.copy(source) -# -# Process a path argument to config.set() that may contain overrides ('::' or -# trailing ':') -# def process_config_path(path): + """Process a path argument to config.set() that may contain overrides ('::' or + trailing ':') + + Note: quoted value path components will be processed as a single value (escaping colons) + quoted path components outside of the value will be considered ill formed and will + raise. + e.g. `this:is:a:path:'value:with:colon'` will yield: + + [this, is, a, path, value:with:colon] + """ result = [] if path.startswith(":"): raise syaml.SpackYAMLError("Illegal leading `:' in path `{0}'".format(path), "") @@ -1261,6 +1271,17 @@ def process_config_path(path): front.append = True result.append(front) + + quote = "['\"]" + not_quote = "[^'\"]" + + if re.match(f"^{quote}", path): + m = re.match(rf"^({quote}{not_quote}+{quote})$", path) + if not m: + raise ValueError("Quotes indicate value, but there are additional path entries") + result.append(m.group(1)) + break + return result @@ -1278,9 +1299,9 @@ def default_modify_scope(section="config"): If this is not 'compilers', a general (non-platform) scope is used. """ if section == "compilers": - return spack.config.config.highest_precedence_scope().name + return CONFIG.highest_precedence_scope().name else: - return spack.config.config.highest_precedence_non_platform_scope().name + return CONFIG.highest_precedence_non_platform_scope().name def default_list_scope(): @@ -1337,18 +1358,18 @@ def use_configuration(*scopes_or_paths): Returns: Configuration object associated with the scopes passed as arguments """ - global config + global CONFIG # Normalize input and construct a Configuration object configuration = _config_from(scopes_or_paths) - config.clear_caches(), configuration.clear_caches() + CONFIG.clear_caches(), configuration.clear_caches() - saved_config, config = config, configuration + saved_config, CONFIG = CONFIG, configuration try: yield configuration finally: - config = saved_config + CONFIG = saved_config @llnl.util.lang.memoized diff --git a/lib/spack/spack/container/images.py b/lib/spack/spack/container/images.py index 130adaffdb3f36..476fbab1db3614 100644 --- a/lib/spack/spack/container/images.py +++ b/lib/spack/spack/container/images.py @@ -5,6 +5,7 @@ """Manages the details on the images used in the various stages.""" import json import os.path +import shlex import sys import llnl.util.filesystem as fs @@ -130,8 +131,11 @@ def checkout_command(url, ref, enforce_sha, verify): if enforce_sha or verify: ref = _verify_ref(url, ref, enforce_sha) - command = ( - "git clone {0} . && git fetch origin {1}:container_branch &&" - " git checkout container_branch " - ).format(url, ref) - return command + return " && ".join( + [ + "git init --quiet", + f"git remote add origin {shlex.quote(url)}", + f"git fetch --depth=1 origin {shlex.quote(ref)}", + "git checkout --detach FETCH_HEAD", + ] + ) diff --git a/lib/spack/spack/container/writers/__init__.py b/lib/spack/spack/container/writers/__init__.py index 8df6d9a043b822..60e6dc61ea4f7d 100644 --- a/lib/spack/spack/container/writers/__init__.py +++ b/lib/spack/spack/container/writers/__init__.py @@ -5,8 +5,8 @@ """Writers for different kind of recipes and related convenience functions. """ -import collections import copy +from collections import namedtuple from typing import Optional import spack.environment as ev @@ -159,13 +159,13 @@ def depfile(self): @tengine.context_property def run(self): """Information related to the run image.""" - Run = collections.namedtuple("Run", ["image"]) + Run = namedtuple("Run", ["image"]) return Run(image=self.final_image) @tengine.context_property def build(self): """Information related to the build image.""" - Build = collections.namedtuple("Build", ["image"]) + Build = namedtuple("Build", ["image"]) return Build(image=self.build_image) @tengine.context_property @@ -176,12 +176,13 @@ def strip(self): @tengine.context_property def paths(self): """Important paths in the image""" - Paths = collections.namedtuple("Paths", ["environment", "store", "hidden_view", "view"]) + Paths = namedtuple("Paths", ["environment", "store", "view_parent", "view", "former_view"]) return Paths( environment="/opt/spack-environment", store="/opt/software", - hidden_view="/opt/._view", - view="/opt/view", + view_parent="/opt/views", + view="/opt/views/view", + former_view="/opt/view", # /opt/view -> /opt/views/view for backward compatibility ) @tengine.context_property @@ -257,7 +258,7 @@ def _package_info_from(self, package_list): update, install, clean = commands_for(os_pkg_manager) - Packages = collections.namedtuple("Packages", ["update", "install", "list", "clean"]) + Packages = namedtuple("Packages", ["update", "install", "list", "clean"]) return Packages(update=update, install=install, list=package_list, clean=clean) def _os_pkg_manager(self): @@ -273,7 +274,7 @@ def _os_pkg_manager(self): @tengine.context_property def extra_instructions(self): - Extras = collections.namedtuple("Extra", ["pre_build", "build", "pre_final", "final"]) + Extras = namedtuple("Extra", ["pre_build", "build", "pre_final", "final"]) extras = self.container_config.get("extra_instructions", {}) pre_build, build, pre_final, final = ( extras.get("pre_build", None), @@ -300,7 +301,7 @@ def bootstrap(self): context = {"bootstrap": {"image": self.bootstrap_image, "spack_checkout": command}} bootstrap_recipe = env.get_template(template_path).render(**context) - Bootstrap = collections.namedtuple("Bootstrap", ["image", "recipe"]) + Bootstrap = namedtuple("Bootstrap", ["image", "recipe"]) return Bootstrap(image=self.bootstrap_image, recipe=bootstrap_recipe) @tengine.context_property @@ -308,7 +309,7 @@ def render_phase(self): render_bootstrap = bool(self.bootstrap_image) render_build = not (self.last_phase == "bootstrap") render_final = self.last_phase in (None, "final") - Render = collections.namedtuple("Render", ["bootstrap", "build", "final"]) + Render = namedtuple("Render", ["bootstrap", "build", "final"]) return Render(bootstrap=render_bootstrap, build=render_build, final=render_final) def __call__(self): diff --git a/lib/spack/spack/context.py b/lib/spack/spack/context.py new file mode 100644 index 00000000000000..de3311da22b0ff --- /dev/null +++ b/lib/spack/spack/context.py @@ -0,0 +1,29 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""This module provides classes used in user and build environment""" + +from enum import Enum + + +class Context(Enum): + """Enum used to indicate the context in which an environment has to be setup: build, + run or test.""" + + BUILD = 1 + RUN = 2 + TEST = 3 + + def __str__(self): + return ("build", "run", "test")[self.value - 1] + + @classmethod + def from_string(cls, s: str): + if s == "build": + return Context.BUILD + elif s == "run": + return Context.RUN + elif s == "test": + return Context.TEST + raise ValueError(f"context should be one of 'build', 'run', 'test', got {s}") diff --git a/lib/spack/spack/cray_manifest.py b/lib/spack/spack/cray_manifest.py index 9a64342434ef23..48ec52d782d3bb 100644 --- a/lib/spack/spack/cray_manifest.py +++ b/lib/spack/spack/cray_manifest.py @@ -4,6 +4,9 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import json +import os +import traceback +import warnings import jsonschema import jsonschema.exceptions @@ -11,6 +14,7 @@ import llnl.util.tty as tty import spack.cmd +import spack.deptypes as dt import spack.error import spack.hash_types as hash_types import spack.platforms @@ -45,9 +49,29 @@ def translated_compiler_name(manifest_compiler_name): ) -def compiler_from_entry(entry): +def compiler_from_entry(entry: dict, manifest_path: str): + # Note that manifest_path is only passed here to compose a + # useful warning message when paths appear to be missing. compiler_name = translated_compiler_name(entry["name"]) - paths = entry["executables"] + + if "prefix" in entry: + prefix = entry["prefix"] + paths = dict( + (lang, os.path.join(prefix, relpath)) + for (lang, relpath) in entry["executables"].items() + ) + else: + paths = entry["executables"] + + # Do a check for missing paths. Note that this isn't possible for + # all compiler entries, since their "paths" might actually be + # exe names like "cc" that depend on modules being loaded. Cray + # manifest entries are always paths though. + missing_paths = [] + for path in paths.values(): + if not os.path.exists(path): + missing_paths.append(path) + # to instantiate a compiler class we may need a concrete version: version = "={}".format(entry["version"]) arch = entry["arch"] @@ -56,8 +80,18 @@ def compiler_from_entry(entry): compiler_cls = spack.compilers.class_for_compiler_name(compiler_name) spec = spack.spec.CompilerSpec(compiler_cls.name, version) - paths = [paths.get(x, None) for x in ("cc", "cxx", "f77", "fc")] - return compiler_cls(spec, operating_system, target, paths) + path_list = [paths.get(x, None) for x in ("cc", "cxx", "f77", "fc")] + + if missing_paths: + warnings.warn( + "Manifest entry refers to nonexistent paths:\n\t" + + "\n\t".join(missing_paths) + + f"\nfor {str(spec)}" + + f"\nin {manifest_path}" + + "\nPlease report this issue" + ) + + return compiler_cls(spec, operating_system, target, path_list) def spec_from_entry(entry): @@ -90,7 +124,7 @@ def spec_from_entry(entry): name=entry["name"], version=entry["version"], compiler=compiler_str, arch=arch_str ) - pkg_cls = spack.repo.path.get_pkg_class(entry["name"]) + pkg_cls = spack.repo.PATH.get_pkg_class(entry["name"]) if "parameters" in entry: variant_strs = list() @@ -158,13 +192,13 @@ def entries_to_specs(entries): dependencies = entry["dependencies"] for name, properties in dependencies.items(): dep_hash = properties["hash"] - deptypes = properties["type"] + depflag = dt.canonicalize(properties["type"]) if dep_hash in spec_dict: if entry["hash"] not in spec_dict: continue parent_spec = spec_dict[entry["hash"]] dep_spec = spec_dict[dep_hash] - parent_spec._add_dependency(dep_spec, deptypes=deptypes, virtuals=()) + parent_spec._add_dependency(dep_spec, depflag=depflag, virtuals=()) for spec in spec_dict.values(): spack.spec.reconstruct_virtuals_on_edges(spec) @@ -186,15 +220,24 @@ def read(path, apply_updates): tty.debug("{0}: {1} specs read from manifest".format(path, str(len(specs)))) compilers = list() if "compilers" in json_data: - compilers.extend(compiler_from_entry(x) for x in json_data["compilers"]) + compilers.extend(compiler_from_entry(x, path) for x in json_data["compilers"]) tty.debug("{0}: {1} compilers read from manifest".format(path, str(len(compilers)))) # Filter out the compilers that already appear in the configuration compilers = spack.compilers.select_new_compilers(compilers) if apply_updates and compilers: - spack.compilers.add_compilers_to_config(compilers, init_config=False) + for compiler in compilers: + try: + spack.compilers.add_compilers_to_config([compiler], init_config=False) + except Exception: + warnings.warn( + f"Could not add compiler {str(compiler.spec)}: " + f"\n\tfrom manifest: {path}" + "\nPlease reexecute with 'spack -d' and include the stack trace" + ) + tty.debug(f"Include this\n{traceback.format_exc()}") if apply_updates: for spec in specs.values(): - spack.store.db.add(spec, directory_layout=None) + spack.store.STORE.db.add(spec, directory_layout=None) class ManifestValidationError(spack.error.SpackError): diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index a8fa85c5586ff8..ecda8c36b0f0ba 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -2,7 +2,6 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - """Spack's installation tracking database. The database serves two purposes: @@ -19,14 +18,16 @@ provides a cache and a sanity checking mechanism for what is in the filesystem. """ - import contextlib import datetime import os +import pathlib import socket import sys import time -from typing import Dict +from typing import Any, Callable, Dict, Generator, List, NamedTuple, Set, Type, Union + +import spack.deptypes as dt try: import uuid @@ -36,14 +37,13 @@ _use_uuid = False pass +from typing import Optional, Tuple + import llnl.util.filesystem as fs -import llnl.util.lang as lang import llnl.util.tty as tty import spack.hash_types as ht -import spack.repo import spack.spec -import spack.store import spack.util.lock as lk import spack.util.spack_json as sjson import spack.version as vn @@ -54,17 +54,17 @@ # TODO: Provide an API automatically retyring a build after detecting and # TODO: clearing a failure. -# DB goes in this directory underneath the root -_db_dirname = ".spack-db" +#: DB goes in this directory underneath the root +_DB_DIRNAME = ".spack-db" -# DB version. This is stuck in the DB file to track changes in format. -# Increment by one when the database format changes. -# Versions before 5 were not integers. -_db_version = vn.Version("7") +#: DB version. This is stuck in the DB file to track changes in format. +#: Increment by one when the database format changes. +#: Versions before 5 were not integers. +_DB_VERSION = vn.Version("7") -# For any version combinations here, skip reindex when upgrading. -# Reindexing can take considerable time and is not always necessary. -_skip_reindex = [ +#: For any version combinations here, skip reindex when upgrading. +#: Reindexing can take considerable time and is not always necessary. +_SKIP_REINDEX = [ # reindexing takes a significant amount of time, and there's # no reason to do it from DB version 0.9.3 to version 5. The # only difference is that v5 can contain "deprecated_for" @@ -75,26 +75,26 @@ (vn.Version("6"), vn.Version("7")), ] -# Default timeout for spack database locks in seconds or None (no timeout). -# A balance needs to be struck between quick turnaround for parallel installs -# (to avoid excess delays) and waiting long enough when the system is busy -# (to ensure the database is updated). -_db_lock_timeout = 120 - -# Default timeout for spack package locks in seconds or None (no timeout). -# A balance needs to be struck between quick turnaround for parallel installs -# (to avoid excess delays when performing a parallel installation) and waiting -# long enough for the next possible spec to install (to avoid excessive -# checking of the last high priority package) or holding on to a lock (to -# ensure a failed install is properly tracked). -_pkg_lock_timeout = None - -# Types of dependencies tracked by the database -# We store by DAG hash, so we track the dependencies that the DAG hash includes. -_tracked_deps = ht.dag_hash.deptype - -# Default list of fields written for each install record -default_install_record_fields = [ +#: Default timeout for spack database locks in seconds or None (no timeout). +#: A balance needs to be struck between quick turnaround for parallel installs +#: (to avoid excess delays) and waiting long enough when the system is busy +#: (to ensure the database is updated). +_DEFAULT_DB_LOCK_TIMEOUT = 120 + +#: Default timeout for spack package locks in seconds or None (no timeout). +#: A balance needs to be struck between quick turnaround for parallel installs +#: (to avoid excess delays when performing a parallel installation) and waiting +#: long enough for the next possible spec to install (to avoid excessive +#: checking of the last high priority package) or holding on to a lock (to +#: ensure a failed install is properly tracked). +_DEFAULT_PKG_LOCK_TIMEOUT = None + +#: Types of dependencies tracked by the database +#: We store by DAG hash, so we track the dependencies that the DAG hash includes. +_TRACKED_DEPENDENCIES = ht.dag_hash.depflag + +#: Default list of fields written for each install record +DEFAULT_INSTALL_RECORD_FIELDS = ( "spec", "ref_count", "path", @@ -102,10 +102,10 @@ "explicit", "installation_time", "deprecated_for", -] +) -def reader(version): +def reader(version: vn.StandardVersion) -> Type["spack.spec.SpecfileReaderBase"]: reader_cls = { vn.Version("5"): spack.spec.SpecfileV1, vn.Version("6"): spack.spec.SpecfileV3, @@ -114,7 +114,7 @@ def reader(version): return reader_cls[version] -def _now(): +def _now() -> float: """Returns the time since the epoch""" return time.time() @@ -144,22 +144,23 @@ class InstallStatuses: def canonicalize(cls, query_arg): if query_arg is True: return [cls.INSTALLED] - elif query_arg is False: + if query_arg is False: return [cls.MISSING] - elif query_arg is any: + if query_arg is any: return [cls.INSTALLED, cls.DEPRECATED, cls.MISSING] - elif isinstance(query_arg, InstallStatus): + if isinstance(query_arg, InstallStatus): return [query_arg] - else: - try: # Try block catches if it is not an iterable at all - if any(type(x) != InstallStatus for x in query_arg): - raise TypeError - except TypeError: - raise TypeError( - "installation query must be `any`, boolean, " - "InstallStatus, or iterable of InstallStatus" - ) - return query_arg + try: + statuses = list(query_arg) + if all(isinstance(x, InstallStatus) for x in statuses): + return statuses + except TypeError: + pass + + raise TypeError( + "installation query must be `any`, boolean, " + "InstallStatus, or iterable of InstallStatus" + ) class InstallRecord: @@ -178,9 +179,9 @@ class InstallRecord: dependents left. Args: - spec (spack.spec.Spec): spec tracked by the install record - path (str): path where the spec has been installed - installed (bool): whether or not the spec is currently installed + spec: spec tracked by the install record + path: path where the spec has been installed + installed: whether or not the spec is currently installed ref_count (int): number of specs that depend on this one explicit (bool or None): whether or not this spec was explicitly installed, or pulled-in as a dependency of something else @@ -189,14 +190,14 @@ class InstallRecord: def __init__( self, - spec, - path, - installed, - ref_count=0, - explicit=False, - installation_time=None, - deprecated_for=None, - in_buildcache=False, + spec: "spack.spec.Spec", + path: str, + installed: bool, + ref_count: int = 0, + explicit: bool = False, + installation_time: Optional[float] = None, + deprecated_for: Optional["spack.spec.Spec"] = None, + in_buildcache: bool = False, origin=None, ): self.spec = spec @@ -218,7 +219,7 @@ def install_type_matches(self, installed): else: return InstallStatuses.MISSING in installed - def to_dict(self, include_fields=default_install_record_fields): + def to_dict(self, include_fields=DEFAULT_INSTALL_RECORD_FIELDS): rec_dict = {} for field_name in include_fields: @@ -254,11 +255,14 @@ class ForbiddenLockError(SpackError): class ForbiddenLock: - def __getattribute__(self, name): + def __getattr__(self, name): raise ForbiddenLockError("Cannot access attribute '{0}' of lock".format(name)) + def __reduce__(self): + return ForbiddenLock, tuple() + -_query_docstring = """ +_QUERY_DOCSTRING = """ Args: query_spec: queries iterate through specs in the database and @@ -307,347 +311,365 @@ def __getattribute__(self, name): """ -class Database: +class LockConfiguration(NamedTuple): + """Data class to configure locks in Database objects - """Per-process lock objects for each install prefix.""" + Args: + enable: whether to enable locks or not. + database_timeout: timeout for the database lock + package_timeout: timeout for the package lock + """ - _prefix_locks: Dict[str, lk.Lock] = {} + enable: bool + database_timeout: Optional[int] + package_timeout: Optional[int] - """Per-process failure (lock) objects for each install prefix.""" - _prefix_failures: Dict[str, lk.Lock] = {} - def __init__( - self, - root, - db_dir=None, - upstream_dbs=None, - is_upstream=False, - enable_transaction_locking=True, - record_fields=default_install_record_fields, - ): - """Create a Database for Spack installations under ``root``. +#: Configure a database to avoid using locks +NO_LOCK: LockConfiguration = LockConfiguration( + enable=False, database_timeout=None, package_timeout=None +) - A Database is a cache of Specs data from ``$prefix/spec.yaml`` - files in Spack installation directories. - By default, Database files (data and lock files) are stored - under ``root/.spack-db``, which is created if it does not - exist. This is the ``db_dir``. +#: Configure the database to use locks without a timeout +NO_TIMEOUT: LockConfiguration = LockConfiguration( + enable=True, database_timeout=None, package_timeout=None +) - The Database will attempt to read an ``index.json`` file in - ``db_dir``. If that does not exist, it will create a database - when needed by scanning the entire Database root for ``spec.yaml`` - files according to Spack's ``DirectoryLayout``. +#: Default configuration for database locks +DEFAULT_LOCK_CFG: LockConfiguration = LockConfiguration( + enable=True, + database_timeout=_DEFAULT_DB_LOCK_TIMEOUT, + package_timeout=_DEFAULT_PKG_LOCK_TIMEOUT, +) - Caller may optionally provide a custom ``db_dir`` parameter - where data will be stored. This is intended to be used for - testing the Database class. - This class supports writing buildcache index files, in which case - certain fields are not needed in each install record, and no - transaction locking is required. To use this feature, provide - ``enable_transaction_locking=False``, and specify a list of needed - fields in ``record_fields``. - """ - self.root = root +def lock_configuration(configuration): + """Return a LockConfiguration from a spack.config.Configuration object.""" + return LockConfiguration( + enable=configuration.get("config:locks", True), + database_timeout=configuration.get("config:db_lock_timeout"), + package_timeout=configuration.get("config:package_lock_timeout"), + ) - # If the db_dir is not provided, default to within the db root. - self._db_dir = db_dir or os.path.join(self.root, _db_dirname) - # Set up layout of database files within the db dir - self._index_path = os.path.join(self._db_dir, "index.json") - self._verifier_path = os.path.join(self._db_dir, "index_verifier") - self._lock_path = os.path.join(self._db_dir, "lock") +def prefix_lock_path(root_dir: Union[str, pathlib.Path]) -> pathlib.Path: + """Returns the path of the prefix lock file, given the root directory. - # This is for other classes to use to lock prefix directories. - self.prefix_lock_path = os.path.join(self._db_dir, "prefix_lock") + Args: + root_dir: root directory containing the database directory + """ + return pathlib.Path(root_dir) / _DB_DIRNAME / "prefix_lock" - # Ensure a persistent location for dealing with parallel installation - # failures (e.g., across near-concurrent processes). - self._failure_dir = os.path.join(self._db_dir, "failures") - # Support special locks for handling parallel installation failures - # of a spec. - self.prefix_fail_path = os.path.join(self._db_dir, "prefix_failures") +def failures_lock_path(root_dir: Union[str, pathlib.Path]) -> pathlib.Path: + """Returns the path of the failures lock file, given the root directory. - # Create needed directories and files - if not is_upstream and not os.path.exists(self._db_dir): - fs.mkdirp(self._db_dir) + Args: + root_dir: root directory containing the database directory + """ + return pathlib.Path(root_dir) / _DB_DIRNAME / "prefix_failures" - if not is_upstream and not os.path.exists(self._failure_dir): - fs.mkdirp(self._failure_dir) - self.is_upstream = is_upstream - self.last_seen_verifier = "" - # Failed write transactions (interrupted by exceptions) will alert - # _write. When that happens, we set this flag to indicate that - # future read/write transactions should re-read the DB. Normally it - # would make more sense to resolve this at the end of the transaction - # but typically a failed transaction will terminate the running - # instance of Spack and we don't want to incur an extra read in that - # case, so we defer the cleanup to when we begin the next transaction - self._state_is_inconsistent = False +class SpecLocker: + """Manages acquiring and releasing read or write locks on concrete specs.""" - # initialize rest of state. - self.db_lock_timeout = spack.config.get("config:db_lock_timeout") or _db_lock_timeout - self.package_lock_timeout = ( - spack.config.get("config:package_lock_timeout") or _pkg_lock_timeout - ) - tty.debug("DATABASE LOCK TIMEOUT: {0}s".format(str(self.db_lock_timeout))) - timeout_format_str = ( - "{0}s".format(str(self.package_lock_timeout)) - if self.package_lock_timeout - else "No timeout" - ) - tty.debug("PACKAGE LOCK TIMEOUT: {0}".format(str(timeout_format_str))) + def __init__(self, lock_path: Union[str, pathlib.Path], default_timeout: Optional[float]): + self.lock_path = pathlib.Path(lock_path) + self.default_timeout = default_timeout - if self.is_upstream: - self.lock = ForbiddenLock() + # Maps (spec.dag_hash(), spec.name) to the corresponding lock object + self.locks: Dict[Tuple[str, str], lk.Lock] = {} + + def lock(self, spec: "spack.spec.Spec", timeout: Optional[float] = None) -> lk.Lock: + """Returns a lock on a concrete spec. + + The lock is a byte range lock on the nth byte of a file. + + The lock file is ``self.lock_path``. + + n is the sys.maxsize-bit prefix of the DAG hash. This makes likelihood of collision is + very low AND it gives us readers-writer lock semantics with just a single lockfile, so + no cleanup required. + """ + assert spec.concrete, "cannot lock a non-concrete spec" + timeout = timeout or self.default_timeout + key = self._lock_key(spec) + + if key not in self.locks: + self.locks[key] = self.raw_lock(spec, timeout=timeout) else: - self.lock = lk.Lock( - self._lock_path, default_timeout=self.db_lock_timeout, desc="database" - ) - self._data = {} + self.locks[key].default_timeout = timeout - # For every installed spec we keep track of its install prefix, so that - # we can answer the simple query whether a given path is already taken - # before installing a different spec. - self._installed_prefixes = set() + return self.locks[key] - self.upstream_dbs = list(upstream_dbs) if upstream_dbs else [] + def raw_lock(self, spec: "spack.spec.Spec", timeout: Optional[float] = None) -> lk.Lock: + """Returns a raw lock for a Spec, but doesn't keep track of it.""" + return lk.Lock( + str(self.lock_path), + start=spec.dag_hash_bit_prefix(bit_length(sys.maxsize)), + length=1, + default_timeout=timeout, + desc=spec.name, + ) - # whether there was an error at the start of a read transaction - self._error = None + def has_lock(self, spec: "spack.spec.Spec") -> bool: + """Returns True if the spec is already managed by this spec locker""" + return self._lock_key(spec) in self.locks - # For testing: if this is true, an exception is thrown when missing - # dependencies are detected (rather than just printing a warning - # message) - self._fail_when_missing_deps = False + def _lock_key(self, spec: "spack.spec.Spec") -> Tuple[str, str]: + return (spec.dag_hash(), spec.name) - if enable_transaction_locking: - self._write_transaction_impl = lk.WriteTransaction - self._read_transaction_impl = lk.ReadTransaction + @contextlib.contextmanager + def write_lock(self, spec: "spack.spec.Spec") -> Generator["SpecLocker", None, None]: + lock = self.lock(spec) + lock.acquire_write() + + try: + yield self + except lk.LockError: + # This addresses the case where a nested lock attempt fails inside + # of this context manager + raise + except (Exception, KeyboardInterrupt): + lock.release_write() + raise else: - self._write_transaction_impl = lang.nullcontext - self._read_transaction_impl = lang.nullcontext + lock.release_write() - self._record_fields = record_fields + def clear(self, spec: "spack.spec.Spec") -> Tuple[bool, Optional[lk.Lock]]: + key = self._lock_key(spec) + lock = self.locks.pop(key, None) + return bool(lock), lock - def write_transaction(self): - """Get a write lock context manager for use in a `with` block.""" - return self._write_transaction_impl(self.lock, acquire=self._read, release=self._write) + def clear_all(self, clear_fn: Optional[Callable[[lk.Lock], Any]] = None) -> None: + if clear_fn is not None: + for lock in self.locks.values(): + clear_fn(lock) + self.locks.clear() - def read_transaction(self): - """Get a read lock context manager for use in a `with` block.""" - return self._read_transaction_impl(self.lock, acquire=self._read) - def _failed_spec_path(self, spec): - """Return the path to the spec's failure file, which may not exist.""" - if not spec.concrete: - raise ValueError("Concrete spec required for failure path for {0}".format(spec.name)) +class FailureTracker: + """Tracks installation failures. - return os.path.join(self._failure_dir, "{0}-{1}".format(spec.name, spec.dag_hash())) + Prefix failure marking takes the form of a byte range lock on the nth + byte of a file for coordinating between concurrent parallel build + processes and a persistent file, named with the full hash and + containing the spec, in a subdirectory of the database to enable + persistence across overlapping but separate related build processes. - def clear_all_failures(self): - """Force remove install failure tracking files.""" - tty.debug("Releasing prefix failure locks") - for pkg_id in list(self._prefix_failures.keys()): - lock = self._prefix_failures.pop(pkg_id, None) - if lock: - lock.release_write() + The failure lock file lives alongside the install DB. - # Remove all failure markings (aka files) - tty.debug("Removing prefix failure tracking files") - for fail_mark in os.listdir(self._failure_dir): - try: - os.remove(os.path.join(self._failure_dir, fail_mark)) - except OSError as exc: - tty.warn( - "Unable to remove failure marking file {0}: {1}".format(fail_mark, str(exc)) - ) + ``n`` is the sys.maxsize-bit prefix of the associated DAG hash to make + the likelihood of collision very low with no cleanup required. + """ - def clear_failure(self, spec, force=False): - """ - Remove any persistent and cached failure tracking for the spec. + def __init__(self, root_dir: Union[str, pathlib.Path], default_timeout: Optional[float]): + #: Ensure a persistent location for dealing with parallel installation + #: failures (e.g., across near-concurrent processes). + self.dir = pathlib.Path(root_dir) / _DB_DIRNAME / "failures" + self.dir.mkdir(parents=True, exist_ok=True) - see `mark_failed()`. + self.locker = SpecLocker(failures_lock_path(root_dir), default_timeout=default_timeout) - Args: - spec (spack.spec.Spec): the spec whose failure indicators are being removed - force (bool): True if the failure information should be cleared - when a prefix failure lock exists for the file or False if - the failure should not be cleared (e.g., it may be - associated with a concurrent build) + def clear(self, spec: "spack.spec.Spec", force: bool = False) -> None: + """Removes any persistent and cached failure tracking for the spec. + + see `mark()`. + Args: + spec: the spec whose failure indicators are being removed + force: True if the failure information should be cleared when a failure lock + exists for the file, or False if the failure should not be cleared (e.g., + it may be associated with a concurrent build) """ - failure_locked = self.prefix_failure_locked(spec) - if failure_locked and not force: - tty.msg("Retaining failure marking for {0} due to lock".format(spec.name)) + locked = self.lock_taken(spec) + if locked and not force: + tty.msg(f"Retaining failure marking for {spec.name} due to lock") return - if failure_locked: - tty.warn("Removing failure marking despite lock for {0}".format(spec.name)) + if locked: + tty.warn(f"Removing failure marking despite lock for {spec.name}") - lock = self._prefix_failures.pop(spec.prefix, None) - if lock: + succeeded, lock = self.locker.clear(spec) + if succeeded and lock is not None: lock.release_write() - if self.prefix_failure_marked(spec): + if self.persistent_mark(spec): + path = self._path(spec) + tty.debug(f"Removing failure marking for {spec.name}") try: - path = self._failed_spec_path(spec) - tty.debug("Removing failure marking for {0}".format(spec.name)) - os.remove(path) + path.unlink() except OSError as err: tty.warn( - "Unable to remove failure marking for {0} ({1}): {2}".format( - spec.name, path, str(err) - ) + f"Unable to remove failure marking for {spec.name} ({str(path)}): {str(err)}" ) - def mark_failed(self, spec): - """ - Mark a spec as failing to install. + def clear_all(self) -> None: + """Force remove install failure tracking files.""" + tty.debug("Releasing prefix failure locks") + self.locker.clear_all( + clear_fn=lambda x: x.release_write() if x.is_write_locked() else True + ) - Prefix failure marking takes the form of a byte range lock on the nth - byte of a file for coordinating between concurrent parallel build - processes and a persistent file, named with the full hash and - containing the spec, in a subdirectory of the database to enable - persistence across overlapping but separate related build processes. + tty.debug("Removing prefix failure tracking files") + try: + for fail_mark in os.listdir(str(self.dir)): + try: + (self.dir / fail_mark).unlink() + except OSError as exc: + tty.warn(f"Unable to remove failure marking file {fail_mark}: {str(exc)}") + except OSError as exc: + tty.warn(f"Unable to remove failure marking files: {str(exc)}") - The failure lock file, ``spack.store.db.prefix_failures``, lives - alongside the install DB. ``n`` is the sys.maxsize-bit prefix of the - associated DAG hash to make the likelihood of collision very low with - no cleanup required. + def mark(self, spec: "spack.spec.Spec") -> lk.Lock: + """Marks a spec as failing to install. + + Args: + spec: spec that failed to install """ # Dump the spec to the failure file for (manual) debugging purposes - path = self._failed_spec_path(spec) - with open(path, "w") as f: - spec.to_json(f) + path = self._path(spec) + path.write_text(spec.to_json()) # Also ensure a failure lock is taken to prevent cleanup removal # of failure status information during a concurrent parallel build. - err = "Unable to mark {0.name} as failed." - - prefix = spec.prefix - if prefix not in self._prefix_failures: - mark = lk.Lock( - self.prefix_fail_path, - start=spec.dag_hash_bit_prefix(bit_length(sys.maxsize)), - length=1, - default_timeout=self.package_lock_timeout, - desc=spec.name, - ) - + if not self.locker.has_lock(spec): try: + mark = self.locker.lock(spec) mark.acquire_write() except lk.LockTimeoutError: # Unlikely that another process failed to install at the same # time but log it anyway. - tty.debug( - "PID {0} failed to mark install failure for {1}".format(os.getpid(), spec.name) - ) - tty.warn(err.format(spec)) - - # Whether we or another process marked it as a failure, track it - # as such locally. - self._prefix_failures[prefix] = mark + tty.debug(f"PID {os.getpid()} failed to mark install failure for {spec.name}") + tty.warn(f"Unable to mark {spec.name} as failed.") - return self._prefix_failures[prefix] + return self.locker.lock(spec) - def prefix_failed(self, spec): - """Return True if the prefix (installation) is marked as failed.""" + def has_failed(self, spec: "spack.spec.Spec") -> bool: + """Return True if the spec is marked as failed.""" # The failure was detected in this process. - if spec.prefix in self._prefix_failures: + if self.locker.has_lock(spec): return True # The failure was detected by a concurrent process (e.g., an srun), # which is expected to be holding a write lock if that is the case. - if self.prefix_failure_locked(spec): + if self.lock_taken(spec): return True # Determine if the spec may have been marked as failed by a separate # spack build process running concurrently. - return self.prefix_failure_marked(spec) - - def prefix_failure_locked(self, spec): - """Return True if a process has a failure lock on the spec.""" - check = lk.Lock( - self.prefix_fail_path, - start=spec.dag_hash_bit_prefix(bit_length(sys.maxsize)), - length=1, - default_timeout=self.package_lock_timeout, - desc=spec.name, - ) + return self.persistent_mark(spec) + def lock_taken(self, spec: "spack.spec.Spec") -> bool: + """Return True if another process has a failure lock on the spec.""" + check = self.locker.raw_lock(spec) return check.is_write_locked() - def prefix_failure_marked(self, spec): + def persistent_mark(self, spec: "spack.spec.Spec") -> bool: """Determine if the spec has a persistent failure marking.""" - return os.path.exists(self._failed_spec_path(spec)) + return self._path(spec).exists() + + def _path(self, spec: "spack.spec.Spec") -> pathlib.Path: + """Return the path to the spec's failure file, which may not exist.""" + assert spec.concrete, "concrete spec required for failure path" + return self.dir / f"{spec.name}-{spec.dag_hash()}" - def prefix_lock(self, spec, timeout=None): - """Get a lock on a particular spec's installation directory. - NOTE: The installation directory **does not** need to exist. +class Database: + #: Fields written for each install record + record_fields: Tuple[str, ...] = DEFAULT_INSTALL_RECORD_FIELDS + + def __init__( + self, + root: str, + upstream_dbs: Optional[List["Database"]] = None, + is_upstream: bool = False, + lock_cfg: LockConfiguration = DEFAULT_LOCK_CFG, + ) -> None: + """Database for Spack installations. + + A Database is a cache of Specs data from ``$prefix/spec.yaml`` files + in Spack installation directories. - Prefix lock is a byte range lock on the nth byte of a file. + Database files (data and lock files) are stored under ``root/.spack-db``, which is + created if it does not exist. This is the "database directory". - The lock file is ``spack.store.db.prefix_lock`` -- the DB - tells us what to call it and it lives alongside the install DB. + The database will attempt to read an ``index.json`` file in the database directory. + If that does not exist, it will create a database when needed by scanning the entire + store root for ``spec.json`` files according to Spack's directory layout. - n is the sys.maxsize-bit prefix of the DAG hash. This makes - likelihood of collision is very low AND it gives us - readers-writer lock semantics with just a single lockfile, so no - cleanup required. + Args: + root: root directory where to create the database directory. + upstream_dbs: upstream databases for this repository. + is_upstream: whether this repository is an upstream. + lock_cfg: configuration for the locks to be used by this repository. + Relevant only if the repository is not an upstream. """ - timeout = timeout or self.package_lock_timeout - prefix = spec.prefix - if prefix not in self._prefix_locks: - self._prefix_locks[prefix] = lk.Lock( - self.prefix_lock_path, - start=spec.dag_hash_bit_prefix(bit_length(sys.maxsize)), - length=1, - default_timeout=timeout, - desc=spec.name, - ) - elif timeout != self._prefix_locks[prefix].default_timeout: - self._prefix_locks[prefix].default_timeout = timeout + self.root = root + self.database_directory = os.path.join(self.root, _DB_DIRNAME) - return self._prefix_locks[prefix] + # Set up layout of database files within the db dir + self._index_path = os.path.join(self.database_directory, "index.json") + self._verifier_path = os.path.join(self.database_directory, "index_verifier") + self._lock_path = os.path.join(self.database_directory, "lock") - @contextlib.contextmanager - def prefix_read_lock(self, spec): - prefix_lock = self.prefix_lock(spec) - prefix_lock.acquire_read() + # Create needed directories and files + if not is_upstream and not os.path.exists(self.database_directory): + fs.mkdirp(self.database_directory) - try: - yield self - except lk.LockError: - # This addresses the case where a nested lock attempt fails inside - # of this context manager - raise - except (Exception, KeyboardInterrupt): - prefix_lock.release_read() - raise - else: - prefix_lock.release_read() + self.is_upstream = is_upstream + self.last_seen_verifier = "" + # Failed write transactions (interrupted by exceptions) will alert + # _write. When that happens, we set this flag to indicate that + # future read/write transactions should re-read the DB. Normally it + # would make more sense to resolve this at the end of the transaction + # but typically a failed transaction will terminate the running + # instance of Spack and we don't want to incur an extra read in that + # case, so we defer the cleanup to when we begin the next transaction + self._state_is_inconsistent = False - @contextlib.contextmanager - def prefix_write_lock(self, spec): - prefix_lock = self.prefix_lock(spec) - prefix_lock.acquire_write() + # initialize rest of state. + self.db_lock_timeout = lock_cfg.database_timeout + tty.debug("DATABASE LOCK TIMEOUT: {0}s".format(str(self.db_lock_timeout))) - try: - yield self - except lk.LockError: - # This addresses the case where a nested lock attempt fails inside - # of this context manager - raise - except (Exception, KeyboardInterrupt): - prefix_lock.release_write() - raise + self.lock: Union[ForbiddenLock, lk.Lock] + if self.is_upstream: + self.lock = ForbiddenLock() else: - prefix_lock.release_write() + self.lock = lk.Lock( + self._lock_path, + default_timeout=self.db_lock_timeout, + desc="database", + enable=lock_cfg.enable, + ) + self._data: Dict[str, InstallRecord] = {} + + # For every installed spec we keep track of its install prefix, so that + # we can answer the simple query whether a given path is already taken + # before installing a different spec. + self._installed_prefixes: Set[str] = set() + + self.upstream_dbs = list(upstream_dbs) if upstream_dbs else [] + + # whether there was an error at the start of a read transaction + self._error = None + + # For testing: if this is true, an exception is thrown when missing + # dependencies are detected (rather than just printing a warning + # message) + self._fail_when_missing_deps = False + + self._write_transaction_impl = lk.WriteTransaction + self._read_transaction_impl = lk.ReadTransaction + + def write_transaction(self): + """Get a write lock context manager for use in a `with` block.""" + return self._write_transaction_impl(self.lock, acquire=self._read, release=self._write) + + def read_transaction(self): + """Get a read lock context manager for use in a `with` block.""" + return self._read_transaction_impl(self.lock, acquire=self._read) def _write_to_file(self, stream): """Write out the database in JSON format to the stream passed @@ -657,7 +679,7 @@ def _write_to_file(self, stream): """ # map from per-spec hash code to installation record. installs = dict( - (k, v.to_dict(include_fields=self._record_fields)) for k, v in self._data.items() + (k, v.to_dict(include_fields=self.record_fields)) for k, v in self._data.items() ) # database includes installation list and version. @@ -670,7 +692,7 @@ def _write_to_file(self, stream): "database": { # TODO: move this to a top-level _meta section if we ever # TODO: bump the DB version to 7 - "version": str(_db_version), + "version": str(_DB_VERSION), # dictionary of installation records, keyed by DAG hash "installs": installs, } @@ -710,7 +732,9 @@ def db_for_spec_hash(self, hash_key): if hash_key in db._data: return db - def query_by_spec_hash(self, hash_key, data=None): + def query_by_spec_hash( + self, hash_key: str, data: Optional[Dict[str, InstallRecord]] = None + ) -> Tuple[bool, Optional[InstallRecord]]: """Get a spec for hash, and whether it's installed upstream. Return: @@ -773,7 +797,7 @@ def _assign_dependencies(self, spec_reader, hash_key, installs, data): tty.warn(msg) continue - spec._add_dependency(child, deptypes=dtypes, virtuals=virtuals) + spec._add_dependency(child, depflag=dt.canonicalize(dtypes), virtuals=virtuals) def _read_from_file(self, filename): """Fill database from file, do not maintain old data. @@ -805,16 +829,16 @@ def check(cond, msg): # TODO: better version checking semantics. version = vn.Version(db["version"]) - if version > _db_version: - raise InvalidDatabaseVersionError(self, _db_version, version) - elif version < _db_version: - if not any(old == version and new == _db_version for old, new in _skip_reindex): + if version > _DB_VERSION: + raise InvalidDatabaseVersionError(self, _DB_VERSION, version) + elif version < _DB_VERSION: + if not any(old == version and new == _DB_VERSION for old, new in _SKIP_REINDEX): tty.warn( "Spack database version changed from %s to %s. Upgrading." - % (version, _db_version) + % (version, _DB_VERSION) ) - self.reindex(spack.store.layout) + self.reindex(spack.store.STORE.layout) installs = dict( (k, v.to_dict(include_fields=self._record_fields)) for k, v in self._data.items() @@ -976,7 +1000,7 @@ def _construct_from_directory_layout(self, directory_layout, old_data): # applications. tty.debug("RECONSTRUCTING FROM OLD DB: {0}".format(entry.spec)) try: - layout = None if entry.spec.external else spack.store.layout + layout = None if entry.spec.external else directory_layout kwargs = { "spec": entry.spec, "directory_layout": layout, @@ -1002,7 +1026,7 @@ def _check_ref_counts(self): counts = {} for key, rec in self._data.items(): counts.setdefault(key, 0) - for dep in rec.spec.dependencies(deptype=_tracked_deps): + for dep in rec.spec.dependencies(deptype=_TRACKED_DEPENDENCIES): dep_key = dep.dag_hash() counts.setdefault(dep_key, 0) counts[dep_key] += 1 @@ -1091,13 +1115,13 @@ def _add( ): """Add an install record for this spec to the database. - Assumes spec is installed in ``layout.path_for_spec(spec)``. + Assumes spec is installed in ``directory_layout.path_for_spec(spec)``. Also ensures dependencies are present and updated in the DB as either installed or missing. Args: - spec: spec to be added + spec (spack.spec.Spec): spec to be added directory_layout: layout of the spec installation explicit: Possible values: True, False, any @@ -1124,7 +1148,7 @@ def _add( # Retrieve optional arguments installation_time = installation_time or _now() - for edge in spec.edges_to_dependencies(deptype=_tracked_deps): + for edge in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES): if edge.spec.dag_hash() in self._data: continue # allow missing build-only deps. This prevents excessive @@ -1132,7 +1156,7 @@ def _add( # is missing a build dep; there's no need to install the # build dep's build dep first, and there's no need to warn # about it missing. - dep_allow_missing = allow_missing or edge.deptypes == ("build",) + dep_allow_missing = allow_missing or edge.depflag == dt.BUILD self._add( edge.spec, directory_layout, @@ -1176,10 +1200,10 @@ def _add( self._data[key] = InstallRecord(new_spec, path, installed, ref_count=0, **extra_args) # Connect dependencies from the DB to the new copy. - for dep in spec.edges_to_dependencies(deptype=_tracked_deps): + for dep in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES): dkey = dep.spec.dag_hash() upstream, record = self.query_by_spec_hash(dkey) - new_spec._add_dependency(record.spec, deptypes=dep.deptypes, virtuals=dep.virtuals) + new_spec._add_dependency(record.spec, depflag=dep.depflag, virtuals=dep.virtuals) if not upstream: record.ref_count += 1 @@ -1239,7 +1263,7 @@ def _decrement_ref_count(self, spec): if rec.ref_count == 0 and not rec.installed: del self._data[key] - for dep in spec.dependencies(deptype=_tracked_deps): + for dep in spec.dependencies(deptype=_TRACKED_DEPENDENCIES): self._decrement_ref_count(dep) def _increment_ref_count(self, spec): @@ -1269,8 +1293,8 @@ def _remove(self, spec): # Remove any reference to this node from dependencies and # decrement the reference count - rec.spec.detach(deptype=_tracked_deps) - for dep in rec.spec.dependencies(deptype=_tracked_deps): + rec.spec.detach(deptype=_TRACKED_DEPENDENCIES) + for dep in rec.spec.dependencies(deptype=_TRACKED_DEPENDENCIES): self._decrement_ref_count(dep) if rec.deprecated_for: @@ -1349,7 +1373,13 @@ def deprecate(self, spec, deprecator): return self._deprecate(spec, deprecator) @_autospec - def installed_relatives(self, spec, direction="children", transitive=True, deptype="all"): + def installed_relatives( + self, + spec, + direction="children", + transitive=True, + deptype: Union[dt.DepFlag, dt.DepTypes] = dt.ALL, + ): """Return installed specs related to this one.""" if direction not in ("parents", "children"): raise ValueError("Invalid direction: %s" % direction) @@ -1386,10 +1416,7 @@ def installed_relatives(self, spec, direction="children", transitive=True, depty @_autospec def installed_extensions_for(self, extendee_spec): - """ - Return the specs of all packages that extend - the given spec - """ + """Returns the specs of all packages that extend the given spec""" for spec in self.query(): if spec.package.extends(extendee_spec): yield spec.package @@ -1416,7 +1443,7 @@ def _get_by_hash_local(self, dag_hash, default=None, installed=any): # nothing found return default - def get_by_hash_local(self, *args, **kwargs): + def get_by_hash_local(self, dag_hash, default=None, installed=any): """Look up a spec in *this DB* by DAG hash, or by a DAG hash prefix. Arguments: @@ -1440,7 +1467,7 @@ def get_by_hash_local(self, *args, **kwargs): """ with self.read_transaction(): - return self._get_by_hash_local(*args, **kwargs) + return self._get_by_hash_local(dag_hash, default=default, installed=installed) def get_by_hash(self, dag_hash, default=None, installed=any): """Look up a spec by DAG hash, or by a DAG hash prefix. @@ -1495,14 +1522,18 @@ def _query( # TODO: like installed and known that can be queried? Or are # TODO: these really special cases that only belong here? - # Just look up concrete specs with hashes; no fancy search. - if isinstance(query_spec, spack.spec.Spec) and query_spec.concrete: - # TODO: handling of hashes restriction is not particularly elegant. - hash_key = query_spec.dag_hash() - if hash_key in self._data and (not hashes or hash_key in hashes): - return [self._data[hash_key].spec] - else: - return [] + if query_spec is not any: + if not isinstance(query_spec, spack.spec.Spec): + query_spec = spack.spec.Spec(query_spec) + + # Just look up concrete specs with hashes; no fancy search. + if query_spec.concrete: + # TODO: handling of hashes restriction is not particularly elegant. + hash_key = query_spec.dag_hash() + if hash_key in self._data and (not hashes or hash_key in hashes): + return [self._data[hash_key].spec] + else: + return [] # Abstract specs require more work -- currently we test # against everything. @@ -1510,6 +1541,9 @@ def _query( start_date = start_date or datetime.datetime.min end_date = end_date or datetime.datetime.max + # save specs whose name doesn't match for last, to avoid a virtual check + deferred = [] + for key, rec in self._data.items(): if hashes is not None and rec.spec.dag_hash() not in hashes: continue @@ -1526,7 +1560,7 @@ def _query( if explicit is not any and rec.explicit != explicit: continue - if known is not any and spack.repo.path.exists(rec.spec.name) != known: + if known is not any and known(rec.spec.name): continue if start_date or end_date: @@ -1534,14 +1568,32 @@ def _query( if not (start_date < inst_date < end_date): continue - if query_spec is any or rec.spec.satisfies(query_spec): + if query_spec is any: results.append(rec.spec) + continue + + # check anon specs and exact name matches first + if not query_spec.name or rec.spec.name == query_spec.name: + if rec.spec.satisfies(query_spec): + results.append(rec.spec) + + # save potential virtual matches for later, but not if we already found a match + elif not results: + deferred.append(rec.spec) + + # Checking for virtuals is expensive, so we save it for last and only if needed. + # If we get here, we didn't find anything in the DB that matched by name. + # If we did fine something, the query spec can't be virtual b/c we matched an actual + # package installation, so skip the virtual check entirely. If we *didn't* find anything, + # check all the deferred specs *if* the query is virtual. + if not results and query_spec is not any and deferred and query_spec.virtual: + results = [spec for spec in deferred if spec.satisfies(query_spec)] return results if _query.__doc__ is None: _query.__doc__ = "" - _query.__doc__ += _query_docstring + _query.__doc__ += _QUERY_DOCSTRING def query_local(self, *args, **kwargs): """Query only the local Spack database. @@ -1555,7 +1607,7 @@ def query_local(self, *args, **kwargs): if query_local.__doc__ is None: query_local.__doc__ = "" - query_local.__doc__ += _query_docstring + query_local.__doc__ += _QUERY_DOCSTRING def query(self, *args, **kwargs): """Query the Spack database including all upstream databases.""" @@ -1574,7 +1626,7 @@ def query(self, *args, **kwargs): if query.__doc__ is None: query.__doc__ = "" - query.__doc__ += _query_docstring + query.__doc__ += _QUERY_DOCSTRING def query_one(self, query_spec, known=any, installed=True): """Query for exactly one spec that matches the query spec. diff --git a/lib/spack/spack/dependency.py b/lib/spack/spack/dependency.py index d1b7fbae623713..3bb2df791b91b3 100644 --- a/lib/spack/spack/dependency.py +++ b/lib/spack/spack/dependency.py @@ -3,64 +3,11 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Data structures that represent Spack's dependency relationships.""" -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Dict, List +import spack.deptypes as dt import spack.spec -#: The types of dependency relationships that Spack understands. -all_deptypes = ("build", "link", "run", "test") - -#: Default dependency type if none is specified -default_deptype = ("build", "link") - -#: Type hint for the arguments accepting a dependency type -DependencyArgument = Union[str, List[str], Tuple[str, ...]] - - -def deptype_chars(*type_tuples: str) -> str: - """Create a string representing deptypes for many dependencies. - - The string will be some subset of 'blrt', like 'bl ', 'b t', or - ' lr ' where each letter in 'blrt' stands for 'build', 'link', - 'run', and 'test' (the dependency types). - - For a single dependency, this just indicates that the dependency has - the indicated deptypes. For a list of dependnecies, this shows - whether ANY dpeendency in the list has the deptypes (so the deptypes - are merged). - """ - types: Set[str] = set() - for t in type_tuples: - if t: - types.update(t) - - return "".join(t[0] if t in types else " " for t in all_deptypes) - - -def canonical_deptype(deptype: DependencyArgument) -> Tuple[str, ...]: - """Convert deptype to a canonical sorted tuple, or raise ValueError. - - Args: - deptype: string representing dependency type, or a list/tuple of such strings. - Can also be the builtin function ``all`` or the string 'all', which result in - a tuple of all dependency types known to Spack. - """ - if deptype in ("all", all): - return all_deptypes - - elif isinstance(deptype, str): - if deptype not in all_deptypes: - raise ValueError("Invalid dependency type: %s" % deptype) - return (deptype,) - - elif isinstance(deptype, (tuple, list, set)): - bad = [d for d in deptype if d not in all_deptypes] - if bad: - raise ValueError("Invalid dependency types: %s" % ",".join(str(t) for t in bad)) - return tuple(sorted(set(deptype))) - - raise ValueError("Invalid dependency type: %s" % repr(deptype)) - class Dependency: """Class representing metadata for a dependency on a package. @@ -93,7 +40,7 @@ def __init__( self, pkg: "spack.package_base.PackageBase", spec: "spack.spec.Spec", - type: Optional[Tuple[str, ...]] = default_deptype, + depflag: dt.DepFlag = dt.DEFAULT, ): """Create a new Dependency. @@ -110,11 +57,7 @@ def __init__( # This dict maps condition specs to lists of Patch objects, just # as the patches dict on packages does. self.patches: Dict[spack.spec.Spec, "List[spack.patch.Patch]"] = {} - - if type is None: - self.type = set(default_deptype) - else: - self.type = set(type) + self.depflag = depflag @property def name(self) -> str: @@ -124,7 +67,7 @@ def name(self) -> str: def merge(self, other: "Dependency"): """Merge constraints, deptypes, and patches of other into self.""" self.spec.constrain(other.spec) - self.type |= other.type + self.depflag |= other.depflag # concatenate patch lists, or just copy them in for cond, p in other.patches.items(): @@ -135,5 +78,5 @@ def merge(self, other: "Dependency"): self.patches[cond] = other.patches[cond] def __repr__(self) -> str: - types = deptype_chars(*self.type) + types = dt.flag_to_chars(self.depflag) return f" {self.spec} [{types}]>" diff --git a/lib/spack/spack/deptypes.py b/lib/spack/spack/deptypes.py new file mode 100644 index 00000000000000..91ac6907908df3 --- /dev/null +++ b/lib/spack/spack/deptypes.py @@ -0,0 +1,123 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Data structures that represent Spack's edge types.""" + +from typing import Iterable, List, Tuple, Union + +#: Type hint for the low-level dependency input (enum.Flag is too slow) +DepFlag = int + +#: Type hint for the high-level dependency input +DepTypes = Union[str, List[str], Tuple[str, ...]] + +#: Individual dependency types +DepType = str # Python 3.8: Literal["build", "link", "run", "test"] + +# Flag values. NOTE: these values are not arbitrary, since hash computation imposes +# the order (link, run, build, test) when depending on the same package multiple times, +# and we rely on default integer comparison to sort dependency types. +# New dependency types should be appended. +LINK = 0b0001 +RUN = 0b0010 +BUILD = 0b0100 +TEST = 0b1000 + +#: The types of dependency relationships that Spack understands. +ALL_TYPES: Tuple[DepType, ...] = ("build", "link", "run", "test") + +#: Default dependency type if none is specified +DEFAULT_TYPES: Tuple[DepType, ...] = ("build", "link") + +#: A flag with all dependency types set +ALL: DepFlag = BUILD | LINK | RUN | TEST + +#: Default dependency type if none is specified +DEFAULT: DepFlag = BUILD | LINK + +#: An iterator of all flag components +ALL_FLAGS: Tuple[DepFlag, DepFlag, DepFlag, DepFlag] = (BUILD, LINK, RUN, TEST) + + +def flag_from_string(s: str) -> DepFlag: + if s == "build": + return BUILD + elif s == "link": + return LINK + elif s == "run": + return RUN + elif s == "test": + return TEST + else: + raise ValueError(f"Invalid dependency type: {s}") + + +def flag_from_strings(deptype: Iterable[str]) -> DepFlag: + """Transform an iterable of deptype strings into a flag.""" + flag = 0 + for deptype_str in deptype: + flag |= flag_from_string(deptype_str) + return flag + + +def canonicalize(deptype: DepTypes) -> DepFlag: + """Convert deptype user input to a DepFlag, or raise ValueError. + + Args: + deptype: string representing dependency type, or a list/tuple of such strings. + Can also be the builtin function ``all`` or the string 'all', which result in + a tuple of all dependency types known to Spack. + """ + if deptype in ("all", all): + return ALL + + if isinstance(deptype, str): + return flag_from_string(deptype) + + if isinstance(deptype, (tuple, list, set)): + return flag_from_strings(deptype) + + raise ValueError(f"Invalid dependency type: {deptype!r}") + + +def flag_to_tuple(x: DepFlag) -> Tuple[DepType, ...]: + deptype: List[DepType] = [] + if x & BUILD: + deptype.append("build") + if x & LINK: + deptype.append("link") + if x & RUN: + deptype.append("run") + if x & TEST: + deptype.append("test") + return tuple(deptype) + + +def flag_to_string(x: DepFlag) -> DepType: + if x == BUILD: + return "build" + elif x == LINK: + return "link" + elif x == RUN: + return "run" + elif x == TEST: + return "test" + else: + raise ValueError(f"Invalid dependency type flag: {x}") + + +def flag_to_chars(depflag: DepFlag) -> str: + """Create a string representing deptypes for many dependencies. + + The string will be some subset of 'blrt', like 'bl ', 'b t', or + ' lr ' where each letter in 'blrt' stands for 'build', 'link', + 'run', and 'test' (the dependency types). + + For a single dependency, this just indicates that the dependency has + the indicated deptypes. For a list of dependnecies, this shows + whether ANY dpeendency in the list has the deptypes (so the deptypes + are merged).""" + return "".join( + t_str[0] if t_flag & depflag else " " for t_str, t_flag in zip(ALL_TYPES, ALL_FLAGS) + ) diff --git a/lib/spack/spack/detection/__init__.py b/lib/spack/spack/detection/__init__.py index 17166cdc099872..7c54fb9d49ba76 100644 --- a/lib/spack/spack/detection/__init__.py +++ b/lib/spack/spack/detection/__init__.py @@ -3,13 +3,14 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from .common import DetectedPackage, executable_prefix, update_configuration -from .path import by_executable, by_library, executables_in_path +from .path import by_path, executables_in_path +from .test import detection_tests __all__ = [ "DetectedPackage", - "by_library", - "by_executable", + "by_path", "executables_in_path", "executable_prefix", "update_configuration", + "detection_tests", ] diff --git a/lib/spack/spack/detection/common.py b/lib/spack/spack/detection/common.py index 41455e43aebf85..6fba021b336b0c 100644 --- a/lib/spack/spack/detection/common.py +++ b/lib/spack/spack/detection/common.py @@ -13,13 +13,13 @@ The module also contains other functions that might be useful across different detection mechanisms. """ -import collections import glob import itertools import os import os.path import re import sys +from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union import llnl.util.tty @@ -29,12 +29,28 @@ import spack.util.spack_yaml import spack.util.windows_registry -#: Information on a package that has been detected -DetectedPackage = collections.namedtuple("DetectedPackage", ["spec", "prefix"]) +class DetectedPackage(NamedTuple): + """Information on a package that has been detected.""" -def _externals_in_packages_yaml(): - """Return all the specs mentioned as externals in packages.yaml""" + #: Spec that was detected + spec: spack.spec.Spec + #: Prefix of the spec + prefix: str + + def __reduce__(self): + return DetectedPackage.restore, (str(self.spec), self.prefix, self.spec.extra_attributes) + + @staticmethod + def restore( + spec_str: str, prefix: str, extra_attributes: Optional[Dict[str, str]] + ) -> "DetectedPackage": + spec = spack.spec.Spec.from_detection(spec_str=spec_str, extra_attributes=extra_attributes) + return DetectedPackage(spec=spec, prefix=prefix) + + +def _externals_in_packages_yaml() -> Set[spack.spec.Spec]: + """Returns all the specs mentioned as externals in packages.yaml""" packages_yaml = spack.config.get("packages") already_defined_specs = set() for pkg_name, package_configuration in packages_yaml.items(): @@ -43,7 +59,12 @@ def _externals_in_packages_yaml(): return already_defined_specs -def _pkg_config_dict(external_pkg_entries): +ExternalEntryType = Union[str, Dict[str, str]] + + +def _pkg_config_dict( + external_pkg_entries: List[DetectedPackage], +) -> Dict[str, Union[bool, List[Dict[str, ExternalEntryType]]]]: """Generate a package specific config dict according to the packages.yaml schema. This does not generate the entire packages.yaml. For example, given some @@ -65,7 +86,10 @@ def _pkg_config_dict(external_pkg_entries): if not _spec_is_valid(e.spec): continue - external_items = [("spec", str(e.spec)), ("prefix", e.prefix)] + external_items: List[Tuple[str, ExternalEntryType]] = [ + ("spec", str(e.spec)), + ("prefix", e.prefix), + ] if e.spec.external_modules: external_items.append(("modules", e.spec.external_modules)) @@ -83,15 +107,14 @@ def _pkg_config_dict(external_pkg_entries): return pkg_dict -def _spec_is_valid(spec): +def _spec_is_valid(spec: spack.spec.Spec) -> bool: try: str(spec) except spack.error.SpackError: - # It is assumed here that we can at least extract the package name from - # the spec so we can look up the implementation of - # determine_spec_details - msg = "Constructed spec for {0} does not have a string representation" - llnl.util.tty.warn(msg.format(spec.name)) + # It is assumed here that we can at least extract the package name from the spec so we + # can look up the implementation of determine_spec_details + msg = f"Constructed spec for {spec.name} does not have a string representation" + llnl.util.tty.warn(msg) return False try: @@ -106,7 +129,7 @@ def _spec_is_valid(spec): return True -def path_to_dict(search_paths): +def path_to_dict(search_paths: List[str]): """Return dictionary[fullpath]: basename from list of paths""" path_to_lib = {} # Reverse order of search directories so that a lib in the first @@ -124,7 +147,7 @@ def path_to_dict(search_paths): return path_to_lib -def is_executable(file_path): +def is_executable(file_path: str) -> bool: """Return True if the path passed as argument is that of an executable""" return os.path.isfile(file_path) and os.access(file_path, os.X_OK) @@ -146,7 +169,7 @@ def _convert_to_iterable(single_val_or_multiple): return [x] -def executable_prefix(executable_dir): +def executable_prefix(executable_dir: str) -> str: """Given a directory where an executable is found, guess the prefix (i.e. the "root" directory of that installation) and return it. @@ -167,12 +190,12 @@ def executable_prefix(executable_dir): return os.sep.join(components[:idx]) -def library_prefix(library_dir): - """Given a directory where an library is found, guess the prefix +def library_prefix(library_dir: str) -> str: + """Given a directory where a library is found, guess the prefix (i.e. the "root" directory of that installation) and return it. Args: - library_dir: directory where an library is found + library_dir: directory where a library is found """ # Given a prefix where an library is found, assuming that prefix # contains /lib/ or /lib64/, strip off the 'lib' or 'lib64' directory @@ -195,13 +218,17 @@ def library_prefix(library_dir): return library_dir -def update_configuration(detected_packages, scope=None, buildable=True): +def update_configuration( + detected_packages: Dict[str, List[DetectedPackage]], + scope: Optional[str] = None, + buildable: bool = True, +) -> List[spack.spec.Spec]: """Add the packages passed as arguments to packages.yaml Args: - detected_packages (list): list of DetectedPackage objects to be added - scope (str): configuration scope where to add the detected packages - buildable (bool): whether the detected packages are buildable or not + detected_packages: list of DetectedPackage objects to be added + scope: configuration scope where to add the detected packages + buildable: whether the detected packages are buildable or not """ predefined_external_specs = _externals_in_packages_yaml() pkg_to_cfg, all_new_specs = {}, [] @@ -209,7 +236,10 @@ def update_configuration(detected_packages, scope=None, buildable=True): new_entries = [e for e in entries if (e.spec not in predefined_external_specs)] pkg_config = _pkg_config_dict(new_entries) - all_new_specs.extend([spack.spec.Spec(x["spec"]) for x in pkg_config.get("externals", [])]) + external_entries = pkg_config.get("externals", []) + assert not isinstance(external_entries, bool), "unexpected value for external entry" + + all_new_specs.extend([spack.spec.Spec(x["spec"]) for x in external_entries]) if buildable is False: pkg_config["buildable"] = False pkg_to_cfg[package_name] = pkg_config @@ -222,24 +252,27 @@ def update_configuration(detected_packages, scope=None, buildable=True): return all_new_specs -def _windows_drive(): - """Return Windows drive string extracted from PROGRAMFILES - env var, which is garunteed to be defined for all logins""" - drive = re.match(r"([a-zA-Z]:)", os.environ["PROGRAMFILES"]).group(1) - return drive +def _windows_drive() -> str: + """Return Windows drive string extracted from the PROGRAMFILES environment variable, + which is guaranteed to be defined for all logins. + """ + match = re.match(r"([a-zA-Z]:)", os.environ["PROGRAMFILES"]) + if match is None: + raise RuntimeError("cannot read the PROGRAMFILES environment variable") + return match.group(1) class WindowsCompilerExternalPaths: @staticmethod - def find_windows_compiler_root_paths(): + def find_windows_compiler_root_paths() -> List[str]: """Helper for Windows compiler installation root discovery At the moment simply returns location of VS install paths from VSWhere But should be extended to include more information as relevant""" - return list(winOs.WindowsOs.vs_install_paths) + return list(winOs.WindowsOs().vs_install_paths) @staticmethod - def find_windows_compiler_cmake_paths(): + def find_windows_compiler_cmake_paths() -> List[str]: """Semi hard-coded search path for cmake bundled with MSVC""" return [ os.path.join( @@ -249,7 +282,7 @@ def find_windows_compiler_cmake_paths(): ] @staticmethod - def find_windows_compiler_ninja_paths(): + def find_windows_compiler_ninja_paths() -> List[str]: """Semi hard-coded search heuristic for locating ninja bundled with MSVC""" return [ os.path.join(path, "Common7", "IDE", "CommonExtensions", "Microsoft", "CMake", "Ninja") @@ -257,7 +290,7 @@ def find_windows_compiler_ninja_paths(): ] @staticmethod - def find_windows_compiler_bundled_packages(): + def find_windows_compiler_bundled_packages() -> List[str]: """Return all MSVC compiler bundled packages""" return ( WindowsCompilerExternalPaths.find_windows_compiler_cmake_paths() @@ -266,36 +299,39 @@ def find_windows_compiler_bundled_packages(): class WindowsKitExternalPaths: - if sys.platform == "win32": - plat_major_ver = str(winOs.windows_version()[0]) - @staticmethod - def find_windows_kit_roots(): + def find_windows_kit_roots() -> List[str]: """Return Windows kit root, typically %programfiles%\\Windows Kits\\10|11\\""" if sys.platform != "win32": return [] program_files = os.environ["PROGRAMFILES(x86)"] - kit_base = os.path.join( - program_files, "Windows Kits", WindowsKitExternalPaths.plat_major_ver - ) - return kit_base + kit_base = os.path.join(program_files, "Windows Kits", "**") + return glob.glob(kit_base) @staticmethod - def find_windows_kit_bin_paths(kit_base=None): + def find_windows_kit_bin_paths(kit_base: Optional[str] = None) -> List[str]: """Returns Windows kit bin directory per version""" kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base - kit_bin = os.path.join(kit_base, "bin") - return glob.glob(os.path.join(kit_bin, "[0-9]*", "*\\")) + assert kit_base, "Unexpectedly empty value for Windows kit base path" + kit_paths = [] + for kit in kit_base: + kit_bin = os.path.join(kit, "bin") + kit_paths.extend(glob.glob(os.path.join(kit_bin, "[0-9]*", "*\\"))) + return kit_paths @staticmethod - def find_windows_kit_lib_paths(kit_base=None): + def find_windows_kit_lib_paths(kit_base: Optional[str] = None) -> List[str]: """Returns Windows kit lib directory per version""" kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base - kit_lib = os.path.join(kit_base, "Lib") - return glob.glob(os.path.join(kit_lib, "[0-9]*", "*", "*\\")) + assert kit_base, "Unexpectedly empty value for Windows kit base path" + kit_paths = [] + for kit in kit_base: + kit_lib = os.path.join(kit, "Lib") + kit_paths.extend(glob.glob(os.path.join(kit_lib, "[0-9]*", "*", "*\\"))) + return kit_paths @staticmethod - def find_windows_driver_development_kit_paths(): + def find_windows_driver_development_kit_paths() -> List[str]: """Provides a list of all installation paths for the WDK by version and architecture """ @@ -303,7 +339,7 @@ def find_windows_driver_development_kit_paths(): return WindowsKitExternalPaths.find_windows_kit_lib_paths(wdk_content_root) @staticmethod - def find_windows_kit_reg_installed_roots_paths(): + def find_windows_kit_reg_installed_roots_paths() -> List[str]: reg = spack.util.windows_registry.WindowsRegistryView( "SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots", root_key=spack.util.windows_registry.HKEY.HKEY_LOCAL_MACHINE, @@ -311,26 +347,33 @@ def find_windows_kit_reg_installed_roots_paths(): if not reg: # couldn't find key, return empty list return [] - return WindowsKitExternalPaths.find_windows_kit_lib_paths( - reg.get_value("KitsRoot%s" % WindowsKitExternalPaths.plat_major_ver).value - ) + kit_root_reg = re.compile(r"KitsRoot[0-9]+") + root_paths = [] + for kit_root in filter(kit_root_reg.match, reg.get_values().keys()): + root_paths.extend( + WindowsKitExternalPaths.find_windows_kit_lib_paths(reg.get_value(kit_root).value) + ) + return root_paths @staticmethod - def find_windows_kit_reg_sdk_paths(): - reg = spack.util.windows_registry.WindowsRegistryView( - "SOFTWARE\\WOW6432Node\\Microsoft\\Microsoft SDKs\\Windows\\v%s.0" - % WindowsKitExternalPaths.plat_major_ver, + def find_windows_kit_reg_sdk_paths() -> List[str]: + sdk_paths = [] + sdk_regex = re.compile(r"v[0-9]+.[0-9]+") + windows_reg = spack.util.windows_registry.WindowsRegistryView( + "SOFTWARE\\WOW6432Node\\Microsoft\\Microsoft SDKs\\Windows", root_key=spack.util.windows_registry.HKEY.HKEY_LOCAL_MACHINE, ) - if not reg: - # couldn't find key, return empty list - return [] - return WindowsKitExternalPaths.find_windows_kit_lib_paths( - reg.get_value("InstallationFolder").value - ) + for key in filter(sdk_regex.match, [x.name for x in windows_reg.get_subkeys()]): + reg = windows_reg.get_subkey(key) + sdk_paths.extend( + WindowsKitExternalPaths.find_windows_kit_lib_paths( + reg.get_value("InstallationFolder").value + ) + ) + return sdk_paths -def find_win32_additional_install_paths(): +def find_win32_additional_install_paths() -> List[str]: """Not all programs on Windows live on the PATH Return a list of other potential install locations. """ @@ -357,13 +400,12 @@ def find_win32_additional_install_paths(): return windows_search_ext -def compute_windows_program_path_for_package(pkg): - """Given a package, attempt to compute its Windows - program files location, return list of best guesses +def compute_windows_program_path_for_package(pkg: "spack.package_base.PackageBase") -> List[str]: + """Given a package, attempts to compute its Windows program files location, + and returns the list of best guesses. Args: - pkg (spack.package_base.PackageBase): package for which - Program Files location is to be computed + pkg: package for which Program Files location is to be computed """ if sys.platform != "win32": return [] @@ -378,7 +420,7 @@ def compute_windows_program_path_for_package(pkg): ] -def compute_windows_user_path_for_package(pkg): +def compute_windows_user_path_for_package(pkg: "spack.package_base.PackageBase") -> List[str]: """Given a package attempt to compute its user scoped install location, return list of potential locations based on common heuristics. For more info on Windows user specific diff --git a/lib/spack/spack/detection/path.py b/lib/spack/spack/detection/path.py index 04cf682f5ceb99..f5da02bede1842 100644 --- a/lib/spack/spack/detection/path.py +++ b/lib/spack/spack/detection/path.py @@ -2,23 +2,28 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -"""Detection of software installed in the system based on paths inspections +"""Detection of software installed in the system, based on paths inspections and running executables. """ import collections +import concurrent.futures import os import os.path import re import sys import warnings +from typing import Dict, List, Optional, Set, Tuple import llnl.util.filesystem +import llnl.util.lang import llnl.util.tty +import spack.util.elf as elf_utils import spack.util.environment +import spack.util.environment as environment import spack.util.ld_so_conf -from .common import ( # find_windows_compiler_bundled_packages, +from .common import ( DetectedPackage, WindowsCompilerExternalPaths, WindowsKitExternalPaths, @@ -31,17 +36,36 @@ path_to_dict, ) +#: Timeout used for package detection (seconds) +DETECTION_TIMEOUT = 60 +if sys.platform == "win32": + DETECTION_TIMEOUT = 120 -def common_windows_package_paths(): + +def common_windows_package_paths(pkg_cls=None) -> List[str]: + """Get the paths for common package installation location on Windows + that are outside the PATH + Returns [] on unix + """ + if sys.platform != "win32": + return [] paths = WindowsCompilerExternalPaths.find_windows_compiler_bundled_packages() paths.extend(find_win32_additional_install_paths()) paths.extend(WindowsKitExternalPaths.find_windows_kit_bin_paths()) paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_installed_roots_paths()) paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_sdk_paths()) + if pkg_cls: + paths.extend(compute_windows_user_path_for_package(pkg_cls)) + paths.extend(compute_windows_program_path_for_package(pkg_cls)) return paths -def executables_in_path(path_hints): +def file_identifier(path): + s = os.stat(path) + return (s.st_dev, s.st_ino) + + +def executables_in_path(path_hints: List[str]) -> Dict[str, str]: """Get the paths of all executables available from the current PATH. For convenience, this is constructed as a dictionary where the keys are @@ -52,19 +76,47 @@ def executables_in_path(path_hints): assumed there are two different instances of the executable. Args: - path_hints (list): list of paths to be searched. If None the list will be + path_hints: list of paths to be searched. If None the list will be constructed based on the PATH environment variable. """ - if sys.platform == "win32": - path_hints.extend(common_windows_package_paths()) search_paths = llnl.util.filesystem.search_paths_for_executables(*path_hints) return path_to_dict(search_paths) -def libraries_in_ld_and_system_library_path(path_hints=None): - """Get the paths of all libraries available from LD_LIBRARY_PATH, - LIBRARY_PATH, DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH, and - standard system library paths. +def get_elf_compat(path): + """For ELF files, get a triplet (EI_CLASS, EI_DATA, e_machine) and see if + it is host-compatible.""" + # On ELF platforms supporting, we try to be a bit smarter when it comes to shared + # libraries, by dropping those that are not host compatible. + with open(path, "rb") as f: + elf = elf_utils.parse_elf(f, only_header=True) + return (elf.is_64_bit, elf.is_little_endian, elf.elf_hdr.e_machine) + + +def accept_elf(path, host_compat): + """Accept an ELF file if the header matches the given compat triplet, + obtained with :py:func:`get_elf_compat`. In case it's not an ELF (e.g. + static library, or some arbitrary file, fall back to is_readable_file).""" + # Fast path: assume libraries at least have .so in their basename. + # Note: don't replace with splitext, because of libsmth.so.1.2.3 file names. + if ".so" not in os.path.basename(path): + return llnl.util.filesystem.is_readable_file(path) + try: + return host_compat == get_elf_compat(path) + except (OSError, elf_utils.ElfParsingError): + return llnl.util.filesystem.is_readable_file(path) + + +def libraries_in_ld_and_system_library_path( + path_hints: Optional[List[str]] = None, +) -> Dict[str, str]: + """Get the paths of all libraries available from ``path_hints`` or the + following defaults: + + - Environment variables (Linux: ``LD_LIBRARY_PATH``, Darwin: ``DYLD_LIBRARY_PATH``, + and ``DYLD_FALLBACK_LIBRARY_PATH``) + - Dynamic linker default paths (glibc: ld.so.conf, musl: ld-musl-.path) + - Default system library paths. For convenience, this is constructed as a dictionary where the keys are the library paths and the values are the names of the libraries @@ -74,250 +126,320 @@ def libraries_in_ld_and_system_library_path(path_hints=None): assumed there are two different instances of the library. Args: - path_hints (list): list of paths to be searched. If None the list will be + path_hints: list of paths to be searched. If None the list will be constructed based on the set of LD_LIBRARY_PATH, LIBRARY_PATH, DYLD_LIBRARY_PATH, and DYLD_FALLBACK_LIBRARY_PATH environment variables as well as the standard system library paths. + path_hints (list): list of paths to be searched. If ``None``, the default + system paths are used. """ - path_hints = ( - path_hints - or spack.util.environment.get_path("LD_LIBRARY_PATH") - + spack.util.environment.get_path("DYLD_LIBRARY_PATH") - + spack.util.environment.get_path("DYLD_FALLBACK_LIBRARY_PATH") - + spack.util.ld_so_conf.host_dynamic_linker_search_paths() - ) - search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints) - return path_to_dict(search_paths) - + if path_hints: + search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints) + else: + search_paths = [] + + # Environment variables + if sys.platform == "darwin": + search_paths.extend(environment.get_path("DYLD_LIBRARY_PATH")) + search_paths.extend(environment.get_path("DYLD_FALLBACK_LIBRARY_PATH")) + elif sys.platform.startswith("linux"): + search_paths.extend(environment.get_path("LD_LIBRARY_PATH")) + + # Dynamic linker paths + search_paths.extend(spack.util.ld_so_conf.host_dynamic_linker_search_paths()) + + # Drop redundant paths + search_paths = list(filter(os.path.isdir, search_paths)) + + # Make use we don't doubly list /usr/lib and /lib etc + search_paths = list(llnl.util.lang.dedupe(search_paths, key=file_identifier)) + + try: + host_compat = get_elf_compat(sys.executable) + accept = lambda path: accept_elf(path, host_compat) + except (OSError, elf_utils.ElfParsingError): + accept = llnl.util.filesystem.is_readable_file + + path_to_lib = {} + # Reverse order of search directories so that a lib in the first + # search path entry overrides later entries + for search_path in reversed(search_paths): + for lib in os.listdir(search_path): + lib_path = os.path.join(search_path, lib) + if accept(lib_path): + path_to_lib[lib_path] = lib + return path_to_lib + + +def libraries_in_windows_paths(path_hints: Optional[List[str]] = None) -> Dict[str, str]: + """Get the paths of all libraries available from the system PATH paths. + + For more details, see `libraries_in_ld_and_system_library_path` regarding + return type and contents. -def libraries_in_windows_paths(path_hints): - path_hints.extend(spack.util.environment.get_path("PATH")) - search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints) + Args: + path_hints: list of paths to be searched. If None the list will be + constructed based on the set of PATH environment + variables as well as the standard system library paths. + """ + search_hints = ( + path_hints if path_hints is not None else spack.util.environment.get_path("PATH") + ) + search_paths = llnl.util.filesystem.search_paths_for_libraries(*search_hints) # on Windows, some libraries (.dlls) are found in the bin directory or sometimes # at the search root. Add both of those options to the search scheme - search_paths.extend(llnl.util.filesystem.search_paths_for_executables(*path_hints)) - search_paths.extend(WindowsKitExternalPaths.find_windows_kit_lib_paths()) - search_paths.extend(WindowsKitExternalPaths.find_windows_kit_bin_paths()) - search_paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_installed_roots_paths()) - search_paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_sdk_paths()) - # SDK and WGL should be handled by above, however on occasion the WDK is in an atypical - # location, so we handle that case specifically. - search_paths.extend(WindowsKitExternalPaths.find_windows_driver_development_kit_paths()) + search_paths.extend(llnl.util.filesystem.search_paths_for_executables(*search_hints)) + if path_hints is None: + # if no user provided path was given, add defaults to the search + search_paths.extend(WindowsKitExternalPaths.find_windows_kit_lib_paths()) + # SDK and WGL should be handled by above, however on occasion the WDK is in an atypical + # location, so we handle that case specifically. + search_paths.extend(WindowsKitExternalPaths.find_windows_driver_development_kit_paths()) return path_to_dict(search_paths) -def _group_by_prefix(paths): +def _group_by_prefix(paths: Set[str]) -> Dict[str, Set[str]]: groups = collections.defaultdict(set) for p in paths: groups[os.path.dirname(p)].add(p) - return groups.items() - - -# TODO consolidate this with by_executable -# Packages should be able to define both .libraries and .executables in the future -# determine_spec_details should get all relevant libraries and executables in one call -def by_library(packages_to_check, path_hints=None): - # Techniques for finding libraries is determined on a per recipe basis in - # the determine_version class method. Some packages will extract the - # version number from a shared libraries filename. - # Other libraries could use the strings function to extract it as described - # in https://unix.stackexchange.com/questions/58846/viewing-linux-library-executable-version-info - """Return the list of packages that have been detected on the system, - searching by LD_LIBRARY_PATH, LIBRARY_PATH, DYLD_LIBRARY_PATH, - DYLD_FALLBACK_LIBRARY_PATH, and standard system library paths. - - Args: - packages_to_check (list): list of packages to be detected - path_hints (list): list of paths to be searched. If None the list will be - constructed based on the LD_LIBRARY_PATH, LIBRARY_PATH, - DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH environment variables - and standard system library paths. - """ - # If no path hints from command line, intialize to empty list so - # we can add default hints on a per package basis - path_hints = [] if path_hints is None else path_hints + return groups - lib_pattern_to_pkgs = collections.defaultdict(list) - for pkg in packages_to_check: - if hasattr(pkg, "libraries"): - for lib in pkg.libraries: - lib_pattern_to_pkgs[lib].append(pkg) - path_hints.extend(compute_windows_user_path_for_package(pkg)) - path_hints.extend(compute_windows_program_path_for_package(pkg)) - - path_to_lib_name = ( - libraries_in_ld_and_system_library_path(path_hints=path_hints) - if sys.platform != "win32" - else libraries_in_windows_paths(path_hints) - ) - pkg_to_found_libs = collections.defaultdict(set) - for lib_pattern, pkgs in lib_pattern_to_pkgs.items(): - compiled_re = re.compile(lib_pattern) - for path, lib in path_to_lib_name.items(): - if compiled_re.search(lib): - for pkg in pkgs: - pkg_to_found_libs[pkg].add(path) +class Finder: + """Inspects the file-system looking for packages. Guesses places where to look using PATH.""" - pkg_to_entries = collections.defaultdict(list) - resolved_specs = {} # spec -> lib found for the spec + def default_path_hints(self) -> List[str]: + return [] - for pkg, libs in pkg_to_found_libs.items(): - if not hasattr(pkg, "determine_spec_details"): - llnl.util.tty.warn( - "{0} must define 'determine_spec_details' in order" - " for Spack to detect externally-provided instances" - " of the package.".format(pkg.name) - ) - continue + def search_patterns(self, *, pkg: "spack.package_base.PackageBase") -> List[str]: + """Returns the list of patterns used to match candidate files. - for prefix, libs_in_prefix in sorted(_group_by_prefix(libs)): - try: - specs = _convert_to_iterable(pkg.determine_spec_details(prefix, libs_in_prefix)) - except Exception as e: - specs = [] - msg = 'error detecting "{0}" from prefix {1} [{2}]' - warnings.warn(msg.format(pkg.name, prefix, str(e))) + Args: + pkg: package being detected + """ + raise NotImplementedError("must be implemented by derived classes") - if not specs: - llnl.util.tty.debug( - "The following libraries in {0} were decidedly not " - "part of the package {1}: {2}".format( - prefix, pkg.name, ", ".join(_convert_to_iterable(libs_in_prefix)) - ) - ) + def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]: + """Returns a list of candidate files found on the system. - for spec in specs: - pkg_prefix = library_prefix(prefix) + Args: + patterns: search patterns to be used for matching files + paths: paths where to search for files + """ + raise NotImplementedError("must be implemented by derived classes") - if not pkg_prefix: - msg = "no lib/ or lib64/ dir found in {0}. Cannot " - "add it as a Spack package" - llnl.util.tty.debug(msg.format(prefix)) - continue + def prefix_from_path(self, *, path: str) -> str: + """Given a path where a file was found, returns the corresponding prefix. - if spec in resolved_specs: - prior_prefix = ", ".join(_convert_to_iterable(resolved_specs[spec])) + Args: + path: path of a detected file + """ + raise NotImplementedError("must be implemented by derived classes") - llnl.util.tty.debug( - "Libraries in {0} and {1} are both associated" - " with the same spec {2}".format(prefix, prior_prefix, str(spec)) - ) - continue - else: - resolved_specs[spec] = prefix + def detect_specs( + self, *, pkg: "spack.package_base.PackageBase", paths: List[str] + ) -> List[DetectedPackage]: + """Given a list of files matching the search patterns, returns a list of detected specs. - try: - spec.validate_detection() - except Exception as e: - msg = ( - '"{0}" has been detected on the system but will ' - "not be added to packages.yaml [reason={1}]" - ) - llnl.util.tty.warn(msg.format(spec, str(e))) - continue - - if spec.external_path: - pkg_prefix = spec.external_path - - pkg_to_entries[pkg.name].append(DetectedPackage(spec=spec, prefix=pkg_prefix)) - - return pkg_to_entries - - -def by_executable(packages_to_check, path_hints=None): - """Return the list of packages that have been detected on the system, - searching by path. - - Args: - packages_to_check (list): list of package classes to be detected - path_hints (list): list of paths to be searched. If None the list will be - constructed based on the PATH environment variable. - """ - path_hints = spack.util.environment.get_path("PATH") if path_hints is None else path_hints - exe_pattern_to_pkgs = collections.defaultdict(list) - for pkg in packages_to_check: - if hasattr(pkg, "executables"): - for exe in pkg.platform_executables(): - exe_pattern_to_pkgs[exe].append(pkg) - # Add Windows specific, package related paths to the search paths - path_hints.extend(compute_windows_user_path_for_package(pkg)) - path_hints.extend(compute_windows_program_path_for_package(pkg)) - - path_to_exe_name = executables_in_path(path_hints=path_hints) - pkg_to_found_exes = collections.defaultdict(set) - for exe_pattern, pkgs in exe_pattern_to_pkgs.items(): - compiled_re = re.compile(exe_pattern) - for path, exe in path_to_exe_name.items(): - if compiled_re.search(exe): - for pkg in pkgs: - pkg_to_found_exes[pkg].add(path) - - pkg_to_entries = collections.defaultdict(list) - resolved_specs = {} # spec -> exe found for the spec - - for pkg, exes in pkg_to_found_exes.items(): + Args: + pkg: package being detected + paths: files matching the package search patterns + """ if not hasattr(pkg, "determine_spec_details"): - llnl.util.tty.warn( - "{0} must define 'determine_spec_details' in order" - " for Spack to detect externally-provided instances" - " of the package.".format(pkg.name) + warnings.warn( + f"{pkg.name} must define 'determine_spec_details' in order" + f" for Spack to detect externally-provided instances" + f" of the package." ) - continue + return [] - for prefix, exes_in_prefix in sorted(_group_by_prefix(exes)): + result = [] + for candidate_path, items_in_prefix in sorted(_group_by_prefix(set(paths)).items()): # TODO: multiple instances of a package can live in the same # prefix, and a package implementation can return multiple specs # for one prefix, but without additional details (e.g. about the # naming scheme which differentiates them), the spec won't be # usable. try: - specs = _convert_to_iterable(pkg.determine_spec_details(prefix, exes_in_prefix)) + specs = _convert_to_iterable( + pkg.determine_spec_details(candidate_path, items_in_prefix) + ) except Exception as e: specs = [] - msg = 'error detecting "{0}" from prefix {1} [{2}]' - warnings.warn(msg.format(pkg.name, prefix, str(e))) + warnings.warn( + f'error detecting "{pkg.name}" from prefix {candidate_path} [{str(e)}]' + ) if not specs: + files = ", ".join(_convert_to_iterable(items_in_prefix)) llnl.util.tty.debug( - "The following executables in {0} were decidedly not " - "part of the package {1}: {2}".format( - prefix, pkg.name, ", ".join(_convert_to_iterable(exes_in_prefix)) - ) + f"The following files in {candidate_path} were decidedly not " + f"part of the package {pkg.name}: {files}" ) + resolved_specs: Dict[spack.spec.Spec, str] = {} # spec -> exe found for the spec for spec in specs: - pkg_prefix = executable_prefix(prefix) - - if not pkg_prefix: - msg = "no bin/ dir found in {0}. Cannot add it as a Spack package" - llnl.util.tty.debug(msg.format(prefix)) + prefix = self.prefix_from_path(path=candidate_path) + if not prefix: continue if spec in resolved_specs: prior_prefix = ", ".join(_convert_to_iterable(resolved_specs[spec])) - llnl.util.tty.debug( - "Executables in {0} and {1} are both associated" - " with the same spec {2}".format(prefix, prior_prefix, str(spec)) + f"Files in {candidate_path} and {prior_prefix} are both associated" + f" with the same spec {str(spec)}" ) continue - else: - resolved_specs[spec] = prefix + resolved_specs[spec] = candidate_path try: spec.validate_detection() except Exception as e: msg = ( - '"{0}" has been detected on the system but will ' - "not be added to packages.yaml [reason={1}]" + f'"{spec}" has been detected on the system but will ' + f"not be added to packages.yaml [reason={str(e)}]" ) - llnl.util.tty.warn(msg.format(spec, str(e))) + warnings.warn(msg) continue if spec.external_path: - pkg_prefix = spec.external_path + prefix = spec.external_path + + result.append(DetectedPackage(spec=spec, prefix=prefix)) + + return result + + def find( + self, *, pkg_name: str, initial_guess: Optional[List[str]] = None + ) -> List[DetectedPackage]: + """For a given package, returns a list of detected specs. + + Args: + pkg_name: package being detected + initial_guess: initial list of paths to search from the caller + if None, default paths are searched. If this + is an empty list, nothing will be searched. + """ + import spack.repo + + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) + patterns = self.search_patterns(pkg=pkg_cls) + if not patterns: + return [] + if initial_guess is None: + initial_guess = self.default_path_hints() + initial_guess.extend(common_windows_package_paths(pkg_cls)) + candidates = self.candidate_files(patterns=patterns, paths=initial_guess) + result = self.detect_specs(pkg=pkg_cls, paths=candidates) + return result + + +class ExecutablesFinder(Finder): + def default_path_hints(self) -> List[str]: + return spack.util.environment.get_path("PATH") + + def search_patterns(self, *, pkg: "spack.package_base.PackageBase") -> List[str]: + result = [] + if hasattr(pkg, "executables") and hasattr(pkg, "platform_executables"): + result = pkg.platform_executables() + return result + + def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]: + executables_by_path = executables_in_path(path_hints=paths) + patterns = [re.compile(x) for x in patterns] + result = [] + for compiled_re in patterns: + for path, exe in executables_by_path.items(): + if compiled_re.search(exe): + result.append(path) + return list(sorted(set(result))) + + def prefix_from_path(self, *, path: str) -> str: + result = executable_prefix(path) + if not result: + msg = f"no bin/ dir found in {path}. Cannot add it as a Spack package" + llnl.util.tty.debug(msg) + return result + + +class LibrariesFinder(Finder): + """Finds libraries on the system, searching by LD_LIBRARY_PATH, LIBRARY_PATH, + DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH, and standard system library paths + """ + + def search_patterns(self, *, pkg: "spack.package_base.PackageBase") -> List[str]: + result = [] + if hasattr(pkg, "libraries"): + result = pkg.libraries + return result + + def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]: + libraries_by_path = ( + libraries_in_ld_and_system_library_path(path_hints=paths) + if sys.platform != "win32" + else libraries_in_windows_paths(path_hints=paths) + ) + patterns = [re.compile(x) for x in patterns] + result = [] + for compiled_re in patterns: + for path, exe in libraries_by_path.items(): + if compiled_re.search(exe): + result.append(path) + return result + + def prefix_from_path(self, *, path: str) -> str: + result = library_prefix(path) + if not result: + msg = f"no lib/ or lib64/ dir found in {path}. Cannot add it as a Spack package" + llnl.util.tty.debug(msg) + return result + + +def by_path( + packages_to_search: List[str], + *, + path_hints: Optional[List[str]] = None, + max_workers: Optional[int] = None, +) -> Dict[str, List[DetectedPackage]]: + """Return the list of packages that have been detected on the system, keyed by + unqualified package name. + + Args: + packages_to_search: list of packages to be detected. Each package can be either unqualified + of fully qualified + path_hints: initial list of paths to be searched + max_workers: maximum number of workers to search for packages in parallel + """ + # TODO: Packages should be able to define both .libraries and .executables in the future + # TODO: determine_spec_details should get all relevant libraries and executables in one call + executables_finder, libraries_finder = ExecutablesFinder(), LibrariesFinder() + detected_specs_by_package: Dict[str, Tuple[concurrent.futures.Future, ...]] = {} + + result = collections.defaultdict(list) + with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: + for pkg in packages_to_search: + executable_future = executor.submit( + executables_finder.find, pkg_name=pkg, initial_guess=path_hints + ) + library_future = executor.submit( + libraries_finder.find, pkg_name=pkg, initial_guess=path_hints + ) + detected_specs_by_package[pkg] = executable_future, library_future - pkg_to_entries[pkg.name].append(DetectedPackage(spec=spec, prefix=pkg_prefix)) + for pkg_name, futures in detected_specs_by_package.items(): + for future in futures: + try: + detected = future.result(timeout=DETECTION_TIMEOUT) + if detected: + _, unqualified_name = spack.repo.partition_package_name(pkg_name) + result[unqualified_name].extend(detected) + except concurrent.futures.TimeoutError: + llnl.util.tty.debug( + f"[EXTERNAL DETECTION] Skipping {pkg_name}: timeout reached" + ) + except Exception as e: + llnl.util.tty.debug( + f"[EXTERNAL DETECTION] Skipping {pkg_name}: exception occured {e}" + ) - return pkg_to_entries + return result diff --git a/lib/spack/spack/detection/test.py b/lib/spack/spack/detection/test.py new file mode 100644 index 00000000000000..f33040f2929e86 --- /dev/null +++ b/lib/spack/spack/detection/test.py @@ -0,0 +1,187 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Create and run mock e2e tests for package detection.""" +import collections +import contextlib +import pathlib +import tempfile +from typing import Any, Deque, Dict, Generator, List, NamedTuple, Tuple + +import jinja2 + +from llnl.util import filesystem + +import spack.repo +import spack.spec +from spack.util import spack_yaml + +from .path import by_path + + +class MockExecutables(NamedTuple): + """Mock executables to be used in detection tests""" + + #: Relative paths for mock executables to be created + executables: List[str] + #: Shell script for the mock executable + script: str + + +class ExpectedTestResult(NamedTuple): + """Data structure to model assertions on detection tests""" + + #: Spec to be detected + spec: str + + +class DetectionTest(NamedTuple): + """Data structure to construct detection tests by PATH inspection. + + Packages may have a YAML file containing the description of one or more detection tests + to be performed. Each test creates a few mock executable scripts in a temporary folder, + and checks that detection by PATH gives the expected results. + """ + + pkg_name: str + layout: List[MockExecutables] + results: List[ExpectedTestResult] + + +class Runner: + """Runs an external detection test""" + + def __init__(self, *, test: DetectionTest, repository: spack.repo.RepoPath) -> None: + self.test = test + self.repository = repository + self.tmpdir = tempfile.TemporaryDirectory() + + def execute(self) -> List[spack.spec.Spec]: + """Executes a test and returns the specs that have been detected. + + This function sets-up a test in a temporary directory, according to the prescriptions + in the test layout, then performs a detection by executables and returns the specs that + have been detected. + """ + with self._mock_layout() as path_hints: + entries = by_path([self.test.pkg_name], path_hints=path_hints) + _, unqualified_name = spack.repo.partition_package_name(self.test.pkg_name) + specs = set(x.spec for x in entries[unqualified_name]) + return list(specs) + + @contextlib.contextmanager + def _mock_layout(self) -> Generator[List[str], None, None]: + hints = set() + try: + for entry in self.test.layout: + exes = self._create_executable_scripts(entry) + + for mock_executable in exes: + hints.add(str(mock_executable.parent)) + + yield list(hints) + finally: + self.tmpdir.cleanup() + + def _create_executable_scripts(self, mock_executables: MockExecutables) -> List[pathlib.Path]: + relative_paths = mock_executables.executables + script = mock_executables.script + script_template = jinja2.Template("#!/bin/bash\n{{ script }}\n") + result = [] + for mock_exe_path in relative_paths: + rel_path = pathlib.Path(mock_exe_path) + abs_path = pathlib.Path(self.tmpdir.name) / rel_path + abs_path.parent.mkdir(parents=True, exist_ok=True) + abs_path.write_text(script_template.render(script=script)) + filesystem.set_executable(abs_path) + result.append(abs_path) + return result + + @property + def expected_specs(self) -> List[spack.spec.Spec]: + return [spack.spec.Spec(r.spec) for r in self.test.results] + + +def detection_tests(pkg_name: str, repository: spack.repo.RepoPath) -> List[Runner]: + """Returns a list of test runners for a given package. + + Currently, detection tests are specified in a YAML file, called ``detection_test.yaml``, + alongside the ``package.py`` file. + + This function reads that file to create a bunch of ``Runner`` objects. + + Args: + pkg_name: name of the package to test + repository: repository where the package lives + """ + result = [] + detection_tests_content = read_detection_tests(pkg_name, repository) + + tests_by_path = detection_tests_content.get("paths", []) + for single_test_data in tests_by_path: + mock_executables = [] + for layout in single_test_data["layout"]: + mock_executables.append( + MockExecutables(executables=layout["executables"], script=layout["script"]) + ) + expected_results = [] + for assertion in single_test_data["results"]: + expected_results.append(ExpectedTestResult(spec=assertion["spec"])) + + current_test = DetectionTest( + pkg_name=pkg_name, layout=mock_executables, results=expected_results + ) + result.append(Runner(test=current_test, repository=repository)) + + return result + + +def read_detection_tests(pkg_name: str, repository: spack.repo.RepoPath) -> Dict[str, Any]: + """Returns the normalized content of the detection_tests.yaml associated with the package + passed in input. + + The content is merged with that of any package that is transitively included using the + "includes" attribute. + + Args: + pkg_name: name of the package to test + repository: repository in which to search for packages + """ + content_stack, seen = [], set() + included_packages: Deque[str] = collections.deque() + + root_detection_yaml, result = _detection_tests_yaml(pkg_name, repository) + included_packages.extend(result.get("includes", [])) + seen |= set(result.get("includes", [])) + + while included_packages: + current_package = included_packages.popleft() + try: + current_detection_yaml, content = _detection_tests_yaml(current_package, repository) + except FileNotFoundError as e: + msg = ( + f"cannot read the detection tests from the '{current_package}' package, " + f"included by {root_detection_yaml}" + ) + raise FileNotFoundError(msg + f"\n\n\t{e}\n") + + content_stack.append((current_package, content)) + included_packages.extend(x for x in content.get("includes", []) if x not in seen) + seen |= set(content.get("includes", [])) + + result.setdefault("paths", []) + for pkg_name, content in content_stack: + result["paths"].extend(content.get("paths", [])) + + return result + + +def _detection_tests_yaml( + pkg_name: str, repository: spack.repo.RepoPath +) -> Tuple[pathlib.Path, Dict[str, Any]]: + pkg_dir = pathlib.Path(repository.filename_for_package_name(pkg_name)).parent + detection_tests_yaml = pkg_dir / "detection_test.yaml" + with open(str(detection_tests_yaml)) as f: + content = spack_yaml.load(f) + return detection_tests_yaml, content diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index b997a190c8ff5d..fcd72d5bfcc12d 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -33,17 +33,19 @@ class OpenMpi(Package): import functools import os.path import re -from typing import List, Optional, Set, Union +from typing import Any, Callable, List, Optional, Set, Tuple, Union import llnl.util.lang import llnl.util.tty.color +import spack.deptypes as dt import spack.error import spack.patch import spack.spec import spack.url +import spack.util.crypto import spack.variant -from spack.dependency import Dependency, canonical_deptype, default_deptype +from spack.dependency import Dependency from spack.fetch_strategy import from_kwargs from spack.resource import Resource from spack.version import ( @@ -62,6 +64,7 @@ class OpenMpi(Package): "depends_on", "extends", "maintainers", + "license", "provides", "patch", "variant", @@ -134,6 +137,7 @@ class DirectiveMeta(type): _directive_dict_names: Set[str] = set() _directives_to_be_executed: List[str] = [] _when_constraints_from_context: List[str] = [] + _default_args: List[dict] = [] def __new__(cls, name, bases, attr_dict): # Initialize the attribute containing the list of directives @@ -196,6 +200,16 @@ def pop_from_context(): """Pop the last constraint from the context""" return DirectiveMeta._when_constraints_from_context.pop() + @staticmethod + def push_default_args(default_args): + """Push default arguments""" + DirectiveMeta._default_args.append(default_args) + + @staticmethod + def pop_default_args(): + """Pop default arguments""" + return DirectiveMeta._default_args.pop() + @staticmethod def directive(dicts=None): """Decorator for Spack directives. @@ -256,7 +270,13 @@ def _decorator(decorated_function): directive_names.append(decorated_function.__name__) @functools.wraps(decorated_function) - def _wrapper(*args, **kwargs): + def _wrapper(*args, **_kwargs): + # First merge default args with kwargs + kwargs = dict() + for default_args in DirectiveMeta._default_args: + kwargs.update(default_args) + kwargs.update(_kwargs) + # Inject when arguments from the context if DirectiveMeta._when_constraints_from_context: # Check that directives not yet supporting the when= argument @@ -407,10 +427,7 @@ def version( def _execute_version(pkg, ver, **kwargs): if ( - any( - s in kwargs - for s in ("sha256", "sha384", "sha512", "md5", "sha1", "sha224", "checksum") - ) + (any(s in kwargs for s in spack.util.crypto.hashes) or "checksum" in kwargs) and hasattr(pkg, "has_code") and not pkg.has_code ): @@ -438,7 +455,7 @@ def _execute_version(pkg, ver, **kwargs): pkg.versions[version] = kwargs -def _depends_on(pkg, spec, when=None, type=default_deptype, patches=None): +def _depends_on(pkg, spec, when=None, type=dt.DEFAULT_TYPES, patches=None): when_spec = make_when_spec(when) if not when_spec: return @@ -449,7 +466,7 @@ def _depends_on(pkg, spec, when=None, type=default_deptype, patches=None): if pkg.name == dep_spec.name: raise CircularReferenceError("Package '%s' cannot depend on itself." % pkg.name) - type = canonical_deptype(type) + depflag = dt.canonicalize(type) conditions = pkg.dependencies.setdefault(dep_spec.name, {}) # call this patches here for clarity -- we want patch to be a list, @@ -479,12 +496,12 @@ def _depends_on(pkg, spec, when=None, type=default_deptype, patches=None): # this is where we actually add the dependency to this package if when_spec not in conditions: - dependency = Dependency(pkg, dep_spec, type=type) + dependency = Dependency(pkg, dep_spec, depflag=depflag) conditions[when_spec] = dependency else: dependency = conditions[when_spec] dependency.spec.constrain(dep_spec, deps=False) - dependency.type |= set(type) + dependency.depflag |= depflag # apply patches to the dependency for execute_patch in patches: @@ -520,13 +537,14 @@ def _execute_conflicts(pkg): # Save in a list the conflicts and the associated custom messages when_spec_list = pkg.conflicts.setdefault(conflict_spec, []) - when_spec_list.append((when_spec, msg)) + msg_with_name = f"{pkg.name}: {msg}" if msg is not None else msg + when_spec_list.append((when_spec, msg_with_name)) return _execute_conflicts @directive(("dependencies")) -def depends_on(spec, when=None, type=default_deptype, patches=None): +def depends_on(spec, when=None, type=dt.DEFAULT_TYPES, patches=None): """Creates a dict of deps with specs defining when they apply. Args: @@ -572,17 +590,21 @@ def _execute_extends(pkg): return _execute_extends -@directive("provided") -def provides(*specs, **kwargs): - """Allows packages to provide a virtual dependency. If a package provides - 'mpi', other packages can declare that they depend on "mpi", and spack - can use the providing package to satisfy the dependency. +@directive(dicts=("provided", "provided_together")) +def provides(*specs, when: Optional[str] = None): + """Allows packages to provide a virtual dependency. + + If a package provides "mpi", other packages can declare that they depend on "mpi", + and spack can use the providing package to satisfy the dependency. + + Args: + *specs: virtual specs provided by this package + when: condition when this provides clause needs to be considered """ def _execute_provides(pkg): import spack.parser # Avoid circular dependency - when = kwargs.get("when") when_spec = make_when_spec(when) if not when_spec: return @@ -590,15 +612,18 @@ def _execute_provides(pkg): # ``when`` specs for ``provides()`` need a name, as they are used # to build the ProviderIndex. when_spec.name = pkg.name + spec_objs = [spack.spec.Spec(x) for x in specs] + spec_names = [x.name for x in spec_objs] + if len(spec_names) > 1: + pkg.provided_together.setdefault(when_spec, []).append(set(spec_names)) - for string in specs: - for provided_spec in spack.parser.parse(string): - if pkg.name == provided_spec.name: - raise CircularReferenceError("Package '%s' cannot provide itself." % pkg.name) + for provided_spec in spec_objs: + if pkg.name == provided_spec.name: + raise CircularReferenceError("Package '%s' cannot provide itself." % pkg.name) - if provided_spec not in pkg.provided: - pkg.provided[provided_spec] = set() - pkg.provided[provided_spec].add(when_spec) + if provided_spec not in pkg.provided: + pkg.provided[provided_spec] = set() + pkg.provided[provided_spec].add(when_spec) return _execute_provides @@ -663,39 +688,35 @@ def _execute_patch(pkg_or_dep): @directive("variants") def variant( - name, - default=None, - description="", - values=None, - multi=None, - validator=None, - when=None, - sticky=False, + name: str, + default: Optional[Any] = None, + description: str = "", + values: Optional[Union[collections.abc.Sequence, Callable[[Any], bool]]] = None, + multi: Optional[bool] = None, + validator: Optional[Callable[[str, str, Tuple[Any, ...]], None]] = None, + when: Optional[Union[str, bool]] = None, + sticky: bool = False, ): - """Define a variant for the package. Packager can specify a default - value as well as a text description. + """Define a variant for the package. + + Packager can specify a default value as well as a text description. Args: - name (str): name of the variant - default (str or bool): default value for the variant, if not - specified otherwise the default will be False for a boolean - variant and 'nothing' for a multi-valued variant - description (str): description of the purpose of the variant - values (tuple or typing.Callable): either a tuple of strings containing the - allowed values, or a callable accepting one value and returning - True if it is valid - multi (bool): if False only one value per spec is allowed for - this variant - validator (typing.Callable): optional group validator to enforce additional - logic. It receives the package name, the variant name and a tuple - of values and should raise an instance of SpackError if the group - doesn't meet the additional constraints - when (spack.spec.Spec, bool): optional condition on which the - variant applies - sticky (bool): the variant should not be changed by the concretizer to - find a valid concrete spec. + name: Name of the variant + default: Default value for the variant, if not specified otherwise the default will be + False for a boolean variant and 'nothing' for a multi-valued variant + description: Description of the purpose of the variant + values: Either a tuple of strings containing the allowed values, or a callable accepting + one value and returning True if it is valid + multi: If False only one value per spec is allowed for this variant + validator: Optional group validator to enforce additional logic. It receives the package + name, the variant name and a tuple of values and should raise an instance of SpackError + if the group doesn't meet the additional constraints + when: Optional condition on which the variant applies + sticky: The variant should not be changed by the concretizer to find a valid concrete spec + Raises: - DirectiveError: if arguments passed to the directive are invalid + DirectiveError: If arguments passed to the directive are invalid """ def format_error(msg, pkg): @@ -763,7 +784,7 @@ def _execute_variant(pkg): when_spec = make_when_spec(when) when_specs = [when_spec] - if not re.match(spack.spec.identifier_re, name): + if not re.match(spack.spec.IDENTIFIER_RE, name): directive = "variant" msg = "Invalid variant name in {0}: '{1}'" raise DirectiveError(directive, msg.format(pkg.name, name)) @@ -866,6 +887,44 @@ def _execute_maintainer(pkg): return _execute_maintainer +def _execute_license(pkg, license_identifier: str, when): + # If when is not specified the license always holds + when_spec = make_when_spec(when) + if not when_spec: + return + + for other_when_spec in pkg.licenses: + if when_spec.intersects(other_when_spec): + when_message = "" + if when_spec != make_when_spec(None): + when_message = f"when {when_spec}" + other_when_message = "" + if other_when_spec != make_when_spec(None): + other_when_message = f"when {other_when_spec}" + err_msg = ( + f"{pkg.name} is specified as being licensed as {license_identifier} " + f"{when_message}, but it is also specified as being licensed under " + f"{pkg.licenses[other_when_spec]} {other_when_message}, which conflict." + ) + raise OverlappingLicenseError(err_msg) + + pkg.licenses[when_spec] = license_identifier + + +@directive("licenses") +def license(license_identifier: str, when=None): + """Add a new license directive, to specify the SPDX identifier the software is + distributed under. + + Args: + license_identifiers: A list of SPDX identifiers specifying the licenses + the software is distributed under. + when: A spec specifying when the license applies. + """ + + return lambda pkg: _execute_license(pkg, license_identifier, when) + + @directive("requirements") def requires(*requirement_specs, policy="one_of", when=None, msg=None): """Allows a package to request a configuration to be present in all valid solutions. @@ -900,7 +959,8 @@ def _execute_requires(pkg): # Save in a list the requirements and the associated custom messages when_spec_list = pkg.requirements.setdefault(tuple(requirement_specs), []) - when_spec_list.append((when_spec, policy, msg)) + msg_with_name = f"{pkg.name}: {msg}" if msg is not None else msg + when_spec_list.append((when_spec, policy, msg_with_name)) return _execute_requires @@ -923,3 +983,7 @@ class DependencyPatchError(DirectiveError): class UnsupportedPackageDirective(DirectiveError): """Raised when an invalid or unsupported package directive is specified.""" + + +class OverlappingLicenseError(DirectiveError): + """Raised when two licenses are declared that apply on overlapping specs.""" diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 4d4952ee38643a..c0741a037c0a03 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -11,6 +11,7 @@ import shutil import sys from contextlib import contextmanager +from pathlib import Path import llnl.util.filesystem as fs import llnl.util.tty as tty @@ -103,8 +104,8 @@ def relative_path_for_spec(self, spec): _check_concrete(spec) projection = spack.projections.get_projection(self.projections, spec) - path = spec.format(projection) - return path + path = spec.format_path(projection) + return str(Path(path)) def write_spec(self, spec, path): """Write a spec out to a file.""" @@ -119,10 +120,8 @@ def write_host_environment(self, spec): versioning. We use it in the case that an analysis later needs to easily access this information. """ - from spack.util.environment import get_host_environment_metadata - env_file = self.env_metadata_path(spec) - environ = get_host_environment_metadata() + environ = spack.spec.get_host_environment_metadata() with open(env_file, "w") as fd: sjson.dump(environ, fd) @@ -325,7 +324,7 @@ def path_for_spec(self, spec): if spec.external: return spec.external_path if self.check_upstream: - upstream, record = spack.store.db.query_by_spec_hash(spec.dag_hash()) + upstream, record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash()) if upstream: raise SpackError( "Internal error: attempted to call path_for_spec on" diff --git a/lib/spack/spack/environment/__init__.py b/lib/spack/spack/environment/__init__.py index 227b48670cafed..2f293d9eb8f81b 100644 --- a/lib/spack/spack/environment/__init__.py +++ b/lib/spack/spack/environment/__init__.py @@ -339,6 +339,7 @@ from .environment import ( TOP_LEVEL_KEY, Environment, + SpackEnvironmentConfigError, SpackEnvironmentError, SpackEnvironmentViewError, activate, @@ -365,12 +366,14 @@ read, root, spack_env_var, + spack_env_view_var, update_yaml, ) __all__ = [ "TOP_LEVEL_KEY", "Environment", + "SpackEnvironmentConfigError", "SpackEnvironmentError", "SpackEnvironmentViewError", "activate", @@ -397,5 +400,6 @@ "read", "root", "spack_env_var", + "spack_env_view_var", "update_yaml", ] diff --git a/lib/spack/spack/environment/depfile.py b/lib/spack/spack/environment/depfile.py index 9f4087d0553272..34e2481fa916c3 100644 --- a/lib/spack/spack/environment/depfile.py +++ b/lib/spack/spack/environment/depfile.py @@ -8,9 +8,11 @@ """ import os +import re from enum import Enum from typing import List, Optional +import spack.deptypes as dt import spack.environment.environment as ev import spack.spec import spack.traverse as traverse @@ -35,7 +37,9 @@ def from_string(s: str) -> "UseBuildCache": def _deptypes(use_buildcache: UseBuildCache): """What edges should we follow for a given node? If it's a cache-only node, then we can drop build type deps.""" - return ("link", "run") if use_buildcache == UseBuildCache.ONLY else ("build", "link", "run") + return ( + dt.LINK | dt.RUN if use_buildcache == UseBuildCache.ONLY else dt.BUILD | dt.LINK | dt.RUN + ) class DepfileNode: @@ -45,8 +49,8 @@ class DepfileNode: def __init__( self, target: spack.spec.Spec, prereqs: List[spack.spec.Spec], buildcache: UseBuildCache ): - self.target = target - self.prereqs = prereqs + self.target = MakefileSpec(target) + self.prereqs = list(MakefileSpec(x) for x in prereqs) if buildcache == UseBuildCache.ONLY: self.buildcache_flag = "--use-buildcache=only" elif buildcache == UseBuildCache.NEVER: @@ -68,13 +72,13 @@ def __init__(self, pkg_buildcache: UseBuildCache, deps_buildcache: UseBuildCache self.adjacency_list: List[DepfileNode] = [] self.pkg_buildcache = pkg_buildcache self.deps_buildcache = deps_buildcache - self.deptypes_root = _deptypes(pkg_buildcache) - self.deptypes_deps = _deptypes(deps_buildcache) + self.depflag_root = _deptypes(pkg_buildcache) + self.depflag_deps = _deptypes(deps_buildcache) def neighbors(self, node): """Produce a list of spec to follow from node""" - deptypes = self.deptypes_root if node.depth == 0 else self.deptypes_deps - return traverse.sort_edges(node.edge.spec.edges_to_dependencies(deptype=deptypes)) + depflag = self.depflag_root if node.depth == 0 else self.depflag_deps + return traverse.sort_edges(node.edge.spec.edges_to_dependencies(depflag=depflag)) def accept(self, node): self.adjacency_list.append( @@ -89,6 +93,32 @@ def accept(self, node): return True +class MakefileSpec(object): + """Limited interface to spec to help generate targets etc. without + introducing unwanted special characters. + """ + + _pattern = None + + def __init__(self, spec): + self.spec = spec + + def safe_name(self): + return self.safe_format("{name}-{version}-{hash}") + + def spec_hash(self): + return self.spec.dag_hash() + + def safe_format(self, format_str): + unsafe_result = self.spec.format(format_str) + if not MakefileSpec._pattern: + MakefileSpec._pattern = re.compile(r"[^A-Za-z0-9_.-]") + return MakefileSpec._pattern.sub("_", unsafe_result) + + def unsafe_format(self, format_str): + return self.spec.format(format_str) + + class MakefileModel: """This class produces all data to render a makefile for specs of an environment.""" @@ -114,7 +144,7 @@ def __init__( self.env_path = env.path # These specs are built in the default target. - self.roots = roots + self.roots = list(MakefileSpec(x) for x in roots) # The SPACK_PACKAGE_IDS variable is "exported", which can be used when including # generated makefiles to add post-install hooks, like pushing to a buildcache, @@ -131,17 +161,19 @@ def __init__( # And here we collect a tuple of (target, prereqs, dag_hash, nice_name, buildcache_flag) self.make_adjacency_list = [ ( - self._safe_name(item.target), - " ".join(self._install_target(self._safe_name(s)) for s in item.prereqs), - item.target.dag_hash(), - item.target.format("{name}{@version}{%compiler}{variants}{arch=architecture}"), + item.target.safe_name(), + " ".join(self._install_target(s.safe_name()) for s in item.prereqs), + item.target.spec_hash(), + item.target.unsafe_format( + "{name}{@version}{%compiler}{variants}{arch=architecture}" + ), item.buildcache_flag, ) for item in adjacency_list ] # Root specs without deps are the prereqs for the environment target - self.root_install_targets = [self._install_target(self._safe_name(s)) for s in roots] + self.root_install_targets = [self._install_target(s.safe_name()) for s in self.roots] self.jobserver_support = "+" if jobserver else "" @@ -157,7 +189,7 @@ def __init__( self.phony_convenience_targets: List[str] = [] for node in adjacency_list: - tgt = self._safe_name(node.target) + tgt = node.target.safe_name() self.all_pkg_identifiers.append(tgt) self.all_install_related_targets.append(self._install_target(tgt)) self.all_install_related_targets.append(self._install_deps_target(tgt)) @@ -165,9 +197,6 @@ def __init__( self.phony_convenience_targets.append(os.path.join("install", tgt)) self.phony_convenience_targets.append(os.path.join("install-deps", tgt)) - def _safe_name(self, spec: spack.spec.Spec) -> str: - return spec.format("{name}-{version}-{hash}") - def _target(self, name: str) -> str: # The `all` and `clean` targets are phony. It doesn't make sense to # have /abs/path/to/env/metadir/{all,clean} targets. But it *does* make @@ -203,6 +232,10 @@ def to_dict(self): "pkg_ids": " ".join(self.all_pkg_identifiers), } + @property + def empty(self): + return len(self.roots) == 0 + @staticmethod def from_env( env: ev.Environment, @@ -225,15 +258,10 @@ def from_env( jobserver: when enabled, make will invoke Spack with jobserver support. For dry-run this should be disabled. """ - # If no specs are provided as a filter, build all the specs in the environment. - if filter_specs: - entrypoints = [env.matching_spec(s) for s in filter_specs] - else: - entrypoints = [s for _, s in env.concretized_specs()] - + roots = env.all_matching_specs(*filter_specs) if filter_specs else env.concrete_roots() visitor = DepfileSpecVisitor(pkg_buildcache, dep_buildcache) traverse.traverse_breadth_first_with_visitor( - entrypoints, traverse.CoverNodesVisitor(visitor, key=lambda s: s.dag_hash()) + roots, traverse.CoverNodesVisitor(visitor, key=lambda s: s.dag_hash()) ) - return MakefileModel(env, entrypoints, visitor.adjacency_list, make_prefix, jobserver) + return MakefileModel(env, roots, visitor.adjacency_list, make_prefix, jobserver) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index a174b4a11ee711..5d6273506ec9c6 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -16,7 +16,7 @@ import urllib.parse import urllib.request import warnings -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union import llnl.util.filesystem as fs import llnl.util.tty as tty @@ -28,7 +28,9 @@ import spack.compilers import spack.concretize import spack.config +import spack.deptypes as dt import spack.error +import spack.fetch_strategy import spack.hash_types as ht import spack.hooks import spack.main @@ -62,6 +64,8 @@ #: environment variable used to indicate the active environment spack_env_var = "SPACK_ENV" +#: environment variable used to indicate the active environment view +spack_env_view_var = "SPACK_ENV_VIEW" #: currently activated environment _active_environment: Optional["Environment"] = None @@ -153,7 +157,7 @@ def installed_specs(): """ env = spack.environment.active_environment() hashes = env.all_hashes() if env else None - return spack.store.db.query(hashes=hashes) + return spack.store.STORE.db.query(hashes=hashes) def valid_env_name(name): @@ -202,7 +206,7 @@ def activate(env, use_env_repo=False): env.store_token = spack.store.reinitialize() if use_env_repo: - spack.repo.path.put_first(env.repo) + spack.repo.PATH.put_first(env.repo) tty.debug("Using environment '%s'" % env.name) @@ -226,7 +230,7 @@ def deactivate(): # use _repo so we only remove if a repo was actually constructed if _active_environment._repo: - spack.repo.path.remove(_active_environment._repo) + spack.repo.PATH.remove(_active_environment._repo) tty.debug("Deactivated environment '%s'" % _active_environment.name) @@ -326,16 +330,21 @@ def create_in_dir( if with_view is None and keep_relative: return Environment(manifest_dir) - manifest = EnvironmentManifestFile(manifest_dir) + try: + manifest = EnvironmentManifestFile(manifest_dir) + + if with_view is not None: + manifest.set_default_view(with_view) - if with_view is not None: - manifest.set_default_view(with_view) + if not keep_relative and init_file is not None and str(init_file).endswith(manifest_name): + init_file = pathlib.Path(init_file) + manifest.absolutify_dev_paths(init_file.parent) - if not keep_relative and init_file is not None and str(init_file).endswith(manifest_name): - init_file = pathlib.Path(init_file) - manifest.absolutify_dev_paths(init_file.parent) + manifest.flush() - manifest.flush() + except (spack.config.ConfigFormatError, SpackEnvironmentConfigError) as e: + shutil.rmtree(manifest_dir) + raise e return Environment(manifest_dir) @@ -387,7 +396,13 @@ def all_environments(): def _read_yaml(str_or_file): """Read YAML from a file for round-trip parsing.""" - data = syaml.load_config(str_or_file) + try: + data = syaml.load_config(str_or_file) + except syaml.SpackYAMLError as e: + raise SpackEnvironmentConfigError( + f"Invalid environment configuration detected: {e.message}" + ) + filename = getattr(str_or_file, "name", None) default_data = spack.config.validate(data, spack.schema.env.schema, filename) return data, default_data @@ -402,7 +417,7 @@ def _write_yaml(data, str_or_file): def _eval_conditional(string): """Evaluate conditional definitions using restricted variable scope.""" - valid_variables = spack.util.environment.get_host_environment() + valid_variables = spack.spec.get_host_environment() valid_variables.update({"re": re, "env": os.environ}) return eval(string, valid_variables) @@ -421,7 +436,7 @@ def _is_dev_spec_and_has_changed(spec): # Not installed -> nothing to compare against return False - _, record = spack.store.db.query_by_spec_hash(spec.dag_hash()) + _, record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash()) mtime = fs.last_modification_time_recursive(dev_path_var.value) return mtime > record.installation_time @@ -582,7 +597,7 @@ def view(self, new=None): raise SpackEnvironmentViewError(msg) return SimpleFilesystemView( root, - spack.store.layout, + spack.store.STORE.layout, ignore_conflicts=True, projections=self.projections, link=self.link_type, @@ -622,7 +637,7 @@ def specs_for_view(self, concretized_root_specs): specs = list(dedupe(concretized_root_specs, key=traverse.by_dag_hash)) # Filter selected, installed specs - with spack.store.db.read_transaction(): + with spack.store.STORE.db.read_transaction(): specs = [s for s in specs if s in self and s.installed] return specs @@ -731,7 +746,7 @@ def __init__(self, manifest_dir: Union[str, pathlib.Path]) -> None: self.txlock = lk.Lock(self._transaction_lock_path) - self.unify = None + self._unify = None self.new_specs: List[Spec] = [] self.new_installs: List[Spec] = [] self.views: Dict[str, ViewDescriptor] = {} @@ -755,6 +770,16 @@ def __init__(self, manifest_dir: Union[str, pathlib.Path]) -> None: self.manifest = EnvironmentManifestFile(manifest_dir) self._read() + @property + def unify(self): + if self._unify is None: + self._unify = spack.config.get("concretizer:unify", False) + return self._unify + + @unify.setter + def unify(self, value): + self._unify = value + def __reduce__(self): return _create_environment, (self.path,) @@ -762,10 +787,18 @@ def _re_read(self): """Reinitialize the environment object.""" self.clear(re_read=True) self.manifest = EnvironmentManifestFile(self.path) - self._read() + self._read(re_read=True) - def _read(self): - self._construct_state_from_manifest() + def _read(self, re_read=False): + # If the manifest has included files, then some of the information + # (e.g., definitions) MAY be in those files. So we need to ensure + # the config is populated with any associated spec lists in order + # to fully construct the manifest state. + includes = self.manifest[TOP_LEVEL_KEY].get("include", []) + if includes and not re_read: + prepare_config_scope(self) + + self._construct_state_from_manifest(re_read) if os.path.exists(self.lock_path): with open(self.lock_path) as f: @@ -779,21 +812,30 @@ def write_transaction(self): """Get a write lock context manager for use in a `with` block.""" return lk.WriteTransaction(self.txlock, acquire=self._re_read) - def _construct_state_from_manifest(self): + def _process_definition(self, item): + """Process a single spec definition item.""" + entry = copy.deepcopy(item) + when = _eval_conditional(entry.pop("when", "True")) + assert len(entry) == 1 + if when: + name, spec_list = next(iter(entry.items())) + user_specs = SpecList(name, spec_list, self.spec_lists.copy()) + if name in self.spec_lists: + self.spec_lists[name].extend(user_specs) + else: + self.spec_lists[name] = user_specs + + def _construct_state_from_manifest(self, re_read=False): """Read manifest file and set up user specs.""" self.spec_lists = collections.OrderedDict() + + if not re_read: + for item in spack.config.get("definitions", []): + self._process_definition(item) + env_configuration = self.manifest[TOP_LEVEL_KEY] for item in env_configuration.get("definitions", []): - entry = copy.deepcopy(item) - when = _eval_conditional(entry.pop("when", "True")) - assert len(entry) == 1 - if when: - name, spec_list = next(iter(entry.items())) - user_specs = SpecList(name, spec_list, self.spec_lists.copy()) - if name in self.spec_lists: - self.spec_lists[name].extend(user_specs) - else: - self.spec_lists[name] = user_specs + self._process_definition(item) spec_list = env_configuration.get(user_speclist_name, []) user_specs = SpecList( @@ -816,9 +858,6 @@ def _construct_state_from_manifest(self): else: self.views = {} - # Retrieve unification scheme for the concretizer - self.unify = spack.config.get("concretizer:unify", False) - # Retrieve dev-build packages: self.dev_specs = copy.deepcopy(env_configuration.get("develop", {})) for name, entry in self.dev_specs.items(): @@ -841,7 +880,9 @@ def clear(self, re_read=False): yaml, and need to be maintained when re-reading an existing environment. """ - self.spec_lists = {user_speclist_name: SpecList()} # specs from yaml + self.spec_lists = collections.OrderedDict() + self.spec_lists[user_speclist_name] = SpecList() + self.dev_specs = {} # dev-build specs from yaml self.concretized_user_specs = [] # user specs from last concretize self.concretized_order = [] # roots of last concretize, in order @@ -990,7 +1031,8 @@ def included_config_scopes(self): elif include_url.scheme: raise ValueError( - "Unsupported URL scheme for environment include: {}".format(config_path) + f"Unsupported URL scheme ({include_url.scheme}) for " + f"environment include: {config_path}" ) # treat relative paths as relative to the environment @@ -1052,8 +1094,10 @@ def update_stale_references(self, from_list=None): from_list = next(iter(self.spec_lists.keys())) index = list(self.spec_lists.keys()).index(from_list) - # spec_lists is an OrderedDict, all list entries after the modified - # list may refer to the modified list. Update stale references + # spec_lists is an OrderedDict to ensure lists read from the manifest + # are maintainted in order, hence, all list entries after the modified + # list may refer to the modified list requiring stale references to be + # updated. for i, (name, speclist) in enumerate( list(self.spec_lists.items())[index + 1 :], index + 1 ): @@ -1076,8 +1120,8 @@ def add(self, user_spec, list_name=user_speclist_name): if list_name == user_speclist_name: if spec.anonymous: raise SpackEnvironmentError("cannot add anonymous specs to an environment") - elif not spack.repo.path.exists(spec.name) and not spec.abstract_hash: - virtuals = spack.repo.path.provider_index.providers.keys() + elif not spack.repo.PATH.exists(spec.name) and not spec.abstract_hash: + virtuals = spack.repo.PATH.provider_index.providers.keys() if spec.name not in virtuals: msg = "no such package: %s" % spec.name raise SpackEnvironmentError(msg) @@ -1151,7 +1195,7 @@ def change_existing_spec( def remove(self, query_spec, list_name=user_speclist_name, force=False): """Remove specs from an environment that match a query_spec""" err_msg_header = ( - f"cannot remove {query_spec} from '{list_name}' definition " + f"Cannot remove '{query_spec}' from '{list_name}' definition " f"in {self.manifest.manifest_file}" ) query_spec = Spec(query_spec) @@ -1182,11 +1226,10 @@ def remove(self, query_spec, list_name=user_speclist_name, force=False): list_to_change.remove(spec) self.update_stale_references(list_name) new_specs = set(self.user_specs) - except spack.spec_list.SpecListError: + except spack.spec_list.SpecListError as e: # define new specs list new_specs = set(self.user_specs) - msg = f"Spec '{spec}' is part of a spec matrix and " - msg += f"cannot be removed from list '{list_to_change}'." + msg = str(e) if force: msg += " It will be removed from the concrete specs." # Mock new specs, so we can remove this spec from concrete spec lists @@ -1249,16 +1292,23 @@ def develop(self, spec: Spec, path: str, clone: bool = False) -> bool: tty.msg("Configuring spec %s for development at path %s" % (spec, path)) if clone: - # "steal" the source code via staging API - abspath = spack.util.path.canonicalize_path(path, default_wd=self.path) - - # Stage, at the moment, requires a concrete Spec, since it needs the - # dag_hash for the stage dir name. Below though we ask for a stage - # to be created, to copy it afterwards somewhere else. It would be + # "steal" the source code via staging API. We ask for a stage + # to be created, then copy it afterwards somewhere else. It would be # better if we can create the `source_path` directly into its final # destination. - pkg_cls = spack.repo.path.get_pkg_class(spec.name) - pkg_cls(spec).stage.steal_source(abspath) + abspath = spack.util.path.canonicalize_path(path, default_wd=self.path) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) + # We construct a package class ourselves, rather than asking for + # Spec.package, since Spec only allows this when it is concrete + package = pkg_cls(spec) + if isinstance(package.fetcher, spack.fetch_strategy.GitFetchStrategy): + package.fetcher.get_full_repo = True + # If we retrieved this version before and cached it, we may have + # done so without cloning the full git repo; likewise, any + # mirror might store an instance with truncated history. + package.stage.disable_mirrors() + + package.stage.steal_source(abspath) # If it wasn't already in the list, append it entry = {"path": path, "spec": str(spec)} @@ -1308,7 +1358,7 @@ def concretize(self, force=False, tests=False): # Remove concrete specs that no longer correlate to a user spec for spec in set(self.concretized_user_specs) - set(self.user_specs): - self.deconcretize(spec) + self.deconcretize(spec, concrete=False) # Pick the right concretization strategy if self.unify == "when_possible": @@ -1323,15 +1373,36 @@ def concretize(self, force=False, tests=False): msg = "concretization strategy not implemented [{0}]" raise SpackEnvironmentError(msg.format(self.unify)) - def deconcretize(self, spec): + def deconcretize(self, spec: spack.spec.Spec, concrete: bool = True): + """ + Remove specified spec from environment concretization + + Arguments: + spec: Spec to deconcretize. This must be a root of the environment + concrete: If True, find all instances of spec as concrete in the environemnt. + If False, find a single instance of the abstract spec as root of the environment. + """ # spec has to be a root of the environment - index = self.concretized_user_specs.index(spec) - dag_hash = self.concretized_order.pop(index) - del self.concretized_user_specs[index] + if concrete: + dag_hash = spec.dag_hash() + + pairs = zip(self.concretized_user_specs, self.concretized_order) + filtered = [(spec, h) for spec, h in pairs if h != dag_hash] + # Cannot use zip and unpack two values; it fails if filtered is empty + self.concretized_user_specs = [s for s, _ in filtered] + self.concretized_order = [h for _, h in filtered] + else: + index = self.concretized_user_specs.index(spec) + dag_hash = self.concretized_order.pop(index) + + del self.concretized_user_specs[index] # If this was the only user spec that concretized to this concrete spec, remove it if dag_hash not in self.concretized_order: - del self.specs_by_hash[dag_hash] + # if we deconcretized a dependency that doesn't correspond to a root, it + # won't be here. + if dag_hash in self.specs_by_hash: + del self.specs_by_hash[dag_hash] def _get_specs_to_concretize( self, @@ -1380,7 +1451,10 @@ def _concretize_together_where_possible( result_by_user_spec = {} solver = spack.solver.asp.Solver() - for result in solver.solve_in_rounds(specs_to_concretize, tests=tests): + allow_deprecated = spack.config.get("config:deprecated", False) + for result in solver.solve_in_rounds( + specs_to_concretize, tests=tests, allow_deprecated=allow_deprecated + ): result_by_user_spec.update(result.specs_by_input) result = [] @@ -1459,11 +1533,12 @@ def _concretize_separately(self, tests=False): self._add_concrete_spec(s, concrete, new=False) # Concretize any new user specs that we haven't concretized yet - arguments, root_specs = [], [] + args, root_specs, i = [], [], 0 for uspec, uspec_constraints in zip(self.user_specs, self.user_specs.specs_as_constraints): if uspec not in old_concretized_user_specs: root_specs.append(uspec) - arguments.append((uspec_constraints, tests)) + args.append((i, [str(x) for x in uspec_constraints], tests)) + i += 1 # Ensure we don't try to bootstrap clingo in parallel if spack.config.get("config:concretizer", "clingo") == "clingo": @@ -1475,42 +1550,54 @@ def _concretize_separately(self, tests=False): # for a write lock. We do this indirectly by retrieving the # provider index, which should in turn trigger the update of # all the indexes if there's any need for that. - _ = spack.repo.path.provider_index + _ = spack.repo.PATH.provider_index # Ensure we have compilers in compilers.yaml to avoid that # processes try to write the config file in parallel _ = spack.compilers.get_compiler_config() # Early return if there is nothing to do - if len(arguments) == 0: + if len(args) == 0: return [] # Solve the environment in parallel on Linux start = time.time() - max_processes = min( - len(arguments), # Number of specs - spack.config.get("config:build_jobs"), # Cap on build jobs - ) + num_procs = min(len(args), spack.util.cpus.determine_number_of_jobs(parallel=True)) - # TODO: revisit this print as soon as darwin is parallel too + # TODO: support parallel concretization on macOS and Windows msg = "Starting concretization" - if sys.platform != "darwin": - pool_size = spack.util.parallel.num_processes(max_processes=max_processes) - if pool_size > 1: - msg = msg + " pool with {0} processes".format(pool_size) + if sys.platform not in ("darwin", "win32") and num_procs > 1: + msg += f" pool with {num_procs} processes" tty.msg(msg) - concretized_root_specs = spack.util.parallel.parallel_map( - _concretize_task, arguments, max_processes=max_processes, debug=tty.is_debug() - ) + batch = [] + for j, (i, concrete, duration) in enumerate( + spack.util.parallel.imap_unordered( + _concretize_task, + args, + processes=num_procs, + debug=tty.is_debug(), + maxtaskperchild=1, + ) + ): + batch.append((i, concrete)) + percentage = (j + 1) / len(args) * 100 + tty.verbose( + f"{duration:6.1f}s [{percentage:3.0f}%] {concrete.cformat('{hash:7}')} " + f"{root_specs[i].colored_str}" + ) + sys.stdout.flush() - finish = time.time() - tty.msg("Environment concretized in %.2f seconds." % (finish - start)) - by_hash = {} - for abstract, concrete in zip(root_specs, concretized_root_specs): - self._add_concrete_spec(abstract, concrete) + # Add specs in original order + batch.sort(key=lambda x: x[0]) + by_hash = {} # for attaching information on test dependencies + for root, (_, concrete) in zip(root_specs, batch): + self._add_concrete_spec(root, concrete) by_hash[concrete.dag_hash()] = concrete + finish = time.time() + tty.msg(f"Environment concretized in {finish - start:.2f} seconds") + # Unify the specs objects, so we get correct references to all parents self._read_lockfile_dict(self._to_lockfile_dict()) @@ -1521,13 +1608,13 @@ def _concretize_separately(self, tests=False): for h in self.specs_by_hash: current_spec, computed_spec = self.specs_by_hash[h], by_hash[h] for node in computed_spec.traverse(): - test_edges = node.edges_to_dependencies(deptype="test") + test_edges = node.edges_to_dependencies(depflag=dt.TEST) for current_edge in test_edges: test_dependency = current_edge.spec if test_dependency in current_spec[node.name]: continue current_spec[node.name].add_dependency_edge( - test_dependency.copy(), deptypes="test", virtuals=current_edge.virtuals + test_dependency.copy(), depflag=dt.TEST, virtuals=current_edge.virtuals ) results = [ @@ -1576,16 +1663,14 @@ def concretize_and_add(self, user_spec, concrete_spec=None, tests=False): @property def default_view(self): - if not self.views: - raise SpackEnvironmentError("{0} does not have a view enabled".format(self.name)) - - if default_view_name not in self.views: - raise SpackEnvironmentError( - "{0} does not have a default view enabled".format(self.name) - ) + if not self.has_view(default_view_name): + raise SpackEnvironmentError(f"{self.name} does not have a default view enabled") return self.views[default_view_name] + def has_view(self, view_name: str) -> bool: + return view_name in self.views + def update_default_view(self, path_or_bool: Union[str, bool]) -> None: """Updates the path of the default view. @@ -1671,62 +1756,37 @@ def check_views(self): "Loading the environment view will require reconcretization." % self.name ) - def _env_modifications_for_default_view(self, reverse=False): - all_mods = spack.util.environment.EnvironmentModifications() - - visited = set() - - errors = [] - for root_spec in self.concrete_roots(): - if root_spec in self.default_view and root_spec.installed and root_spec.package: - for spec in root_spec.traverse(deptype="run", root=True): - if spec.name in visited: - # It is expected that only one instance of the package - # can be added to the environment - do not attempt to - # add multiple. - tty.debug( - "Not adding {0} to shell modifications: " - "this package has already been added".format( - spec.format("{name}/{hash:7}") - ) - ) - continue - else: - visited.add(spec.name) - - try: - mods = uenv.environment_modifications_for_spec(spec, self.default_view) - except Exception as e: - msg = "couldn't get environment settings for %s" % spec.format( - "{name}@{version} /{hash:7}" - ) - errors.append((msg, str(e))) - continue - - all_mods.extend(mods.reversed() if reverse else mods) - - return all_mods, errors + def _env_modifications_for_view( + self, view: ViewDescriptor, reverse: bool = False + ) -> spack.util.environment.EnvironmentModifications: + try: + with spack.store.STORE.db.read_transaction(): + installed_roots = [s for s in self.concrete_roots() if s.installed] + mods = uenv.environment_modifications_for_specs(*installed_roots, view=view) + except Exception as e: + # Failing to setup spec-specific changes shouldn't be a hard error. + tty.warn( + f"could not {'unload' if reverse else 'load'} runtime environment due " + f"to {e.__class__.__name__}: {e}" + ) + return spack.util.environment.EnvironmentModifications() + return mods.reversed() if reverse else mods - def add_default_view_to_env(self, env_mod): - """ - Collect the environment modifications to activate an environment using the - default view. Removes duplicate paths. + def add_view_to_env( + self, env_mod: spack.util.environment.EnvironmentModifications, view: str + ) -> spack.util.environment.EnvironmentModifications: + """Collect the environment modifications to activate an environment using the provided + view. Removes duplicate paths. Args: - env_mod (spack.util.environment.EnvironmentModifications): the environment - modifications object that is modified. - """ - if default_view_name not in self.views: - # No default view to add to shell + env_mod: the environment modifications object that is modified. + view: the name of the view to activate.""" + descriptor = self.views.get(view) + if not descriptor: return env_mod - env_mod.extend(uenv.unconditional_environment_modifications(self.default_view)) - - mods, errors = self._env_modifications_for_default_view() - env_mod.extend(mods) - if errors: - for err in errors: - tty.warn(*err) + env_mod.extend(uenv.unconditional_environment_modifications(descriptor)) + env_mod.extend(self._env_modifications_for_view(descriptor)) # deduplicate paths from specs mapped to the same location for env_var in env_mod.group_by_name(): @@ -1734,23 +1794,21 @@ def add_default_view_to_env(self, env_mod): return env_mod - def rm_default_view_from_env(self, env_mod): - """ - Collect the environment modifications to deactivate an environment using the - default view. Reverses the action of ``add_default_view_to_env``. + def rm_view_from_env( + self, env_mod: spack.util.environment.EnvironmentModifications, view: str + ) -> spack.util.environment.EnvironmentModifications: + """Collect the environment modifications to deactivate an environment using the provided + view. Reverses the action of ``add_view_to_env``. Args: - env_mod (spack.util.environment.EnvironmentModifications): the environment - modifications object that is modified. - """ - if default_view_name not in self.views: - # No default view to add to shell + env_mod: the environment modifications object that is modified. + view: the name of the view to deactivate.""" + descriptor = self.views.get(view) + if not descriptor: return env_mod - env_mod.extend(uenv.unconditional_environment_modifications(self.default_view).reversed()) - - mods, _ = self._env_modifications_for_default_view(reverse=True) - env_mod.extend(mods) + env_mod.extend(uenv.unconditional_environment_modifications(descriptor).reversed()) + env_mod.extend(self._env_modifications_for_view(descriptor, reverse=True)) return env_mod @@ -1826,7 +1884,7 @@ def _partition_roots_by_install_status(self): specs. This is done in a single read transaction per environment instead of per spec.""" installed, uninstalled = [], [] - with spack.store.db.read_transaction(): + with spack.store.STORE.db.read_transaction(): for concretized_hash in self.concretized_order: spec = self.specs_by_hash[concretized_hash] if not spec.installed or ( @@ -1871,9 +1929,9 @@ def install_specs(self, specs=None, **install_args): # Already installed root specs should be marked explicitly installed in the # database. if specs_dropped: - with spack.store.db.write_transaction(): # do all in one transaction + with spack.store.STORE.db.write_transaction(): # do all in one transaction for spec in specs_dropped: - spack.store.db.update_explicit(spec, True) + spack.store.STORE.db.update_explicit(spec, True) if not specs_to_install: tty.msg("All of the packages are already installed") @@ -1906,16 +1964,17 @@ def install_specs(self, specs=None, **install_args): "Could not install log links for {0}: {1}".format(spec.name, str(e)) ) - def all_specs(self): - """Return all specs, even those a user spec would shadow.""" - roots = [self.specs_by_hash[h] for h in self.concretized_order] - specs = [s for s in traverse.traverse_nodes(roots, key=traverse.by_dag_hash)] - specs.sort() - return specs + def all_specs_generator(self) -> Iterable[Spec]: + """Returns a generator for all concrete specs""" + return traverse.traverse_nodes(self.concrete_roots(), key=traverse.by_dag_hash) + + def all_specs(self) -> List[Spec]: + """Returns a list of all concrete specs""" + return list(self.all_specs_generator()) def all_hashes(self): """Return hashes of all specs.""" - return [s.dag_hash() for s in self.all_specs()] + return [s.dag_hash() for s in self.all_specs_generator()] def roots(self): """Specs explicitly requested by the user *in this environment*. @@ -1936,7 +1995,7 @@ def added_specs(self): """ # use a transaction to avoid overhead of repeated calls # to `package.spec.installed` - with spack.store.db.read_transaction(): + with spack.store.STORE.db.read_transaction(): concretized = dict(self.concretized_specs()) for spec in self.user_specs: concrete = concretized.get(spec) @@ -1978,14 +2037,10 @@ def get_one_by_hash(self, dag_hash): def all_matching_specs(self, *specs: spack.spec.Spec) -> List[Spec]: """Returns all concretized specs in the environment satisfying any of the input specs""" - # Look up abstract hashes ahead of time, to avoid O(n^2) traversal. - specs = [s.lookup_hash() for s in specs] - - # Avoid double lookup by directly calling _satisfies. return [ s for s in traverse.traverse_nodes(self.concrete_roots(), key=traverse.by_dag_hash) - if any(s._satisfies(t) for t in specs) + if any(s.satisfies(t) for t in specs) ] @spack.repo.autospec @@ -2046,7 +2101,7 @@ def matching_spec(self, spec): # If multiple root specs match, it is assumed that the abstract # spec will most-succinctly summarize the difference between them # (and the user can enter one of these to disambiguate) - fmt_str = "{hash:7} " + spack.spec.default_format + fmt_str = "{hash:7} " + spack.spec.DEFAULT_FORMAT color = clr.get_color_when() match_strings = [ f"Root spec {abstract.format(color=color)}\n {concrete.format(fmt_str, color=color)}" @@ -2063,7 +2118,7 @@ def matching_spec(self, spec): def removed_specs(self): """Tuples of (user spec, concrete spec) for all specs that will be - removed on nexg concretize.""" + removed on next concretize.""" needed = set() for s, c in self.concretized_specs(): if s in self.user_specs: @@ -2178,7 +2233,7 @@ def _read_lockfile_dict(self, d): name, data = reader.name_and_data(node_dict) for _, dep_hash, deptypes, _, virtuals in reader.dependencies_from_node_dict(data): specs_by_hash[lockfile_key]._add_dependency( - specs_by_hash[dep_hash], deptypes=deptypes, virtuals=virtuals + specs_by_hash[dep_hash], depflag=dt.canonicalize(deptypes), virtuals=virtuals ) # Traverse the root specs one at a time in the order they appear. @@ -2264,7 +2319,7 @@ def _add_to_environment_repository(self, spec_node: Spec) -> None: repository = spack.repo.create_or_construct(repository_dir, spec_node.namespace) pkg_dir = repository.dirname_for_package_name(spec_node.name) fs.mkdirp(pkg_dir) - spack.repo.path.dump_provenance(spec_node, pkg_dir) + spack.repo.PATH.dump_provenance(spec_node, pkg_dir) def manifest_uptodate_or_warn(self): """Emits a warning if the manifest file is not up-to-date.""" @@ -2354,7 +2409,7 @@ def display_specs(concretized_specs): def _tree_to_display(spec): return spec.tree( recurse_dependencies=True, - format=spack.spec.display_format, + format=spack.spec.DISPLAY_FORMAT, status_fn=spack.spec.Spec.install_status, hashlen=7, hashes=True, @@ -2406,10 +2461,13 @@ def _concretize_from_constraints(spec_constraints, tests=False): invalid_constraints.extend(inv_variant_constraints) -def _concretize_task(packed_arguments): - spec_constraints, tests = packed_arguments +def _concretize_task(packed_arguments) -> Tuple[int, Spec, float]: + index, spec_constraints, tests = packed_arguments + spec_constraints = [Spec(x) for x in spec_constraints] with tty.SuppressOutput(msg_enabled=False): - return _concretize_from_constraints(spec_constraints, tests) + start = time.time() + spec = _concretize_from_constraints(spec_constraints, tests) + return index, spec, time.time() - start def make_repo_path(root): @@ -2432,13 +2490,13 @@ def make_repo_path(root): def prepare_config_scope(env): """Add env's scope to the global configuration search path.""" for scope in env.config_scopes(): - spack.config.config.push_scope(scope) + spack.config.CONFIG.push_scope(scope) def deactivate_config_scope(env): """Remove any scopes from env from the global config path.""" for scope in env.config_scopes(): - spack.config.config.remove_scope(scope.name) + spack.config.CONFIG.remove_scope(scope.name) def manifest_file(env_name_or_dir): @@ -2652,6 +2710,26 @@ def __init__(self, manifest_dir: Union[pathlib.Path, str]) -> None: self.yaml_content = with_defaults_added self.changed = False + def _all_matches(self, user_spec: str) -> List[str]: + """Maps the input string to the first equivalent user spec in the manifest, + and returns it. + + Args: + user_spec: user spec to be found + + Raises: + ValueError: if no equivalent match is found + """ + result = [] + for yaml_spec_str in self.pristine_configuration["specs"]: + if Spec(yaml_spec_str) == Spec(user_spec): + result.append(yaml_spec_str) + + if not result: + raise ValueError(f"cannot find a spec equivalent to {user_spec}") + + return result + def add_user_spec(self, user_spec: str) -> None: """Appends the user spec passed as input to the list of root specs. @@ -2672,8 +2750,9 @@ def remove_user_spec(self, user_spec: str) -> None: SpackEnvironmentError: when the user spec is not in the list """ try: - self.pristine_configuration["specs"].remove(user_spec) - self.configuration["specs"].remove(user_spec) + for key in self._all_matches(user_spec): + self.pristine_configuration["specs"].remove(key) + self.configuration["specs"].remove(key) except ValueError as e: msg = f"cannot remove {user_spec} from {self}, no such spec exists" raise SpackEnvironmentError(msg) from e @@ -2698,7 +2777,7 @@ def override_user_spec(self, user_spec: str, idx: int) -> None: self.changed = True def add_definition(self, user_spec: str, list_name: str) -> None: - """Appends a user spec to the first active definition mathing the name passed as argument. + """Appends a user spec to the first active definition matching the name passed as argument. Args: user_spec: user spec to be appended @@ -2911,3 +2990,7 @@ class SpackEnvironmentError(spack.error.SpackError): class SpackEnvironmentViewError(SpackEnvironmentError): """Class for errors regarding view generation.""" + + +class SpackEnvironmentConfigError(SpackEnvironmentError): + """Class for Spack environment-specific configuration errors.""" diff --git a/lib/spack/spack/environment/shell.py b/lib/spack/spack/environment/shell.py index c6d2b06c98c5f7..a4f9634a8da051 100644 --- a/lib/spack/spack/environment/shell.py +++ b/lib/spack/spack/environment/shell.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os +from typing import Optional import llnl.util.tty as tty from llnl.util.tty.color import colorize @@ -13,12 +14,14 @@ from spack.util.environment import EnvironmentModifications -def activate_header(env, shell, prompt=None): +def activate_header(env, shell, prompt=None, view: Optional[str] = None): # Construct the commands to run cmds = "" if shell == "csh": # TODO: figure out how to make color work for csh cmds += "setenv SPACK_ENV %s;\n" % env.path + if view: + cmds += "setenv SPACK_ENV_VIEW %s;\n" % view cmds += 'alias despacktivate "spack env deactivate";\n' if prompt: cmds += "if (! $?SPACK_OLD_PROMPT ) " @@ -29,6 +32,8 @@ def activate_header(env, shell, prompt=None): prompt = colorize("@G{%s} " % prompt, color=True) cmds += "set -gx SPACK_ENV %s;\n" % env.path + if view: + cmds += "set -gx SPACK_ENV_VIEW %s;\n" % view cmds += "function despacktivate;\n" cmds += " spack env deactivate;\n" cmds += "end;\n" @@ -40,15 +45,21 @@ def activate_header(env, shell, prompt=None): elif shell == "bat": # TODO: Color cmds += 'set "SPACK_ENV=%s"\n' % env.path + if view: + cmds += 'set "SPACK_ENV_VIEW=%s"\n' % view # TODO: despacktivate # TODO: prompt elif shell == "pwsh": - cmds += "$Env:SPACK_ENV=%s\n" % env.path + cmds += "$Env:SPACK_ENV='%s'\n" % env.path + if view: + cmds += "$Env:SPACK_ENV_VIEW='%s'\n" % view else: if "color" in os.getenv("TERM", "") and prompt: prompt = colorize("@G{%s}" % prompt, color=True, enclose=True) cmds += "export SPACK_ENV=%s;\n" % env.path + if view: + cmds += "export SPACK_ENV_VIEW=%s;\n" % view cmds += "alias despacktivate='spack env deactivate';\n" if prompt: cmds += "if [ -z ${SPACK_OLD_PS1+x} ]; then\n" @@ -66,12 +77,14 @@ def deactivate_header(shell): cmds = "" if shell == "csh": cmds += "unsetenv SPACK_ENV;\n" + cmds += "unsetenv SPACK_ENV_VIEW;\n" cmds += "if ( $?SPACK_OLD_PROMPT ) " cmds += ' eval \'set prompt="$SPACK_OLD_PROMPT" &&' cmds += " unsetenv SPACK_OLD_PROMPT';\n" cmds += "unalias despacktivate;\n" elif shell == "fish": cmds += "set -e SPACK_ENV;\n" + cmds += "set -e SPACK_ENV_VIEW;\n" cmds += "functions -e despacktivate;\n" # # NOTE: Not changing fish_prompt (above) => no need to restore it here. @@ -79,14 +92,19 @@ def deactivate_header(shell): elif shell == "bat": # TODO: Color cmds += 'set "SPACK_ENV="\n' + cmds += 'set "SPACK_ENV_VIEW="\n' # TODO: despacktivate # TODO: prompt elif shell == "pwsh": - cmds += "Remove-Item Env:SPACK_ENV" + cmds += "Set-Item -Path Env:SPACK_ENV\n" + cmds += "Set-Item -Path Env:SPACK_ENV_VIEW\n" else: cmds += "if [ ! -z ${SPACK_ENV+x} ]; then\n" cmds += "unset SPACK_ENV; export SPACK_ENV;\n" cmds += "fi;\n" + cmds += "if [ ! -z ${SPACK_ENV_VIEW+x} ]; then\n" + cmds += "unset SPACK_ENV_VIEW; export SPACK_ENV_VIEW;\n" + cmds += "fi;\n" cmds += "alias despacktivate > /dev/null 2>&1 && unalias despacktivate;\n" cmds += "if [ ! -z ${SPACK_OLD_PS1+x} ]; then\n" cmds += " if [ \"$SPACK_OLD_PS1\" = '$$$$' ]; then\n" @@ -100,24 +118,23 @@ def deactivate_header(shell): return cmds -def activate(env, use_env_repo=False, add_view=True): - """ - Activate an environment and append environment modifications +def activate( + env: ev.Environment, use_env_repo=False, view: Optional[str] = "default" +) -> EnvironmentModifications: + """Activate an environment and append environment modifications To activate an environment, we add its configuration scope to the existing Spack configuration, and we set active to the current environment. Arguments: - env (spack.environment.Environment): the environment to activate - use_env_repo (bool): use the packages exactly as they appear in the - environment's repository - add_view (bool): generate commands to add view to path variables + env: the environment to activate + use_env_repo: use the packages exactly as they appear in the environment's repository + view: generate commands to add runtime environment variables for named view Returns: spack.util.environment.EnvironmentModifications: Environment variables - modifications to activate environment. - """ + modifications to activate environment.""" ev.activate(env, use_env_repo=use_env_repo) env_mods = EnvironmentModifications() @@ -129,9 +146,9 @@ def activate(env, use_env_repo=False, add_view=True): # become PATH variables. # try: - if add_view and ev.default_view_name in env.views: - with spack.store.db.read_transaction(): - env.add_default_view_to_env(env_mods) + if view and env.has_view(view): + with spack.store.STORE.db.read_transaction(): + env.add_view_to_env(env_mods, view) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.error(e) tty.die( @@ -145,17 +162,15 @@ def activate(env, use_env_repo=False, add_view=True): return env_mods -def deactivate(): - """ - Deactivate an environment and collect corresponding environment modifications. +def deactivate() -> EnvironmentModifications: + """Deactivate an environment and collect corresponding environment modifications. Note: unloads the environment in its current state, not in the state it was loaded in, meaning that specs that were removed from the spack environment after activation are not unloaded. Returns: - spack.util.environment.EnvironmentModifications: Environment variables - modifications to activate environment. + Environment variables modifications to activate environment. """ env_mods = EnvironmentModifications() active = ev.active_environment() @@ -163,10 +178,12 @@ def deactivate(): if active is None: return env_mods - if ev.default_view_name in active.views: + active_view = os.getenv(ev.spack_env_view_var) + + if active_view and active.has_view(active_view): try: - with spack.store.db.read_transaction(): - active.rm_default_view_from_env(env_mods) + with spack.store.STORE.db.read_transaction(): + active.rm_view_from_env(env_mods, active_view) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.warn(e) tty.warn( diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index 33986c9cde9fc9..8c9015ed6d9de7 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -128,3 +128,7 @@ def __init__(self, provided, required, constraint_type): self.provided = provided self.required = required self.constraint_type = constraint_type + + +class FetchError(SpackError): + """Superclass for fetch-related errors.""" diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py index af900722cc9cfd..0ee01a22a12a99 100644 --- a/lib/spack/spack/extensions.py +++ b/lib/spack/spack/extensions.py @@ -5,6 +5,7 @@ """Service functions and classes to implement the hooks for Spack's command extensions. """ +import difflib import importlib import os import re @@ -176,10 +177,19 @@ class CommandNotFoundError(spack.error.SpackError): """ def __init__(self, cmd_name): - super().__init__( + msg = ( "{0} is not a recognized Spack command or extension command;" " check with `spack commands`.".format(cmd_name) ) + long_msg = None + + similar = difflib.get_close_matches(cmd_name, spack.cmd.all_commands()) + + if 1 <= len(similar) <= 5: + long_msg = "\nDid you mean one of the following commands?\n " + long_msg += "\n ".join(similar) + + super().__init__(msg, long_msg) class ExtensionNamingError(spack.error.SpackError): diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 077a5d2ae4797b..a922d9caf4c085 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -28,27 +28,30 @@ import os.path import re import shutil +import urllib.error import urllib.parse from typing import List, Optional +import llnl.url import llnl.util import llnl.util.filesystem as fs import llnl.util.tty as tty +from llnl.string import comma_and, quote from llnl.util.filesystem import get_single_file, mkdirp, temp_cwd, temp_rename, working_dir from llnl.util.symlink import symlink import spack.config import spack.error +import spack.oci.opener import spack.url import spack.util.crypto as crypto import spack.util.git -import spack.util.pattern as pattern import spack.util.url as url_util import spack.util.web as web_util import spack.version -from spack.util.compression import decompressor_for, extension_from_path +import spack.version.git_ref_lookup +from spack.util.compression import decompressor_for from spack.util.executable import CommandNotFoundError, which -from spack.util.string import comma_and, quote #: List of all fetch strategies, created by FetchStrategy metaclass. all_strategies = [] @@ -228,24 +231,6 @@ def mirror_id(self): """BundlePackages don't have a mirror id.""" -class FetchStrategyComposite(pattern.Composite): - """Composite for a FetchStrategy object.""" - - matches = FetchStrategy.matches - - def __init__(self): - super().__init__(["fetch", "check", "expand", "reset", "archive", "cachable", "mirror_id"]) - - def source_id(self): - component_ids = tuple(i.source_id() for i in self) - if all(component_ids): - return component_ids - - def set_package(self, package): - for item in self: - item.package = package - - @fetcher class URLFetchStrategy(FetchStrategy): """URLFetchStrategy pulls source code from a URL for an archive, check the @@ -418,7 +403,7 @@ def _fetch_curl(self, url): try: web_util.check_curl_code(curl.returncode) - except web_util.FetchError as err: + except spack.error.FetchError as err: raise spack.fetch_strategy.FailedDownloadError(url, str(err)) self._check_headers(headers) @@ -459,7 +444,7 @@ def expand(self): # TODO: replace this by mime check. if not self.extension: - self.extension = spack.url.determine_url_file_extension(self.url) + self.extension = llnl.url.determine_url_file_extension(self.url) if self.stage.expanded: tty.debug("Source already staged to %s" % self.stage.source_path) @@ -488,17 +473,7 @@ def check(self): if not self.digest: raise NoDigestError("Attempt to check URLFetchStrategy with no digest.") - checker = crypto.Checker(self.digest) - if not checker.check(self.archive_file): - # On failure, provide some information about the file size and - # contents, so that we can quickly see what the issue is (redirect - # was not followed, empty file, text instead of binary, ...) - size, contents = fs.filesummary(self.archive_file) - raise ChecksumError( - f"{checker.hash_name} checksum failed for {self.archive_file}", - f"Expected {self.digest} but got {checker.sum}. " - f"File size = {size} bytes. Contents = {contents!r}", - ) + verify_checksum(self.archive_file, self.digest) @_needs_stage def reset(self): @@ -564,6 +539,34 @@ def fetch(self): tty.msg("Using cached archive: {0}".format(path)) +class OCIRegistryFetchStrategy(URLFetchStrategy): + def __init__(self, url=None, checksum=None, **kwargs): + super().__init__(url, checksum, **kwargs) + + self._urlopen = kwargs.get("_urlopen", spack.oci.opener.urlopen) + + @_needs_stage + def fetch(self): + file = self.stage.save_filename + tty.msg(f"Fetching {self.url}") + + try: + response = self._urlopen(self.url) + except urllib.error.URLError as e: + # clean up archive on failure. + if self.archive_file: + os.remove(self.archive_file) + if os.path.lexists(file): + os.remove(file) + raise FailedDownloadError(self.url, f"Failed to fetch {self.url}: {e}") from e + + if os.path.lexists(file): + os.remove(file) + + with open(file, "wb") as f: + shutil.copyfileobj(response, f) + + class VCSFetchStrategy(FetchStrategy): """Superclass for version control system fetch strategies. @@ -598,7 +601,7 @@ def expand(self): @_needs_stage def archive(self, destination, **kwargs): - assert extension_from_path(destination) == "tar.gz" + assert llnl.url.extension_from_path(destination) == "tar.gz" assert self.stage.source_path.startswith(self.stage.path) tar = which("tar", required=True) @@ -761,13 +764,16 @@ def version_from_git(git_exe): @property def git(self): if not self._git: - self._git = spack.util.git.git() + try: + self._git = spack.util.git.git(required=True) + except CommandNotFoundError as exc: + tty.error(str(exc)) + raise # Disable advice for a quieter fetch # https://github.com/git/git/blob/master/Documentation/RelNotes/1.7.2.txt if self.git_version >= spack.version.Version("1.7.2"): - self._git.add_default_arg("-c") - self._git.add_default_arg("advice.detachedHead=false") + self._git.add_default_arg("-c", "advice.detachedHead=false") # If the user asked for insecure fetching, make that work # with git as well. @@ -1317,7 +1323,7 @@ def fetch(self): parsed_url = urllib.parse.urlparse(self.url) if parsed_url.scheme != "s3": - raise web_util.FetchError("S3FetchStrategy can only fetch from s3:// urls.") + raise spack.error.FetchError("S3FetchStrategy can only fetch from s3:// urls.") tty.debug("Fetching {0}".format(self.url)) @@ -1364,7 +1370,7 @@ def fetch(self): parsed_url = urllib.parse.urlparse(self.url) if parsed_url.scheme != "gs": - raise web_util.FetchError("GCSFetchStrategy can only fetch from gs:// urls.") + raise spack.error.FetchError("GCSFetchStrategy can only fetch from gs:// urls.") tty.debug("Fetching {0}".format(self.url)) @@ -1388,6 +1394,45 @@ def fetch(self): raise FailedDownloadError(self.url) +@fetcher +class FetchAndVerifyExpandedFile(URLFetchStrategy): + """Fetch strategy that verifies the content digest during fetching, + as well as after expanding it.""" + + def __init__(self, url, archive_sha256: str, expanded_sha256: str): + super().__init__(url, archive_sha256) + self.expanded_sha256 = expanded_sha256 + + def expand(self): + """Verify checksum after expanding the archive.""" + + # Expand the archive + super().expand() + + # Ensure a single patch file. + src_dir = self.stage.source_path + files = os.listdir(src_dir) + + if len(files) != 1: + raise ChecksumError(self, f"Expected a single file in {src_dir}.") + + verify_checksum(os.path.join(src_dir, files[0]), self.expanded_sha256) + + +def verify_checksum(file, digest): + checker = crypto.Checker(digest) + if not checker.check(file): + # On failure, provide some information about the file size and + # contents, so that we can quickly see what the issue is (redirect + # was not followed, empty file, text instead of binary, ...) + size, contents = fs.filesummary(file) + raise ChecksumError( + f"{checker.hash_name} checksum failed for {file}", + f"Expected {digest} but got {checker.sum}. " + f"File size = {size} bytes. Contents = {contents!r}", + ) + + def stable_target(fetcher): """Returns whether the fetcher target is expected to have a stable checksum. This is only true if the target is a preexisting archive @@ -1419,7 +1464,7 @@ def from_kwargs(**kwargs): on attribute names (e.g., ``git``, ``hg``, etc.) Raises: - spack.util.web.FetchError: If no ``fetch_strategy`` matches the args. + spack.error.FetchError: If no ``fetch_strategy`` matches the args. """ for fetcher in all_strategies: if fetcher.matches(kwargs): @@ -1526,11 +1571,11 @@ def for_package_version(pkg, version=None): # if it's a commit, we must use a GitFetchStrategy if isinstance(version, spack.version.GitVersion): if not hasattr(pkg, "git"): - raise web_util.FetchError( + raise spack.error.FetchError( f"Cannot fetch git version for {pkg.name}. Package has no 'git' attribute" ) # Populate the version with comparisons to other commits - version.attach_git_lookup_from_package(pkg.name) + version.attach_lookup(spack.version.git_ref_lookup.GitRefLookup(pkg.name)) # For GitVersion, we have no way to determine whether a ref is a branch or tag # Fortunately, we handle branches and tags identically, except tags are @@ -1676,11 +1721,11 @@ def destroy(self): shutil.rmtree(self.root, ignore_errors=True) -class NoCacheError(web_util.FetchError): +class NoCacheError(spack.error.FetchError): """Raised when there is no cached archive for a package.""" -class FailedDownloadError(web_util.FetchError): +class FailedDownloadError(spack.error.FetchError): """Raised when a download fails.""" def __init__(self, url, msg=""): @@ -1688,23 +1733,23 @@ def __init__(self, url, msg=""): self.url = url -class NoArchiveFileError(web_util.FetchError): +class NoArchiveFileError(spack.error.FetchError): """Raised when an archive file is expected but none exists.""" -class NoDigestError(web_util.FetchError): +class NoDigestError(spack.error.FetchError): """Raised after attempt to checksum when URL has no digest.""" -class ExtrapolationError(web_util.FetchError): +class ExtrapolationError(spack.error.FetchError): """Raised when we can't extrapolate a version for a package.""" -class FetcherConflict(web_util.FetchError): +class FetcherConflict(spack.error.FetchError): """Raised for packages with invalid fetch attributes.""" -class InvalidArgsError(web_util.FetchError): +class InvalidArgsError(spack.error.FetchError): """Raised when a version can't be deduced from a set of arguments.""" def __init__(self, pkg=None, version=None, **args): @@ -1717,11 +1762,11 @@ def __init__(self, pkg=None, version=None, **args): super().__init__(msg, long_msg) -class ChecksumError(web_util.FetchError): +class ChecksumError(spack.error.FetchError): """Raised when archive fails to checksum.""" -class NoStageError(web_util.FetchError): +class NoStageError(spack.error.FetchError): """Raised when fetch operations are called before set_stage().""" def __init__(self, method): diff --git a/lib/spack/spack/filesystem_view.py b/lib/spack/spack/filesystem_view.py index 4df8df3cef3882..e6631fecbf66a4 100644 --- a/lib/spack/spack/filesystem_view.py +++ b/lib/spack/spack/filesystem_view.py @@ -88,7 +88,7 @@ def view_copy(src: str, dst: str, view, spec: Optional[spack.spec.Spec] = None): elif spack.relocate.is_binary(dst): spack.relocate.relocate_text_bin(binaries=[dst], prefixes=prefix_to_projection) else: - prefix_to_projection[spack.store.layout.root] = view._root + prefix_to_projection[spack.store.STORE.layout.root] = view._root # This is vestigial code for the *old* location of sbang. prefix_to_projection[ @@ -379,7 +379,7 @@ def needs_file(spec, file): # check if this spec owns a file of that name (through the # manifest in the metadata dir, which we have in the view). manifest_file = os.path.join( - self.get_path_meta_folder(spec), spack.store.layout.manifest_file_name + self.get_path_meta_folder(spec), spack.store.STORE.layout.manifest_file_name ) try: with open(manifest_file, "r") as f: @@ -500,20 +500,22 @@ def get_projection_for_spec(self, spec): proj = spack.projections.get_projection(self.projections, locator_spec) if proj: - return os.path.join(self._root, locator_spec.format(proj)) + return os.path.join(self._root, locator_spec.format_path(proj)) return self._root def get_all_specs(self): md_dirs = [] for root, dirs, files in os.walk(self._root): - if spack.store.layout.metadata_dir in dirs: - md_dirs.append(os.path.join(root, spack.store.layout.metadata_dir)) + if spack.store.STORE.layout.metadata_dir in dirs: + md_dirs.append(os.path.join(root, spack.store.STORE.layout.metadata_dir)) specs = [] for md_dir in md_dirs: if os.path.exists(md_dir): for name_dir in os.listdir(md_dir): - filename = os.path.join(md_dir, name_dir, spack.store.layout.spec_file_name) + filename = os.path.join( + md_dir, name_dir, spack.store.STORE.layout.spec_file_name + ) spec = get_spec_from_file(filename) if spec: specs.append(spec) @@ -531,18 +533,18 @@ def get_path_meta_folder(self, spec): "Get path to meta folder for either spec or spec name." return os.path.join( self.get_projection_for_spec(spec), - spack.store.layout.metadata_dir, + spack.store.STORE.layout.metadata_dir, getattr(spec, "name", spec), ) def get_spec(self, spec): dotspack = self.get_path_meta_folder(spec) - filename = os.path.join(dotspack, spack.store.layout.spec_file_name) + filename = os.path.join(dotspack, spack.store.STORE.layout.spec_file_name) return get_spec_from_file(filename) def link_meta_folder(self, spec): - src = spack.store.layout.metadata_path(spec) + src = spack.store.STORE.layout.metadata_path(spec) tgt = self.get_path_meta_folder(spec) tree = LinkTree(src) @@ -588,9 +590,9 @@ def print_status(self, *specs, **kwargs): print() header = "%s{%s} / %s{%s}" % ( - spack.spec.architecture_color, + spack.spec.ARCHITECTURE_COLOR, architecture, - spack.spec.compiler_color, + spack.spec.COMPILER_COLOR, compiler, ) tty.hline(colorize(header), char="-") @@ -673,7 +675,7 @@ def add_specs(self, *specs, **kwargs): # Ignore spack meta data folder. def skip_list(file): - return os.path.basename(file) == spack.store.layout.metadata_dir + return os.path.basename(file) == spack.store.STORE.layout.metadata_dir visitor = SourceMergeVisitor(ignore=skip_list) @@ -735,14 +737,18 @@ def _source_merge_visitor_to_merge_map(self, visitor: SourceMergeVisitor): def relative_metadata_dir_for_spec(self, spec): return os.path.join( - self.get_relative_projection_for_spec(spec), spack.store.layout.metadata_dir, spec.name + self.get_relative_projection_for_spec(spec), + spack.store.STORE.layout.metadata_dir, + spec.name, ) def link_metadata(self, specs): metadata_visitor = SourceMergeVisitor() for spec in specs: - src_prefix = os.path.join(spec.package.view_source(), spack.store.layout.metadata_dir) + src_prefix = os.path.join( + spec.package.view_source(), spack.store.STORE.layout.metadata_dir + ) proj = self.relative_metadata_dir_for_spec(spec) metadata_visitor.set_projection(proj) visit_directory_tree(src_prefix, metadata_visitor) @@ -770,7 +776,7 @@ def get_relative_projection_for_spec(self, spec): spec = spec.package.extendee_spec p = spack.projections.get_projection(self.projections, spec) - return spec.format(p) if p else "" + return spec.format_path(p) if p else "" def get_projection_for_spec(self, spec): """ @@ -785,7 +791,7 @@ def get_projection_for_spec(self, spec): proj = spack.projections.get_projection(self.projections, spec) if proj: - return os.path.join(self._root, spec.format(proj)) + return os.path.join(self._root, spec.format_path(proj)) return self._root diff --git a/lib/spack/spack/gcs_handler.py b/lib/spack/spack/gcs_handler.py deleted file mode 100644 index b002fa70ac4fe5..00000000000000 --- a/lib/spack/spack/gcs_handler.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -import urllib.parse -import urllib.response -from urllib.error import URLError -from urllib.request import BaseHandler - - -def gcs_open(req, *args, **kwargs): - """Open a reader stream to a blob object on GCS""" - import spack.util.gcs as gcs_util - - url = urllib.parse.urlparse(req.get_full_url()) - gcsblob = gcs_util.GCSBlob(url) - - if not gcsblob.exists(): - raise URLError("GCS blob {0} does not exist".format(gcsblob.blob_path)) - stream = gcsblob.get_blob_byte_stream() - headers = gcsblob.get_blob_headers() - - return urllib.response.addinfourl(stream, headers, url) - - -class GCSHandler(BaseHandler): - def gs_open(self, req): - return gcs_open(req) diff --git a/lib/spack/spack/graph.py b/lib/spack/spack/graph.py index 73fffaeaa67c41..78bf38ec0e7e43 100644 --- a/lib/spack/spack/graph.py +++ b/lib/spack/spack/graph.py @@ -38,11 +38,12 @@ """ import enum import sys -from typing import List, Optional, Set, TextIO, Tuple, Union +from typing import List, Optional, Set, TextIO, Tuple import llnl.util.tty.color -import spack.dependency +import spack.deptypes as dt +import spack.repo import spack.spec import spack.tengine @@ -78,7 +79,7 @@ def __init__(self): self.node_character = "o" self.debug = False self.indent = 0 - self.deptype = spack.dependency.all_deptypes + self.depflag = dt.ALL # These are colors in the order they'll be used for edges. # See llnl.util.tty.color for details on color characters. @@ -326,7 +327,7 @@ def write(self, spec, color=None, out=None): nodes_in_topological_order = [ edge.spec for edge in spack.traverse.traverse_edges_topo( - [spec], direction="children", deptype=self.deptype + [spec], direction="children", deptype=self.depflag ) ] nodes_in_topological_order.reverse() @@ -424,7 +425,7 @@ def write(self, spec, color=None, out=None): # Replace node with its dependencies self._frontier.pop(i) - edges = sorted(node.edges_to_dependencies(deptype=self.deptype), reverse=True) + edges = sorted(node.edges_to_dependencies(depflag=self.depflag), reverse=True) if edges: deps = [e.spec.dag_hash() for e in edges] self._connect_deps(i, deps, "new-deps") # anywhere. @@ -433,13 +434,14 @@ def write(self, spec, color=None, out=None): self._collapse_line(i) -def graph_ascii(spec, node="o", out=None, debug=False, indent=0, color=None, deptype="all"): +def graph_ascii( + spec, node="o", out=None, debug=False, indent=0, color=None, depflag: dt.DepFlag = dt.ALL +): graph = AsciiGraph() graph.debug = debug graph.indent = indent graph.node_character = node - if deptype: - graph.deptype = spack.dependency.canonical_deptype(deptype) + graph.depflag = depflag graph.write(spec, color=color, out=out) @@ -513,7 +515,7 @@ def __init__(self): def visit(self, edge): if edge.parent is None: - for node in spack.traverse.traverse_nodes([edge.spec], deptype=("link", "run")): + for node in spack.traverse.traverse_nodes([edge.spec], deptype=dt.LINK | dt.RUN): self.main_unified_space.add(node.dag_hash()) super().visit(edge) @@ -526,43 +528,46 @@ def node_entry(self, node): def edge_entry(self, edge): colormap = {"build": "dodgerblue", "link": "crimson", "run": "goldenrod"} + label = "" + if edge.virtuals: + label = f" xlabel=\"virtuals={','.join(edge.virtuals)}\"" return ( edge.parent.dag_hash(), edge.spec.dag_hash(), - f"[color=\"{':'.join(colormap[x] for x in edge.deptypes)}\"]", + f"[color=\"{':'.join(colormap[x] for x in dt.flag_to_tuple(edge.depflag))}\"" + + label + + "]", ) -def _static_edges(specs, deptype): +def _static_edges(specs, depflag): for spec in specs: - pkg_cls = spack.repo.path.get_pkg_class(spec.name) - possible = pkg_cls.possible_dependencies(expand_virtuals=True, deptype=deptype) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) + possible = pkg_cls.possible_dependencies(expand_virtuals=True, depflag=depflag) for parent_name, dependencies in possible.items(): for dependency_name in dependencies: yield spack.spec.DependencySpec( spack.spec.Spec(parent_name), spack.spec.Spec(dependency_name), - deptypes=deptype, + depflag=depflag, virtuals=(), ) def static_graph_dot( - specs: List[spack.spec.Spec], - deptype: Optional[Union[str, Tuple[str, ...]]] = "all", - out: Optional[TextIO] = None, + specs: List[spack.spec.Spec], depflag: dt.DepFlag = dt.ALL, out: Optional[TextIO] = None ): """Static DOT graph with edges to all possible dependencies. Args: specs: abstract specs to be represented - deptype: dependency types to consider + depflag: dependency types to consider out: optional output stream. If None sys.stdout is used """ out = out or sys.stdout builder = StaticDag() - for edge in _static_edges(specs, deptype): + for edge in _static_edges(specs, depflag): builder.visit(edge) out.write(builder.render()) @@ -570,7 +575,7 @@ def static_graph_dot( def graph_dot( specs: List[spack.spec.Spec], builder: Optional[DotGraphBuilder] = None, - deptype: spack.dependency.DependencyArgument = "all", + depflag: dt.DepFlag = dt.ALL, out: Optional[TextIO] = None, ): """DOT graph of the concrete specs passed as input. @@ -578,7 +583,7 @@ def graph_dot( Args: specs: specs to be represented builder: builder to use to render the graph - deptype: dependency types to consider + depflag: dependency types to consider out: optional output stream. If None sys.stdout is used """ if not specs: @@ -587,10 +592,9 @@ def graph_dot( if out is None: out = sys.stdout - deptype = spack.dependency.canonical_deptype(deptype) builder = builder or SimpleDAG() for edge in spack.traverse.traverse_edges( - specs, cover="edges", order="breadth", deptype=deptype + specs, cover="edges", order="breadth", deptype=depflag ): builder.visit(edge) diff --git a/lib/spack/spack/hash_types.py b/lib/spack/spack/hash_types.py index 9e0276df528b0c..c1e25198cb3242 100644 --- a/lib/spack/spack/hash_types.py +++ b/lib/spack/spack/hash_types.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Definitions that control how Spack creates Spec hashes.""" -import spack.dependency as dp +import spack.deptypes as dt import spack.repo hashes = [] @@ -20,8 +20,8 @@ class SpecHashDescriptor: We currently use different hashes for different use cases.""" - def __init__(self, deptype, package_hash, name, override=None): - self.deptype = dp.canonical_deptype(deptype) + def __init__(self, depflag: dt.DepFlag, package_hash, name, override=None): + self.depflag = depflag self.package_hash = package_hash self.name = name hashes.append(self) @@ -39,24 +39,24 @@ def __call__(self, spec): #: Spack's deployment hash. Includes all inputs that can affect how a package is built. -dag_hash = SpecHashDescriptor(deptype=("build", "link", "run"), package_hash=True, name="hash") +dag_hash = SpecHashDescriptor(depflag=dt.BUILD | dt.LINK | dt.RUN, package_hash=True, name="hash") #: Hash descriptor used only to transfer a DAG, as is, across processes process_hash = SpecHashDescriptor( - deptype=("build", "link", "run", "test"), package_hash=True, name="process_hash" + depflag=dt.BUILD | dt.LINK | dt.RUN | dt.TEST, package_hash=True, name="process_hash" ) def _content_hash_override(spec): - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) pkg = pkg_cls(spec) return pkg.content_hash() #: Package hash used as part of dag hash package_hash = SpecHashDescriptor( - deptype=(), package_hash=True, name="package_hash", override=_content_hash_override + depflag=0, package_hash=True, name="package_hash", override=_content_hash_override ) @@ -64,10 +64,10 @@ def _content_hash_override(spec): # spec formats full_hash = SpecHashDescriptor( - deptype=("build", "link", "run"), package_hash=True, name="full_hash" + depflag=dt.BUILD | dt.LINK | dt.RUN, package_hash=True, name="full_hash" ) build_hash = SpecHashDescriptor( - deptype=("build", "link", "run"), package_hash=False, name="build_hash" + depflag=dt.BUILD | dt.LINK | dt.RUN, package_hash=False, name="build_hash" ) diff --git a/lib/spack/spack/hooks/drop_redundant_rpaths.py b/lib/spack/spack/hooks/drop_redundant_rpaths.py new file mode 100644 index 00000000000000..a32d435e388617 --- /dev/null +++ b/lib/spack/spack/hooks/drop_redundant_rpaths.py @@ -0,0 +1,123 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +from typing import IO, Optional, Tuple + +import llnl.util.tty as tty +from llnl.util.filesystem import BaseDirectoryVisitor, visit_directory_tree + +from spack.util.elf import ElfParsingError, parse_elf + + +def should_keep(path: bytes) -> bool: + """Return True iff path starts with $ (typically for $ORIGIN/${ORIGIN}) or is + absolute and exists.""" + return path.startswith(b"$") or (os.path.isabs(path) and os.path.lexists(path)) + + +def _drop_redundant_rpaths(f: IO) -> Optional[Tuple[bytes, bytes]]: + """Drop redundant entries from rpath. + + Args: + f: File object to patch opened in r+b mode. + + Returns: + A tuple of the old and new rpath if the rpath was patched, None otherwise. + """ + try: + elf = parse_elf(f, interpreter=False, dynamic_section=True) + except ElfParsingError: + return None + + # Nothing to do. + if not elf.has_rpath: + return None + + old_rpath_str = elf.dt_rpath_str + new_rpath_str = b":".join(p for p in old_rpath_str.split(b":") if should_keep(p)) + + # Nothing to write. + if old_rpath_str == new_rpath_str: + return None + + # Pad with 0 bytes. + pad = len(old_rpath_str) - len(new_rpath_str) + + # This can never happen since we're filtering, but lets be defensive. + if pad < 0: + return None + + # The rpath is at a given offset in the string table used by the + # dynamic section. + rpath_offset = elf.pt_dynamic_strtab_offset + elf.rpath_strtab_offset + + f.seek(rpath_offset) + f.write(new_rpath_str + b"\x00" * pad) + return old_rpath_str, new_rpath_str + + +def drop_redundant_rpaths(path: str) -> Optional[Tuple[bytes, bytes]]: + """Drop redundant entries from rpath. + + Args: + path: Path to a potential ELF file to patch. + + Returns: + A tuple of the old and new rpath if the rpath was patched, None otherwise. + """ + try: + with open(path, "r+b") as f: + return _drop_redundant_rpaths(f) + except OSError: + return None + + +class ElfFilesWithRPathVisitor(BaseDirectoryVisitor): + """Visitor that collects all elf files that have an rpath""" + + def __init__(self): + # Keep track of what hardlinked files we've already visited. + self.visited = set() + + def visit_file(self, root, rel_path, depth): + filepath = os.path.join(root, rel_path) + s = os.lstat(filepath) + identifier = (s.st_ino, s.st_dev) + + # We're hitting a hardlink or symlink of an excluded lib, no need to parse. + if s.st_nlink > 1: + if identifier in self.visited: + return + self.visited.add(identifier) + + result = drop_redundant_rpaths(filepath) + + if result is not None: + old, new = result + tty.debug(f"Patched rpath in {rel_path} from {old!r} to {new!r}") + + def visit_symlinked_file(self, root, rel_path, depth): + pass + + def before_visit_dir(self, root, rel_path, depth): + # Always enter dirs + return True + + def before_visit_symlinked_dir(self, root, rel_path, depth): + # Never enter symlinked dirs + return False + + +def post_install(spec, explicit=None): + # Skip externals + if spec.external: + return + + # Only enable on platforms using ELF. + if not spec.satisfies("platform=linux") and not spec.satisfies("platform=cray"): + return + + visit_directory_tree(spec.prefix, ElfFilesWithRPathVisitor()) diff --git a/lib/spack/spack/hooks/module_file_generation.py b/lib/spack/spack/hooks/module_file_generation.py index dc86c43205a8d5..1a2bbfdfe42d5c 100644 --- a/lib/spack/spack/hooks/module_file_generation.py +++ b/lib/spack/spack/hooks/module_file_generation.py @@ -3,18 +3,22 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from typing import Optional, Set + from llnl.util import tty import spack.config import spack.modules +import spack.spec -def _for_each_enabled(spec, method_name, explicit=None): +def _for_each_enabled( + spec: spack.spec.Spec, method_name: str, explicit: Optional[bool] = None +) -> None: """Calls a method for each enabled module""" - spack.modules.ensure_modules_are_enabled_or_warn() - set_names = set(spack.config.get("modules", {}).keys()) + set_names: Set[str] = set(spack.config.get("modules", {}).keys()) for name in set_names: - enabled = spack.config.get("modules:%s:enable" % name) + enabled = spack.config.get(f"modules:{name}:enable") if not enabled: tty.debug("NO MODULE WRITTEN: list of enabled module files is empty") continue @@ -29,7 +33,7 @@ def _for_each_enabled(spec, method_name, explicit=None): tty.warn(msg.format(method_name, str(e))) -def post_install(spec, explicit): +def post_install(spec, explicit: bool): import spack.environment as ev # break import cycle if ev.active_environment(): diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 7a6334dcc2e3e2..11e2d12f74b74b 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -41,7 +41,7 @@ def sbang_install_path(): """Location sbang should be installed within Spack's ``install_tree``.""" - sbang_root = str(spack.store.unpadded_root) + sbang_root = str(spack.store.STORE.unpadded_root) install_path = os.path.join(sbang_root, "bin", "sbang") path_length = len(install_path) if path_length > system_shebang_limit: diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index 2cdde481cc050e..662a1536c4b827 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -17,6 +17,7 @@ import llnl.util.filesystem as fs import llnl.util.tty as tty +from llnl.string import plural from llnl.util.lang import nullcontext from llnl.util.tty.color import colorize @@ -26,7 +27,6 @@ from spack.installer import InstallError from spack.spec import Spec from spack.util.prefix import Prefix -from spack.util.string import plural #: Stand-alone test failure info type TestFailureType = Tuple[BaseException, str] @@ -1039,7 +1039,7 @@ def test_pkg_id(cls, spec): Returns: str: the install test package identifier """ - return spec.format("{name}-{version}-{hash:7}") + return spec.format_path("{name}-{version}-{hash:7}") @classmethod def test_log_name(cls, spec): @@ -1147,12 +1147,12 @@ def write_test_result(self, spec, result): def write_reproducibility_data(self): for spec in self.specs: repo_cache_path = self.stage.repo.join(spec.name) - spack.repo.path.dump_provenance(spec, repo_cache_path) + spack.repo.PATH.dump_provenance(spec, repo_cache_path) for vspec in spec.package.virtuals_provided: repo_cache_path = self.stage.repo.join(vspec.name) if not os.path.exists(repo_cache_path): try: - spack.repo.path.dump_provenance(vspec, repo_cache_path) + spack.repo.PATH.dump_provenance(vspec, repo_cache_path) except spack.repo.UnknownPackageError: pass # not all virtuals have package files diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index bc3e17d41d72a1..51f70341761835 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -36,6 +36,7 @@ import sys import time from collections import defaultdict +from typing import Dict, Iterator, List, Optional, Set, Tuple import llnl.util.filesystem as fs import llnl.util.lock as lk @@ -45,7 +46,11 @@ from llnl.util.tty.log import log_output import spack.binary_distribution as binary_distribution +import spack.build_environment import spack.compilers +import spack.config +import spack.database +import spack.deptypes as dt import spack.error import spack.hooks import spack.mirror @@ -86,6 +91,16 @@ STATUS_REMOVED = "removed" +def _write_timer_json(pkg, timer, cache): + extra_attributes = {"name": pkg.name, "cache": cache, "hash": pkg.spec.dag_hash()} + try: + with open(pkg.times_log_path, "w") as timelog: + timer.write_json(timelog, extra_attributes=extra_attributes) + except Exception as e: + tty.debug(str(e)) + return + + class InstallAction: #: Don't perform an install NONE = 0 @@ -95,7 +110,76 @@ class InstallAction: OVERWRITE = 2 -def _check_last_phase(pkg): +class InstallStatus: + def __init__(self, pkg_count: int): + # Counters used for showing status information + self.pkg_num: int = 0 + self.pkg_count: int = pkg_count + self.pkg_ids: Set[str] = set() + + def next_pkg(self, pkg: "spack.package_base.PackageBase"): + pkg_id = package_id(pkg) + + if pkg_id not in self.pkg_ids: + self.pkg_num += 1 + self.pkg_ids.add(pkg_id) + + def set_term_title(self, text: str): + if not spack.config.get("config:install_status", True): + return + + if not sys.stdout.isatty(): + return + + status = f"{text} {self.get_progress()}" + sys.stdout.write(f"\x1b]0;Spack: {status}\x07") + sys.stdout.flush() + + def get_progress(self) -> str: + return f"[{self.pkg_num}/{self.pkg_count}]" + + +class TermStatusLine: + """ + This class is used in distributed builds to inform the user that other packages are + being installed by another process. + """ + + def __init__(self, enabled: bool): + self.enabled: bool = enabled + self.pkg_set: Set[str] = set() + self.pkg_list: List[str] = [] + + def add(self, pkg_id: str): + """Add a package to the waiting list, and if it is new, update the status line.""" + if not self.enabled or pkg_id in self.pkg_set: + return + + self.pkg_set.add(pkg_id) + self.pkg_list.append(pkg_id) + tty.msg(colorize("@*{Waiting for} @*g{%s}" % pkg_id)) + sys.stdout.flush() + + def clear(self): + """Clear the status line.""" + if not self.enabled: + return + + lines = len(self.pkg_list) + + if lines == 0: + return + + self.pkg_set.clear() + self.pkg_list = [] + + # Move the cursor to the beginning of the first "Waiting for" message and clear + # everything after it. + sys.stdout.write(f"\x1b[{lines}F\x1b[J") + sys.stdout.flush() + + +def _check_last_phase(pkg: "spack.package_base.PackageBase") -> None: """ Ensures the specified package has a valid last phase before proceeding with its installation. @@ -104,48 +188,45 @@ def _check_last_phase(pkg): package already. Args: - pkg (spack.package_base.PackageBase): the package being installed + pkg: the package being installed Raises: ``BadInstallPhase`` if stop_before or last phase is invalid """ - phases = pkg.builder.phases - if pkg.stop_before_phase and pkg.stop_before_phase not in phases: - raise BadInstallPhase(pkg.name, pkg.stop_before_phase) + phases = pkg.builder.phases # type: ignore[attr-defined] + if pkg.stop_before_phase and pkg.stop_before_phase not in phases: # type: ignore[attr-defined] + raise BadInstallPhase(pkg.name, pkg.stop_before_phase) # type: ignore[attr-defined] - if pkg.last_phase and pkg.last_phase not in phases: - raise BadInstallPhase(pkg.name, pkg.last_phase) + if pkg.last_phase and pkg.last_phase not in phases: # type: ignore[attr-defined] + raise BadInstallPhase(pkg.name, pkg.last_phase) # type: ignore[attr-defined] # If we got a last_phase, make sure it's not already last - if pkg.last_phase and pkg.last_phase == phases[-1]: - pkg.last_phase = None + if pkg.last_phase and pkg.last_phase == phases[-1]: # type: ignore[attr-defined] + pkg.last_phase = None # type: ignore[attr-defined] -def _handle_external_and_upstream(pkg, explicit): +def _handle_external_and_upstream(pkg: "spack.package_base.PackageBase", explicit: bool) -> bool: """ Determine if the package is external or upstream and register it in the database if it is external package. Args: - pkg (spack.package_base.PackageBase): the package whose installation is under - consideration - explicit (bool): the package was explicitly requested by the user + pkg: the package whose installation is under consideration + explicit: the package was explicitly requested by the user Return: - bool: ``True`` if the package is external or upstream (so not to - be installed locally), otherwise, ``True`` + ``True`` if the package is not to be installed locally, otherwise ``False`` """ # For external packages the workflow is simplified, and basically # consists in module file generation and registration in the DB. if pkg.spec.external: _process_external_package(pkg, explicit) - _print_installed_pkg("{0} (external {1})".format(pkg.prefix, package_id(pkg))) + _print_installed_pkg(f"{pkg.prefix} (external {package_id(pkg)})") return True if pkg.spec.installed_upstream: tty.verbose( - "{0} is installed in an upstream Spack instance at {1}".format( - package_id(pkg), pkg.spec.prefix - ) + f"{package_id(pkg)} is installed in an upstream Spack instance at " + f"{pkg.spec.prefix}" ) _print_installed_pkg(pkg.prefix) @@ -157,7 +238,7 @@ def _handle_external_and_upstream(pkg, explicit): return False -def _do_fake_install(pkg): +def _do_fake_install(pkg: "spack.package_base.PackageBase") -> None: """Make a fake install directory with fake executables, headers, and libraries.""" command = pkg.name header = pkg.name @@ -190,11 +271,13 @@ def _do_fake_install(pkg): # Install fake man page fs.mkdirp(pkg.prefix.man.man1) - packages_dir = spack.store.layout.build_packages_path(pkg.spec) + packages_dir = spack.store.STORE.layout.build_packages_path(pkg.spec) dump_packages(pkg.spec, packages_dir) -def _packages_needed_to_bootstrap_compiler(compiler, architecture, pkgs): +def _packages_needed_to_bootstrap_compiler( + compiler: "spack.spec.CompilerSpec", architecture: "spack.spec.ArchSpec", pkgs: list +) -> List[Tuple["spack.package_base.PackageBase", bool]]: """ Return a list of packages required to bootstrap `pkg`s compiler @@ -202,19 +285,17 @@ def _packages_needed_to_bootstrap_compiler(compiler, architecture, pkgs): matches the package spec. Args: - compiler (CompilerSpec): the compiler to bootstrap - architecture (ArchSpec): the architecture for which to boostrap the - compiler - pkgs (list): the packages that may need their compiler - installed + compiler: the compiler to bootstrap + architecture: the architecture for which to boostrap the compiler + pkgs: the packages that may need their compiler installed Return: - list: list of tuples, (PackageBase, bool), for concretized compiler-related + list of tuples of packages and a boolean, for concretized compiler-related packages that need to be installed and bool values specify whether the package is the bootstrap compiler (``True``) or one of its dependencies (``False``). The list will be empty if there are no compilers. """ - tty.debug("Bootstrapping {0} compiler".format(compiler)) + tty.debug(f"Bootstrapping {compiler} compiler") compilers = spack.compilers.compilers_for_spec(compiler, arch_spec=architecture) if compilers: return [] @@ -223,16 +304,16 @@ def _packages_needed_to_bootstrap_compiler(compiler, architecture, pkgs): # Set the architecture for the compiler package in a way that allows the # concretizer to back off if needed for the older bootstrapping compiler - dep.constrain("platform=%s" % str(architecture.platform)) - dep.constrain("os=%s" % str(architecture.os)) - dep.constrain("target=%s:" % architecture.target.microarchitecture.family.name) + dep.constrain(f"platform={str(architecture.platform)}") + dep.constrain(f"os={str(architecture.os)}") + dep.constrain(f"target={architecture.target.microarchitecture.family.name}:") # concrete CompilerSpec has less info than concrete Spec # concretize as Spec to add that information dep.concretize() # mark compiler as depended-on by the packages that use it for pkg in pkgs: dep._dependents.add( - spack.spec.DependencySpec(pkg.spec, dep, deptypes=("build",), virtuals=()) + spack.spec.DependencySpec(pkg.spec, dep, depflag=dt.BUILD, virtuals=()) ) packages = [(s.package, False) for s in dep.traverse(order="post", root=False)] @@ -240,15 +321,14 @@ def _packages_needed_to_bootstrap_compiler(compiler, architecture, pkgs): return packages -def _hms(seconds): +def _hms(seconds: int) -> str: """ Convert seconds to hours, minutes, seconds Args: - seconds (int): time to be converted in seconds + seconds: time to be converted in seconds - Return: - (str) String representation of the time as #h #m #.##s + Return: String representation of the time as #h #m #.##s """ m, s = divmod(seconds, 60) h, m = divmod(m, 60) @@ -259,18 +339,18 @@ def _hms(seconds): if m: parts.append("%dm" % m) if s: - parts.append("%.2fs" % s) + parts.append(f"{s:.2f}s") return " ".join(parts) -def _log_prefix(pkg_name): +def _log_prefix(pkg_name) -> str: """Prefix of the form "[pid]: [pkg name]: ..." when printing a status update during the build.""" - pid = "{0}: ".format(os.getpid()) if tty.show_pid() else "" - return "{0}{1}:".format(pid, pkg_name) + pid = f"{os.getpid()}: " if tty.show_pid() else "" + return f"{pid}{pkg_name}:" -def _print_installed_pkg(message): +def _print_installed_pkg(message: str) -> None: """ Output a message with a package icon. @@ -280,7 +360,7 @@ def _print_installed_pkg(message): print(colorize("@*g{[+]} ") + spack.util.path.debug_padded_filter(message)) -def print_install_test_log(pkg: "spack.package_base.PackageBase"): +def print_install_test_log(pkg: "spack.package_base.PackageBase") -> None: """Output install test log file path but only if have test failures. Args: @@ -293,27 +373,27 @@ def print_install_test_log(pkg: "spack.package_base.PackageBase"): pkg.tester.print_log_path() -def _print_timer(pre, pkg_id, timer): - phases = ["{}: {}.".format(p.capitalize(), _hms(timer.duration(p))) for p in timer.phases] - phases.append("Total: {}".format(_hms(timer.duration()))) - tty.msg("{0} Successfully installed {1}".format(pre, pkg_id), " ".join(phases)) +def _print_timer(pre: str, pkg_id: str, timer: timer.BaseTimer) -> None: + phases = [f"{p.capitalize()}: {_hms(timer.duration(p))}." for p in timer.phases] + phases.append(f"Total: {_hms(timer.duration())}") + tty.msg(f"{pre} Successfully installed {pkg_id}", " ".join(phases)) -def _install_from_cache(pkg, cache_only, explicit, unsigned=False): +def _install_from_cache( + pkg: "spack.package_base.PackageBase", cache_only: bool, explicit: bool, unsigned: bool = False +) -> bool: """ Extract the package from binary cache Args: - pkg (spack.package_base.PackageBase): package to install from the binary cache - cache_only (bool): only extract from binary cache - explicit (bool): ``True`` if installing the package was explicitly + pkg: package to install from the binary cache + cache_only: only extract from binary cache + explicit: ``True`` if installing the package was explicitly requested by the user, otherwise, ``False`` - unsigned (bool): ``True`` if binary package signatures to be checked, + unsigned: ``True`` if binary package signatures to be checked, otherwise, ``False`` - Return: - bool: ``True`` if the package was extract from binary cache, - ``False`` otherwise + Return: ``True`` if the package was extract from binary cache, ``False`` otherwise """ t = timer.Timer() installed_from_cache = _try_install_from_binary_cache( @@ -321,75 +401,81 @@ def _install_from_cache(pkg, cache_only, explicit, unsigned=False): ) pkg_id = package_id(pkg) if not installed_from_cache: - pre = "No binary for {0} found".format(pkg_id) + pre = f"No binary for {pkg_id} found" if cache_only: - tty.die("{0} when cache-only specified".format(pre)) + tty.die(f"{pre} when cache-only specified") - tty.msg("{0}: installing from source".format(pre)) + tty.msg(f"{pre}: installing from source") return False t.stop() - tty.debug("Successfully extracted {0} from binary cache".format(pkg_id)) + tty.debug(f"Successfully extracted {pkg_id} from binary cache") + + _write_timer_json(pkg, t, True) _print_timer(pre=_log_prefix(pkg.name), pkg_id=pkg_id, timer=t) _print_installed_pkg(pkg.spec.prefix) spack.hooks.post_install(pkg.spec, explicit) return True -def _process_external_package(pkg, explicit): +def _process_external_package(pkg: "spack.package_base.PackageBase", explicit: bool) -> None: """ Helper function to run post install hooks and register external packages. Args: - pkg (Package): the external package - explicit (bool): if the package was requested explicitly by the user, + pkg: the external package + explicit: if the package was requested explicitly by the user, ``False`` if it was pulled in as a dependency of an explicit package. """ assert pkg.spec.external, "Expected to post-install/register an external package." - pre = "{s.name}@{s.version} :".format(s=pkg.spec) + pre = f"{pkg.spec.name}@{pkg.spec.version} :" spec = pkg.spec if spec.external_modules: - tty.msg("{0} has external module in {1}".format(pre, spec.external_modules)) - tty.debug("{0} is actually installed in {1}".format(pre, spec.external_path)) + tty.msg(f"{pre} has external module in {spec.external_modules}") + tty.debug(f"{pre} is actually installed in {spec.external_path}") else: - tty.debug("{0} externally installed in {1}".format(pre, spec.external_path)) + tty.debug(f"{pre} externally installed in {spec.external_path}") try: # Check if the package was already registered in the DB. # If this is the case, then only make explicit if required. - tty.debug("{0} already registered in DB".format(pre)) - record = spack.store.db.get_record(spec) + tty.debug(f"{pre} already registered in DB") + record = spack.store.STORE.db.get_record(spec) if explicit and not record.explicit: - spack.store.db.update_explicit(spec, explicit) + spack.store.STORE.db.update_explicit(spec, explicit) except KeyError: # If not, register it and generate the module file. # For external packages we just need to run # post-install hooks to generate module files. - tty.debug("{0} generating module file".format(pre)) + tty.debug(f"{pre} generating module file") spack.hooks.post_install(spec, explicit) # Add to the DB - tty.debug("{0} registering into DB".format(pre)) - spack.store.db.add(spec, None, explicit=explicit) + tty.debug(f"{pre} registering into DB") + spack.store.STORE.db.add(spec, None, explicit=explicit) def _process_binary_cache_tarball( - pkg, explicit, unsigned, mirrors_for_spec=None, timer=timer.NULL_TIMER -): + pkg: "spack.package_base.PackageBase", + explicit: bool, + unsigned: bool, + mirrors_for_spec: Optional[list] = None, + timer: timer.BaseTimer = timer.NULL_TIMER, +) -> bool: """ Process the binary cache tarball. Args: - pkg (spack.package_base.PackageBase): the package being installed - explicit (bool): the package was explicitly requested by the user - unsigned (bool): ``True`` if binary package signatures to be checked, + pkg: the package being installed + explicit: the package was explicitly requested by the user + unsigned: ``True`` if binary package signatures to be checked, otherwise, ``False`` - mirrors_for_spec (list): Optional list of concrete specs and mirrors + mirrors_for_spec: Optional list of concrete specs and mirrors obtained by calling binary_distribution.get_mirrors_for_spec(). - timer (Timer): timer to keep track of binary install phases. + timer: timer to keep track of binary install phases. Return: bool: ``True`` if the package was extracted from binary cache, @@ -403,34 +489,39 @@ def _process_binary_cache_tarball( if download_result is None: return False - tty.msg("Extracting {0} from binary cache".format(package_id(pkg))) + tty.msg(f"Extracting {package_id(pkg)} from binary cache") with timer.measure("install"), spack.util.path.filter_padding(): binary_distribution.extract_tarball( - pkg.spec, download_result, unsigned=unsigned, force=False + pkg.spec, download_result, unsigned=unsigned, force=False, timer=timer ) pkg.installed_from_binary_cache = True - spack.store.db.add(pkg.spec, spack.store.layout, explicit=explicit) + spack.store.STORE.db.add(pkg.spec, spack.store.STORE.layout, explicit=explicit) return True -def _try_install_from_binary_cache(pkg, explicit, unsigned=False, timer=timer.NULL_TIMER): +def _try_install_from_binary_cache( + pkg: "spack.package_base.PackageBase", + explicit: bool, + unsigned: bool = False, + timer: timer.BaseTimer = timer.NULL_TIMER, +) -> bool: """ Try to extract the package from binary cache. Args: - pkg (spack.package_base.PackageBase): package to be extracted from binary cache - explicit (bool): the package was explicitly requested by the user - unsigned (bool): ``True`` if binary package signatures to be checked, + pkg: package to be extracted from binary cache + explicit: the package was explicitly requested by the user + unsigned: ``True`` if binary package signatures to be checked, otherwise, ``False`` - timer (Timer): + timer: timer to keep track of binary install phases. """ - # Early exit if no mirrors are configured. - if not spack.mirror.MirrorCollection(): + # Early exit if no binary mirrors are configured. + if not spack.mirror.MirrorCollection(binary=True): return False - tty.debug("Searching for binary cache of {0}".format(package_id(pkg))) + tty.debug(f"Searching for binary cache of {package_id(pkg)}") with timer.measure("search"): matches = binary_distribution.get_mirrors_for_spec(pkg.spec, index_only=True) @@ -440,14 +531,7 @@ def _try_install_from_binary_cache(pkg, explicit, unsigned=False, timer=timer.NU ) -def clear_failures(): - """ - Remove all failure tracking markers for the Spack instance. - """ - spack.store.db.clear_all_failures() - - -def combine_phase_logs(phase_log_files, log_path): +def combine_phase_logs(phase_log_files: List[str], log_path: str) -> None: """ Read set or list of logs and combine them into one file. @@ -456,8 +540,8 @@ def combine_phase_logs(phase_log_files, log_path): generally to accept some list of files, and a log path to combine them to. Args: - phase_log_files (list): a list or iterator of logs to combine - log_path (str): the path to combine them to + phase_log_files: a list or iterator of logs to combine + log_path: the path to combine them to """ with open(log_path, "bw") as log_file: for phase_log_file in phase_log_files: @@ -465,7 +549,7 @@ def combine_phase_logs(phase_log_files, log_path): shutil.copyfileobj(phase_log, log_file) -def dump_packages(spec, path): +def dump_packages(spec: "spack.spec.Spec", path: str) -> None: """ Dump all package information for a spec and its dependencies. @@ -474,8 +558,8 @@ def dump_packages(spec, path): node in the DAG. Args: - spec (spack.spec.Spec): the Spack spec whose package information is to be dumped - path (str): the path to the build packages directory + spec: the Spack spec whose package information is to be dumped + path: the path to the build packages directory """ fs.mkdirp(path) @@ -487,7 +571,7 @@ def dump_packages(spec, path): if node is not spec: # Locate the dependency package in the install tree and find # its provenance information. - source = spack.store.layout.build_packages_path(node) + source = spack.store.STORE.layout.build_packages_path(node) source_repo_root = os.path.join(source, node.namespace) # If there's no provenance installed for the package, skip it. @@ -505,9 +589,9 @@ def dump_packages(spec, path): source_repo = spack.repo.Repo(source_repo_root) source_pkg_dir = source_repo.dirname_for_package_name(node.name) except spack.repo.RepoError as err: - tty.debug("Failed to create source repo for {0}: {1}".format(node.name, str(err))) + tty.debug(f"Failed to create source repo for {node.name}: {str(err)}") source_pkg_dir = None - tty.warn("Warning: Couldn't copy in provenance for {0}".format(node.name)) + tty.warn(f"Warning: Couldn't copy in provenance for {node.name}") # Create a destination repository dest_repo_root = os.path.join(path, node.namespace) @@ -518,45 +602,50 @@ def dump_packages(spec, path): # Get the location of the package in the dest repo. dest_pkg_dir = repo.dirname_for_package_name(node.name) if node is spec: - spack.repo.path.dump_provenance(node, dest_pkg_dir) + spack.repo.PATH.dump_provenance(node, dest_pkg_dir) elif source_pkg_dir: - fs.install_tree(source_pkg_dir, dest_pkg_dir) + fs.install_tree( + source_pkg_dir, dest_pkg_dir, allow_broken_symlinks=(sys.platform != "win32") + ) -def get_dependent_ids(spec): +def get_dependent_ids(spec: "spack.spec.Spec") -> List[str]: """ Return a list of package ids for the spec's dependents Args: - spec (spack.spec.Spec): Concretized spec + spec: Concretized spec - Returns: - list: list of package ids + Returns: list of package ids """ return [package_id(d.package) for d in spec.dependents()] -def install_msg(name, pid): +def install_msg(name: str, pid: int, install_status: InstallStatus) -> str: """ Colorize the name/id of the package being installed Args: - name (str): Name/id of the package being installed - pid (int): id of the installer process + name: Name/id of the package being installed + pid: id of the installer process - Return: - str: Colorized installing message + Return: Colorized installing message """ - pre = "{0}: ".format(pid) if tty.show_pid() else "" - return pre + colorize("@*{Installing} @*g{%s}" % name) + pre = f"{pid}: " if tty.show_pid() else "" + post = ( + " @*{%s}" % install_status.get_progress() + if install_status and spack.config.get("config:install_status", True) + else "" + ) + return pre + colorize("@*{Installing} @*g{%s}%s" % (name, post)) -def archive_install_logs(pkg, phase_log_dir): +def archive_install_logs(pkg: "spack.package_base.PackageBase", phase_log_dir: str) -> None: """ Copy install logs to their destination directory(ies) Args: - pkg (spack.package_base.PackageBase): the package that was built and installed - phase_log_dir (str): path to the archive directory + pkg: the package that was built and installed + phase_log_dir: path to the archive directory """ # Archive the whole stdout + stderr for the package fs.install(pkg.log_path, pkg.install_log_path) @@ -570,14 +659,14 @@ def archive_install_logs(pkg, phase_log_dir): pkg.archive_install_test_log() -def log(pkg): +def log(pkg: "spack.package_base.PackageBase") -> None: """ Copy provenance into the install directory on success Args: - pkg (spack.package_base.PackageBase): the package that was built and installed + pkg: the package that was built and installed """ - packages_dir = spack.store.layout.build_packages_path(pkg.spec) + packages_dir = spack.store.STORE.layout.build_packages_path(pkg.spec) # Remove first if we're overwriting another build try: @@ -599,14 +688,16 @@ def log(pkg): # Finally, archive files that are specific to each package with fs.working_dir(pkg.stage.path): errors = io.StringIO() - target_dir = os.path.join(spack.store.layout.metadata_path(pkg.spec), "archived-files") + target_dir = os.path.join( + spack.store.STORE.layout.metadata_path(pkg.spec), "archived-files" + ) for glob_expr in pkg.builder.archive_files: # Check that we are trying to copy things that are # in the stage tree (not arbitrary files) abs_expr = os.path.realpath(glob_expr) if os.path.realpath(pkg.stage.path) not in abs_expr: - errors.write("[OUTSIDE SOURCE PATH]: {0}\n".format(glob_expr)) + errors.write(f"[OUTSIDE SOURCE PATH]: {glob_expr}\n") continue # Now that we are sure that the path is within the correct # folder, make it relative and check for matches @@ -626,19 +717,19 @@ def log(pkg): # Here try to be conservative, and avoid discarding # the whole install procedure because of copying a # single file failed - errors.write("[FAILED TO ARCHIVE]: {0}".format(f)) + errors.write(f"[FAILED TO ARCHIVE]: {f}") if errors.getvalue(): error_file = os.path.join(target_dir, "errors.txt") fs.mkdirp(target_dir) with open(error_file, "w") as err: err.write(errors.getvalue()) - tty.warn("Errors occurred when archiving files.\n\t" "See: {0}".format(error_file)) + tty.warn(f"Errors occurred when archiving files.\n\tSee: {error_file}") dump_packages(pkg.spec, packages_dir) -def package_id(pkg): +def package_id(pkg: "spack.package_base.PackageBase") -> str: """A "unique" package identifier for installation purposes The identifier is used to track build tasks, locks, install, and @@ -648,197 +739,508 @@ def package_id(pkg): and packages for combinatorial environments. Args: - pkg (spack.package_base.PackageBase): the package from which the identifier is - derived + pkg: the package from which the identifier is derived """ if not pkg.spec.concrete: raise ValueError("Cannot provide a unique, readable id when the spec is not concretized.") - return "{0}-{1}-{2}".format(pkg.name, pkg.version, pkg.spec.dag_hash()) + return f"{pkg.name}-{pkg.version}-{pkg.spec.dag_hash()}" -class TermTitle: - def __init__(self, pkg_count): - # Counters used for showing status information in the terminal title - self.pkg_num = 0 - self.pkg_count = pkg_count - self.pkg_ids = set() +class BuildRequest: + """Class for representing an installation request.""" - def next_pkg(self, pkg): - pkg_id = package_id(pkg) + def __init__(self, pkg: "spack.package_base.PackageBase", install_args: dict): + """ + Instantiate a build request for a package. - if pkg_id not in self.pkg_ids: - self.pkg_num += 1 - self.pkg_ids.add(pkg_id) + Args: + pkg: the package to be built and installed + install_args: the install arguments associated with ``pkg`` + """ + # Ensure dealing with a package that has a concrete spec + if not isinstance(pkg, spack.package_base.PackageBase): + raise ValueError(f"{str(pkg)} must be a package") - def set(self, text): - if not spack.config.get("config:terminal_title", False): - return + self.pkg = pkg + if not self.pkg.spec.concrete: + raise ValueError(f"{self.pkg.name} must have a concrete spec") - if not sys.stdout.isatty(): - return + # Cache the package phase options with the explicit package, + # popping the options to ensure installation of associated + # dependencies is NOT affected by these options. - status = "{0} [{1}/{2}]".format(text, self.pkg_num, self.pkg_count) - sys.stdout.write("\033]0;Spack: {0}\007".format(status)) - sys.stdout.flush() + self.pkg.stop_before_phase = install_args.pop("stop_before", None) # type: ignore[attr-defined] # noqa: E501 + self.pkg.last_phase = install_args.pop("stop_at", None) # type: ignore[attr-defined] + # Cache the package id for convenience + self.pkg_id = package_id(pkg) -class TermStatusLine: - """ - This class is used in distributed builds to inform the user that other packages are - being installed by another process. - """ + # Save off the original install arguments plus standard defaults + # since they apply to the requested package *and* dependencies. + self.install_args = install_args if install_args else {} + self._add_default_args() - def __init__(self, enabled): - self.enabled = enabled - self.pkg_set = set() - self.pkg_list = [] + # Cache overwrite information + self.overwrite = set(self.install_args.get("overwrite", [])) + self.overwrite_time = time.time() - def add(self, pkg_id): - """ - Add a package to the waiting list, and if it is new, update the status line. - """ - if not self.enabled or pkg_id in self.pkg_set: - return + # Save off dependency package ids for quick checks since traversals + # are not able to return full dependents for all packages across + # environment specs. + self.dependencies = set( + package_id(d.package) + for d in self.pkg.spec.dependencies(deptype=self.get_depflags(self.pkg)) + if package_id(d.package) != self.pkg_id + ) - self.pkg_set.add(pkg_id) - self.pkg_list.append(pkg_id) - tty.msg(colorize("@*{Waiting for} @*g{%s}" % pkg_id)) - sys.stdout.flush() + def __repr__(self) -> str: + """Returns a formal representation of the build request.""" + rep = f"{self.__class__.__name__}(" + for attr, value in self.__dict__.items(): + rep += f"{attr}={value.__repr__()}, " + return f"{rep.strip(', ')})" - def clear(self): - """ - Clear the status line. - """ - if not self.enabled: - return + def __str__(self) -> str: + """Returns a printable version of the build request.""" + return f"package={self.pkg.name}, install_args={self.install_args}" - lines = len(self.pkg_list) + def _add_default_args(self) -> None: + """Ensure standard install options are set to at least the default.""" + for arg, default in [ + ("context", "build"), # installs *always* build + ("dependencies_cache_only", False), + ("dependencies_use_cache", True), + ("dirty", False), + ("fail_fast", False), + ("fake", False), + ("install_deps", True), + ("install_package", True), + ("install_source", False), + ("package_cache_only", False), + ("package_use_cache", True), + ("keep_prefix", False), + ("keep_stage", False), + ("restage", False), + ("skip_patch", False), + ("tests", False), + ("unsigned", False), + ("verbose", False), + ]: + _ = self.install_args.setdefault(arg, default) - if lines == 0: - return + def get_depflags(self, pkg: "spack.package_base.PackageBase") -> int: + """Determine the required dependency types for the associated package. - self.pkg_set.clear() - self.pkg_list = [] + Args: + pkg: explicit or implicit package being installed - # Move the cursor to the beginning of the first "Waiting for" message and clear - # everything after it. - sys.stdout.write("\x1b[%sF\x1b[J" % lines) - sys.stdout.flush() + Returns: + tuple: required dependency type(s) for the package + """ + depflag = dt.LINK | dt.RUN + include_build_deps = self.install_args.get("include_build_deps") + if self.pkg_id == package_id(pkg): + cache_only = self.install_args.get("package_cache_only") + else: + cache_only = self.install_args.get("dependencies_cache_only") -class PackageInstaller: - """ - Class for managing the install process for a Spack instance based on a - bottom-up DAG approach. + # Include build dependencies if pkg is going to be built from sources, or + # if build deps are explicitly requested. + if include_build_deps or not ( + cache_only or pkg.spec.installed and not pkg.spec.dag_hash() in self.overwrite + ): + depflag |= dt.BUILD + if self.run_tests(pkg): + depflag |= dt.TEST + return depflag - This installer can coordinate concurrent batch and interactive, local - and distributed (on a shared file system) builds for the same Spack - instance. - """ + def has_dependency(self, dep_id) -> bool: + """Returns ``True`` if the package id represents a known dependency + of the requested package, ``False`` otherwise.""" + return dep_id in self.dependencies - def __init__(self, installs=[]): - """Initialize the installer. + def run_tests(self, pkg: "spack.package_base.PackageBase") -> bool: + """Determine if the tests should be run for the provided packages Args: - installs (list): list of tuples, where each - tuple consists of a package (PackageBase) and its associated - install arguments (dict) - Return: - PackageInstaller: instance - """ - # List of build requests - self.build_requests = [BuildRequest(pkg, install_args) for pkg, install_args in installs] - - # Priority queue of build tasks - self.build_pq = [] - - # Mapping of unique package ids to build task - self.build_tasks = {} - - # Cache of package locks for failed packages, keyed on package's ids - self.failed = {} + pkg: explicit or implicit package being installed - # Cache the PID for distributed build messaging - self.pid = os.getpid() + Returns: + bool: ``True`` if they should be run; ``False`` otherwise + """ + tests = self.install_args.get("tests", False) + return tests is True or (tests and pkg.name in tests) - # Cache of installed packages' unique ids - self.installed = set() + @property + def spec(self) -> "spack.spec.Spec": + """The specification associated with the package.""" + return self.pkg.spec - # Data store layout - self.layout = spack.store.layout + def traverse_dependencies(self, spec=None, visited=None) -> Iterator["spack.spec.Spec"]: + """Yield any dependencies of the appropriate type(s)""" + # notice: deptype is not constant across nodes, so we cannot use + # spec.traverse_edges(deptype=...). - # Locks on specs being built, keyed on the package's unique id - self.locks = {} + if spec is None: + spec = self.spec + if visited is None: + visited = set() - # Cache fail_fast option to ensure if one build request asks to fail - # fast then that option applies to all build requests. - self.fail_fast = False + for dep in spec.dependencies(deptype=self.get_depflags(spec.package)): + hash = dep.dag_hash() + if hash in visited: + continue + visited.add(hash) + # In Python 3: yield from self.traverse_dependencies(dep, visited) + for s in self.traverse_dependencies(dep, visited): + yield s + yield dep - def __repr__(self): - """Returns a formal representation of the package installer.""" - rep = "{0}(".format(self.__class__.__name__) - for attr, value in self.__dict__.items(): - rep += "{0}={1}, ".format(attr, value.__repr__()) - return "{0})".format(rep.strip(", ")) - def __str__(self): - """Returns a printable version of the package installer.""" - requests = "#requests={0}".format(len(self.build_requests)) - tasks = "#tasks={0}".format(len(self.build_tasks)) - failed = "failed ({0}) = {1}".format(len(self.failed), self.failed) - installed = "installed ({0}) = {1}".format(len(self.installed), self.installed) - return "{0}: {1}; {2}; {3}; {4}".format(self.pid, requests, tasks, installed, failed) +class BuildTask: + """Class for representing the build task for a package.""" - def _add_bootstrap_compilers(self, compiler, architecture, pkgs, request, all_deps): + def __init__( + self, + pkg: "spack.package_base.PackageBase", + request: Optional[BuildRequest], + compiler: bool, + start: float, + attempts: int, + status: str, + installed: Set[str], + ): """ - Add bootstrap compilers and dependencies to the build queue. + Instantiate a build task for a package. Args: - compiler: the compiler to boostrap - architecture: the architecture for which to bootstrap the compiler - pkgs (spack.package_base.PackageBase): the package with possible compiler - dependencies - request (BuildRequest): the associated install request - all_deps (defaultdict(set)): dictionary of all dependencies and - associated dependents + pkg: the package to be built and installed + request: the associated install request where ``None`` can be + used to indicate the package was explicitly requested by the user + compiler: whether task is for a bootstrap compiler + start: the initial start time for the package, in seconds + attempts: the number of attempts to install the package + status: the installation status + installed: the identifiers of packages that have + been installed so far """ - packages = _packages_needed_to_bootstrap_compiler(compiler, architecture, pkgs) - for comp_pkg, is_compiler in packages: - pkgid = package_id(comp_pkg) - if pkgid not in self.build_tasks: - self._add_init_task(comp_pkg, request, is_compiler, all_deps) - elif is_compiler: - # ensure it's queued as a compiler - self._modify_existing_task(pkgid, "compiler", True) - def _modify_existing_task(self, pkgid, attr, value): - """ - Update a task in-place to modify its behavior. + # Ensure dealing with a package that has a concrete spec + if not isinstance(pkg, spack.package_base.PackageBase): + raise ValueError(f"{str(pkg)} must be a package") - Currently used to update the ``compiler`` field on tasks - that were originally created as a dependency of a compiler, - but are compilers in their own right. + self.pkg = pkg + if not self.pkg.spec.concrete: + raise ValueError(f"{self.pkg.name} must have a concrete spec") - For example, ``intel-oneapi-compilers-classic`` depends on - ``intel-oneapi-compilers``, which can cause the latter to be - queued first as a non-compiler, and only later as a compiler. - """ - for i, tup in enumerate(self.build_pq): + # The "unique" identifier for the task's package + self.pkg_id = package_id(self.pkg) + + # The explicit build request associated with the package + if not isinstance(request, BuildRequest): + raise ValueError(f"{str(pkg)} must have a build request") + + self.request = request + + # Initialize the status to an active state. The status is used to + # ensure priority queue invariants when tasks are "removed" from the + # queue. + if status == STATUS_REMOVED: + raise InstallError( + f"Cannot create a build task for {self.pkg_id} with status '{status}'", pkg=pkg + ) + + self.status = status + + # Package is associated with a bootstrap compiler + self.compiler = compiler + + # The initial start time for processing the spec + self.start = start + + # Set of dependents, which needs to include the requesting package + # to support tracking of parallel, multi-spec, environment installs. + self.dependents = set(get_dependent_ids(self.pkg.spec)) + + tty.debug(f"Pkg id {self.pkg_id} has the following dependents:") + for dep_id in self.dependents: + tty.debug(f"- {dep_id}") + + # Set of dependencies + # + # Be consistent wrt use of dependents and dependencies. That is, + # if use traverse for transitive dependencies, then must remove + # transitive dependents on failure. + self.dependencies = set( + package_id(d.package) + for d in self.pkg.spec.dependencies(deptype=self.request.get_depflags(self.pkg)) + if package_id(d.package) != self.pkg_id + ) + + # Handle bootstrapped compiler + # + # The bootstrapped compiler is not a dependency in the spec, but it is + # a dependency of the build task. Here we add it to self.dependencies + compiler_spec = self.pkg.spec.compiler + arch_spec = self.pkg.spec.architecture + if not spack.compilers.compilers_for_spec(compiler_spec, arch_spec=arch_spec): + # The compiler is in the queue, identify it as dependency + dep = spack.compilers.pkg_spec_for_compiler(compiler_spec) + dep.constrain(f"platform={str(arch_spec.platform)}") + dep.constrain(f"os={str(arch_spec.os)}") + dep.constrain(f"target={arch_spec.target.microarchitecture.family.name}:") + dep.concretize() + dep_id = package_id(dep.package) + self.dependencies.add(dep_id) + + # List of uninstalled dependencies, which is used to establish + # the priority of the build task. + # + self.uninstalled_deps = set( + pkg_id for pkg_id in self.dependencies if pkg_id not in installed + ) + + # Ensure key sequence-related properties are updated accordingly. + self.attempts = 0 + self._update() + + def __eq__(self, other): + return self.key == other.key + + def __ge__(self, other): + return self.key >= other.key + + def __gt__(self, other): + return self.key > other.key + + def __le__(self, other): + return self.key <= other.key + + def __lt__(self, other): + return self.key < other.key + + def __ne__(self, other): + return self.key != other.key + + def __repr__(self) -> str: + """Returns a formal representation of the build task.""" + rep = f"{self.__class__.__name__}(" + for attr, value in self.__dict__.items(): + rep += f"{attr}={value.__repr__()}, " + return f"{rep.strip(', ')})" + + def __str__(self) -> str: + """Returns a printable version of the build task.""" + dependencies = f"#dependencies={len(self.dependencies)}" + return "priority={0}, status={1}, start={2}, {3}".format( + self.priority, self.status, self.start, dependencies + ) + + def _update(self) -> None: + """Update properties associated with a new instance of a task.""" + # Number of times the task has/will be queued + self.attempts = self.attempts + 1 + + # Ensure the task gets a unique sequence number to preserve the + # order in which it is added. + self.sequence = next(_counter) + + def add_dependent(self, pkg_id: str) -> None: + """ + Ensure the dependent package id is in the task's list so it will be + properly updated when this package is installed. + + Args: + pkg_id: package identifier of the dependent package + """ + if pkg_id != self.pkg_id and pkg_id not in self.dependents: + tty.debug(f"Adding {pkg_id} as a dependent of {self.pkg_id}") + self.dependents.add(pkg_id) + + def flag_installed(self, installed: List[str]) -> None: + """ + Ensure the dependency is not considered to still be uninstalled. + + Args: + installed: the identifiers of packages that have been installed so far + """ + now_installed = self.uninstalled_deps & set(installed) + for pkg_id in now_installed: + self.uninstalled_deps.remove(pkg_id) + tty.debug( + f"{self.pkg_id}: Removed {pkg_id} from uninstalled deps list: " + f"{self.uninstalled_deps}", + level=2, + ) + + @property + def explicit(self) -> bool: + """The package was explicitly requested by the user.""" + return self.is_root and self.request.install_args.get("explicit", True) + + @property + def is_root(self) -> bool: + """The package was requested directly, but may or may not be explicit + in an environment.""" + return self.pkg == self.request.pkg + + @property + def use_cache(self) -> bool: + _use_cache = True + if self.is_root: + return self.request.install_args.get("package_use_cache", _use_cache) + else: + return self.request.install_args.get("dependencies_use_cache", _use_cache) + + @property + def cache_only(self) -> bool: + _cache_only = False + if self.is_root: + return self.request.install_args.get("package_cache_only", _cache_only) + else: + return self.request.install_args.get("dependencies_cache_only", _cache_only) + + @property + def key(self) -> Tuple[int, int]: + """The key is the tuple (# uninstalled dependencies, sequence).""" + return (self.priority, self.sequence) + + def next_attempt(self, installed) -> "BuildTask": + """Create a new, updated task for the next installation attempt.""" + task = copy.copy(self) + task._update() + task.start = self.start or time.time() + task.flag_installed(installed) + return task + + @property + def priority(self): + """The priority is based on the remaining uninstalled dependencies.""" + return len(self.uninstalled_deps) + + +class PackageInstaller: + """ + Class for managing the install process for a Spack instance based on a + bottom-up DAG approach. + + This installer can coordinate concurrent batch and interactive, local + and distributed (on a shared file system) builds for the same Spack + instance. + """ + + def __init__(self, installs: List[Tuple["spack.package_base.PackageBase", dict]] = []) -> None: + """Initialize the installer. + + Args: + installs (list): list of tuples, where each + tuple consists of a package (PackageBase) and its associated + install arguments (dict) + """ + # List of build requests + self.build_requests = [BuildRequest(pkg, install_args) for pkg, install_args in installs] + + # Priority queue of build tasks + self.build_pq: List[Tuple[Tuple[int, int], BuildTask]] = [] + + # Mapping of unique package ids to build task + self.build_tasks: Dict[str, BuildTask] = {} + + # Cache of package locks for failed packages, keyed on package's ids + self.failed: Dict[str, Optional[lk.Lock]] = {} + + # Cache the PID for distributed build messaging + self.pid: int = os.getpid() + + # Cache of installed packages' unique ids + self.installed: Set[str] = set() + + # Data store layout + self.layout = spack.store.STORE.layout + + # Locks on specs being built, keyed on the package's unique id + self.locks: Dict[str, Tuple[str, Optional[lk.Lock]]] = {} + + # Cache fail_fast option to ensure if one build request asks to fail + # fast then that option applies to all build requests. + self.fail_fast = False + + def __repr__(self) -> str: + """Returns a formal representation of the package installer.""" + rep = f"{self.__class__.__name__}(" + for attr, value in self.__dict__.items(): + rep += f"{attr}={value.__repr__()}, " + return f"{rep.strip(', ')})" + + def __str__(self) -> str: + """Returns a printable version of the package installer.""" + requests = f"#requests={len(self.build_requests)}" + tasks = f"#tasks={len(self.build_tasks)}" + failed = f"failed ({len(self.failed)}) = {self.failed}" + installed = f"installed ({len(self.installed)}) = {self.installed}" + return f"{self.pid}: {requests}; {tasks}; {installed}; {failed}" + + def _add_bootstrap_compilers( + self, + compiler: "spack.spec.CompilerSpec", + architecture: "spack.spec.ArchSpec", + pkgs: List["spack.package_base.PackageBase"], + request: BuildRequest, + all_deps, + ) -> None: + """ + Add bootstrap compilers and dependencies to the build queue. + + Args: + compiler: the compiler to boostrap + architecture: the architecture for which to bootstrap the compiler + pkgs: the package list with possible compiler dependencies + request: the associated install request + all_deps (defaultdict(set)): dictionary of all dependencies and + associated dependents + """ + packages = _packages_needed_to_bootstrap_compiler(compiler, architecture, pkgs) + for comp_pkg, is_compiler in packages: + pkgid = package_id(comp_pkg) + if pkgid not in self.build_tasks: + self._add_init_task(comp_pkg, request, is_compiler, all_deps) + elif is_compiler: + # ensure it's queued as a compiler + self._modify_existing_task(pkgid, "compiler", True) + + def _modify_existing_task(self, pkgid: str, attr, value) -> None: + """ + Update a task in-place to modify its behavior. + + Currently used to update the ``compiler`` field on tasks + that were originally created as a dependency of a compiler, + but are compilers in their own right. + + For example, ``intel-oneapi-compilers-classic`` depends on + ``intel-oneapi-compilers``, which can cause the latter to be + queued first as a non-compiler, and only later as a compiler. + """ + for i, tup in enumerate(self.build_pq): key, task = tup if task.pkg_id == pkgid: - tty.debug( - "Modifying task for {0} to treat it as a compiler".format(pkgid), level=2 - ) + tty.debug(f"Modifying task for {pkgid} to treat it as a compiler", level=2) setattr(task, attr, value) self.build_pq[i] = (key, task) - def _add_init_task(self, pkg, request, is_compiler, all_deps): + def _add_init_task( + self, + pkg: "spack.package_base.PackageBase", + request: Optional[BuildRequest], + is_compiler: bool, + all_deps: Dict[str, Set[str]], + ) -> None: """ Creates and queus the initial build task for the package. Args: - pkg (spack.package_base.PackageBase): the package to be built and installed + pkg: the package to be built and installed request (BuildRequest or None): the associated install request where ``None`` can be used to indicate the package was explicitly requested by the user @@ -852,20 +1254,20 @@ def _add_init_task(self, pkg, request, is_compiler, all_deps): self._push_task(task) - def _check_db(self, spec): + def _check_db( + self, spec: "spack.spec.Spec" + ) -> Tuple[Optional[spack.database.InstallRecord], bool]: """Determine if the spec is flagged as installed in the database Args: - spec (spack.spec.Spec): spec whose database install status is being checked + spec: spec whose database install status is being checked Return: - (rec, installed_in_db) tuple where rec is the database record, or - None, if there is no matching spec, and installed_in_db is - ``True`` if the spec is considered installed and ``False`` - otherwise + Tuple of optional database record, and a boolean installed_in_db + that's ``True`` iff the spec is considered installed """ try: - rec = spack.store.db.get_record(spec) + rec = spack.store.STORE.db.get_record(spec) installed_in_db = rec.installed if rec else False except KeyError: # KeyError is raised if there is no matching spec in the database @@ -874,11 +1276,11 @@ def _check_db(self, spec): installed_in_db = False return rec, installed_in_db - def _check_deps_status(self, request): + def _check_deps_status(self, request: BuildRequest) -> None: """Check the install status of the requested package Args: - request (BuildRequest): the associated install request + request: the associated install request """ err = "Cannot proceed with {0}: {1}" for dep in request.traverse_dependencies(): @@ -886,9 +1288,9 @@ def _check_deps_status(self, request): dep_id = package_id(dep_pkg) # Check for failure since a prefix lock is not required - if spack.store.db.prefix_failed(dep): + if spack.store.STORE.failure_tracker.has_failed(dep): action = "'spack install' the dependency" - msg = "{0} is marked as an install failure: {1}".format(dep_id, action) + msg = f"{dep_id} is marked as an install failure: {action}" raise InstallError(err.format(request.pkg_id, msg), pkg=dep_pkg) # Attempt to get a read lock to ensure another process does not @@ -896,7 +1298,7 @@ def _check_deps_status(self, request): # installed ltype, lock = self._ensure_locked("read", dep_pkg) if lock is None: - msg = "{0} is write locked by another process".format(dep_id) + msg = f"{dep_id} is write locked by another process" raise InstallError(err.format(request.pkg_id, msg), pkg=request.pkg) # Flag external and upstream packages as being installed @@ -907,20 +1309,23 @@ def _check_deps_status(self, request): # Check the database to see if the dependency has been installed # and flag as such if appropriate rec, installed_in_db = self._check_db(dep) - if installed_in_db and ( - dep.dag_hash() not in request.overwrite - or rec.installation_time > request.overwrite_time + if ( + rec + and installed_in_db + and ( + dep.dag_hash() not in request.overwrite + or rec.installation_time > request.overwrite_time + ) ): - tty.debug("Flagging {0} as installed per the database".format(dep_id)) + tty.debug(f"Flagging {dep_id} as installed per the database") self._flag_installed(dep_pkg) else: lock.release_read() - def _prepare_for_install(self, task): + def _prepare_for_install(self, task: BuildTask) -> None: """ Check the database and leftover installation directories/files and prepare for a new install attempt for an uninstalled package. - Preparation includes cleaning up installation and stage directories and ensuring the database is up-to-date. @@ -946,11 +1351,11 @@ def _prepare_for_install(self, task): if not installed_in_db: # Ensure there is no other installed spec with the same prefix dir - if spack.store.db.is_occupied_install_prefix(task.pkg.spec.prefix): + if spack.store.STORE.db.is_occupied_install_prefix(task.pkg.spec.prefix): raise InstallError( - "Install prefix collision for {0}".format(task.pkg_id), - long_msg="Prefix directory {0} already used by another " - "installed spec.".format(task.pkg.spec.prefix), + f"Install prefix collision for {task.pkg_id}", + long_msg=f"Prefix directory {task.pkg.spec.prefix} already " + "used by another installed spec.", pkg=task.pkg, ) @@ -960,23 +1365,27 @@ def _prepare_for_install(self, task): if not keep_prefix: task.pkg.remove_prefix() else: - tty.debug("{0} is partially installed".format(task.pkg_id)) + tty.debug(f"{task.pkg_id} is partially installed") # Destroy the stage for a locally installed, non-DIYStage, package if restage and task.pkg.stage.managed_by_spack: task.pkg.stage.destroy() - if installed_in_db and ( - rec.spec.dag_hash() not in task.request.overwrite - or rec.installation_time > task.request.overwrite_time + if ( + rec + and installed_in_db + and ( + rec.spec.dag_hash() not in task.request.overwrite + or rec.installation_time > task.request.overwrite_time + ) ): self._update_installed(task) # Only update the explicit entry once for the explicit package if task.explicit: - spack.store.db.update_explicit(task.pkg.spec, True) + spack.store.STORE.db.update_explicit(task.pkg.spec, True) - def _cleanup_all_tasks(self): + def _cleanup_all_tasks(self) -> None: """Cleanup all build tasks to include releasing their locks.""" for pkg_id in self.locks: self._release_lock(pkg_id) @@ -991,7 +1400,7 @@ def _cleanup_all_tasks(self): except Exception: pass - def _cleanup_failed(self, pkg_id): + def _cleanup_failed(self, pkg_id: str) -> None: """ Cleanup any failed markers for the package @@ -1001,19 +1410,18 @@ def _cleanup_failed(self, pkg_id): lock = self.failed.get(pkg_id, None) if lock is not None: err = "{0} exception when removing failure tracking for {1}: {2}" - msg = "Removing failure mark on {0}" try: - tty.verbose(msg.format(pkg_id)) + tty.verbose(f"Removing failure mark on {pkg_id}") lock.release_write() except Exception as exc: tty.warn(err.format(exc.__class__.__name__, pkg_id, str(exc))) - def _cleanup_task(self, pkg): + def _cleanup_task(self, pkg: "spack.package_base.PackageBase") -> None: """ Cleanup the build task for the spec Args: - pkg (spack.package_base.PackageBase): the package being installed + pkg: the package being installed """ self._remove_task(package_id(pkg)) @@ -1021,30 +1429,32 @@ def _cleanup_task(self, pkg): # spec during our installation. self._ensure_locked("read", pkg) - def _ensure_install_ready(self, pkg): + def _ensure_install_ready(self, pkg: "spack.package_base.PackageBase") -> None: """ Ensure the package is ready to install locally, which includes already locked. Args: - pkg (spack.package_base.PackageBase): the package being locally installed + pkg: the package being locally installed """ pkg_id = package_id(pkg) - pre = "{0} cannot be installed locally:".format(pkg_id) + pre = f"{pkg_id} cannot be installed locally:" # External packages cannot be installed locally. if pkg.spec.external: - raise ExternalPackageError("{0} {1}".format(pre, "is external")) + raise ExternalPackageError(f"{pre} is external") # Upstream packages cannot be installed locally. if pkg.spec.installed_upstream: - raise UpstreamPackageError("{0} {1}".format(pre, "is upstream")) + raise UpstreamPackageError(f"{pre} is upstream") # The package must have a prefix lock at this stage. if pkg_id not in self.locks: - raise InstallLockError("{0} {1}".format(pre, "not locked")) + raise InstallLockError(f"{pre} not locked") - def _ensure_locked(self, lock_type, pkg): + def _ensure_locked( + self, lock_type: str, pkg: "spack.package_base.PackageBase" + ) -> Tuple[str, Optional[lk.Lock]]: """ Add a prefix lock of the specified type for the package spec @@ -1058,25 +1468,23 @@ def _ensure_locked(self, lock_type, pkg): the next spec. Args: - lock_type (str): 'read' for a read lock, 'write' for a write lock - pkg (spack.package_base.PackageBase): the package whose spec is being - installed + lock_type: 'read' for a read lock, 'write' for a write lock + pkg: the package whose spec is being installed Return: - (lock_type, lock) tuple where lock will be None if it could not - be obtained + (lock_type, lock) tuple where lock will be None if it could not be obtained """ assert lock_type in [ "read", "write", - ], '"{0}" is not a supported package management lock type'.format(lock_type) + ], f'"{lock_type}" is not a supported package management lock type' pkg_id = package_id(pkg) ltype, lock = self.locks.get(pkg_id, (lock_type, None)) if lock and ltype == lock_type: return ltype, lock - desc = "{0} lock".format(lock_type) + desc = f"{lock_type} lock" msg = "{0} a {1} on {2} with timeout {3}" err = "Failed to {0} a {1} for {2} due to {3}: {4}" @@ -1085,7 +1493,7 @@ def _ensure_locked(self, lock_type, pkg): # build tasks with priority 0 (i.e., with no uninstalled # dependencies). no_p0 = len(self.build_tasks) == 0 or not self._next_is_pri0() - timeout = None if no_p0 else 3 + timeout = None if no_p0 else 3.0 else: timeout = 1e-9 # Near 0 to iterate through install specs quickly @@ -1093,13 +1501,9 @@ def _ensure_locked(self, lock_type, pkg): if lock is None: tty.debug(msg.format("Acquiring", desc, pkg_id, pretty_seconds(timeout or 0))) op = "acquire" - lock = spack.store.db.prefix_lock(pkg.spec, timeout) + lock = spack.store.STORE.prefix_locker.lock(pkg.spec, timeout) if timeout != lock.default_timeout: - tty.warn( - "Expected prefix lock timeout {0}, not {1}".format( - timeout, lock.default_timeout - ) - ) + tty.warn(f"Expected prefix lock timeout {timeout}, not {lock.default_timeout}") if lock_type == "read": lock.acquire_read() else: @@ -1124,7 +1528,7 @@ def _ensure_locked(self, lock_type, pkg): tty.debug(msg.format("Upgrading to", desc, pkg_id, pretty_seconds(timeout or 0))) op = "upgrade to" lock.upgrade_read_to_write(timeout) - tty.debug("{0} is now {1} locked".format(pkg_id, lock_type)) + tty.debug(f"{pkg_id} is now {lock_type} locked") except (lk.LockDowngradeError, lk.LockTimeoutError) as exc: tty.debug(err.format(op, desc, pkg_id, exc.__class__.__name__, str(exc))) @@ -1138,7 +1542,7 @@ def _ensure_locked(self, lock_type, pkg): self.locks[pkg_id] = (lock_type, lock) return self.locks[pkg_id] - def _add_tasks(self, request, all_deps): + def _add_tasks(self, request: BuildRequest, all_deps): """Add tasks to the priority queue for the given build request. It also tracks all dependents associated with each dependency in @@ -1149,14 +1553,14 @@ def _add_tasks(self, request, all_deps): all_deps (defaultdict(set)): dictionary of all dependencies and associated dependents """ - tty.debug("Initializing the build queue for {0}".format(request.pkg.name)) + tty.debug(f"Initializing the build queue for {request.pkg.name}") # Ensure not attempting to perform an installation when user didn't # want to go that far for the requested package. try: _check_last_phase(request.pkg) except BadInstallPhase as err: - tty.warn("Installation request refused: {0}".format(str(err))) + tty.warn(f"Installation request refused: {str(err)}") return # Skip out early if the spec is not being installed locally (i.e., if @@ -1175,7 +1579,10 @@ def _add_tasks(self, request, all_deps): install_deps = request.install_args.get("install_deps") # Bootstrap compilers first if install_deps and install_compilers: - packages_per_compiler = {} + packages_per_compiler: Dict[ + "spack.spec.CompilerSpec", + Dict["spack.spec.ArchSpec", List["spack.package_base.PackageBase"]], + ] = {} for dep in request.traverse_dependencies(): dep_pkg = dep.package @@ -1215,12 +1622,12 @@ def _add_tasks(self, request, all_deps): # Clear any persistent failure markings _unless_ they are # associated with another process in this parallel build # of the spec. - spack.store.db.clear_failure(dep, force=False) + spack.store.STORE.failure_tracker.clear(dep, force=False) install_package = request.install_args.get("install_package") if install_package and request.pkg_id not in self.build_tasks: # Be sure to clear any previous failure - spack.store.db.clear_failure(request.spec, force=True) + spack.store.STORE.failure_tracker.clear(request.spec, force=True) # If not installing dependencies, then determine their # installation status before proceeding @@ -1231,33 +1638,35 @@ def _add_tasks(self, request, all_deps): self._add_init_task(request.pkg, request, False, all_deps) # Ensure if one request is to fail fast then all requests will. - fail_fast = request.install_args.get("fail_fast") + fail_fast = bool(request.install_args.get("fail_fast")) self.fail_fast = self.fail_fast or fail_fast - def _add_compiler_package_to_config(self, pkg): + def _add_compiler_package_to_config(self, pkg: "spack.package_base.PackageBase") -> None: compiler_search_prefix = getattr(pkg, "compiler_search_prefix", pkg.spec.prefix) spack.compilers.add_compilers_to_config( spack.compilers.find_compilers([compiler_search_prefix]) ) - def _install_task(self, task): + def _install_task(self, task: BuildTask, install_status: InstallStatus) -> None: """ Perform the installation of the requested spec and/or dependency represented by the build task. Args: - task (BuildTask): the installation build task for a package""" + task: the installation build task for a package + install_status: the installation status for the package""" explicit = task.explicit install_args = task.request.install_args cache_only = task.cache_only use_cache = task.use_cache - tests = install_args.get("tests") - unsigned = install_args.get("unsigned") + tests = install_args.get("tests", False) + assert isinstance(tests, (bool, list)) # make mypy happy. + unsigned = bool(install_args.get("unsigned")) pkg, pkg_id = task.pkg, task.pkg_id - tty.msg(install_msg(pkg_id, self.pid)) + tty.msg(install_msg(pkg_id, self.pid, install_status)) task.start = task.start or time.time() task.status = STATUS_INSTALLING @@ -1268,7 +1677,7 @@ def _install_task(self, task): self._add_compiler_package_to_config(pkg) return - pkg.run_tests = tests is True or tests and pkg.name in tests + pkg.run_tests = tests if isinstance(tests, bool) else pkg.name in tests # hook that allows tests to inspect the Package before installation # see unit_test_check() docs. @@ -1293,7 +1702,7 @@ def _install_task(self, task): pkg.windows_establish_runtime_linkage() # Note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. - spack.store.db.add(pkg.spec, spack.store.layout, explicit=explicit) + spack.store.STORE.db.add(pkg.spec, spack.store.STORE.layout, explicit=explicit) # If a compiler, ensure it is added to the configuration if task.compiler: @@ -1302,11 +1711,11 @@ def _install_task(self, task): # A StopPhase exception means that do_install was asked to # stop early from clients, and is not an error at this point spack.hooks.on_install_failure(task.request.pkg.spec) - pid = "{0}: ".format(self.pid) if tty.show_pid() else "" - tty.debug("{0}{1}".format(pid, str(e))) - tty.debug("Package stage directory: {0}".format(pkg.stage.source_path)) + pid = f"{self.pid}: " if tty.show_pid() else "" + tty.debug(f"{pid}{str(e)}") + tty.debug(f"Package stage directory: {pkg.stage.source_path}") - def _next_is_pri0(self): + def _next_is_pri0(self) -> bool: """ Determine if the next build task has priority 0 @@ -1318,7 +1727,7 @@ def _next_is_pri0(self): task = self.build_pq[0][1] return task.priority == 0 - def _pop_task(self): + def _pop_task(self) -> Optional[BuildTask]: """ Remove and return the lowest priority build task. @@ -1332,7 +1741,7 @@ def _pop_task(self): return task return None - def _push_task(self, task): + def _push_task(self, task: BuildTask) -> None: """ Push (or queue) the specified build task for the package. @@ -1340,7 +1749,7 @@ def _push_task(self, task): docs.python.org/2/library/heapq.html Args: - task (BuildTask): the installation build task for a package + task: the installation build task for a package """ msg = "{0} a build task for {1} with status '{2}'" skip = "Skipping requeue of task for {0}: {1}" @@ -1367,7 +1776,7 @@ def _push_task(self, task): self.build_tasks[task.pkg_id] = task heapq.heappush(self.build_pq, (task.key, task)) - def _release_lock(self, pkg_id): + def _release_lock(self, pkg_id: str) -> None: """ Release any lock on the package @@ -1388,7 +1797,7 @@ def _release_lock(self, pkg_id): except Exception as exc: tty.warn(err.format(exc.__class__.__name__, ltype, pkg_id, str(exc))) - def _remove_task(self, pkg_id): + def _remove_task(self, pkg_id: str) -> Optional[BuildTask]: """ Mark the existing package build task as being removed and return it. Raises KeyError if not found. @@ -1396,17 +1805,17 @@ def _remove_task(self, pkg_id): Source: Variant of function at docs.python.org/2/library/heapq.html Args: - pkg_id (str): identifier for the package to be removed + pkg_id: identifier for the package to be removed """ if pkg_id in self.build_tasks: - tty.debug("Removing build task for {0} from list".format(pkg_id)) + tty.debug(f"Removing build task for {pkg_id} from list") task = self.build_tasks.pop(pkg_id) task.status = STATUS_REMOVED return task else: return None - def _requeue_task(self, task): + def _requeue_task(self, task: BuildTask, install_status: InstallStatus) -> None: """ Requeues a task that appears to be in progress by another process. @@ -1415,27 +1824,26 @@ def _requeue_task(self, task): """ if task.status not in [STATUS_INSTALLED, STATUS_INSTALLING]: tty.debug( - "{0} {1}".format( - install_msg(task.pkg_id, self.pid), "in progress by another process" - ) + f"{install_msg(task.pkg_id, self.pid, install_status)} " + "in progress by another process" ) new_task = task.next_attempt(self.installed) new_task.status = STATUS_INSTALLING self._push_task(new_task) - def _setup_install_dir(self, pkg): + def _setup_install_dir(self, pkg: "spack.package_base.PackageBase") -> None: """ Create and ensure proper access controls for the install directory. Write a small metadata file with the current spack environment. Args: - pkg (spack.package_base.PackageBase): the package to be built and installed + pkg: the package to be built and installed """ if not os.path.exists(pkg.spec.prefix): path = spack.util.path.debug_padded_filter(pkg.spec.prefix) - tty.debug("Creating the installation directory {0}".format(path)) - spack.store.layout.create_install_directory(pkg.spec) + tty.debug(f"Creating the installation directory {path}") + spack.store.STORE.layout.create_install_directory(pkg.spec) else: # Set the proper group for the prefix group = prefs.get_package_group(pkg.spec) @@ -1451,43 +1859,45 @@ def _setup_install_dir(self, pkg): os.chmod(pkg.spec.prefix, perms) # Ensure the metadata path exists as well - fs.mkdirp(spack.store.layout.metadata_path(pkg.spec), mode=perms) + fs.mkdirp(spack.store.STORE.layout.metadata_path(pkg.spec), mode=perms) # Always write host environment - we assume this can change - spack.store.layout.write_host_environment(pkg.spec) + spack.store.STORE.layout.write_host_environment(pkg.spec) - def _update_failed(self, task, mark=False, exc=None): + def _update_failed( + self, task: BuildTask, mark: bool = False, exc: Optional[BaseException] = None + ) -> None: """ Update the task and transitive dependents as failed; optionally mark externally as failed; and remove associated build tasks. Args: - task (BuildTask): the build task for the failed package - mark (bool): ``True`` if the package and its dependencies are to + task: the build task for the failed package + mark: ``True`` if the package and its dependencies are to be marked as "failed", otherwise, ``False`` - exc (Exception): optional exception if associated with the failure + exc: optional exception if associated with the failure """ pkg_id = task.pkg_id - err = "" if exc is None else ": {0}".format(str(exc)) - tty.debug("Flagging {0} as failed{1}".format(pkg_id, err)) + err = "" if exc is None else f": {str(exc)}" + tty.debug(f"Flagging {pkg_id} as failed{err}") if mark: - self.failed[pkg_id] = spack.store.db.mark_failed(task.pkg.spec) + self.failed[pkg_id] = spack.store.STORE.failure_tracker.mark(task.pkg.spec) else: self.failed[pkg_id] = None task.status = STATUS_FAILED for dep_id in task.dependents: if dep_id in self.build_tasks: - tty.warn("Skipping build of {0} since {1} failed".format(dep_id, pkg_id)) + tty.warn(f"Skipping build of {dep_id} since {pkg_id} failed") # Ensure the dependent's uninstalled dependents are # up-to-date and their build tasks removed. dep_task = self.build_tasks[dep_id] self._update_failed(dep_task, mark) self._remove_task(dep_id) else: - tty.debug("No build task for {0} to skip since {1} failed".format(dep_id, pkg_id)) + tty.debug(f"No build task for {dep_id} to skip since {pkg_id} failed") - def _update_installed(self, task): + def _update_installed(self, task: BuildTask) -> None: """ Mark the task as installed and ensure dependent build tasks are aware. @@ -1497,17 +1907,17 @@ def _update_installed(self, task): task.status = STATUS_INSTALLED self._flag_installed(task.pkg, task.dependents) - def _flag_installed(self, pkg, dependent_ids=None): + def _flag_installed( + self, pkg: "spack.package_base.PackageBase", dependent_ids: Optional[Set[str]] = None + ) -> None: """ Flag the package as installed and ensure known by all build tasks of known dependents. Args: - pkg (spack.package_base.PackageBase): Package that has been installed - locally, externally or upstream - dependent_ids (list or None): list of the package's - dependent ids, or None if the dependent ids are limited to - those maintained in the package (dependency DAG) + pkg: Package that has been installed locally, externally or upstream + dependent_ids: set of the package's dependent ids, or None if the dependent ids are + limited to those maintained in the package (dependency DAG) """ pkg_id = package_id(pkg) @@ -1515,27 +1925,25 @@ def _flag_installed(self, pkg, dependent_ids=None): # Already determined the package has been installed return - tty.debug("Flagging {0} as installed".format(pkg_id)) + tty.debug(f"Flagging {pkg_id} as installed") self.installed.add(pkg_id) # Update affected dependents dependent_ids = dependent_ids or get_dependent_ids(pkg.spec) for dep_id in set(dependent_ids): - tty.debug("Removing {0} from {1}'s uninstalled dependencies.".format(pkg_id, dep_id)) + tty.debug(f"Removing {pkg_id} from {dep_id}'s uninstalled dependencies.") if dep_id in self.build_tasks: # Ensure the dependent's uninstalled dependencies are # up-to-date. This will require requeueing the task. dep_task = self.build_tasks[dep_id] self._push_task(dep_task.next_attempt(self.installed)) else: - tty.debug( - "{0} has no build task to update for {1}'s success".format(dep_id, pkg_id) - ) + tty.debug(f"{dep_id} has no build task to update for {pkg_id}'s success") - def _init_queue(self): + def _init_queue(self) -> None: """Initialize the build queue from the list of build requests.""" - all_dependencies = defaultdict(set) + all_dependencies: Dict[str, Set[str]] = defaultdict(set) tty.debug("Initializing the build queue from the build requests") for request in self.build_requests: @@ -1551,7 +1959,7 @@ def _init_queue(self): for dependent_id in dependents.difference(task.dependents): task.add_dependent(dependent_id) - def _install_action(self, task): + def _install_action(self, task: BuildTask) -> int: """ Determine whether the installation should be overwritten (if it already exists) or skipped (if has been handled by another process). @@ -1570,7 +1978,7 @@ def _install_action(self, task): return InstallAction.INSTALL # Ensure install_tree projections have not changed. - assert task.pkg.prefix == rec.path + assert rec and task.pkg.prefix == rec.path # If another process has overwritten this, we shouldn't install at all if rec.installation_time >= task.request.overwrite_time: @@ -1587,7 +1995,7 @@ def _install_action(self, task): # back on failure return InstallAction.OVERWRITE - def install(self): + def install(self) -> None: """Install the requested package(s) and or associated dependencies.""" self._init_queue() @@ -1595,7 +2003,7 @@ def install(self): single_explicit_spec = len(self.build_requests) == 1 failed_explicits = [] - term_title = TermTitle(len(self.build_pq)) + install_status = InstallStatus(len(self.build_pq)) # Only enable the terminal status line when we're in a tty without debug info # enabled, so that the output does not get cluttered. @@ -1611,9 +2019,9 @@ def install(self): keep_prefix = install_args.get("keep_prefix") pkg, pkg_id, spec = task.pkg, task.pkg_id, task.pkg.spec - term_title.next_pkg(pkg) - term_title.set("Processing {0}".format(pkg.name)) - tty.debug("Processing {0}: task={1}".format(pkg_id, task)) + install_status.next_pkg(pkg) + install_status.set_term_title(f"Processing {pkg.name}") + tty.debug(f"Processing {pkg_id}: task={task}") # Ensure that the current spec has NO uninstalled dependencies, # which is assumed to be reflected directly in its priority. # @@ -1625,24 +2033,19 @@ def install(self): if task.priority != 0: term_status.clear() tty.error( - "Detected uninstalled dependencies for {0}: {1}".format( - pkg_id, task.uninstalled_deps - ) + f"Detected uninstalled dependencies for {pkg_id}: " f"{task.uninstalled_deps}" ) left = [dep_id for dep_id in task.uninstalled_deps if dep_id not in self.installed] if not left: - tty.warn( - "{0} does NOT actually have any uninstalled deps" " left".format(pkg_id) - ) + tty.warn(f"{pkg_id} does NOT actually have any uninstalled deps left") dep_str = "dependencies" if task.priority > 1 else "dependency" # Hook to indicate task failure, but without an exception spack.hooks.on_install_failure(task.request.pkg.spec) raise InstallError( - "Cannot proceed with {0}: {1} uninstalled {2}: {3}".format( - pkg_id, task.priority, dep_str, ",".join(task.uninstalled_deps) - ), + f"Cannot proceed with {pkg_id}: {task.priority} uninstalled " + f"{dep_str}: {','.join(task.uninstalled_deps)}", pkg=pkg, ) @@ -1657,9 +2060,9 @@ def install(self): # Flag a failed spec. Do not need an (install) prefix lock since # assume using a separate (failed) prefix lock file. - if pkg_id in self.failed or spack.store.db.prefix_failed(spec): + if pkg_id in self.failed or spack.store.STORE.failure_tracker.has_failed(spec): term_status.clear() - tty.warn("{0} failed to install".format(pkg_id)) + tty.warn(f"{pkg_id} failed to install") self._update_failed(task) # Mark that the package failed @@ -1676,7 +2079,7 @@ def install(self): # another process is likely (un)installing the spec or has # determined the spec has already been installed (though the # other process may be hung). - term_title.set("Acquiring lock for {0}".format(pkg.name)) + install_status.set_term_title(f"Acquiring lock for {pkg.name}") term_status.add(pkg_id) ltype, lock = self._ensure_locked("write", pkg) if lock is None: @@ -1684,12 +2087,11 @@ def install(self): # another process has a write lock so must be (un)installing # the spec (or that process is hung). ltype, lock = self._ensure_locked("read", pkg) - # Requeue the spec if we cannot get at least a read lock so we # can check the status presumably established by another process # -- failed, installed, or uninstalled -- on the next pass. if lock is None: - self._requeue_task(task) + self._requeue_task(task, install_status) continue term_status.clear() @@ -1700,7 +2102,7 @@ def install(self): task.request.overwrite_time = time.time() # Determine state of installation artifacts and adjust accordingly. - term_title.set("Preparing {0}".format(pkg.name)) + install_status.set_term_title(f"Preparing {pkg.name}") self._prepare_for_install(task) # Flag an already installed package @@ -1728,7 +2130,7 @@ def install(self): # established by the other process -- failed, installed, # or uninstalled -- on the next pass. self.installed.remove(pkg_id) - self._requeue_task(task) + self._requeue_task(task, install_status) continue # Having a read lock on an uninstalled pkg may mean another @@ -1741,19 +2143,21 @@ def install(self): # uninstalled -- on the next pass. if ltype == "read": lock.release_read() - self._requeue_task(task) + self._requeue_task(task, install_status) continue # Proceed with the installation since we have an exclusive write # lock on the package. - term_title.set("Installing {0}".format(pkg.name)) + install_status.set_term_title(f"Installing {pkg.name}") try: action = self._install_action(task) if action == InstallAction.INSTALL: - self._install_task(task) + self._install_task(task, install_status) elif action == InstallAction.OVERWRITE: - OverwriteInstall(self, spack.store.db, task).install() + # spack.store.STORE.db is not really a Database object, but a small + # wrapper -- silence mypy + OverwriteInstall(self, spack.store.STORE.db, task, install_status).install() # type: ignore[arg-type] # noqa: E501 self._update_installed(task) @@ -1765,8 +2169,9 @@ def install(self): except KeyboardInterrupt as exc: # The build has been terminated with a Ctrl-C so terminate # regardless of the number of remaining specs. - err = "Failed to install {0} due to {1}: {2}" - tty.error(err.format(pkg.name, exc.__class__.__name__, str(exc))) + tty.error( + f"Failed to install {pkg.name} due to " f"{exc.__class__.__name__}: {str(exc)}" + ) spack.hooks.on_install_cancel(task.request.pkg.spec) raise @@ -1775,11 +2180,13 @@ def install(self): raise # Checking hash on downloaded binary failed. - err = "Failed to install {0} from binary cache due to {1}:" - err += " Requeueing to install from source." - tty.error(err.format(pkg.name, str(exc))) - task.use_cache = False - self._requeue_task(task) + tty.error( + f"Failed to install {pkg.name} from binary cache due " + f"to {str(exc)}: Requeueing to install from source." + ) + # this overrides a full method, which is ugly. + task.use_cache = False # type: ignore[misc] + self._requeue_task(task, install_status) continue except (Exception, SystemExit) as exc: @@ -1788,19 +2195,18 @@ def install(self): # Best effort installs suppress the exception and mark the # package as a failure. - if not isinstance(exc, spack.error.SpackError) or not exc.printed: - exc.printed = True + if not isinstance(exc, spack.error.SpackError) or not exc.printed: # type: ignore[union-attr] # noqa: E501 + exc.printed = True # type: ignore[union-attr] # SpackErrors can be printed by the build process or at # lower levels -- skip printing if already printed. # TODO: sort out this and SpackError.print_context() tty.error( - "Failed to install {0} due to {1}: {2}".format( - pkg.name, exc.__class__.__name__, str(exc) - ) + f"Failed to install {pkg.name} due to " + f"{exc.__class__.__name__}: {str(exc)}" ) # Terminate if requested to do so on the first failure. if self.fail_fast: - raise InstallError("{0}: {1}".format(fail_fast_err, str(exc)), pkg=pkg) + raise InstallError(f"{fail_fast_err}: {str(exc)}", pkg=pkg) # Terminate at this point if the single explicit spec has # failed to install. @@ -1839,26 +2245,25 @@ def install(self): if failed_explicits or missing: for _, pkg_id, err in failed_explicits: - tty.error("{0}: {1}".format(pkg_id, err)) + tty.error(f"{pkg_id}: {err}") for _, pkg_id in missing: - tty.error("{0}: Package was not installed".format(pkg_id)) + tty.error(f"{pkg_id}: Package was not installed") - pkg = None if len(failed_explicits) > 0: pkg = failed_explicits[0][0] ids = [pkg_id for _, pkg_id, _ in failed_explicits] tty.debug( "Associating installation failure with first failed " - "explicit package ({0}) from {1}".format(ids[0], ", ".join(ids)) + f"explicit package ({ids[0]}) from {', '.join(ids)}" ) - if not pkg and len(missing) > 0: + elif len(missing) > 0: pkg = missing[0][0] ids = [pkg_id for _, pkg_id in missing] tty.debug( "Associating installation failure with first " - "missing package ({0}) from {1}".format(ids[0], ", ".join(ids)) + f"missing package ({ids[0]}) from {', '.join(ids)}" ) raise InstallError( @@ -1870,15 +2275,15 @@ def install(self): class BuildProcessInstaller: """This class implements the part installation that happens in the child process.""" - def __init__(self, pkg, install_args): + def __init__(self, pkg: "spack.package_base.PackageBase", install_args: dict): """Create a new BuildProcessInstaller. It is assumed that the lifecycle of this object is the same as the child process in the build. Arguments: - pkg (spack.package_base.PackageBase) the package being installed. - install_args (dict) arguments to do_install() from parent process. + pkg: the package being installed. + install_args: arguments to do_install() from parent process. """ self.pkg = pkg @@ -1896,7 +2301,7 @@ def __init__(self, pkg, install_args): self.skip_patch = install_args.get("skip_patch", False) # whether to enable echoing of build output initially or not - self.verbose = install_args.get("verbose", False) + self.verbose = bool(install_args.get("verbose", False)) # whether installation was explicitly requested by the user self.explicit = install_args.get("is_root", False) and install_args.get("explicit", True) @@ -1919,31 +2324,31 @@ def __init__(self, pkg, install_args): self.pre = _log_prefix(pkg.name) self.pkg_id = package_id(pkg) - def run(self): + def run(self) -> bool: """Main entry point from ``build_process`` to kick off install in child.""" - self.timer.start("stage") + self.pkg.stage.keep = self.keep_stage - if not self.fake: - if not self.skip_patch: - self.pkg.do_patch() - else: - self.pkg.do_stage() + with self.pkg.stage: + self.timer.start("stage") - self.timer.stop("stage") + if not self.fake: + if not self.skip_patch: + self.pkg.do_patch() + else: + self.pkg.do_stage() - tty.debug( - "{0} Building {1} [{2}]".format(self.pre, self.pkg_id, self.pkg.build_system_class) - ) + self.timer.stop("stage") - # get verbosity from do_install() parameter or saved value - self.echo = self.verbose - if spack.package_base.PackageBase._verbose is not None: - self.echo = spack.package_base.PackageBase._verbose + tty.debug( + f"{self.pre} Building {self.pkg_id} [{self.pkg.build_system_class}]" # type: ignore[attr-defined] # noqa: E501 + ) - self.pkg.stage.keep = self.keep_stage + # get verbosity from do_install() parameter or saved value + self.echo = self.verbose + if spack.package_base.PackageBase._verbose is not None: + self.echo = spack.package_base.PackageBase._verbose - with self.pkg.stage: # Run the pre-install hook in the child process after # the directory is created. spack.hooks.pre_install(self.pkg.spec) @@ -1962,8 +2367,7 @@ def run(self): # Stop the timer and save results self.timer.stop() - with open(self.pkg.times_log_path, "w") as timelog: - self.timer.write_json(timelog) + _write_timer_json(self.pkg, self.timer, False) print_install_test_log(self.pkg) _print_timer(pre=self.pre, pkg_id=self.pkg_id, timer=self.timer) @@ -1975,18 +2379,20 @@ def run(self): # preserve verbosity across runs return self.echo - def _install_source(self): + def _install_source(self) -> None: """Install source code from stage into share/pkg/src if necessary.""" pkg = self.pkg if not os.path.isdir(pkg.stage.source_path): return src_target = os.path.join(pkg.spec.prefix, "share", pkg.name, "src") - tty.debug("{0} Copying source to {1}".format(self.pre, src_target)) + tty.debug(f"{self.pre} Copying source to {src_target}") - fs.install_tree(pkg.stage.source_path, src_target) + fs.install_tree( + pkg.stage.source_path, src_target, allow_broken_symlinks=(sys.platform != "win32") + ) - def _real_install(self): + def _real_install(self) -> None: import spack.builder pkg = self.pkg @@ -2042,8 +2448,7 @@ def _real_install(self): with logger.force_echo(): inner_debug_level = tty.debug_level() tty.set_debug(debug_level) - msg = "{0} Executing phase: '{1}'" - tty.msg(msg.format(self.pre, phase_fn.name)) + tty.msg(f"{self.pre} Executing phase: '{phase_fn.name}'") tty.set_debug(inner_debug_level) # Catch any errors to report to logging @@ -2068,7 +2473,7 @@ def _real_install(self): log(pkg) -def build_process(pkg, install_args): +def build_process(pkg: "spack.package_base.PackageBase", install_args: dict) -> bool: """Perform the installation/build of the package. This runs in a separate child process, and has its own process and @@ -2080,8 +2485,8 @@ def build_process(pkg, install_args): This function's return value is returned to the parent process. Arguments: - pkg (spack.package_base.PackageBase): the package being installed. - install_args (dict): arguments to do_install() from parent process. + pkg: the package being installed. + install_args: arguments to do_install() from parent process. """ installer = BuildProcessInstaller(pkg, install_args) @@ -2092,10 +2497,17 @@ def build_process(pkg, install_args): class OverwriteInstall: - def __init__(self, installer, database, task): + def __init__( + self, + installer: PackageInstaller, + database: spack.database.Database, + task: BuildTask, + install_status: InstallStatus, + ): self.installer = installer self.database = database self.task = task + self.install_status = install_status def install(self): """ @@ -2106,397 +2518,19 @@ def install(self): """ try: with fs.replace_directory_transaction(self.task.pkg.prefix): - self.installer._install_task(self.task) + self.installer._install_task(self.task, self.install_status) except fs.CouldNotRestoreDirectoryBackup as e: self.database.remove(self.task.pkg.spec) tty.error( - "Recovery of install dir of {0} failed due to " - "{1}: {2}. The spec is now uninstalled.".format( - self.task.pkg.name, - e.outer_exception.__class__.__name__, - str(e.outer_exception), - ) + f"Recovery of install dir of {self.task.pkg.name} failed due to " + f"{e.outer_exception.__class__.__name__}: {str(e.outer_exception)}. " + "The spec is now uninstalled." ) # Unwrap the actual installation exception. raise e.inner_exception -class BuildTask: - """Class for representing the build task for a package.""" - - def __init__(self, pkg, request, compiler, start, attempts, status, installed): - """ - Instantiate a build task for a package. - - Args: - pkg (spack.package_base.PackageBase): the package to be built and installed - request (BuildRequest or None): the associated install request - where ``None`` can be used to indicate the package was - explicitly requested by the user - compiler (bool): whether task is for a bootstrap compiler - start (int): the initial start time for the package, in seconds - attempts (int): the number of attempts to install the package - status (str): the installation status - installed (list): the identifiers of packages that have - been installed so far - """ - - # Ensure dealing with a package that has a concrete spec - if not isinstance(pkg, spack.package_base.PackageBase): - raise ValueError("{0} must be a package".format(str(pkg))) - - self.pkg = pkg - if not self.pkg.spec.concrete: - raise ValueError("{0} must have a concrete spec".format(self.pkg.name)) - - # The "unique" identifier for the task's package - self.pkg_id = package_id(self.pkg) - - # The explicit build request associated with the package - if not isinstance(request, BuildRequest): - raise ValueError("{0} must have a build request".format(str(pkg))) - - self.request = request - - # Initialize the status to an active state. The status is used to - # ensure priority queue invariants when tasks are "removed" from the - # queue. - if status == STATUS_REMOVED: - msg = "Cannot create a build task for {0} with status '{1}'" - raise InstallError(msg.format(self.pkg_id, status), pkg=pkg) - - self.status = status - - # Package is associated with a bootstrap compiler - self.compiler = compiler - - # The initial start time for processing the spec - self.start = start - - # Set of dependents, which needs to include the requesting package - # to support tracking of parallel, multi-spec, environment installs. - self.dependents = set(get_dependent_ids(self.pkg.spec)) - - tty.debug("Pkg id {0} has the following dependents:".format(self.pkg_id)) - for dep_id in self.dependents: - tty.debug("- {0}".format(dep_id)) - - # Set of dependencies - # - # Be consistent wrt use of dependents and dependencies. That is, - # if use traverse for transitive dependencies, then must remove - # transitive dependents on failure. - deptypes = self.request.get_deptypes(self.pkg) - self.dependencies = set( - package_id(d.package) - for d in self.pkg.spec.dependencies(deptype=deptypes) - if package_id(d.package) != self.pkg_id - ) - - # Handle bootstrapped compiler - # - # The bootstrapped compiler is not a dependency in the spec, but it is - # a dependency of the build task. Here we add it to self.dependencies - compiler_spec = self.pkg.spec.compiler - arch_spec = self.pkg.spec.architecture - if not spack.compilers.compilers_for_spec(compiler_spec, arch_spec=arch_spec): - # The compiler is in the queue, identify it as dependency - dep = spack.compilers.pkg_spec_for_compiler(compiler_spec) - dep.constrain("platform=%s" % str(arch_spec.platform)) - dep.constrain("os=%s" % str(arch_spec.os)) - dep.constrain("target=%s:" % arch_spec.target.microarchitecture.family.name) - dep.concretize() - dep_id = package_id(dep.package) - self.dependencies.add(dep_id) - - # List of uninstalled dependencies, which is used to establish - # the priority of the build task. - # - self.uninstalled_deps = set( - pkg_id for pkg_id in self.dependencies if pkg_id not in installed - ) - - # Ensure key sequence-related properties are updated accordingly. - self.attempts = 0 - self._update() - - def __eq__(self, other): - return self.key == other.key - - def __ge__(self, other): - return self.key >= other.key - - def __gt__(self, other): - return self.key > other.key - - def __le__(self, other): - return self.key <= other.key - - def __lt__(self, other): - return self.key < other.key - - def __ne__(self, other): - return self.key != other.key - - def __repr__(self): - """Returns a formal representation of the build task.""" - rep = "{0}(".format(self.__class__.__name__) - for attr, value in self.__dict__.items(): - rep += "{0}={1}, ".format(attr, value.__repr__()) - return "{0})".format(rep.strip(", ")) - - def __str__(self): - """Returns a printable version of the build task.""" - dependencies = "#dependencies={0}".format(len(self.dependencies)) - return "priority={0}, status={1}, start={2}, {3}".format( - self.priority, self.status, self.start, dependencies - ) - - def _update(self): - """Update properties associated with a new instance of a task.""" - # Number of times the task has/will be queued - self.attempts = self.attempts + 1 - - # Ensure the task gets a unique sequence number to preserve the - # order in which it is added. - self.sequence = next(_counter) - - def add_dependent(self, pkg_id): - """ - Ensure the dependent package id is in the task's list so it will be - properly updated when this package is installed. - - Args: - pkg_id (str): package identifier of the dependent package - """ - if pkg_id != self.pkg_id and pkg_id not in self.dependents: - tty.debug("Adding {0} as a dependent of {1}".format(pkg_id, self.pkg_id)) - self.dependents.add(pkg_id) - - def flag_installed(self, installed): - """ - Ensure the dependency is not considered to still be uninstalled. - - Args: - installed (list): the identifiers of packages that have - been installed so far - """ - now_installed = self.uninstalled_deps & set(installed) - for pkg_id in now_installed: - self.uninstalled_deps.remove(pkg_id) - tty.debug( - "{0}: Removed {1} from uninstalled deps list: {2}".format( - self.pkg_id, pkg_id, self.uninstalled_deps - ), - level=2, - ) - - @property - def explicit(self): - """The package was explicitly requested by the user.""" - return self.is_root and self.request.install_args.get("explicit", True) - - @property - def is_root(self): - """The package was requested directly, but may or may not be explicit - in an environment.""" - return self.pkg == self.request.pkg - - @property - def use_cache(self): - _use_cache = True - if self.is_root: - return self.request.install_args.get("package_use_cache", _use_cache) - else: - return self.request.install_args.get("dependencies_use_cache", _use_cache) - - @property - def cache_only(self): - _cache_only = False - if self.is_root: - return self.request.install_args.get("package_cache_only", _cache_only) - else: - return self.request.install_args.get("dependencies_cache_only", _cache_only) - - @property - def key(self): - """The key is the tuple (# uninstalled dependencies, sequence).""" - return (self.priority, self.sequence) - - def next_attempt(self, installed): - """Create a new, updated task for the next installation attempt.""" - task = copy.copy(self) - task._update() - task.start = self.start or time.time() - task.flag_installed(installed) - return task - - @property - def priority(self): - """The priority is based on the remaining uninstalled dependencies.""" - return len(self.uninstalled_deps) - - -class BuildRequest: - """Class for representing an installation request.""" - - def __init__(self, pkg, install_args): - """ - Instantiate a build request for a package. - - Args: - pkg (spack.package_base.PackageBase): the package to be built and installed - install_args (dict): the install arguments associated with ``pkg`` - """ - # Ensure dealing with a package that has a concrete spec - if not isinstance(pkg, spack.package_base.PackageBase): - raise ValueError("{0} must be a package".format(str(pkg))) - - self.pkg = pkg - if not self.pkg.spec.concrete: - raise ValueError("{0} must have a concrete spec".format(self.pkg.name)) - - # Cache the package phase options with the explicit package, - # popping the options to ensure installation of associated - # dependencies is NOT affected by these options. - self.pkg.stop_before_phase = install_args.pop("stop_before", None) - self.pkg.last_phase = install_args.pop("stop_at", None) - - # Cache the package id for convenience - self.pkg_id = package_id(pkg) - - # Save off the original install arguments plus standard defaults - # since they apply to the requested package *and* dependencies. - self.install_args = install_args if install_args else {} - self._add_default_args() - - # Cache overwrite information - self.overwrite = set(self.install_args.get("overwrite", [])) - self.overwrite_time = time.time() - - # Save off dependency package ids for quick checks since traversals - # are not able to return full dependents for all packages across - # environment specs. - deptypes = self.get_deptypes(self.pkg) - self.dependencies = set( - package_id(d.package) - for d in self.pkg.spec.dependencies(deptype=deptypes) - if package_id(d.package) != self.pkg_id - ) - - def __repr__(self): - """Returns a formal representation of the build request.""" - rep = "{0}(".format(self.__class__.__name__) - for attr, value in self.__dict__.items(): - rep += "{0}={1}, ".format(attr, value.__repr__()) - return "{0})".format(rep.strip(", ")) - - def __str__(self): - """Returns a printable version of the build request.""" - return "package={0}, install_args={1}".format(self.pkg.name, self.install_args) - - def _add_default_args(self): - """Ensure standard install options are set to at least the default.""" - for arg, default in [ - ("context", "build"), # installs *always* build - ("dependencies_cache_only", False), - ("dependencies_use_cache", True), - ("dirty", False), - ("fail_fast", False), - ("fake", False), - ("install_deps", True), - ("install_package", True), - ("install_source", False), - ("package_cache_only", False), - ("package_use_cache", True), - ("keep_prefix", False), - ("keep_stage", False), - ("restage", False), - ("skip_patch", False), - ("tests", False), - ("unsigned", False), - ("verbose", False), - ]: - _ = self.install_args.setdefault(arg, default) - - def get_deptypes(self, pkg): - """Determine the required dependency types for the associated package. - - Args: - pkg (spack.package_base.PackageBase): explicit or implicit package being - installed - - Returns: - tuple: required dependency type(s) for the package - """ - deptypes = ["link", "run"] - include_build_deps = self.install_args.get("include_build_deps") - - if self.pkg_id == package_id(pkg): - cache_only = self.install_args.get("package_cache_only") - else: - cache_only = self.install_args.get("dependencies_cache_only") - - # Include build dependencies if pkg is not installed and cache_only - # is False, or if build depdencies are explicitly called for - # by include_build_deps. - if include_build_deps or not (cache_only or pkg.spec.installed): - deptypes.append("build") - if self.run_tests(pkg): - deptypes.append("test") - return tuple(sorted(deptypes)) - - def has_dependency(self, dep_id): - """Returns ``True`` if the package id represents a known dependency - of the requested package, ``False`` otherwise.""" - return dep_id in self.dependencies - - def run_tests(self, pkg): - """Determine if the tests should be run for the provided packages - - Args: - pkg (spack.package_base.PackageBase): explicit or implicit package being - installed - - Returns: - bool: ``True`` if they should be run; ``False`` otherwise - """ - tests = self.install_args.get("tests", False) - return tests is True or (tests and pkg.name in tests) - - @property - def spec(self): - """The specification associated with the package.""" - return self.pkg.spec - - def traverse_dependencies(self, spec=None, visited=None): - """ - Yield any dependencies of the appropriate type(s) - - Yields: - (Spec) The next child spec in the DAG - """ - # notice: deptype is not constant across nodes, so we cannot use - # spec.traverse_edges(deptype=...). - - if spec is None: - spec = self.spec - if visited is None: - visited = set() - deptype = self.get_deptypes(spec.package) - - for dep in spec.dependencies(deptype=deptype): - hash = dep.dag_hash() - if hash in visited: - continue - visited.add(hash) - # In Python 3: yield from self.traverse_dependencies(dep, visited) - for s in self.traverse_dependencies(dep, visited): - yield s - yield dep - - class InstallError(spack.error.SpackError): """Raised when something goes wrong during install or uninstall. @@ -2513,7 +2547,7 @@ class BadInstallPhase(InstallError): """Raised for an install phase option is not allowed for a package.""" def __init__(self, pkg_name, phase): - super().__init__("'{0}' is not a valid phase for package {1}".format(phase, pkg_name)) + super().__init__(f"'{phase}' is not a valid phase for package {pkg_name}") class ExternalPackageError(InstallError): diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 273dc6ba0d6a05..5f28ab480cb02a 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -16,11 +16,13 @@ import os.path import pstats import re +import shlex import signal import subprocess as sp import sys import traceback import warnings +from typing import List, Tuple import archspec.cpu @@ -30,7 +32,6 @@ import llnl.util.tty.color as color from llnl.util.tty.log import log_output -import spack import spack.cmd import spack.config import spack.environment as ev @@ -50,9 +51,6 @@ #: names of profile statistics stat_names = pstats.Stats.sort_arg_dict_default -#: top-level aliases for Spack commands -aliases = {"rm": "remove"} - #: help levels in order of detail (i.e., number of commands shown) levels = ["short", "long"] @@ -360,7 +358,10 @@ def add_command(self, cmd_name): module = spack.cmd.get_module(cmd_name) # build a list of aliases - alias_list = [k for k, v in aliases.items() if v == cmd_name] + alias_list = [] + aliases = spack.config.get("config:aliases") + if aliases: + alias_list = [k for k, v in aliases.items() if shlex.split(v)[0] == cmd_name] subparser = self.subparsers.add_parser( cmd_name, @@ -436,7 +437,7 @@ def make_argument_parser(**kwargs): default=None, action="append", dest="config_vars", - help="add one or more custom, one off config settings.", + help="add one or more custom, one off config settings", ) parser.add_argument( "-C", @@ -451,9 +452,9 @@ def make_argument_parser(**kwargs): "--debug", action="count", default=0, - help="write out debug messages " "(more d's for more verbosity: -d, -dd, -ddd, etc.)", + help="write out debug messages\n\n(more d's for more verbosity: -d, -dd, -ddd, etc.)", ) - parser.add_argument("--timestamp", action="store_true", help="Add a timestamp to tty output") + parser.add_argument("--timestamp", action="store_true", help="add a timestamp to tty output") parser.add_argument("--pdb", action="store_true", help="run spack under the pdb debugger") env_group = parser.add_mutually_exclusive_group() @@ -527,8 +528,7 @@ def make_argument_parser(**kwargs): "--sorted-profile", default=None, metavar="STAT", - help="profile and sort by one or more of:\n[%s]" - % ",\n ".join([", ".join(line) for line in stat_lines]), + help=f"profile and sort\n\none or more of: {stat_lines[0]}", ) parser.add_argument( "--lines", @@ -555,7 +555,7 @@ def make_argument_parser(**kwargs): "-V", "--version", action="store_true", help="show version number and exit" ) parser.add_argument( - "--print-shell-vars", action="store", help="print info needed by setup-env.[c]sh" + "--print-shell-vars", action="store", help="print info needed by setup-env.*sh" ) return parser @@ -603,10 +603,10 @@ def setup_main_options(args): key = syaml.syaml_str("repos") key.override = True - spack.config.config.scopes["command_line"].sections["repos"] = syaml.syaml_dict( + spack.config.CONFIG.scopes["command_line"].sections["repos"] = syaml.syaml_dict( [(key, [spack.paths.mock_packages_path])] ) - spack.repo.path = spack.repo.create(spack.config.config) + spack.repo.PATH = spack.repo.create(spack.config.CONFIG) # If the user asked for it, don't check ssl certs. if args.insecure: @@ -672,7 +672,6 @@ def __init__(self, command_name, subprocess=False): Windows, where it is always False. """ self.parser = make_argument_parser() - self.command = self.parser.add_command(command_name) self.command_name = command_name # TODO: figure out how to support this on windows self.subprocess = subprocess if sys.platform != "win32" else False @@ -704,21 +703,22 @@ def __call__(self, *argv, **kwargs): if self.subprocess: p = sp.Popen( - [spack.paths.spack_script, self.command_name] + prepend + list(argv), + [spack.paths.spack_script] + prepend + [self.command_name] + list(argv), stdout=sp.PIPE, stderr=sp.STDOUT, ) out, self.returncode = p.communicate() out = out.decode() else: + command = self.parser.add_command(self.command_name) args, unknown = self.parser.parse_known_args( prepend + [self.command_name] + list(argv) ) out = io.StringIO() try: - with log_output(out): - self.returncode = _invoke_command(self.command, self.parser, args, unknown) + with log_output(out, echo=True): + self.returncode = _invoke_command(command, self.parser, args, unknown) except SystemExit as e: self.returncode = e.code @@ -776,7 +776,7 @@ def _profile_wrapper(command, parser, args, unknown_args): pr.disable() # print out profile stats. - stats = pstats.Stats(pr) + stats = pstats.Stats(pr, stream=sys.stderr) stats.sort_stats(*sortby) stats.print_stats(nlines) @@ -848,7 +848,7 @@ def shell_set(var, value): if "modules" in info: generic_arch = archspec.cpu.host().family module_spec = "environment-modules target={0}".format(generic_arch) - specs = spack.store.db.query(module_spec) + specs = spack.store.STORE.db.query(module_spec) if specs: shell_set("_sp_module_prefix", specs[-1].prefix) else: @@ -872,6 +872,46 @@ def restore_macos_dyld_vars(): os.environ[dyld_var] = os.environ[stored_var_name] +def resolve_alias(cmd_name: str, cmd: List[str]) -> Tuple[str, List[str]]: + """Resolves aliases in the given command. + + Args: + cmd_name: command name. + cmd: command line arguments. + + Returns: + new command name and arguments. + """ + all_commands = spack.cmd.all_commands() + aliases = spack.config.get("config:aliases") + + if aliases: + for key, value in aliases.items(): + if " " in key: + tty.warn( + f"Alias '{key}' (mapping to '{value}') contains a space" + ", which is not supported." + ) + if key in all_commands: + tty.warn( + f"Alias '{key}' (mapping to '{value}') attempts to override" + " built-in command." + ) + + if cmd_name not in all_commands: + alias = None + + if aliases: + alias = aliases.get(cmd_name) + + if alias is not None: + alias_parts = shlex.split(alias) + cmd_name = alias_parts[0] + cmd = alias_parts + cmd[1:] + + return cmd_name, cmd + + def _main(argv=None): """Logic for the main entry point for the Spack command. @@ -931,7 +971,7 @@ def _main(argv=None): # make spack.config aware of any command line configuration scopes if args.config_scopes: - spack.config.command_line_scopes = args.config_scopes + spack.config.COMMAND_LINE_SCOPES = args.config_scopes # ensure options on spack command come before everything setup_main_options(args) @@ -964,7 +1004,7 @@ def _main(argv=None): # Try to load the particular command the caller asked for. cmd_name = args.command[0] - cmd_name = aliases.get(cmd_name, cmd_name) + cmd_name, args.command = resolve_alias(cmd_name, args.command) # set up a bootstrap context, if asked. # bootstrap context needs to include parsing the command, b/c things @@ -976,10 +1016,10 @@ def _main(argv=None): bootstrap_context = bootstrap.ensure_bootstrap_configuration() with bootstrap_context: - return finish_parse_and_run(parser, cmd_name, env_format_error) + return finish_parse_and_run(parser, cmd_name, args.command, env_format_error) -def finish_parse_and_run(parser, cmd_name, env_format_error): +def finish_parse_and_run(parser, cmd_name, cmd, env_format_error): """Finish parsing after we know the command to run.""" # add the found command to the parser and re-run then re-parse command = parser.add_command(cmd_name) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 65a1379455e50f..d5425772cdd3be 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -18,35 +18,27 @@ import sys import traceback import urllib.parse +from typing import List, Optional, Union +import llnl.url import llnl.util.tty as tty from llnl.util.filesystem import mkdirp import spack.caches import spack.config import spack.error -import spack.fetch_strategy as fs +import spack.fetch_strategy import spack.mirror +import spack.oci.image import spack.spec -import spack.url as url import spack.util.path import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml import spack.util.url as url_util -from spack.util.spack_yaml import syaml_dict -from spack.version import VersionList +import spack.version #: What schemes do we support -supported_url_schemes = ("file", "http", "https", "sftp", "ftp", "s3", "gs") - - -def _display_mirror_entry(size, name, url, type_=None): - if type_: - type_ = "".join((" (", type_, ")")) - else: - type_ = "" - - print("%-*s%s%s" % (size + 4, name, url, type_)) +supported_url_schemes = ("file", "http", "https", "sftp", "ftp", "s3", "gs", "oci") def _url_or_path_to_url(url_or_path: str) -> str: @@ -71,36 +63,24 @@ class Mirror: to them. These two URLs are usually the same. """ - def __init__(self, fetch_url, push_url=None, name=None): - self._fetch_url = fetch_url - self._push_url = push_url + def __init__(self, data: Union[str, dict], name: Optional[str] = None): + self._data = data self._name = name - def __eq__(self, other): - return self._fetch_url == other._fetch_url and self._push_url == other._push_url - - def to_json(self, stream=None): - return sjson.dump(self.to_dict(), stream) - - def to_yaml(self, stream=None): - return syaml.dump(self.to_dict(), stream) - @staticmethod def from_yaml(stream, name=None): - data = syaml.load(stream) - return Mirror.from_dict(data, name) + return Mirror(syaml.load(stream), name) @staticmethod def from_json(stream, name=None): try: - d = sjson.load(stream) - return Mirror.from_dict(d, name) + return Mirror(sjson.load(stream), name) except Exception as e: raise sjson.SpackJSONError("error parsing JSON mirror:", str(e)) from e @staticmethod def from_local_path(path: str): - return Mirror(fetch_url=url_util.path_to_file_url(path)) + return Mirror(url_util.path_to_file_url(path)) @staticmethod def from_url(url: str): @@ -111,165 +91,223 @@ def from_url(url: str): url, ", ".join(supported_url_schemes) ) ) - return Mirror(fetch_url=url) - - def to_dict(self): - # Keep it a key-value pair : when possible. - if isinstance(self._fetch_url, str) and self._push_url is None: - return self._fetch_url + return Mirror(url) - if self._push_url is None: - return syaml_dict([("fetch", self._fetch_url), ("push", self._fetch_url)]) - else: - return syaml_dict([("fetch", self._fetch_url), ("push", self._push_url)]) + def __eq__(self, other): + if not isinstance(other, Mirror): + return NotImplemented + return self._data == other._data and self._name == other._name - @staticmethod - def from_dict(d, name=None): - if isinstance(d, str): - return Mirror(d, name=name) - else: - return Mirror(d["fetch"], d["push"], name=name) + def __str__(self): + return f"{self._name}: {self.push_url} {self.fetch_url}" - def display(self, max_len=0): - if self._push_url is None: - _display_mirror_entry(max_len, self._name, self.fetch_url) - else: - _display_mirror_entry(max_len, self._name, self.fetch_url, "fetch") - _display_mirror_entry(max_len, self._name, self.push_url, "push") + def __repr__(self): + return f"Mirror(name={self._name!r}, data={self._data!r})" - def __str__(self): - name = self._name - if name is None: - name = "" - else: - name = ' "%s"' % name + def to_json(self, stream=None): + return sjson.dump(self.to_dict(), stream) - if self._push_url is None: - return "[Mirror%s (%s)]" % (name, self._fetch_url) + def to_yaml(self, stream=None): + return syaml.dump(self.to_dict(), stream) - return "[Mirror%s (fetch: %s, push: %s)]" % (name, self._fetch_url, self._push_url) + def to_dict(self): + return self._data - def __repr__(self): - return "".join( - ( - "Mirror(", - ", ".join( - "%s=%s" % (k, repr(v)) - for k, v in ( - ("fetch_url", self._fetch_url), - ("push_url", self._push_url), - ("name", self._name), - ) - if k == "fetch_url" or v - ), - ")", - ) - ) + def display(self, max_len=0): + fetch, push = self.fetch_url, self.push_url + # don't print the same URL twice + url = fetch if fetch == push else f"fetch: {fetch} push: {push}" + source = "s" if self.source else " " + binary = "b" if self.binary else " " + print(f"{self.name: <{max_len}} [{source}{binary}] {url}") @property def name(self): return self._name or "" - def get_profile(self, url_type): - if isinstance(self._fetch_url, dict): - if url_type == "push": - return self._push_url.get("profile", None) - return self._fetch_url.get("profile", None) - else: - return None + @property + def binary(self): + return isinstance(self._data, str) or self._data.get("binary", True) - def set_profile(self, url_type, profile): - if url_type == "push": - self._push_url["profile"] = profile - else: - self._fetch_url["profile"] = profile + @property + def source(self): + return isinstance(self._data, str) or self._data.get("source", True) - def get_access_pair(self, url_type): - if isinstance(self._fetch_url, dict): - if url_type == "push": - return self._push_url.get("access_pair", None) - return self._fetch_url.get("access_pair", None) - else: + @property + def fetch_url(self): + """Get the valid, canonicalized fetch URL""" + return self.get_url("fetch") + + @property + def push_url(self): + """Get the valid, canonicalized fetch URL""" + return self.get_url("push") + + def _update_connection_dict(self, current_data: dict, new_data: dict, top_level: bool): + keys = ["url", "access_pair", "access_token", "profile", "endpoint_url"] + if top_level: + keys += ["binary", "source"] + changed = False + for key in keys: + if key in new_data and current_data.get(key) != new_data[key]: + current_data[key] = new_data[key] + changed = True + return changed + + def update(self, data: dict, direction: Optional[str] = None) -> bool: + """Modify the mirror with the given data. This takes care + of expanding trivial mirror definitions by URL to something more + rich with a dict if necessary + + Args: + data (dict): The data to update the mirror with. + direction (str): The direction to update the mirror in (fetch + or push or None for top-level update) + + Returns: + bool: True if the mirror was updated, False otherwise.""" + + # Modify the top-level entry when no direction is given. + if not data: + return False + + # If we only update a URL, there's typically no need to expand things to a dict. + set_url = data["url"] if len(data) == 1 and "url" in data else None + + if direction is None: + # First deal with the case where the current top-level entry is just a string. + if isinstance(self._data, str): + # Can we replace that string with something new? + if set_url: + if self._data == set_url: + return False + self._data = set_url + return True + + # Otherwise promote to a dict + self._data = {"url": self._data} + + # And update the dictionary accordingly. + return self._update_connection_dict(self._data, data, top_level=True) + + # Otherwise, update the fetch / push entry; turn top-level + # url string into a dict if necessary. + if isinstance(self._data, str): + self._data = {"url": self._data} + + # Create a new fetch / push entry if necessary + if direction not in self._data: + # Keep config minimal if we're just setting the URL. + if set_url: + self._data[direction] = set_url + return True + self._data[direction] = {} + + entry = self._data[direction] + + # Keep the entry simple if we're just swapping out the URL. + if isinstance(entry, str): + if set_url: + if entry == set_url: + return False + self._data[direction] = set_url + return True + + # Otherwise promote to a dict + self._data[direction] = {"url": entry} + + return self._update_connection_dict(self._data[direction], data, top_level=False) + + def _get_value(self, attribute: str, direction: str): + """Returns the most specific value for a given attribute (either push/fetch or global)""" + if direction not in ("fetch", "push"): + raise ValueError(f"direction must be either 'fetch' or 'push', not {direction}") + + if isinstance(self._data, str): return None - def set_access_pair(self, url_type, connection_tuple): - if url_type == "push": - self._push_url["access_pair"] = connection_tuple - else: - self._fetch_url["access_pair"] = connection_tuple + # Either a string (url) or a dictionary, we care about the dict here. + value = self._data.get(direction, {}) - def get_endpoint_url(self, url_type): - if isinstance(self._fetch_url, dict): - if url_type == "push": - return self._push_url.get("endpoint_url", None) - return self._fetch_url.get("endpoint_url", None) - else: - return None + # Return top-level entry if only a URL was set. + if isinstance(value, str) or attribute not in value: + return self._data.get(attribute) - def set_endpoint_url(self, url_type, url): - if url_type == "push": - self._push_url["endpoint_url"] = url - else: - self._fetch_url["endpoint_url"] = url + return value[attribute] - def get_access_token(self, url_type): - if isinstance(self._fetch_url, dict): - if url_type == "push": - return self._push_url.get("access_token", None) - return self._fetch_url.get("access_token", None) - else: - return None + def get_url(self, direction: str) -> str: + if direction not in ("fetch", "push"): + raise ValueError(f"direction must be either 'fetch' or 'push', not {direction}") - def set_access_token(self, url_type, connection_token): - if url_type == "push": - self._push_url["access_token"] = connection_token - else: - self._fetch_url["access_token"] = connection_token + # Whole mirror config is just a url. + if isinstance(self._data, str): + return _url_or_path_to_url(self._data) - @property - def fetch_url(self): - """Get the valid, canonicalized fetch URL""" - url_or_path = ( - self._fetch_url if isinstance(self._fetch_url, str) else self._fetch_url["url"] - ) - return _url_or_path_to_url(url_or_path) + # Default value + url = self._data.get("url") - @fetch_url.setter - def fetch_url(self, url): - self._fetch_url["url"] = url - self._normalize() + # Override it with a direction-specific value + if direction in self._data: + # Either a url as string or a dict with url key + info = self._data[direction] + if isinstance(info, str): + url = info + elif "url" in info: + url = info["url"] - @property - def push_url(self): - """Get the valid, canonicalized push URL. Returns fetch URL if no custom - push URL is defined""" - if self._push_url is None: - return self.fetch_url - url_or_path = self._push_url if isinstance(self._push_url, str) else self._push_url["url"] - return _url_or_path_to_url(url_or_path) + if not url: + raise ValueError(f"Mirror {self.name} has no URL configured") + + return _url_or_path_to_url(url) + + def get_access_token(self, direction: str) -> Optional[str]: + return self._get_value("access_token", direction) + + def get_access_pair(self, direction: str) -> Optional[List]: + return self._get_value("access_pair", direction) - @push_url.setter - def push_url(self, url): - self._push_url["url"] = url - self._normalize() + def get_profile(self, direction: str) -> Optional[str]: + return self._get_value("profile", direction) - def _normalize(self): - if self._push_url is not None and self._push_url == self._fetch_url: - self._push_url = None + def get_endpoint_url(self, direction: str) -> Optional[str]: + return self._get_value("endpoint_url", direction) class MirrorCollection(collections.abc.Mapping): """A mapping of mirror names to mirrors.""" - def __init__(self, mirrors=None, scope=None): - self._mirrors = collections.OrderedDict( - (name, Mirror.from_dict(mirror, name)) + def __init__( + self, + mirrors=None, + scope=None, + binary: Optional[bool] = None, + source: Optional[bool] = None, + ): + """Initialize a mirror collection. + + Args: + mirrors: A name-to-mirror mapping to initialize the collection with. + scope: The scope to use when looking up mirrors from the config. + binary: If True, only include binary mirrors. + If False, omit binary mirrors. + If None, do not filter on binary mirrors. + source: If True, only include source mirrors. + If False, omit source mirrors. + If None, do not filter on source mirrors.""" + self._mirrors = { + name: Mirror(data=mirror, name=name) for name, mirror in ( mirrors.items() if mirrors is not None else spack.config.get("mirrors", scope=scope).items() ) - ) + } + + if source is not None: + self._mirrors = {k: v for k, v in self._mirrors.items() if v.source == source} + + if binary is not None: + self._mirrors = {k: v for k, v in self._mirrors.items() if v.binary == binary} def __eq__(self, other): return self._mirrors == other._mirrors @@ -295,7 +333,7 @@ def from_json(stream, name=None): raise sjson.SpackJSONError("error parsing JSON mirror collection:", str(e)) from e def to_dict(self, recursive=False): - return syaml_dict( + return syaml.syaml_dict( sorted( ((k, (v.to_dict() if recursive else v)) for (k, v) in self._mirrors.items()), key=operator.itemgetter(0), @@ -325,7 +363,7 @@ def lookup(self, name_or_url): result = self.get(name_or_url) if result is None: - result = Mirror(fetch_url=name_or_url) + result = Mirror(fetch=name_or_url) return result @@ -337,10 +375,10 @@ def __len__(self): def _determine_extension(fetcher): - if isinstance(fetcher, fs.URLFetchStrategy): + if isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy): if fetcher.expand_archive: # If we fetch with a URLFetchStrategy, use URL's archive type - ext = url.determine_url_file_extension(fetcher.url) + ext = llnl.url.determine_url_file_extension(fetcher.url) if ext: # Remove any leading dots @@ -402,12 +440,25 @@ def __iter__(self): yield self.cosmetic_path +class OCIImageLayout: + """Follow the OCI Image Layout Specification to archive blobs + + Paths are of the form `blobs//` + """ + + def __init__(self, digest: spack.oci.image.Digest) -> None: + self.storage_path = os.path.join("blobs", digest.algorithm, digest.digest) + + def __iter__(self): + yield self.storage_path + + def mirror_archive_paths(fetcher, per_package_ref, spec=None): """Returns a ``MirrorReference`` object which keeps track of the relative storage path of the resource associated with the specified ``fetcher``.""" ext = None if spec: - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) versions = pkg_cls.versions.get(spec.version, {}) ext = versions.get("extension", None) # If the spec does not explicitly specify an extension (the default case), @@ -439,7 +490,7 @@ def get_all_versions(specs): """ version_specs = [] for spec in specs: - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) # Skip any package that has no known versions. if not pkg_cls.versions: tty.msg("No safe (checksummed) versions for package %s" % pkg_cls.name) @@ -447,7 +498,7 @@ def get_all_versions(specs): for version in pkg_cls.versions: version_spec = spack.spec.Spec(pkg_cls.name) - version_spec.versions = VersionList([version]) + version_spec.versions = spack.version.VersionList([version]) version_specs.append(version_spec) return version_specs @@ -486,7 +537,7 @@ def get_matching_versions(specs, num_versions=1): # Generate only versions that satisfy the spec. if spec.concrete or v.intersects(spec.versions): s = spack.spec.Spec(pkg.name) - s.versions = VersionList([v]) + s.versions = spack.version.VersionList([v]) s.variants = spec.variants.copy() # This is needed to avoid hanging references during the # concretization phase @@ -556,14 +607,14 @@ def add(mirror: Mirror, scope=None): """Add a named mirror in the given scope""" mirrors = spack.config.get("mirrors", scope=scope) if not mirrors: - mirrors = syaml_dict() + mirrors = syaml.syaml_dict() if mirror.name in mirrors: tty.die("Mirror with name {} already exists.".format(mirror.name)) items = [(n, u) for n, u in mirrors.items()] items.insert(0, (mirror.name, mirror.to_dict())) - mirrors = syaml_dict(items) + mirrors = syaml.syaml_dict(items) spack.config.set("mirrors", mirrors, scope=scope) @@ -571,29 +622,13 @@ def remove(name, scope): """Remove the named mirror in the given scope""" mirrors = spack.config.get("mirrors", scope=scope) if not mirrors: - mirrors = syaml_dict() + mirrors = syaml.syaml_dict() if name not in mirrors: tty.die("No mirror with name %s" % name) - old_value = mirrors.pop(name) + mirrors.pop(name) spack.config.set("mirrors", mirrors, scope=scope) - - debug_msg_url = "url %s" - debug_msg = ["Removed mirror %s with"] - values = [name] - - try: - fetch_value = old_value["fetch"] - push_value = old_value["push"] - - debug_msg.extend(("fetch", debug_msg_url, "and push", debug_msg_url)) - values.extend((fetch_value, push_value)) - except TypeError: - debug_msg.append(debug_msg_url) - values.append(old_value) - - tty.debug(" ".join(debug_msg) % tuple(values)) tty.msg("Removed mirror %s." % name) @@ -656,12 +691,9 @@ def create_mirror_from_package_object(pkg_obj, mirror_cache, mirror_stats): num_retries = 3 while num_retries > 0: try: + # Includes patches and resources with pkg_obj.stage as pkg_stage: pkg_stage.cache_mirror(mirror_cache, mirror_stats) - for patch in pkg_obj.all_patches(): - if patch.stage: - patch.stage.cache_mirror(mirror_cache, mirror_stats) - patch.clean() exception = None break except Exception as e: diff --git a/lib/spack/spack/modules/__init__.py b/lib/spack/spack/modules/__init__.py index ccd800cdecda11..dde8b74a5c285e 100644 --- a/lib/spack/spack/modules/__init__.py +++ b/lib/spack/spack/modules/__init__.py @@ -7,15 +7,15 @@ include Tcl non-hierarchical modules, Lua hierarchical modules, and others. """ -from .common import disable_modules, ensure_modules_are_enabled_or_warn +from typing import Dict, Type + +from .common import BaseModuleFileWriter, disable_modules from .lmod import LmodModulefileWriter from .tcl import TclModulefileWriter -__all__ = [ - "TclModulefileWriter", - "LmodModulefileWriter", - "disable_modules", - "ensure_modules_are_enabled_or_warn", -] +__all__ = ["TclModulefileWriter", "LmodModulefileWriter", "disable_modules"] -module_types = {"tcl": TclModulefileWriter, "lmod": LmodModulefileWriter} +module_types: Dict[str, Type[BaseModuleFileWriter]] = { + "tcl": TclModulefileWriter, + "lmod": LmodModulefileWriter, +} diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index 58c27e288ac29f..2ad5bb51bc002b 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -33,10 +33,9 @@ import datetime import inspect import os.path -import pathlib import re -import warnings -from typing import Optional +import string +from typing import List, Optional import llnl.util.filesystem import llnl.util.tty as tty @@ -51,12 +50,14 @@ import spack.projections as proj import spack.repo import spack.schema.environment +import spack.spec import spack.store import spack.tengine as tengine import spack.util.environment import spack.util.file_permissions as fp import spack.util.path import spack.util.spack_yaml as syaml +from spack.context import Context #: config section for this file @@ -177,7 +178,7 @@ def merge_config_rules(configuration, spec): if spec.satisfies(constraint): if hasattr(constraint, "override") and constraint.override: spec_configuration = {} - update_dictionary_extending_lists(spec_configuration, action) + update_dictionary_extending_lists(spec_configuration, copy.deepcopy(action)) # Transform keywords for dependencies or prerequisites into a list of spec @@ -248,7 +249,7 @@ def generate_module_index(root, modules, overwrite=False): def _generate_upstream_module_index(): module_indices = read_module_indices() - return UpstreamModuleIndex(spack.store.db, module_indices) + return UpstreamModuleIndex(spack.store.STORE.db, module_indices) upstream_module_index = llnl.util.lang.Singleton(_generate_upstream_module_index) @@ -353,7 +354,7 @@ def get_module(module_type, spec, get_full_path, module_set_name="default", requ try: upstream = spec.installed_upstream except spack.repo.UnknownPackageError: - upstream, record = spack.store.db.query_by_spec_hash(spec.dag_hash()) + upstream, record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash()) if upstream: module = spack.modules.common.upstream_module_index.upstream_module(spec, module_type) if not module: @@ -395,16 +396,14 @@ class BaseConfiguration: default_projections = {"all": "{name}/{version}-{compiler.name}-{compiler.version}"} - def __init__(self, spec, module_set_name, explicit=None): + def __init__(self, spec: spack.spec.Spec, module_set_name: str, explicit: bool) -> None: # Module where type(self) is defined - self.module = inspect.getmodule(self) + m = inspect.getmodule(self) + assert m is not None # make mypy happy + self.module = m # Spec for which we want to generate a module file self.spec = spec self.name = module_set_name - # Software installation has been explicitly asked (get this information from - # db when querying an existing module, like during a refresh or rm operations) - if explicit is None: - explicit = spec._installed_explicitly() self.explicit = explicit # Dictionary of configuration options that should be applied # to the spec @@ -458,7 +457,11 @@ def suffixes(self): if constraint in self.spec: suffixes.append(suffix) suffixes = list(dedupe(suffixes)) - if self.hash: + # For hidden modules we can always add a fixed length hash as suffix, since it guards + # against file name clashes, and the module is not exposed to the user anyways. + if self.hidden: + suffixes.append(self.spec.dag_hash(length=7)) + elif self.hash: suffixes.append(self.hash) return suffixes @@ -470,6 +473,11 @@ def hash(self): return self.spec.dag_hash(length=hash_length) return None + @property + def conflicts(self): + """Conflicts for this module file""" + return self.conf.get("conflict", []) + @property def excluded(self): """Returns True if the module has been excluded, False otherwise.""" @@ -478,37 +486,37 @@ def excluded(self): spec = self.spec conf = self.module.configuration(self.name) - # Compute the list of include rules that match - include_rules = conf.get("include", []) - include_matches = [x for x in include_rules if spec.satisfies(x)] - - # Compute the list of exclude rules that match - exclude_rules = conf.get("exclude", []) - exclude_matches = [x for x in exclude_rules if spec.satisfies(x)] - - # Should I exclude the module because it's implicit? - exclude_implicits = conf.get("exclude_implicits", None) - excluded_as_implicit = exclude_implicits and not self.explicit + # Compute the list of matching include / exclude rules, and whether excluded as implicit + include_matches = [x for x in conf.get("include", []) if spec.satisfies(x)] + exclude_matches = [x for x in conf.get("exclude", []) if spec.satisfies(x)] + excluded_as_implicit = not self.explicit and conf.get("exclude_implicits", False) def debug_info(line_header, match_list): if match_list: - msg = "\t{0} : {1}".format(line_header, spec.cshort_spec) - tty.debug(msg) + tty.debug(f"\t{line_header} : {spec.cshort_spec}") for rule in match_list: - tty.debug("\t\tmatches rule: {0}".format(rule)) + tty.debug(f"\t\tmatches rule: {rule}") debug_info("INCLUDE", include_matches) debug_info("EXCLUDE", exclude_matches) if excluded_as_implicit: - msg = "\tEXCLUDED_AS_IMPLICIT : {0}".format(spec.cshort_spec) - tty.debug(msg) + tty.debug(f"\tEXCLUDED_AS_IMPLICIT : {spec.cshort_spec}") - is_excluded = exclude_matches or excluded_as_implicit - if not include_matches and is_excluded: - return True + return not include_matches and (exclude_matches or excluded_as_implicit) - return False + @property + def hidden(self): + """Returns True if the module has been hidden, False otherwise.""" + + conf = self.module.configuration(self.name) + + hidden_as_implicit = not self.explicit and conf.get("hide_implicits", False) + + if hidden_as_implicit: + tty.debug(f"\tHIDDEN_AS_IMPLICIT : {self.spec.cshort_spec}") + + return hidden_as_implicit @property def context(self): @@ -538,8 +546,7 @@ def exclude_env_vars(self): def _create_list_for(self, what): include = [] for item in self.conf[what]: - conf = type(self)(item, self.name) - if not conf.excluded: + if not self.module.make_configuration(item, self.name).excluded: include.append(item) return include @@ -582,7 +589,7 @@ def use_name(self): if not projection: projection = self.conf.default_projections["all"] - name = self.spec.format(projection) + name = self.spec.format_path(projection) # Not everybody is working on linux... parts = name.split("/") name = os.path.join(*parts) @@ -713,10 +720,18 @@ def environment_modifications(self): ) # Let the extendee/dependency modify their extensions/dependencies - # before asking for package-specific modifications - env.extend(spack.build_environment.modifications_from_dependencies(spec, context="run")) - # Package specific modifications - spack.build_environment.set_module_variables_for_package(spec.package) + + # The only thing we care about is `setup_dependent_run_environment`, but + # for that to work, globals have to be set on the package modules, and the + # whole chain of setup_dependent_package has to be followed from leaf to spec. + # So: just run it here, but don't collect env mods. + spack.build_environment.SetupContext( + spec, context=Context.RUN + ).set_all_package_py_globals() + + # Then run setup_dependent_run_environment before setup_run_environment. + for dep in spec.dependencies(deptype=("link", "run")): + dep.package.setup_dependent_run_environment(env, spec) spec.package.setup_run_environment(env) # Modifications required from modules.yaml @@ -763,6 +778,36 @@ def has_manpath_modifications(self): else: return False + @tengine.context_property + def conflicts(self): + """List of conflicts for the module file.""" + fmts = [] + projection = proj.get_projection(self.conf.projections, self.spec) + for item in self.conf.conflicts: + self._verify_conflict_naming_consistency_or_raise(item, projection) + item = self.spec.format(item) + fmts.append(item) + return fmts + + def _verify_conflict_naming_consistency_or_raise(self, item, projection): + f = string.Formatter() + errors = [] + if len([x for x in f.parse(item)]) > 1: + for naming_dir, conflict_dir in zip(projection.split("/"), item.split("/")): + if naming_dir != conflict_dir: + errors.extend( + [ + f"spec={self.spec.cshort_spec}", + f"conflict_scheme={item}", + f"naming_scheme={projection}", + ] + ) + if errors: + raise ModulesError( + message="conflict scheme does not match naming scheme", + long_message="\n ".join(errors), + ) + @tengine.context_property def autoload(self): """List of modules that needs to be loaded automatically.""" @@ -775,8 +820,7 @@ def autoload(self): def _create_module_list_of(self, what): m = self.conf.module name = self.conf.name - explicit = self.conf.explicit - return [m.make_layout(x, name, explicit).use_name for x in getattr(self.conf, what)] + return [m.make_layout(x, name).use_name for x in getattr(self.conf, what)] @tengine.context_property def verbose(self): @@ -784,50 +828,20 @@ def verbose(self): return self.conf.verbose -def ensure_modules_are_enabled_or_warn(): - """Ensures that, if a custom configuration file is found with custom configuration for the - default tcl module set, then tcl module file generation is enabled. Otherwise, a warning - is emitted. - """ - - # TODO (v0.21 - Remove this function) - # Check if TCL module generation is enabled, return early if it is - enabled = spack.config.get("modules:default:enable", []) - if "tcl" in enabled: - return - - # Check if we have custom TCL module sections - for scope in spack.config.config.file_scopes: - # Skip default configuration - if scope.name.startswith("default"): - continue - - data = spack.config.get("modules:default:tcl", scope=scope.name) - if data: - config_file = pathlib.Path(scope.path) - if not scope.name.startswith("env"): - config_file = config_file / "modules.yaml" - break - else: - return - - # If we are here we have a custom "modules" section in "config_file" - msg = ( - f"detected custom TCL modules configuration in {config_file}, while TCL module file " - f"generation for the default module set is disabled. " - f"In Spack v0.20 module file generation has been disabled by default. To enable " - f"it run:\n\n\t$ spack config add 'modules:default:enable:[tcl]'\n" - ) - warnings.warn(msg) - - class BaseModuleFileWriter: - def __init__(self, spec, module_set_name, explicit=None): + default_template: str + hide_cmd_format: str + modulerc_header: List[str] + + def __init__( + self, spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None + ) -> None: self.spec = spec # This class is meant to be derived. Get the module of the # actual writer. self.module = inspect.getmodule(self) + assert self.module is not None # make mypy happy m = self.module # Create the triplet of configuration/layout/context @@ -845,6 +859,26 @@ def __init__(self, spec, module_set_name, explicit=None): name = type(self).__name__ raise DefaultTemplateNotDefined(msg.format(name)) + # Check if format for module hide command has been defined, + # throw if not found + try: + self.hide_cmd_format + except AttributeError: + msg = "'{0}' object has no attribute 'hide_cmd_format'\n" + msg += "Did you forget to define it in the class?" + name = type(self).__name__ + raise HideCmdFormatNotDefined(msg.format(name)) + + # Check if modulerc header content has been defined, + # throw if not found + try: + self.modulerc_header + except AttributeError: + msg = "'{0}' object has no attribute 'modulerc_header'\n" + msg += "Did you forget to define it in the class?" + name = type(self).__name__ + raise ModulercHeaderNotDefined(msg.format(name)) + def _get_template(self): """Gets the template that will be rendered for this spec.""" # Get templates and put them in the order of importance: @@ -939,6 +973,9 @@ def write(self, overwrite=False): # Symlink defaults if needed self.update_module_defaults() + # record module hiddenness if implicit + self.update_module_hiddenness() + def update_module_defaults(self): if any(self.spec.satisfies(default) for default in self.conf.defaults): # This spec matches a default, it needs to be symlinked to default @@ -949,6 +986,60 @@ def update_module_defaults(self): os.symlink(self.layout.filename, default_tmp) os.rename(default_tmp, default_path) + def update_module_hiddenness(self, remove=False): + """Update modulerc file corresponding to module to add or remove + command that hides module depending on its hidden state. + + Args: + remove (bool): if True, hiddenness information for module is + removed from modulerc. + """ + modulerc_path = self.layout.modulerc + hide_module_cmd = self.hide_cmd_format % self.layout.use_name + hidden = self.conf.hidden and not remove + modulerc_exists = os.path.exists(modulerc_path) + updated = False + + if modulerc_exists: + # retrieve modulerc content + with open(modulerc_path, "r") as f: + content = f.readlines() + content = "".join(content).split("\n") + # remove last empty item if any + if len(content[-1]) == 0: + del content[-1] + already_hidden = hide_module_cmd in content + + # remove hide command if module not hidden + if already_hidden and not hidden: + content.remove(hide_module_cmd) + updated = True + + # add hide command if module is hidden + elif not already_hidden and hidden: + if len(content) == 0: + content = self.modulerc_header.copy() + content.append(hide_module_cmd) + updated = True + else: + content = self.modulerc_header.copy() + if hidden: + content.append(hide_module_cmd) + updated = True + + # no modulerc file change if no content update + if updated: + is_empty = content == self.modulerc_header or len(content) == 0 + # remove existing modulerc if empty + if modulerc_exists and is_empty: + os.remove(modulerc_path) + # create or update modulerc + elif content != self.modulerc_header: + # ensure file ends with a newline character + content.append("") + with open(modulerc_path, "w") as f: + f.write("\n".join(content)) + def remove(self): """Deletes the module file.""" mod_file = self.layout.filename @@ -956,6 +1047,7 @@ def remove(self): try: os.remove(mod_file) # Remove the module file self.remove_module_defaults() # Remove default targeting module file + self.update_module_hiddenness(remove=True) # Remove hide cmd in modulerc os.removedirs( os.path.dirname(mod_file) ) # Remove all the empty directories from the leaf up @@ -999,5 +1091,17 @@ class DefaultTemplateNotDefined(AttributeError, ModulesError): """ +class HideCmdFormatNotDefined(AttributeError, ModulesError): + """Raised if the attribute 'hide_cmd_format' has not been specified + in the derived classes. + """ + + +class ModulercHeaderNotDefined(AttributeError, ModulesError): + """Raised if the attribute 'modulerc_header' has not been specified + in the derived classes. + """ + + class ModulesTemplateNotFoundError(ModulesError, RuntimeError): """Raised if the template for a module file was not found.""" diff --git a/lib/spack/spack/modules/lmod.py b/lib/spack/spack/modules/lmod.py index 9b0cf4822f6a18..5e042259c3a418 100644 --- a/lib/spack/spack/modules/lmod.py +++ b/lib/spack/spack/modules/lmod.py @@ -6,9 +6,9 @@ import collections import itertools import os.path -import posixpath -from typing import Any, Dict, List +from typing import Dict, List, Optional, Tuple +import llnl.util.filesystem as fs import llnl.util.lang as lang import spack.compilers @@ -23,10 +23,8 @@ #: lmod specific part of the configuration -def configuration(module_set_name): - config_path = "modules:%s:lmod" % module_set_name - config = spack.config.get(config_path, {}) - return config +def configuration(module_set_name: str) -> dict: + return spack.config.get(f"modules:{module_set_name}:lmod", {}) # DH* this is not the best way, but works for now @@ -41,11 +39,14 @@ def get_hash_length(): # Caches the configuration {spec_hash: configuration} -configuration_registry: Dict[str, Any] = {} +configuration_registry: Dict[Tuple[str, str, bool], BaseConfiguration] = {} -def make_configuration(spec, module_set_name, explicit): +def make_configuration( + spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None +) -> BaseConfiguration: """Returns the lmod configuration for spec""" + explicit = bool(spec._installed_explicitly()) if explicit is None else explicit key = (spec.dag_hash(), module_set_name, explicit) try: return configuration_registry[key] @@ -55,16 +56,18 @@ def make_configuration(spec, module_set_name, explicit): ) -def make_layout(spec, module_set_name, explicit): +def make_layout( + spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None +) -> BaseFileLayout: """Returns the layout information for spec""" - conf = make_configuration(spec, module_set_name, explicit) - return LmodFileLayout(conf) + return LmodFileLayout(make_configuration(spec, module_set_name, explicit)) -def make_context(spec, module_set_name, explicit): +def make_context( + spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None +) -> BaseContext: """Returns the context information for spec""" - conf = make_configuration(spec, module_set_name, explicit) - return LmodContext(conf) + return LmodContext(make_configuration(spec, module_set_name, explicit)) def guess_core_compilers(name, store=False) -> List[spack.spec.CompilerSpec]: @@ -107,10 +110,7 @@ def guess_core_compilers(name, store=False) -> List[spack.spec.CompilerSpec]: class LmodConfiguration(BaseConfiguration): """Configuration class for lmod module files.""" - # Note: Posixpath is used here as well as below as opposed to - # os.path.join due to spack.spec.Spec.format - # requiring forward slash path seperators at this stage - default_projections = {"all": posixpath.join("{name}", "{version}")} + default_projections = {"all": "{name}/{version}"} @property def core_compilers(self) -> List[spack.spec.CompilerSpec]: @@ -154,7 +154,7 @@ def hierarchy_tokens(self): # Check if all the tokens in the hierarchy are virtual specs. # If not warn the user and raise an error. - not_virtual = [t for t in tokens if t != "compiler" and not spack.repo.path.is_virtual(t)] + not_virtual = [t for t in tokens if t != "compiler" and not spack.repo.PATH.is_virtual(t)] if not_virtual: msg = "Non-virtual specs in 'hierarchy' list for lmod: {0}\n" msg += "Please check the 'modules.yaml' configuration files" @@ -242,6 +242,13 @@ def missing(self): """Returns the list of tokens that are not available.""" return [x for x in self.hierarchy_tokens if x not in self.available] + @property + def hidden(self): + # Never hide a module that opens a hierarchy + if any(self.spec.package.provides(x) for x in self.hierarchy_tokens): + return False + return super().hidden + class LmodFileLayout(BaseFileLayout): """File layout for lmod module files.""" @@ -277,12 +284,16 @@ def filename(self): hierarchy_name = os.path.join(*parts) # Compute the absolute path - fullname = os.path.join( + return os.path.join( self.arch_dirname, # root for lmod files on this architecture hierarchy_name, # relative path - ".".join([self.use_name, self.extension]), # file name + f"{self.use_name}.{self.extension}", # file name ) - return fullname + + @property + def modulerc(self): + """Returns the modulerc file associated with current module file""" + return os.path.join(os.path.dirname(self.filename), f".modulerc.{self.extension}") def token_to_path(self, name, value): """Transforms a hierarchy token into the corresponding path part. @@ -294,8 +305,10 @@ def token_to_path(self, name, value): Returns: str: part of the path associated with the service """ + # General format for the path part - path_part_fmt = os.path.join("{token.name}", "{token.version}") + def path_part_fmt(token): + return fs.polite_path([f"{token.name}", f"{token.version}"]) # If we are dealing with a core compiler, return 'Core' core_compilers = self.conf.core_compilers @@ -307,13 +320,13 @@ def token_to_path(self, name, value): # CompilerSpec does not have a hash, as we are not allowed to # use different flavors of the same compiler if name == "compiler": - return path_part_fmt.format(token=value) + return path_part_fmt(token=value) # In case the hierarchy token refers to a virtual provider # we need to append a hash to the version to distinguish # among flavors of the same library (e.g. openblas~openmp vs. # openblas+openmp) - path = path_part_fmt.format(token=value) + path = path_part_fmt(token=value) # Use the hash length that the user specied, not something else if get_hash_length() > 0: @@ -332,8 +345,7 @@ def available_path_parts(self): # List of services that are part of the hierarchy hierarchy = self.conf.hierarchy_tokens # Tokenize each part that is both in the hierarchy and available - parts = [self.token_to_path(x, available[x]) for x in hierarchy if x in available] - return parts + return [self.token_to_path(x, available[x]) for x in hierarchy if x in available] @property @lang.memoized @@ -457,7 +469,7 @@ def missing(self): @lang.memoized def unlocked_paths(self): """Returns the list of paths that are unlocked unconditionally.""" - layout = make_layout(self.spec, self.conf.name, self.conf.explicit) + layout = make_layout(self.spec, self.conf.name) return [os.path.join(*parts) for parts in layout.unlocked_paths[None]] @tengine.context_property @@ -465,7 +477,7 @@ def conditionally_unlocked_paths(self): """Returns the list of paths that are unlocked conditionally. Each item in the list is a tuple with the structure (condition, path). """ - layout = make_layout(self.spec, self.conf.name, self.conf.explicit) + layout = make_layout(self.spec, self.conf.name) value = [] conditional_paths = layout.unlocked_paths conditional_paths.pop(None) @@ -487,7 +499,11 @@ def manipulate_path(token): class LmodModulefileWriter(BaseModuleFileWriter): """Writer class for lmod module files.""" - default_template = posixpath.join("modules", "modulefile.lua") + default_template = "modules/modulefile.lua" + + modulerc_header = [] + + hide_cmd_format = 'hide_version("%s")' class CoreCompilersNotFoundError(spack.error.SpackError, KeyError): diff --git a/lib/spack/spack/modules/tcl.py b/lib/spack/spack/modules/tcl.py index 634426c357fbb7..6d7f49b3309f33 100644 --- a/lib/spack/spack/modules/tcl.py +++ b/lib/spack/spack/modules/tcl.py @@ -6,32 +6,30 @@ """This module implements the classes necessary to generate Tcl non-hierarchical modules. """ -import posixpath -import string -from typing import Any, Dict - -import llnl.util.tty as tty +import os.path +from typing import Dict, Optional, Tuple import spack.config -import spack.projections as proj +import spack.spec import spack.tengine as tengine from .common import BaseConfiguration, BaseContext, BaseFileLayout, BaseModuleFileWriter #: Tcl specific part of the configuration -def configuration(module_set_name): - config_path = "modules:%s:tcl" % module_set_name - config = spack.config.get(config_path, {}) - return config +def configuration(module_set_name: str) -> dict: + return spack.config.get(f"modules:{module_set_name}:tcl", {}) # Caches the configuration {spec_hash: configuration} -configuration_registry: Dict[str, Any] = {} +configuration_registry: Dict[Tuple[str, str, bool], BaseConfiguration] = {} -def make_configuration(spec, module_set_name, explicit): +def make_configuration( + spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None +) -> BaseConfiguration: """Returns the tcl configuration for spec""" + explicit = bool(spec._installed_explicitly()) if explicit is None else explicit key = (spec.dag_hash(), module_set_name, explicit) try: return configuration_registry[key] @@ -41,30 +39,32 @@ def make_configuration(spec, module_set_name, explicit): ) -def make_layout(spec, module_set_name, explicit): +def make_layout( + spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None +) -> BaseFileLayout: """Returns the layout information for spec""" - conf = make_configuration(spec, module_set_name, explicit) - return TclFileLayout(conf) + return TclFileLayout(make_configuration(spec, module_set_name, explicit)) -def make_context(spec, module_set_name, explicit): +def make_context( + spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None +) -> BaseContext: """Returns the context information for spec""" - conf = make_configuration(spec, module_set_name, explicit) - return TclContext(conf) + return TclContext(make_configuration(spec, module_set_name, explicit)) class TclConfiguration(BaseConfiguration): """Configuration class for tcl module files.""" - @property - def conflicts(self): - """Conflicts for this module file""" - return self.conf.get("conflict", []) - class TclFileLayout(BaseFileLayout): """File layout for tcl module files.""" + @property + def modulerc(self): + """Returns the modulerc file associated with current module file""" + return os.path.join(os.path.dirname(self.filename), ".modulerc") + class TclContext(BaseContext): """Context class for tcl module files.""" @@ -74,34 +74,12 @@ def prerequisites(self): """List of modules that needs to be loaded automatically.""" return self._create_module_list_of("specs_to_prereq") - @tengine.context_property - def conflicts(self): - """List of conflicts for the tcl module file.""" - fmts = [] - projection = proj.get_projection(self.conf.projections, self.spec) - f = string.Formatter() - for item in self.conf.conflicts: - if len([x for x in f.parse(item)]) > 1: - for naming_dir, conflict_dir in zip(projection.split("/"), item.split("/")): - if naming_dir != conflict_dir: - message = "conflict scheme does not match naming " - message += "scheme [{spec}]\n\n" - message += 'naming scheme : "{nformat}"\n' - message += 'conflict scheme : "{cformat}"\n\n' - message += "** You may want to check your " - message += "`modules.yaml` configuration file **\n" - tty.error(message.format(spec=self.spec, nformat=projection, cformat=item)) - raise SystemExit("Module generation aborted.") - item = self.spec.format(item) - fmts.append(item) - # Substitute spec tokens if present - return [self.spec.format(x) for x in fmts] - class TclModulefileWriter(BaseModuleFileWriter): """Writer class for tcl module files.""" - # Note: Posixpath is used here as opposed to - # os.path.join due to spack.spec.Spec.format - # requiring forward slash path seperators at this stage - default_template = posixpath.join("modules", "modulefile.tcl") + default_template = "modules/modulefile.tcl" + + modulerc_header = ["#%Module4.7"] + + hide_cmd_format = "module-hide --soft --hidden-loaded %s" diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index c268b6caba7ed8..0c661172424751 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -26,6 +26,7 @@ """ import functools import inspect +from contextlib import contextmanager from llnl.util.lang import caller_locals @@ -236,7 +237,7 @@ def install(self, prefix): # Create a multimethod with this name if there is not one already original_method = MultiMethodMeta._locals.get(method.__name__) - if not type(original_method) == SpecMultiMethod: + if not isinstance(original_method, SpecMultiMethod): original_method = SpecMultiMethod(original_method) if self.spec is not None: @@ -271,6 +272,13 @@ def __exit__(self, exc_type, exc_val, exc_tb): spack.directives.DirectiveMeta.pop_from_context() +@contextmanager +def default_args(**kwargs): + spack.directives.DirectiveMeta.push_default_args(kwargs) + yield + spack.directives.DirectiveMeta.pop_default_args() + + class MultiMethodError(spack.error.SpackError): """Superclass for multimethod dispatch errors""" diff --git a/lib/spack/spack/oci/__init__.py b/lib/spack/spack/oci/__init__.py new file mode 100644 index 00000000000000..af304aecb70f37 --- /dev/null +++ b/lib/spack/spack/oci/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/oci/image.py b/lib/spack/spack/oci/image.py new file mode 100644 index 00000000000000..b61591b7bed0d0 --- /dev/null +++ b/lib/spack/spack/oci/image.py @@ -0,0 +1,235 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import re +import urllib.parse +from typing import Optional, Union + +import spack.spec + +# notice: Docker is more strict (no uppercase allowed). We parse image names *with* uppercase +# and normalize, so: example.com/Organization/Name -> example.com/organization/name. Tags are +# case sensitive though. +alphanumeric_with_uppercase = r"[a-zA-Z0-9]+" +separator = r"(?:[._]|__|[-]+)" +localhost = r"localhost" +domainNameComponent = r"(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])" +optionalPort = r"(?::[0-9]+)?" +tag = r"[\w][\w.-]{0,127}" +digestPat = r"[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][0-9a-fA-F]{32,}" +ipv6address = r"\[(?:[a-fA-F0-9:]+)\]" + +# domain name +domainName = rf"{domainNameComponent}(?:\.{domainNameComponent})*" +host = rf"(?:{domainName}|{ipv6address})" +domainAndPort = rf"{host}{optionalPort}" + +# image name +pathComponent = rf"{alphanumeric_with_uppercase}(?:{separator}{alphanumeric_with_uppercase})*" +remoteName = rf"{pathComponent}(?:\/{pathComponent})*" +namePat = rf"(?:{domainAndPort}\/)?{remoteName}" + +# Regex for a full image reference, with 3 groups: name, tag, digest +referencePat = re.compile(rf"^({namePat})(?::({tag}))?(?:@({digestPat}))?$") + +# Regex for splitting the name into domain and path components +anchoredNameRegexp = re.compile(rf"^(?:({domainAndPort})\/)?({remoteName})$") + + +def ensure_sha256_checksum(oci_blob: str): + """Validate that the reference is of the format sha256: + Return the checksum if valid, raise ValueError otherwise.""" + if ":" not in oci_blob: + raise ValueError(f"Invalid OCI blob format: {oci_blob}") + alg, checksum = oci_blob.split(":", 1) + if alg != "sha256": + raise ValueError(f"Unsupported OCI blob checksum algorithm: {alg}") + if len(checksum) != 64: + raise ValueError(f"Invalid OCI blob checksum length: {len(checksum)}") + return checksum + + +class Digest: + """Represents a digest in the format :. + Currently only supports sha256 digests.""" + + __slots__ = ["algorithm", "digest"] + + def __init__(self, *, algorithm: str, digest: str) -> None: + self.algorithm = algorithm + self.digest = digest + + def __eq__(self, __value: object) -> bool: + if not isinstance(__value, Digest): + return NotImplemented + return self.algorithm == __value.algorithm and self.digest == __value.digest + + @classmethod + def from_string(cls, string: str) -> "Digest": + return cls(algorithm="sha256", digest=ensure_sha256_checksum(string)) + + @classmethod + def from_sha256(cls, digest: str) -> "Digest": + return cls(algorithm="sha256", digest=digest) + + def __str__(self) -> str: + return f"{self.algorithm}:{self.digest}" + + +class ImageReference: + """A parsed image of the form domain/name:tag[@digest]. + The digest is optional, and domain and tag are automatically + filled out with defaults when parsed from string.""" + + __slots__ = ["domain", "name", "tag", "digest"] + + def __init__( + self, *, domain: str, name: str, tag: str = "latest", digest: Optional[Digest] = None + ): + self.domain = domain + self.name = name + self.tag = tag + self.digest = digest + + @classmethod + def from_string(cls, string) -> "ImageReference": + match = referencePat.match(string) + if not match: + raise ValueError(f"Invalid image reference: {string}") + + image, tag, digest = match.groups() + + assert isinstance(image, str) + assert isinstance(tag, (str, type(None))) + assert isinstance(digest, (str, type(None))) + + match = anchoredNameRegexp.match(image) + + # This can never happen, since the regex is implied + # by the regex above. It's just here to make mypy happy. + assert match, f"Invalid image reference: {string}" + + domain, name = match.groups() + + assert isinstance(domain, (str, type(None))) + assert isinstance(name, str) + + # Fill out defaults like docker would do... + # Based on github.com/distribution/distribution: allow short names like "ubuntu" + # and "user/repo" to be interpreted as "library/ubuntu" and "user/repo:latest + # Not sure if Spack should follow Docker, but it's what people expect... + if not domain: + domain = "index.docker.io" + name = f"library/{name}" + elif ( + "." not in domain + and ":" not in domain + and domain != "localhost" + and domain == domain.lower() + ): + name = f"{domain}/{name}" + domain = "index.docker.io" + + # Lowercase the image name. This is enforced by Docker, although the OCI spec isn't clear? + # We do this anyways, cause for example in Github Actions the / + # part can have uppercase, and may be interpolated when specifying the relevant OCI image. + name = name.lower() + + if not tag: + tag = "latest" + + # sha256 is currently the only algorithm that + # we implement, even though the spec allows for more + if isinstance(digest, str): + digest = Digest.from_string(digest) + + return cls(domain=domain, name=name, tag=tag, digest=digest) + + def manifest_url(self) -> str: + digest_or_tag = self.digest or self.tag + return f"https://{self.domain}/v2/{self.name}/manifests/{digest_or_tag}" + + def blob_url(self, digest: Union[str, Digest]) -> str: + if isinstance(digest, str): + digest = Digest.from_string(digest) + return f"https://{self.domain}/v2/{self.name}/blobs/{digest}" + + def with_digest(self, digest: Union[str, Digest]) -> "ImageReference": + if isinstance(digest, str): + digest = Digest.from_string(digest) + return ImageReference(domain=self.domain, name=self.name, tag=self.tag, digest=digest) + + def with_tag(self, tag: str) -> "ImageReference": + return ImageReference(domain=self.domain, name=self.name, tag=tag, digest=self.digest) + + def uploads_url(self, digest: Optional[Digest] = None) -> str: + url = f"https://{self.domain}/v2/{self.name}/blobs/uploads/" + if digest: + url += f"?digest={digest}" + return url + + def tags_url(self) -> str: + return f"https://{self.domain}/v2/{self.name}/tags/list" + + def endpoint(self, path: str = "") -> str: + return urllib.parse.urljoin(f"https://{self.domain}/v2/", path) + + def __str__(self) -> str: + s = f"{self.domain}/{self.name}" + if self.tag: + s += f":{self.tag}" + if self.digest: + s += f"@{self.digest}" + return s + + def __eq__(self, __value: object) -> bool: + if not isinstance(__value, ImageReference): + return NotImplemented + return ( + self.domain == __value.domain + and self.name == __value.name + and self.tag == __value.tag + and self.digest == __value.digest + ) + + +def _ensure_valid_tag(tag: str) -> str: + """Ensure a tag is valid for an OCI registry.""" + sanitized = re.sub(r"[^\w.-]", "_", tag) + if len(sanitized) > 128: + return sanitized[:64] + sanitized[-64:] + return sanitized + + +def default_tag(spec: "spack.spec.Spec") -> str: + """Return a valid, default image tag for a spec.""" + return _ensure_valid_tag(f"{spec.name}-{spec.version}-{spec.dag_hash()}.spack") + + +#: Default OCI index tag +default_index_tag = "index.spack" + + +def tag_is_spec(tag: str) -> bool: + """Check if a tag is likely a Spec""" + return tag.endswith(".spack") and tag != default_index_tag + + +def default_config(architecture: str, os: str): + return { + "architecture": architecture, + "os": os, + "rootfs": {"type": "layers", "diff_ids": []}, + "config": {"Env": []}, + } + + +def default_manifest(): + return { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "schemaVersion": 2, + "config": {"mediaType": "application/vnd.oci.image.config.v1+json"}, + "layers": [], + } diff --git a/lib/spack/spack/oci/oci.py b/lib/spack/spack/oci/oci.py new file mode 100644 index 00000000000000..4e5e196cd10db9 --- /dev/null +++ b/lib/spack/spack/oci/oci.py @@ -0,0 +1,381 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import hashlib +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from http.client import HTTPResponse +from typing import NamedTuple, Tuple +from urllib.request import Request + +import llnl.util.tty as tty + +import spack.binary_distribution +import spack.config +import spack.error +import spack.fetch_strategy +import spack.mirror +import spack.oci.opener +import spack.repo +import spack.spec +import spack.stage +import spack.traverse +import spack.util.crypto + +from .image import Digest, ImageReference + + +class Blob(NamedTuple): + compressed_digest: Digest + uncompressed_digest: Digest + size: int + + +def create_tarball(spec: spack.spec.Spec, tarfile_path): + buildinfo = spack.binary_distribution.get_buildinfo_dict(spec) + return spack.binary_distribution._do_create_tarball(tarfile_path, spec.prefix, buildinfo) + + +def _log_upload_progress(digest: Digest, size: int, elapsed: float): + elapsed = max(elapsed, 0.001) # guard against division by zero + tty.info(f"Uploaded {digest} ({elapsed:.2f}s, {size / elapsed / 1024 / 1024:.2f} MB/s)") + + +def with_query_param(url: str, param: str, value: str) -> str: + """Add a query parameter to a URL + + Args: + url: The URL to add the parameter to. + param: The parameter name. + value: The parameter value. + + Returns: + The URL with the parameter added. + """ + parsed = urllib.parse.urlparse(url) + query = urllib.parse.parse_qs(parsed.query) + if param in query: + query[param].append(value) + else: + query[param] = [value] + return urllib.parse.urlunparse( + parsed._replace(query=urllib.parse.urlencode(query, doseq=True)) + ) + + +def upload_blob( + ref: ImageReference, + file: str, + digest: Digest, + force: bool = False, + small_file_size: int = 0, + _urlopen: spack.oci.opener.MaybeOpen = None, +) -> bool: + """Uploads a blob to an OCI registry + + We only do monolithic uploads, even though it's very simple to do chunked. + Observed problems with chunked uploads: + (1) it's slow, many sequential requests, (2) some registries set an *unknown* + max chunk size, and the spec doesn't say how to obtain it + + Args: + ref: The image reference. + file: The file to upload. + digest: The digest of the file. + force: Whether to force upload the blob, even if it already exists. + small_file_size: For files at most this size, attempt + to do a single POST request instead of POST + PUT. + Some registries do no support single requests, and others + do not specify what size they support in single POST. + For now this feature is disabled by default (0KB) + + Returns: + True if the blob was uploaded, False if it already existed. + """ + _urlopen = _urlopen or spack.oci.opener.urlopen + + # Test if the blob already exists, if so, early exit. + if not force and blob_exists(ref, digest, _urlopen): + return False + + start = time.time() + + with open(file, "rb") as f: + file_size = os.fstat(f.fileno()).st_size + + # For small blobs, do a single POST request. + # The spec says that registries MAY support this + if file_size <= small_file_size: + request = Request( + url=ref.uploads_url(digest), + method="POST", + data=f, + headers={ + "Content-Type": "application/octet-stream", + "Content-Length": str(file_size), + }, + ) + else: + request = Request( + url=ref.uploads_url(), method="POST", headers={"Content-Length": "0"} + ) + + response = _urlopen(request) + + # Created the blob in one go. + if response.status == 201: + _log_upload_progress(digest, file_size, time.time() - start) + return True + + # Otherwise, do another PUT request. + spack.oci.opener.ensure_status(response, 202) + assert "Location" in response.headers + + # Can be absolute or relative, joining handles both + upload_url = with_query_param( + ref.endpoint(response.headers["Location"]), "digest", str(digest) + ) + f.seek(0) + + response = _urlopen( + Request( + url=upload_url, + method="PUT", + data=f, + headers={ + "Content-Type": "application/octet-stream", + "Content-Length": str(file_size), + }, + ) + ) + + spack.oci.opener.ensure_status(response, 201) + + # print elapsed time and # MB/s + _log_upload_progress(digest, file_size, time.time() - start) + return True + + +def upload_manifest( + ref: ImageReference, + oci_manifest: dict, + tag: bool = True, + _urlopen: spack.oci.opener.MaybeOpen = None, +): + """Uploads a manifest/index to a registry + + Args: + ref: The image reference. + oci_manifest: The OCI manifest or index. + tag: When true, use the tag, otherwise use the digest, + this is relevant for multi-arch images, where the + tag is an index, referencing the manifests by digest. + + Returns: + The digest and size of the uploaded manifest. + """ + _urlopen = _urlopen or spack.oci.opener.urlopen + + data = json.dumps(oci_manifest, separators=(",", ":")).encode() + digest = Digest.from_sha256(hashlib.sha256(data).hexdigest()) + size = len(data) + + if not tag: + ref = ref.with_digest(digest) + + response = _urlopen( + Request( + url=ref.manifest_url(), + method="PUT", + data=data, + headers={"Content-Type": oci_manifest["mediaType"]}, + ) + ) + + spack.oci.opener.ensure_status(response, 201) + return digest, size + + +def image_from_mirror(mirror: spack.mirror.Mirror) -> ImageReference: + """Given an OCI based mirror, extract the URL and image name from it""" + url = mirror.push_url + if not url.startswith("oci://"): + raise ValueError(f"Mirror {mirror} is not an OCI mirror") + return ImageReference.from_string(url[6:]) + + +def blob_exists( + ref: ImageReference, digest: Digest, _urlopen: spack.oci.opener.MaybeOpen = None +) -> bool: + """Checks if a blob exists in an OCI registry""" + try: + _urlopen = _urlopen or spack.oci.opener.urlopen + response = _urlopen(Request(url=ref.blob_url(digest), method="HEAD")) + return response.status == 200 + except urllib.error.HTTPError as e: + if e.getcode() == 404: + return False + raise + + +def copy_missing_layers( + src: ImageReference, + dst: ImageReference, + architecture: str, + _urlopen: spack.oci.opener.MaybeOpen = None, +) -> Tuple[dict, dict]: + """Copy image layers from src to dst for given architecture. + + Args: + src: The source image reference. + dst: The destination image reference. + architecture: The architecture (when referencing an index) + + Returns: + Tuple of manifest and config of the base image. + """ + _urlopen = _urlopen or spack.oci.opener.urlopen + manifest, config = get_manifest_and_config(src, architecture, _urlopen=_urlopen) + + # Get layer digests + digests = [Digest.from_string(layer["digest"]) for layer in manifest["layers"]] + + # Filter digests that are don't exist in the registry + missing_digests = [ + digest for digest in digests if not blob_exists(dst, digest, _urlopen=_urlopen) + ] + + if not missing_digests: + return manifest, config + + # Pull missing blobs, push them to the registry + with spack.stage.StageComposite.from_iterable( + make_stage(url=src.blob_url(digest), digest=digest, _urlopen=_urlopen) + for digest in missing_digests + ) as stages: + stages.fetch() + stages.check() + stages.cache_local() + + for stage, digest in zip(stages, missing_digests): + # No need to check existince again, force=True. + upload_blob( + dst, file=stage.save_filename, force=True, digest=digest, _urlopen=_urlopen + ) + + return manifest, config + + +#: OCI manifest content types (including docker type) +manifest_content_type = [ + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +] + +#: OCI index content types (including docker type) +index_content_type = [ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", +] + +#: All OCI manifest / index content types +all_content_type = manifest_content_type + index_content_type + + +def get_manifest_and_config( + ref: ImageReference, + architecture="amd64", + recurse=3, + _urlopen: spack.oci.opener.MaybeOpen = None, +) -> Tuple[dict, dict]: + """Recursively fetch manifest and config for a given image reference + with a given architecture. + + Args: + ref: The image reference. + architecture: The architecture (when referencing an index) + recurse: How many levels of index to recurse into. + + Returns: + A tuple of (manifest, config)""" + + _urlopen = _urlopen or spack.oci.opener.urlopen + + # Get manifest + response: HTTPResponse = _urlopen( + Request(url=ref.manifest_url(), headers={"Accept": ", ".join(all_content_type)}) + ) + + # Recurse when we find an index + if response.headers["Content-Type"] in index_content_type: + if recurse == 0: + raise Exception("Maximum recursion depth reached while fetching OCI manifest") + + index = json.load(response) + manifest_meta = next( + manifest + for manifest in index["manifests"] + if manifest["platform"]["architecture"] == architecture + ) + + return get_manifest_and_config( + ref.with_digest(manifest_meta["digest"]), + architecture=architecture, + recurse=recurse - 1, + _urlopen=_urlopen, + ) + + # Otherwise, require a manifest + if response.headers["Content-Type"] not in manifest_content_type: + raise Exception(f"Unknown content type {response.headers['Content-Type']}") + + manifest = json.load(response) + + # Download, verify and cache config file + config_digest = Digest.from_string(manifest["config"]["digest"]) + with make_stage(ref.blob_url(config_digest), config_digest, _urlopen=_urlopen) as stage: + stage.fetch() + stage.check() + stage.cache_local() + with open(stage.save_filename, "rb") as f: + config = json.load(f) + + return manifest, config + + +#: Same as upload_manifest, but with retry wrapper +upload_manifest_with_retry = spack.oci.opener.default_retry(upload_manifest) + +#: Same as upload_blob, but with retry wrapper +upload_blob_with_retry = spack.oci.opener.default_retry(upload_blob) + +#: Same as get_manifest_and_config, but with retry wrapper +get_manifest_and_config_with_retry = spack.oci.opener.default_retry(get_manifest_and_config) + +#: Same as copy_missing_layers, but with retry wrapper +copy_missing_layers_with_retry = spack.oci.opener.default_retry(copy_missing_layers) + + +def make_stage( + url: str, digest: Digest, keep: bool = False, _urlopen: spack.oci.opener.MaybeOpen = None +) -> spack.stage.Stage: + _urlopen = _urlopen or spack.oci.opener.urlopen + fetch_strategy = spack.fetch_strategy.OCIRegistryFetchStrategy( + url, checksum=digest.digest, _urlopen=_urlopen + ) + # Use blobs// as the cache path, which follows + # the OCI Image Layout Specification. What's missing though, + # is the `oci-layout` and `index.json` files, which are + # required by the spec. + return spack.stage.Stage( + fetch_strategy, + mirror_paths=spack.mirror.OCIImageLayout(digest), + name=digest.digest, + keep=keep, + ) diff --git a/lib/spack/spack/oci/opener.py b/lib/spack/spack/oci/opener.py new file mode 100644 index 00000000000000..792598578d3204 --- /dev/null +++ b/lib/spack/spack/oci/opener.py @@ -0,0 +1,442 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""All the logic for OCI fetching and authentication""" + +import base64 +import json +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from enum import Enum, auto +from http.client import HTTPResponse +from typing import Callable, Dict, Iterable, List, NamedTuple, Optional, Tuple +from urllib.request import Request + +import llnl.util.lang + +import spack.config +import spack.mirror +import spack.parser +import spack.repo +import spack.util.web + +from .image import ImageReference + + +def _urlopen(): + opener = create_opener() + + def dispatch_open(fullurl, data=None, timeout=None): + timeout = timeout or spack.config.get("config:connect_timeout", 10) + return opener.open(fullurl, data, timeout) + + return dispatch_open + + +OpenType = Callable[..., HTTPResponse] +MaybeOpen = Optional[OpenType] + +#: Opener that automatically uses OCI authentication based on mirror config +urlopen: OpenType = llnl.util.lang.Singleton(_urlopen) + + +SP = r" " +OWS = r"[ \t]*" +BWS = OWS +HTAB = r"\t" +VCHAR = r"\x21-\x7E" +tchar = r"[!#$%&'*+\-.^_`|~0-9A-Za-z]" +token = rf"{tchar}+" +obs_text = r"\x80-\xFF" +qdtext = rf"[{HTAB}{SP}\x21\x23-\x5B\x5D-\x7E{obs_text}]" +quoted_pair = rf"\\([{HTAB}{SP}{VCHAR}{obs_text}])" +quoted_string = rf'"(?:({qdtext}*)|{quoted_pair})*"' + + +class TokenType(spack.parser.TokenBase): + AUTH_PARAM = rf"({token}){BWS}={BWS}({token}|{quoted_string})" + # TOKEN68 = r"([A-Za-z0-9\-._~+/]+=*)" # todo... support this? + TOKEN = rf"{tchar}+" + EQUALS = rf"{BWS}={BWS}" + COMMA = rf"{OWS},{OWS}" + SPACE = r" +" + EOF = r"$" + ANY = r"." + + +TOKEN_REGEXES = [rf"(?P<{token}>{token.regex})" for token in TokenType] + +ALL_TOKENS = re.compile("|".join(TOKEN_REGEXES)) + + +class State(Enum): + CHALLENGE = auto() + AUTH_PARAM_LIST_START = auto() + AUTH_PARAM = auto() + NEXT_IN_LIST = auto() + AUTH_PARAM_OR_SCHEME = auto() + + +def tokenize(input: str): + scanner = ALL_TOKENS.scanner(input) # type: ignore[attr-defined] + + for match in iter(scanner.match, None): # type: ignore[var-annotated] + yield spack.parser.Token( + TokenType.__members__[match.lastgroup], # type: ignore[attr-defined] + match.group(), # type: ignore[attr-defined] + match.start(), # type: ignore[attr-defined] + match.end(), # type: ignore[attr-defined] + ) + + +class Challenge: + __slots__ = ["scheme", "params"] + + def __init__( + self, scheme: Optional[str] = None, params: Optional[List[Tuple[str, str]]] = None + ) -> None: + self.scheme = scheme or "" + self.params = params or [] + + def __repr__(self) -> str: + return f"Challenge({self.scheme}, {self.params})" + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, Challenge) + and self.scheme == other.scheme + and self.params == other.params + ) + + +def parse_www_authenticate(input: str): + """Very basic parsing of www-authenticate parsing (RFC7235 section 4.1) + Notice: this omits token68 support.""" + + # auth-scheme = token + # auth-param = token BWS "=" BWS ( token / quoted-string ) + # challenge = auth-scheme [ 1*SP ( token68 / #auth-param ) ] + # WWW-Authenticate = 1#challenge + + challenges: List[Challenge] = [] + + _unquote = re.compile(quoted_pair).sub + unquote = lambda s: _unquote(r"\1", s[1:-1]) + + mode: State = State.CHALLENGE + tokens = tokenize(input) + + current_challenge = Challenge() + + def extract_auth_param(input: str) -> Tuple[str, str]: + key, value = input.split("=", 1) + key = key.rstrip() + value = value.lstrip() + if value.startswith('"'): + value = unquote(value) + return key, value + + while True: + token: spack.parser.Token = next(tokens) + + if mode == State.CHALLENGE: + if token.kind == TokenType.EOF: + raise ValueError(token) + elif token.kind == TokenType.TOKEN: + current_challenge.scheme = token.value + mode = State.AUTH_PARAM_LIST_START + else: + raise ValueError(token) + + elif mode == State.AUTH_PARAM_LIST_START: + if token.kind == TokenType.EOF: + challenges.append(current_challenge) + break + elif token.kind == TokenType.COMMA: + # Challenge without param list, followed by another challenge. + challenges.append(current_challenge) + current_challenge = Challenge() + mode = State.CHALLENGE + elif token.kind == TokenType.SPACE: + # A space means it must be followed by param list + mode = State.AUTH_PARAM + else: + raise ValueError(token) + + elif mode == State.AUTH_PARAM: + if token.kind == TokenType.EOF: + raise ValueError(token) + elif token.kind == TokenType.AUTH_PARAM: + key, value = extract_auth_param(token.value) + current_challenge.params.append((key, value)) + mode = State.NEXT_IN_LIST + else: + raise ValueError(token) + + elif mode == State.NEXT_IN_LIST: + if token.kind == TokenType.EOF: + challenges.append(current_challenge) + break + elif token.kind == TokenType.COMMA: + mode = State.AUTH_PARAM_OR_SCHEME + else: + raise ValueError(token) + + elif mode == State.AUTH_PARAM_OR_SCHEME: + if token.kind == TokenType.EOF: + raise ValueError(token) + elif token.kind == TokenType.TOKEN: + challenges.append(current_challenge) + current_challenge = Challenge(token.value) + mode = State.AUTH_PARAM_LIST_START + elif token.kind == TokenType.AUTH_PARAM: + key, value = extract_auth_param(token.value) + current_challenge.params.append((key, value)) + mode = State.NEXT_IN_LIST + + return challenges + + +class RealmServiceScope(NamedTuple): + realm: str + service: str + scope: str + + +class UsernamePassword(NamedTuple): + username: str + password: str + + +def get_bearer_challenge(challenges: List[Challenge]) -> Optional[RealmServiceScope]: + # Find a challenge that we can handle (currently only Bearer) + challenge = next((c for c in challenges if c.scheme == "Bearer"), None) + + if challenge is None: + return None + + # Get realm / service / scope from challenge + realm = next((v for k, v in challenge.params if k == "realm"), None) + service = next((v for k, v in challenge.params if k == "service"), None) + scope = next((v for k, v in challenge.params if k == "scope"), None) + + if realm is None or service is None or scope is None: + return None + + return RealmServiceScope(realm, service, scope) + + +class OCIAuthHandler(urllib.request.BaseHandler): + def __init__(self, credentials_provider: Callable[[str], Optional[UsernamePassword]]): + """ + Args: + credentials_provider: A function that takes a domain and may return a UsernamePassword. + """ + self.credentials_provider = credentials_provider + + # Cached bearer tokens for a given domain. + self.cached_tokens: Dict[str, str] = {} + + def obtain_bearer_token(self, registry: str, challenge: RealmServiceScope, timeout) -> str: + # See https://docs.docker.com/registry/spec/auth/token/ + + query = urllib.parse.urlencode( + {"service": challenge.service, "scope": challenge.scope, "client_id": "spack"} + ) + + parsed = urllib.parse.urlparse(challenge.realm)._replace( + query=query, fragment="", params="" + ) + + # Don't send credentials over insecure transport. + if parsed.scheme != "https": + raise ValueError( + f"Cannot login to {registry} over insecure {parsed.scheme} connection" + ) + + request = Request(urllib.parse.urlunparse(parsed)) + + # I guess we shouldn't cache this, since we don't know + # the context in which it's used (may depend on config) + pair = self.credentials_provider(registry) + + if pair is not None: + encoded = base64.b64encode(f"{pair.username}:{pair.password}".encode("utf-8")).decode( + "utf-8" + ) + request.add_unredirected_header("Authorization", f"Basic {encoded}") + + # Do a GET request. + response = self.parent.open(request, timeout=timeout) + + # Read the response and parse the JSON + response_json = json.load(response) + + # Get the token from the response + token = response_json["token"] + + # Remember the last obtained token for this registry + # Note: we should probably take into account realm, service and scope + # so we can store multiple tokens for the same registry. + self.cached_tokens[registry] = token + + return token + + def https_request(self, req: Request): + # Eagerly add the bearer token to the request if no + # auth header is set yet, to avoid 401s in multiple + # requests to the same registry. + + # Use has_header, not .headers, since there are two + # types of headers (redirected and unredirected) + if req.has_header("Authorization"): + return req + + parsed = urllib.parse.urlparse(req.full_url) + token = self.cached_tokens.get(parsed.netloc) + + if not token: + return req + + req.add_unredirected_header("Authorization", f"Bearer {token}") + return req + + def http_error_401(self, req: Request, fp, code, msg, headers): + # Login failed, avoid infinite recursion where we go back and + # forth between auth server and registry + if hasattr(req, "login_attempted"): + raise urllib.error.HTTPError( + req.full_url, code, f"Failed to login to {req.full_url}: {msg}", headers, fp + ) + + # On 401 Unauthorized, parse the WWW-Authenticate header + # to determine what authentication is required + if "WWW-Authenticate" not in headers: + raise urllib.error.HTTPError( + req.full_url, + code, + "Cannot login to registry, missing WWW-Authenticate header", + headers, + fp, + ) + + header_value = headers["WWW-Authenticate"] + + try: + challenge = get_bearer_challenge(parse_www_authenticate(header_value)) + except ValueError as e: + raise urllib.error.HTTPError( + req.full_url, + code, + f"Cannot login to registry, malformed WWW-Authenticate header: {header_value}", + headers, + fp, + ) from e + + # If there is no bearer challenge, we can't handle it + if not challenge: + raise urllib.error.HTTPError( + req.full_url, + code, + f"Cannot login to registry, unsupported authentication scheme: {header_value}", + headers, + fp, + ) + + # Get the token from the auth handler + try: + token = self.obtain_bearer_token( + registry=urllib.parse.urlparse(req.get_full_url()).netloc, + challenge=challenge, + timeout=req.timeout, + ) + except ValueError as e: + raise urllib.error.HTTPError( + req.full_url, + code, + f"Cannot login to registry, failed to obtain bearer token: {e}", + headers, + fp, + ) from e + + # Add the token to the request + req.add_unredirected_header("Authorization", f"Bearer {token}") + setattr(req, "login_attempted", True) + + return self.parent.open(req, timeout=req.timeout) + + +def credentials_from_mirrors( + domain: str, *, mirrors: Optional[Iterable[spack.mirror.Mirror]] = None +) -> Optional[UsernamePassword]: + """Filter out OCI registry credentials from a list of mirrors.""" + + mirrors = mirrors or spack.mirror.MirrorCollection().values() + + for mirror in mirrors: + # Prefer push credentials over fetch. Unlikely that those are different + # but our config format allows it. + for direction in ("push", "fetch"): + pair = mirror.get_access_pair(direction) + if pair is None: + continue + url = mirror.get_url(direction) + if not url.startswith("oci://"): + continue + try: + parsed = ImageReference.from_string(url[6:]) + except ValueError: + continue + if parsed.domain == domain: + return UsernamePassword(*pair) + return None + + +def create_opener(): + """Create an opener that can handle OCI authentication.""" + opener = urllib.request.OpenerDirector() + for handler in [ + urllib.request.UnknownHandler(), + urllib.request.HTTPSHandler(), + spack.util.web.SpackHTTPDefaultErrorHandler(), + urllib.request.HTTPRedirectHandler(), + urllib.request.HTTPErrorProcessor(), + OCIAuthHandler(credentials_from_mirrors), + ]: + opener.add_handler(handler) + return opener + + +def ensure_status(response: HTTPResponse, status: int): + """Raise an error if the response status is not the expected one.""" + if response.status == status: + return + + raise urllib.error.HTTPError( + response.geturl(), response.status, response.reason, response.info(), None + ) + + +def default_retry(f, retries: int = 3, sleep=None): + sleep = sleep or time.sleep + + def wrapper(*args, **kwargs): + for i in range(retries): + try: + return f(*args, **kwargs) + except urllib.error.HTTPError as e: + # Retry on internal server errors, and rate limit errors + # Potentially this could take into account the Retry-After header + # if registries support it + if i + 1 != retries and (500 <= e.code < 600 or e.code == 429): + # Exponential backoff + sleep(2**i) + continue + raise + + return wrapper diff --git a/lib/spack/spack/operating_systems/mac_os.py b/lib/spack/spack/operating_systems/mac_os.py index 65d2317d4651a2..d52885fd5460e5 100644 --- a/lib/spack/spack/operating_systems/mac_os.py +++ b/lib/spack/spack/operating_systems/mac_os.py @@ -142,6 +142,7 @@ def __init__(self): "11": "bigsur", "12": "monterey", "13": "ventura", + "14": "sonoma", } version = macos_version() diff --git a/lib/spack/spack/operating_systems/windows_os.py b/lib/spack/spack/operating_systems/windows_os.py index 0c3930e99c48f1..fa767d71fb1c22 100755 --- a/lib/spack/spack/operating_systems/windows_os.py +++ b/lib/spack/spack/operating_systems/windows_os.py @@ -5,10 +5,12 @@ import glob import os +import pathlib import platform import subprocess from spack.error import SpackError +from spack.util import windows_registry as winreg from spack.version import Version from ._operating_system import OperatingSystem @@ -31,43 +33,6 @@ class WindowsOs(OperatingSystem): 10. """ - # Find MSVC directories using vswhere - comp_search_paths = [] - vs_install_paths = [] - root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") - if root: - try: - extra_args = {"encoding": "mbcs", "errors": "strict"} - paths = subprocess.check_output( # type: ignore[call-overload] # novermin - [ - os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), - "-prerelease", - "-requires", - "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", - "-property", - "installationPath", - "-products", - "*", - ], - **extra_args, - ).strip() - vs_install_paths = paths.split("\n") - msvc_paths = [os.path.join(path, "VC", "Tools", "MSVC") for path in vs_install_paths] - for p in msvc_paths: - comp_search_paths.extend(glob.glob(os.path.join(p, "*", "bin", "Hostx64", "x64"))) - if os.getenv("ONEAPI_ROOT"): - comp_search_paths.extend( - glob.glob( - os.path.join( - str(os.getenv("ONEAPI_ROOT")), "compiler", "*", "windows", "bin" - ) - ) - ) - except (subprocess.CalledProcessError, OSError, UnicodeDecodeError): - pass - if comp_search_paths: - compiler_search_paths = comp_search_paths - def __init__(self): plat_ver = windows_version() if plat_ver < Version("10"): @@ -76,3 +41,71 @@ def __init__(self): def __str__(self): return self.name + + @property + def vs_install_paths(self): + vs_install_paths = [] + root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") + if root: + try: + extra_args = {"encoding": "mbcs", "errors": "strict"} + paths = subprocess.check_output( # type: ignore[call-overload] # novermin + [ + os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), + "-prerelease", + "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", + "installationPath", + "-products", + "*", + ], + **extra_args, + ).strip() + vs_install_paths = paths.split("\n") + except (subprocess.CalledProcessError, OSError, UnicodeDecodeError): + pass + return vs_install_paths + + @property + def msvc_paths(self): + return [os.path.join(path, "VC", "Tools", "MSVC") for path in self.vs_install_paths] + + @property + def compiler_search_paths(self): + # First Strategy: Find MSVC directories using vswhere + _compiler_search_paths = [] + for p in self.msvc_paths: + _compiler_search_paths.extend(glob.glob(os.path.join(p, "*", "bin", "Hostx64", "x64"))) + if os.getenv("ONEAPI_ROOT"): + _compiler_search_paths.extend( + glob.glob( + os.path.join(str(os.getenv("ONEAPI_ROOT")), "compiler", "*", "windows", "bin") + ) + ) + # Second strategy: Find MSVC via the registry + msft = winreg.WindowsRegistryView( + "SOFTWARE\\WOW6432Node\\Microsoft", winreg.HKEY.HKEY_LOCAL_MACHINE + ) + vs_entries = msft.find_subkeys(r"VisualStudio_.*") + vs_paths = [] + + def clean_vs_path(path): + path = path.split(",")[0].lstrip("@") + return str((pathlib.Path(path).parent / "..\\..").resolve()) + + for entry in vs_entries: + try: + val = entry.get_subkey("Capabilities").get_value("ApplicationDescription").value + vs_paths.append(clean_vs_path(val)) + except FileNotFoundError as e: + if hasattr(e, "winerror"): + if e.winerror == 2: + pass + else: + raise + else: + raise + + _compiler_search_paths.extend(vs_paths) + return _compiler_search_paths diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 1e0a9eb6553492..16b1216acb6e1b 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -49,6 +49,7 @@ from spack.build_systems.nmake import NMakePackage from spack.build_systems.octave import OctavePackage from spack.build_systems.oneapi import ( + INTEL_MATH_LIBRARIES, IntelOneApiLibraryPackage, IntelOneApiPackage, IntelOneApiStaticLibraryList, @@ -67,7 +68,7 @@ from spack.build_systems.waf import WafPackage from spack.build_systems.xorg import XorgPackage from spack.builder import run_after, run_before -from spack.dependency import all_deptypes +from spack.deptypes import ALL_TYPES as all_deptypes from spack.directives import * from spack.install_test import ( SkipTest, @@ -85,7 +86,7 @@ UpstreamPackageError, ) from spack.mixins import filter_compiler_wrappers -from spack.multimethod import when +from spack.multimethod import default_args, when from spack.package_base import ( DependencyConflictError, build_system_flags, @@ -96,6 +97,7 @@ on_package_attributes, ) from spack.spec import InvalidSpecDetected, Spec +from spack.util.cpus import determine_number_of_jobs from spack.util.executable import * from spack.variant import ( any_combination_of, diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py index 2a31d5161f47a7..416b16cefc0621 100644 --- a/lib/spack/spack/package_base.py +++ b/lib/spack/spack/package_base.py @@ -11,7 +11,6 @@ import base64 import collections -import contextlib import copy import functools import glob @@ -35,7 +34,7 @@ import spack.compilers import spack.config -import spack.dependency +import spack.deptypes as dt import spack.directives import spack.directory_layout import spack.environment @@ -45,6 +44,7 @@ import spack.mirror import spack.mixins import spack.multimethod +import spack.patch import spack.paths import spack.repo import spack.spec @@ -63,10 +63,9 @@ install_test_root, ) from spack.installer import InstallError, PackageInstaller -from spack.stage import ResourceStage, Stage, StageComposite, compute_stage_name +from spack.stage import DIYStage, ResourceStage, Stage, StageComposite, compute_stage_name from spack.util.executable import ProcessError, which from spack.util.package_hash import package_hash -from spack.util.web import FetchError from spack.version import GitVersion, StandardVersion, Version FLAG_HANDLER_RETURN_TYPE = Tuple[ @@ -175,11 +174,13 @@ def windows_establish_runtime_linkage(self): detectable_packages = collections.defaultdict(list) -class DetectablePackageMeta: +class DetectablePackageMeta(type): """Check if a package is detectable and add default implementations for the detection function. """ + TAG = "detectable" + def __init__(cls, name, bases, attr_dict): if hasattr(cls, "executables") and hasattr(cls, "libraries"): msg = "a package can have either an 'executables' or 'libraries' attribute" @@ -195,6 +196,11 @@ def __init__(cls, name, bases, attr_dict): # If a package has the executables or libraries attribute then it's # assumed to be detectable if hasattr(cls, "executables") or hasattr(cls, "libraries"): + # Append a tag to each detectable package, so that finding them is faster + if hasattr(cls, "tags"): + getattr(cls, "tags").append(DetectablePackageMeta.TAG) + else: + setattr(cls, "tags", [DetectablePackageMeta.TAG]) @classmethod def platform_executables(cls): @@ -514,6 +520,13 @@ class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta): # These are default values for instance variables. # + # Declare versions dictionary as placeholder for values. + # This allows analysis tools to correctly interpret the class attributes. + versions: dict + + # Same for dependencies + dependencies: dict + #: By default, packages are not virtual #: Virtual packages override this attribute virtual = False @@ -528,10 +541,6 @@ class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta): #: By default do not run tests within package's install() run_tests = False - # FIXME: this is a bad object-oriented design, should be moved to Clang. - #: By default do not setup mockup XCode on macOS with Clang - use_xcode = False - #: Keep -Werror flags, matches config:flags:keep_werror to override config # NOTE: should be type Optional[Literal['all', 'specific', 'none']] in 3.8+ keep_werror: Optional[str] = None @@ -639,7 +648,7 @@ class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta): def __init__(self, spec): # this determines how the package should be built. - self.spec = spec + self.spec: "spack.spec.Spec" = spec # Allow custom staging paths for packages self.path = None @@ -665,7 +674,7 @@ def __init__(self, spec): self.win_rpath = fsys.WindowsSimulatedRPath(self) if self.is_extension: - pkg_cls = spack.repo.path.get_pkg_class(self.extendee_spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(self.extendee_spec.name) pkg_cls(self.extendee_spec)._check_extendable() super().__init__() @@ -675,7 +684,7 @@ def possible_dependencies( cls, transitive=True, expand_virtuals=True, - deptype="all", + depflag: dt.DepFlag = dt.ALL, visited=None, missing=None, virtuals=None, @@ -687,7 +696,7 @@ def possible_dependencies( True, only direct dependencies if False (default True).. expand_virtuals (bool or None): expand virtual dependencies into all possible implementations (default True) - deptype (str or tuple or None): dependency types to consider + depflag: dependency types to consider visited (dict or None): dict of names of dependencies visited so far, mapped to their immediate dependencies' names. missing (dict or None): dict to populate with packages and their @@ -713,8 +722,6 @@ def possible_dependencies( Note: the returned dict *includes* the package itself. """ - deptype = spack.dependency.canonical_deptype(deptype) - visited = {} if visited is None else visited missing = {} if missing is None else missing @@ -722,17 +729,18 @@ def possible_dependencies( for name, conditions in cls.dependencies.items(): # check whether this dependency could be of the type asked for - deptypes = [dep.type for cond, dep in conditions.items()] - deptypes = set.union(*deptypes) - if not any(d in deptypes for d in deptype): + depflag_union = 0 + for dep in conditions.values(): + depflag_union |= dep.depflag + if not (depflag & depflag_union): continue # expand virtuals if enabled, otherwise just stop at virtuals - if spack.repo.path.is_virtual(name): + if spack.repo.PATH.is_virtual(name): if virtuals is not None: virtuals.add(name) if expand_virtuals: - providers = spack.repo.path.providers_for(name) + providers = spack.repo.PATH.providers_for(name) dep_names = [spec.name for spec in providers] else: visited.setdefault(cls.name, set()).add(name) @@ -756,14 +764,14 @@ def possible_dependencies( continue try: - dep_cls = spack.repo.path.get_pkg_class(dep_name) + dep_cls = spack.repo.PATH.get_pkg_class(dep_name) except spack.repo.UnknownPackageError: # log unknown packages missing.setdefault(cls.name, set()).add(dep_name) continue dep_cls.possible_dependencies( - transitive, expand_virtuals, deptype, visited, missing, virtuals + transitive, expand_virtuals, depflag, visited, missing, virtuals ) return visited @@ -982,20 +990,18 @@ def find_valid_url_for_version(self, version): return None - def _make_resource_stage(self, root_stage, fetcher, resource): - resource_stage_folder = self._resource_stage(resource) - mirror_paths = spack.mirror.mirror_archive_paths( - fetcher, os.path.join(self.name, "%s-%s" % (resource.name, self.version)) - ) - stage = ResourceStage( + def _make_resource_stage(self, root_stage, resource): + pretty_resource_name = fsys.polite_filename(f"{resource.name}-{self.version}") + return ResourceStage( resource.fetcher, root=root_stage, resource=resource, - name=resource_stage_folder, - mirror_paths=mirror_paths, + name=self._resource_stage(resource), + mirror_paths=spack.mirror.mirror_archive_paths( + resource.fetcher, os.path.join(self.name, pretty_resource_name) + ), path=self.path, ) - return stage def _download_search(self): dynamic_fetcher = fs.from_list_url(self) @@ -1003,8 +1009,10 @@ def _download_search(self): def _make_root_stage(self, fetcher): # Construct a mirror path (TODO: get this out of package.py) + format_string = "{name}-{version}" + pretty_name = self.spec.format_path(format_string) mirror_paths = spack.mirror.mirror_archive_paths( - fetcher, os.path.join(self.name, "%s-%s" % (self.name, self.version)), self.spec + fetcher, os.path.join(self.name, pretty_name), self.spec ) # Construct a path where the stage should build.. s = self.spec @@ -1022,24 +1030,21 @@ def _make_stage(self): # If it's a dev package (not transitively), use a DIY stage object dev_path_var = self.spec.variants.get("dev_path", None) if dev_path_var: - return spack.stage.DIYStage(dev_path_var.value) - - # Construct a composite stage on top of the composite FetchStrategy - composite_fetcher = self.fetcher - composite_stage = StageComposite() - resources = self._get_needed_resources() - for ii, fetcher in enumerate(composite_fetcher): - if ii == 0: - # Construct root stage first - stage = self._make_root_stage(fetcher) - else: - # Construct resource stage - resource = resources[ii - 1] # ii == 0 is root! - stage = self._make_resource_stage(composite_stage[0], fetcher, resource) - # Append the item to the composite - composite_stage.append(stage) + return DIYStage(dev_path_var.value) + + # To fetch the current version + source_stage = self._make_root_stage(self.fetcher) - return composite_stage + # Extend it with all resources and patches + all_stages = StageComposite() + all_stages.append(source_stage) + all_stages.extend( + self._make_resource_stage(source_stage, r) for r in self._get_needed_resources() + ) + all_stages.extend( + p.stage for p in self.spec.patches if isinstance(p, spack.patch.UrlPatch) + ) + return all_stages @property def stage(self): @@ -1082,7 +1087,7 @@ def env_mods_path(self): @property def metadata_dir(self): """Return the install metadata directory.""" - return spack.store.layout.metadata_path(self.spec) + return spack.store.STORE.layout.metadata_path(self.spec) @property def install_env_path(self): @@ -1152,7 +1157,7 @@ def install_test_root(self): """Return the install test root directory.""" tty.warn( "The 'pkg.install_test_root' property is deprecated with removal " - "expected v0.21. Use 'install_test_root(pkg)' instead." + "expected v0.22. Use 'install_test_root(pkg)' instead." ) return install_test_root(self) @@ -1188,26 +1193,12 @@ def installed_upstream(self): warnings.warn(msg) return self.spec.installed_upstream - def _make_fetcher(self): - # Construct a composite fetcher that always contains at least - # one element (the root package). In case there are resources - # associated with the package, append their fetcher to the - # composite. - root_fetcher = fs.for_package_version(self) - fetcher = fs.FetchStrategyComposite() # Composite fetcher - fetcher.append(root_fetcher) # Root fetcher is always present - resources = self._get_needed_resources() - for resource in resources: - fetcher.append(resource.fetcher) - fetcher.set_package(self) - return fetcher - @property def fetcher(self): if not self.spec.versions.concrete: raise ValueError("Cannot retrieve fetcher for package without concrete version.") if not self._fetcher: - self._fetcher = self._make_fetcher() + self._fetcher = fs.for_package_version(self) return self._fetcher @fetcher.setter @@ -1216,7 +1207,7 @@ def fetcher(self, f): self._fetcher.set_package(self) @classmethod - def dependencies_of_type(cls, *deptypes): + def dependencies_of_type(cls, deptypes: dt.DepFlag): """Get dependencies that can possibly have these deptypes. This analyzes the package and determines which dependencies *can* @@ -1228,7 +1219,7 @@ def dependencies_of_type(cls, *deptypes): return dict( (name, conds) for name, conds in cls.dependencies.items() - if any(dt in cls.dependencies[name][cond].type for cond in conds for dt in deptypes) + if any(deptypes & cls.dependencies[name][cond].depflag for cond in conds) ) # TODO: allow more than one active extendee. @@ -1353,7 +1344,7 @@ def remove_prefix(self): Removes the prefix for a package along with any empty parent directories """ - spack.store.layout.remove_install_directory(self.spec) + spack.store.STORE.layout.remove_install_directory(self.spec) @property def download_instr(self): @@ -1405,7 +1396,7 @@ def do_fetch(self, mirror_only=False): tty.debug("Fetching with no checksum. {0}".format(ck_msg)) if not ignore_checksum: - raise FetchError( + raise spack.error.FetchError( "Will not fetch %s" % self.spec.format("{name}{@version}"), ck_msg ) @@ -1431,7 +1422,7 @@ def do_fetch(self, mirror_only=False): tty.debug("Fetching deprecated version. {0}".format(dp_msg)) if not ignore_deprecation: - raise FetchError( + raise spack.error.FetchError( "Will not fetch {0}".format(self.spec.format("{name}{@version}")), dp_msg ) @@ -1446,11 +1437,6 @@ def do_fetch(self, mirror_only=False): self.stage.cache_local() - for patch in self.spec.patches: - patch.fetch() - if patch.stage: - patch.stage.cache_local() - def do_stage(self, mirror_only=False): """Unpacks and expands the fetched tarball.""" # Always create the stage directory at this point. Why? A no-code @@ -1463,7 +1449,7 @@ def do_stage(self, mirror_only=False): self.stage.expand_archive() if not os.listdir(self.stage.path): - raise FetchError("Archive was empty for %s" % self.name) + raise spack.error.FetchError("Archive was empty for %s" % self.name) else: # Support for post-install hooks requires a stage.source_path fsys.mkdirp(self.stage.source_path) @@ -1785,13 +1771,6 @@ def _resource_stage(self, resource): resource_stage_folder = "-".join(pieces) return resource_stage_folder - @contextlib.contextmanager - def _stage_and_write_lock(self): - """Prefix lock nested in a stage.""" - with self.stage: - with spack.store.db.prefix_write_lock(self.spec): - yield - def do_install(self, **kwargs): """Called by commands to install a package and or its dependencies. @@ -1829,14 +1808,7 @@ def do_install(self, **kwargs): verbose (bool): Display verbose build output (by default, suppresses it) """ - # Non-transitive dev specs need to keep the dev stage and be built from - # source every time. Transitive ones just need to be built from source. - dev_path_var = self.spec.variants.get("dev_path", None) - if dev_path_var: - kwargs["keep_stage"] = True - - builder = PackageInstaller([(self, kwargs)]) - builder.install() + PackageInstaller([(self, kwargs)]).install() # TODO (post-34236): Update tests and all packages that use this as a # TODO (post-34236): package method to the routine made available to @@ -1857,7 +1829,7 @@ def cache_extra_test_sources(self, srcs): """ msg = ( "'pkg.cache_extra_test_sources(srcs) is deprecated with removal " - "expected in v0.21. Use 'cache_extra_test_sources(pkg, srcs)' " + "expected in v0.22. Use 'cache_extra_test_sources(pkg, srcs)' " "instead." ) warnings.warn(msg) @@ -2215,20 +2187,20 @@ def uninstall_by_spec(spec, force=False, deprecator=None): if not os.path.isdir(spec.prefix): # prefix may not exist, but DB may be inconsistent. Try to fix by # removing, but omit hooks. - specs = spack.store.db.query(spec, installed=True) + specs = spack.store.STORE.db.query(spec, installed=True) if specs: if deprecator: - spack.store.db.deprecate(specs[0], deprecator) + spack.store.STORE.db.deprecate(specs[0], deprecator) tty.debug("Deprecating stale DB entry for {0}".format(spec.short_spec)) else: - spack.store.db.remove(specs[0]) + spack.store.STORE.db.remove(specs[0]) tty.debug("Removed stale DB entry for {0}".format(spec.short_spec)) return else: raise InstallError(str(spec) + " is not installed.") if not force: - dependents = spack.store.db.installed_relatives( + dependents = spack.store.STORE.db.installed_relatives( spec, direction="parents", transitive=True, deptype=("link", "run") ) if dependents: @@ -2241,7 +2213,7 @@ def uninstall_by_spec(spec, force=False, deprecator=None): pkg = None # Pre-uninstall hook runs first. - with spack.store.db.prefix_write_lock(spec): + with spack.store.STORE.prefix_locker.write_lock(spec): if pkg is not None: try: spack.hooks.pre_uninstall(spec) @@ -2267,17 +2239,17 @@ def uninstall_by_spec(spec, force=False, deprecator=None): tty.debug(msg.format(spec.short_spec)) # test if spec is already deprecated, not whether we want to # deprecate it now - deprecated = bool(spack.store.db.deprecator(spec)) - spack.store.layout.remove_install_directory(spec, deprecated) + deprecated = bool(spack.store.STORE.db.deprecator(spec)) + spack.store.STORE.layout.remove_install_directory(spec, deprecated) # Delete DB entry if deprecator: msg = "deprecating DB entry [{0}] in favor of [{1}]" tty.debug(msg.format(spec.short_spec, deprecator.short_spec)) - spack.store.db.deprecate(spec, deprecator) + spack.store.STORE.db.deprecate(spec, deprecator) else: msg = "Deleting DB entry [{0}]" tty.debug(msg.format(spec.short_spec)) - spack.store.db.remove(spec) + spack.store.STORE.db.remove(spec) if pkg is not None: try: @@ -2308,24 +2280,24 @@ def do_deprecate(self, deprecator, link_fn): spec = self.spec # Install deprecator if it isn't installed already - if not spack.store.db.query(deprecator): + if not spack.store.STORE.db.query(deprecator): deprecator.package.do_install() - old_deprecator = spack.store.db.deprecator(spec) + old_deprecator = spack.store.STORE.db.deprecator(spec) if old_deprecator: # Find this specs yaml file from its old deprecation - self_yaml = spack.store.layout.deprecated_file_path(spec, old_deprecator) + self_yaml = spack.store.STORE.layout.deprecated_file_path(spec, old_deprecator) else: - self_yaml = spack.store.layout.spec_file_path(spec) + self_yaml = spack.store.STORE.layout.spec_file_path(spec) # copy spec metadata to "deprecated" dir of deprecator - depr_yaml = spack.store.layout.deprecated_file_path(spec, deprecator) + depr_yaml = spack.store.STORE.layout.deprecated_file_path(spec, deprecator) fsys.mkdirp(os.path.dirname(depr_yaml)) shutil.copy2(self_yaml, depr_yaml) # Any specs deprecated in favor of this spec are re-deprecated in # favor of its new deprecator - for deprecated in spack.store.db.specs_deprecated_by(spec): + for deprecated in spack.store.STORE.db.specs_deprecated_by(spec): deprecated.package.do_deprecate(deprecator, link_fn) # Now that we've handled metadata, uninstall and replace with link @@ -2341,7 +2313,7 @@ def view(self): Extensions added to this view will modify the installation prefix of this package. """ - return YamlFilesystemView(self.prefix, spack.store.layout) + return YamlFilesystemView(self.prefix, spack.store.STORE.layout) def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" @@ -2349,9 +2321,6 @@ def do_restage(self): def do_clean(self): """Removes the package's build stage and source tarball.""" - for patch in self.spec.patches: - patch.clean() - self.stage.destroy() @classmethod @@ -2391,7 +2360,7 @@ def all_urls(self): urls.append(args["url"]) return urls - def fetch_remote_versions(self, concurrency=128): + def fetch_remote_versions(self, concurrency=None): """Find remote versions of this package. Uses ``list_url`` and any other URLs listed in the package file. @@ -2403,7 +2372,7 @@ def fetch_remote_versions(self, concurrency=128): return {} try: - return spack.util.web.find_versions_of_archive( + return spack.url.find_versions_of_archive( self.all_urls, self.list_url, self.list_depth, concurrency, reference_package=self ) except spack.util.web.NoNetworkConnectionError as e: @@ -2468,7 +2437,7 @@ def flatten_dependencies(spec, flat_dir): for dep in spec.traverse(root=False): name = dep.name - dep_path = spack.store.layout.path_for_spec(dep) + dep_path = spack.store.STORE.layout.path_for_spec(dep) dep_files = LinkTree(dep_path) os.mkdir(flat_dir + "/" + name) @@ -2494,8 +2463,8 @@ def possible_dependencies(*pkg_or_spec, **kwargs): if not isinstance(pos, spack.spec.Spec): pos = spack.spec.Spec(pos) - if spack.repo.path.is_virtual(pos.name): - packages.extend(p.package_class for p in spack.repo.path.providers_for(pos.name)) + if spack.repo.PATH.is_virtual(pos.name): + packages.extend(p.package_class for p in spack.repo.PATH.providers_for(pos.name)) continue else: packages.append(pos.package_class) diff --git a/lib/spack/spack/package_prefs.py b/lib/spack/spack/package_prefs.py index ca217839b1487a..c2997034feeaf4 100644 --- a/lib/spack/spack/package_prefs.py +++ b/lib/spack/spack/package_prefs.py @@ -147,7 +147,7 @@ def preferred_variants(cls, pkg_name): variants = " ".join(variants) # Only return variants that are actually supported by the package - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) spec = spack.spec.Spec("%s %s" % (pkg_name, variants)) return dict( (name, variant) for name, variant in spec.variants.items() if name in pkg_cls.variants @@ -162,7 +162,7 @@ def spec_externals(spec): from spack.util.module_cmd import path_from_modules # noqa: F401 def _package(maybe_abstract_spec): - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) return pkg_cls(maybe_abstract_spec) allpkgs = spack.config.get("packages") @@ -199,7 +199,7 @@ def is_spec_buildable(spec): so_far = all_buildable # the default "so far" def _package(s): - pkg_cls = spack.repo.path.get_pkg_class(s.name) + pkg_cls = spack.repo.PATH.get_pkg_class(s.name) return pkg_cls(s) # check whether any providers for this package override the default diff --git a/lib/spack/spack/parser.py b/lib/spack/spack/parser.py index 3416f2540bce4c..c69918b41905b5 100644 --- a/lib/spack/spack/parser.py +++ b/lib/spack/spack/parser.py @@ -6,7 +6,7 @@ Here is the EBNF grammar for a spec:: - spec = [name] [node_options] { ^ node } | + spec = [name] [node_options] { ^[edge_properties] node } | [name] [node_options] hash | filename @@ -14,7 +14,8 @@ [name] [node_options] hash | filename - node_options = [@(version_list|version_pair)] [%compiler] { variant } + node_options = [@(version_list|version_pair)] [%compiler] { variant } + edge_properties = [ { bool_variant | key_value } ] hash = / id filename = (.|/|[a-zA-Z0-9-_]*/)([a-zA-Z0-9-_./]*)(.json|.yaml) @@ -64,19 +65,21 @@ from llnl.util.tty import color +import spack.deptypes import spack.error import spack.spec -import spack.variant import spack.version IS_WINDOWS = sys.platform == "win32" #: Valid name for specs and variants. Here we are not using #: the previous "w[\w.-]*" since that would match most #: characters that can be part of a word in any language -IDENTIFIER = r"([a-zA-Z_0-9][a-zA-Z_0-9\-]*)" -DOTTED_IDENTIFIER = rf"({IDENTIFIER}(\.{IDENTIFIER})+)" -GIT_HASH = r"([A-Fa-f0-9]{40})" -GIT_VERSION = rf"((git\.({DOTTED_IDENTIFIER}|{IDENTIFIER}))|({GIT_HASH}))" +IDENTIFIER = r"(?:[a-zA-Z_0-9][a-zA-Z_0-9\-]*)" +DOTTED_IDENTIFIER = rf"(?:{IDENTIFIER}(?:\.{IDENTIFIER})+)" +GIT_HASH = r"(?:[A-Fa-f0-9]{40})" +#: Git refs include branch names, and can contain "." and "/" +GIT_REF = r"(?:[a-zA-Z_0-9][a-zA-Z_0-9./\-]*)" +GIT_VERSION_PATTERN = rf"(?:(?:git\.(?:{GIT_REF}))|(?:{GIT_HASH}))" NAME = r"[a-zA-Z_0-9][a-zA-Z_0-9\-.]*" @@ -85,19 +88,19 @@ #: A filename starts either with a "." or a "/" or a "{name}/, # or on Windows, a drive letter followed by a colon and "\" # or "." or {name}\ -WINDOWS_FILENAME = r"(\.|[a-zA-Z0-9-_]*\\|[a-zA-Z]:\\)([a-zA-Z0-9-_\.\\]*)(\.json|\.yaml)" -UNIX_FILENAME = r"(\.|\/|[a-zA-Z0-9-_]*\/)([a-zA-Z0-9-_\.\/]*)(\.json|\.yaml)" +WINDOWS_FILENAME = r"(?:\.|[a-zA-Z0-9-_]*\\|[a-zA-Z]:\\)(?:[a-zA-Z0-9-_\.\\]*)(?:\.json|\.yaml)" +UNIX_FILENAME = r"(?:\.|\/|[a-zA-Z0-9-_]*\/)(?:[a-zA-Z0-9-_\.\/]*)(?:\.json|\.yaml)" if not IS_WINDOWS: FILENAME = UNIX_FILENAME else: FILENAME = WINDOWS_FILENAME -VALUE = r"([a-zA-Z_0-9\-+\*.,:=\~\/\\]+)" -QUOTED_VALUE = r"[\"']+([a-zA-Z_0-9\-+\*.,:=\~\/\\\s]+)[\"']+" +VALUE = r"(?:[a-zA-Z_0-9\-+\*.,:=\~\/\\]+)" +QUOTED_VALUE = r"[\"']+(?:[a-zA-Z_0-9\-+\*.,:=\~\/\\\s]+)[\"']+" -VERSION = r"=?([a-zA-Z0-9_][a-zA-Z_0-9\-\.]*\b)" -VERSION_RANGE = rf"({VERSION}\s*:\s*{VERSION}(?!\s*=)|:\s*{VERSION}(?!\s*=)|{VERSION}\s*:|:)" -VERSION_LIST = rf"({VERSION_RANGE}|{VERSION})(\s*[,]\s*({VERSION_RANGE}|{VERSION}))*" +VERSION = r"=?(?:[a-zA-Z0-9_][a-zA-Z_0-9\-\.]*\b)" +VERSION_RANGE = rf"(?:(?:{VERSION})?:(?:{VERSION}(?!\s*=))?)" +VERSION_LIST = rf"(?:{VERSION_RANGE}|{VERSION})(?:\s*,\s*(?:{VERSION_RANGE}|{VERSION}))*" class TokenBase(enum.Enum): @@ -125,34 +128,37 @@ class TokenType(TokenBase): """ # Dependency - DEPENDENCY = r"(\^)" + START_EDGE_PROPERTIES = r"(?:\^\[)" + END_EDGE_PROPERTIES = r"(?:\])" + DEPENDENCY = r"(?:\^)" # Version - VERSION_HASH_PAIR = rf"(@({GIT_VERSION})=({VERSION}))" - VERSION = rf"(@\s*({VERSION_LIST}))" + VERSION_HASH_PAIR = rf"(?:@(?:{GIT_VERSION_PATTERN})=(?:{VERSION}))" + GIT_VERSION = rf"@(?:{GIT_VERSION_PATTERN})" + VERSION = rf"(?:@\s*(?:{VERSION_LIST}))" # Variants - PROPAGATED_BOOL_VARIANT = rf"((\+\+|~~|--)\s*{NAME})" - BOOL_VARIANT = rf"([~+-]\s*{NAME})" - PROPAGATED_KEY_VALUE_PAIR = rf"({NAME}\s*==\s*({VALUE}|{QUOTED_VALUE}))" - KEY_VALUE_PAIR = rf"({NAME}\s*=\s*({VALUE}|{QUOTED_VALUE}))" + PROPAGATED_BOOL_VARIANT = rf"(?:(?:\+\+|~~|--)\s*{NAME})" + BOOL_VARIANT = rf"(?:[~+-]\s*{NAME})" + PROPAGATED_KEY_VALUE_PAIR = rf"(?:{NAME}\s*==\s*(?:{VALUE}|{QUOTED_VALUE}))" + KEY_VALUE_PAIR = rf"(?:{NAME}\s*=\s*(?:{VALUE}|{QUOTED_VALUE}))" # Compilers - COMPILER_AND_VERSION = rf"(%\s*({NAME})([\s]*)@\s*({VERSION_LIST}))" - COMPILER = rf"(%\s*({NAME}))" + COMPILER_AND_VERSION = rf"(?:%\s*(?:{NAME})(?:[\s]*)@\s*(?:{VERSION_LIST}))" + COMPILER = rf"(?:%\s*(?:{NAME}))" # FILENAME - FILENAME = rf"({FILENAME})" + FILENAME = rf"(?:{FILENAME})" # Package name - FULLY_QUALIFIED_PACKAGE_NAME = rf"({DOTTED_IDENTIFIER})" - UNQUALIFIED_PACKAGE_NAME = rf"({IDENTIFIER})" + FULLY_QUALIFIED_PACKAGE_NAME = rf"(?:{DOTTED_IDENTIFIER})" + UNQUALIFIED_PACKAGE_NAME = rf"(?:{IDENTIFIER})" # DAG hash - DAG_HASH = rf"(/({HASH}))" + DAG_HASH = rf"(?:/(?:{HASH}))" # White spaces - WS = r"(\s+)" + WS = r"(?:\s+)" class ErrorTokenType(TokenBase): """Enum with regexes for error analysis""" # Unexpected character - UNEXPECTED = r"(.[\s]*)" + UNEXPECTED = r"(?:.[\s]*)" class Token: @@ -161,7 +167,7 @@ class Token: __slots__ = "kind", "value", "start", "end" def __init__( - self, kind: TokenType, value: str, start: Optional[int] = None, end: Optional[int] = None + self, kind: TokenBase, value: str, start: Optional[int] = None, end: Optional[int] = None ): self.kind = kind self.value = value @@ -261,8 +267,8 @@ def tokens(self) -> List[Token]: return list(filter(lambda x: x.kind != TokenType.WS, tokenize(self.literal_str))) def next_spec( - self, initial_spec: Optional[spack.spec.Spec] = None - ) -> Optional[spack.spec.Spec]: + self, initial_spec: Optional["spack.spec.Spec"] = None + ) -> Optional["spack.spec.Spec"]: """Return the next spec parsed from text. Args: @@ -278,27 +284,35 @@ def next_spec( initial_spec = initial_spec or spack.spec.Spec() root_spec = SpecNodeParser(self.ctx).parse(initial_spec) while True: - if self.ctx.accept(TokenType.DEPENDENCY): - dependency = SpecNodeParser(self.ctx).parse() + if self.ctx.accept(TokenType.START_EDGE_PROPERTIES): + edge_properties = EdgeAttributeParser(self.ctx, self.literal_str).parse() + edge_properties.setdefault("depflag", 0) + edge_properties.setdefault("virtuals", ()) + dependency = self._parse_node(root_spec) + root_spec._add_dependency(dependency, **edge_properties) - if dependency is None: - msg = ( - "this dependency sigil needs to be followed by a package name " - "or a node attribute (version, variant, etc.)" - ) - raise SpecParsingError(msg, self.ctx.current_token, self.literal_str) - - if root_spec.concrete: - raise spack.spec.RedundantSpecError(root_spec, "^" + str(dependency)) - - root_spec._add_dependency(dependency, deptypes=(), virtuals=()) + elif self.ctx.accept(TokenType.DEPENDENCY): + dependency = self._parse_node(root_spec) + root_spec._add_dependency(dependency, depflag=0, virtuals=()) else: break return root_spec - def all_specs(self) -> List[spack.spec.Spec]: + def _parse_node(self, root_spec): + dependency = SpecNodeParser(self.ctx).parse() + if dependency is None: + msg = ( + "the dependency sigil and any optional edge attributes must be followed by a " + "package name or a node attribute (version, variant, etc.)" + ) + raise SpecParsingError(msg, self.ctx.current_token, self.literal_str) + if root_spec.concrete: + raise spack.spec.RedundantSpecError(root_spec, "^" + str(dependency)) + return dependency + + def all_specs(self) -> List["spack.spec.Spec"]: """Return all the specs that remain to be parsed""" return list(iter(self.next_spec, None)) @@ -306,15 +320,16 @@ def all_specs(self) -> List[spack.spec.Spec]: class SpecNodeParser: """Parse a single spec node from a stream of tokens""" - __slots__ = "ctx", "has_compiler", "has_version", "has_hash" + __slots__ = "ctx", "has_compiler", "has_version" def __init__(self, ctx): self.ctx = ctx self.has_compiler = False self.has_version = False - self.has_hash = False - def parse(self, initial_spec: Optional[spack.spec.Spec] = None) -> Optional[spack.spec.Spec]: + def parse( + self, initial_spec: Optional["spack.spec.Spec"] = None + ) -> Optional["spack.spec.Spec"]: """Parse a single spec node from a stream of tokens Args: @@ -343,7 +358,6 @@ def parse(self, initial_spec: Optional[spack.spec.Spec] = None) -> Optional[spac while True: if self.ctx.accept(TokenType.COMPILER): - self.hash_not_parsed_or_raise(initial_spec, self.ctx.current_token.value) if self.has_compiler: raise spack.spec.DuplicateCompilerSpecError( f"{initial_spec} cannot have multiple compilers" @@ -353,7 +367,6 @@ def parse(self, initial_spec: Optional[spack.spec.Spec] = None) -> Optional[spac initial_spec.compiler = spack.spec.CompilerSpec(compiler_name.strip(), ":") self.has_compiler = True elif self.ctx.accept(TokenType.COMPILER_AND_VERSION): - self.hash_not_parsed_or_raise(initial_spec, self.ctx.current_token.value) if self.has_compiler: raise spack.spec.DuplicateCompilerSpecError( f"{initial_spec} cannot have multiple compilers" @@ -364,10 +377,11 @@ def parse(self, initial_spec: Optional[spack.spec.Spec] = None) -> Optional[spac compiler_name.strip(), compiler_version ) self.has_compiler = True - elif self.ctx.accept(TokenType.VERSION) or self.ctx.accept( - TokenType.VERSION_HASH_PAIR + elif ( + self.ctx.accept(TokenType.VERSION_HASH_PAIR) + or self.ctx.accept(TokenType.GIT_VERSION) + or self.ctx.accept(TokenType.VERSION) ): - self.hash_not_parsed_or_raise(initial_spec, self.ctx.current_token.value) if self.has_version: raise spack.spec.MultipleVersionError( f"{initial_spec} cannot have multiple versions" @@ -378,25 +392,21 @@ def parse(self, initial_spec: Optional[spack.spec.Spec] = None) -> Optional[spac initial_spec.attach_git_version_lookup() self.has_version = True elif self.ctx.accept(TokenType.BOOL_VARIANT): - self.hash_not_parsed_or_raise(initial_spec, self.ctx.current_token.value) variant_value = self.ctx.current_token.value[0] == "+" initial_spec._add_flag( self.ctx.current_token.value[1:].strip(), variant_value, propagate=False ) elif self.ctx.accept(TokenType.PROPAGATED_BOOL_VARIANT): - self.hash_not_parsed_or_raise(initial_spec, self.ctx.current_token.value) variant_value = self.ctx.current_token.value[0:2] == "++" initial_spec._add_flag( self.ctx.current_token.value[2:].strip(), variant_value, propagate=True ) elif self.ctx.accept(TokenType.KEY_VALUE_PAIR): - self.hash_not_parsed_or_raise(initial_spec, self.ctx.current_token.value) name, value = self.ctx.current_token.value.split("=", maxsplit=1) name = name.strip("'\" ") value = value.strip("'\" ") initial_spec._add_flag(name, value, propagate=False) elif self.ctx.accept(TokenType.PROPAGATED_KEY_VALUE_PAIR): - self.hash_not_parsed_or_raise(initial_spec, self.ctx.current_token.value) name, value = self.ctx.current_token.value.split("==", maxsplit=1) name = name.strip("'\" ") value = value.strip("'\" ") @@ -411,12 +421,6 @@ def parse(self, initial_spec: Optional[spack.spec.Spec] = None) -> Optional[spac return initial_spec - def hash_not_parsed_or_raise(self, spec, addition): - if not self.has_hash: - return - - raise spack.spec.RedundantSpecError(spec, addition) - class FileParser: """Parse a single spec from a JSON or YAML file""" @@ -426,7 +430,7 @@ class FileParser: def __init__(self, ctx): self.ctx = ctx - def parse(self, initial_spec: spack.spec.Spec) -> spack.spec.Spec: + def parse(self, initial_spec: "spack.spec.Spec") -> "spack.spec.Spec": """Parse a spec tree from a specfile. Args: @@ -449,7 +453,42 @@ def parse(self, initial_spec: spack.spec.Spec) -> spack.spec.Spec: return initial_spec -def parse(text: str) -> List[spack.spec.Spec]: +class EdgeAttributeParser: + __slots__ = "ctx", "literal_str" + + def __init__(self, ctx, literal_str): + self.ctx = ctx + self.literal_str = literal_str + + def parse(self): + attributes = {} + while True: + if self.ctx.accept(TokenType.KEY_VALUE_PAIR): + name, value = self.ctx.current_token.value.split("=", maxsplit=1) + name = name.strip("'\" ") + value = value.strip("'\" ").split(",") + attributes[name] = value + if name not in ("deptypes", "virtuals"): + msg = ( + "the only edge attributes that are currently accepted " + 'are "deptypes" and "virtuals"' + ) + raise SpecParsingError(msg, self.ctx.current_token, self.literal_str) + # TODO: Add code to accept bool variants here as soon as use variants are implemented + elif self.ctx.accept(TokenType.END_EDGE_PROPERTIES): + break + else: + msg = "unexpected token in edge attributes" + raise SpecParsingError(msg, self.ctx.next_token, self.literal_str) + + # Turn deptypes=... to depflag representation + if "deptypes" in attributes: + deptype_string = attributes.pop("deptypes") + attributes["depflag"] = spack.deptypes.canonicalize(deptype_string) + return attributes + + +def parse(text: str) -> List["spack.spec.Spec"]: """Parse text into a list of strings Args: @@ -462,8 +501,8 @@ def parse(text: str) -> List[spack.spec.Spec]: def parse_one_or_raise( - text: str, initial_spec: Optional[spack.spec.Spec] = None -) -> spack.spec.Spec: + text: str, initial_spec: Optional["spack.spec.Spec"] = None +) -> "spack.spec.Spec": """Parse exactly one spec from text and return it, or raise Args: diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index e761102d2bc003..7e2fcaff103ef3 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -7,18 +7,20 @@ import inspect import os import os.path +import pathlib import sys import llnl.util.filesystem import llnl.util.lang +from llnl.url import allowed_archive import spack import spack.error import spack.fetch_strategy as fs +import spack.mirror import spack.repo import spack.stage import spack.util.spack_json as sjson -from spack.util.compression import allowed_archive from spack.util.crypto import Checker, checksum from spack.util.executable import which, which_string @@ -35,10 +37,12 @@ def apply_patch(stage, patch_path, level=1, working_dir="."): """ git_utils_path = os.environ.get("PATH", "") if sys.platform == "win32": - git = which_string("git", required=True) - git_root = git.split("\\")[:-2] - git_root.extend(["usr", "bin"]) - git_utils_path = os.sep.join(git_root) + git = which_string("git") + if git: + git = pathlib.Path(git) + git_root = git.parent.parent + git_root = git_root / "usr" / "bin" + git_utils_path = os.pathsep.join([str(git_root), git_utils_path]) # TODO: Decouple Spack's patch support on Windows from Git # for Windows, and instead have Spack directly fetch, install, and @@ -75,22 +79,14 @@ def __init__(self, pkg, path_or_url, level, working_dir): self.level = level self.working_dir = working_dir - def fetch(self): - """Fetch the patch in case of a UrlPatch""" - - def clean(self): - """Clean up the patch stage in case of a UrlPatch""" - - def apply(self, stage): + def apply(self, stage: "spack.stage.Stage"): """Apply a patch to source in a stage. Arguments: stage (spack.stage.Stage): stage where source code lives """ - assert self.path, "Path for patch not set in apply: %s" % self.path_or_url - - if not os.path.isfile(self.path): - raise NoSuchPatchError("No such patch: %s" % self.path) + if not self.path or not os.path.isfile(self.path): + raise NoSuchPatchError(f"No such patch: {self.path}") apply_patch(stage, self.path, self.level, self.working_dir) @@ -197,70 +193,44 @@ def __init__(self, pkg, url, level=1, working_dir=".", ordering_key=None, **kwar if not self.sha256: raise PatchDirectiveError("URL patches require a sha256 checksum") - def fetch(self): - """Retrieve the patch in a temporary stage and compute self.path + def apply(self, stage: "spack.stage.Stage"): + assert self.stage.expanded, "Stage must be expanded before applying patches" - Args: - stage: stage for the package that needs to be patched - """ - self.stage.create() - self.stage.fetch() - self.stage.check() + # Get the patch file. + files = os.listdir(self.stage.source_path) + assert len(files) == 1, "Expected one file in stage source path, found %s" % files + self.path = os.path.join(self.stage.source_path, files[0]) - root = self.stage.path - if self.archive_sha256: - self.stage.expand_archive() - root = self.stage.source_path - - files = os.listdir(root) - if not files: - if self.archive_sha256: - raise NoSuchPatchError("Archive was empty: %s" % self.url) - else: - raise NoSuchPatchError("Patch failed to download: %s" % self.url) - - self.path = os.path.join(root, files.pop()) - - if not os.path.isfile(self.path): - raise NoSuchPatchError("Archive %s contains no patch file!" % self.url) - - # for a compressed archive, Need to check the patch sha256 again - # and the patch is in a directory, not in the same place - if self.archive_sha256 and spack.config.get("config:checksum"): - checker = Checker(self.sha256) - if not checker.check(self.path): - raise fs.ChecksumError( - "sha256 checksum failed for %s" % self.path, - "Expected %s but got %s" % (self.sha256, checker.sum), - ) + return super().apply(stage) @property def stage(self): if self._stage: return self._stage - # use archive digest for compressed archives - fetch_digest = self.sha256 - if self.archive_sha256: - fetch_digest = self.archive_sha256 + fetch_digest = self.archive_sha256 or self.sha256 - fetcher = fs.URLFetchStrategy(self.url, fetch_digest, expand=bool(self.archive_sha256)) + # Two checksums, one for compressed file, one for its contents + if self.archive_sha256: + fetcher = fs.FetchAndVerifyExpandedFile( + self.url, archive_sha256=self.archive_sha256, expanded_sha256=self.sha256 + ) + else: + fetcher = fs.URLFetchStrategy(self.url, sha256=self.sha256, expand=False) # The same package can have multiple patches with the same name but # with different contents, therefore apply a subset of the hash. name = "{0}-{1}".format(os.path.basename(self.url), fetch_digest[:7]) per_package_ref = os.path.join(self.owner.split(".")[-1], name) - # Reference starting with "spack." is required to avoid cyclic imports mirror_ref = spack.mirror.mirror_archive_paths(fetcher, per_package_ref) - - self._stage = spack.stage.Stage(fetcher, mirror_paths=mirror_ref) - self._stage.create() + self._stage = spack.stage.Stage( + fetcher, + name=f"{spack.stage.stage_prefix}patch-{fetch_digest}", + mirror_paths=mirror_ref, + ) return self._stage - def clean(self): - self.stage.destroy() - def to_dict(self): data = super().to_dict() data["url"] = self.url @@ -271,7 +241,7 @@ def to_dict(self): def from_dict(dictionary, repository=None): """Create a patch from json dictionary.""" - repository = repository or spack.repo.path + repository = repository or spack.repo.PATH owner = dictionary.get("owner") if "owner" not in dictionary: raise ValueError("Invalid patch dictionary: %s" % dictionary) @@ -345,21 +315,19 @@ def from_json(cls, stream, repository): def to_json(self, stream): sjson.dump({"patches": self.index}, stream) - def patch_for_package(self, sha256, pkg): + def patch_for_package(self, sha256: str, pkg): """Look up a patch in the index and build a patch object for it. Arguments: - sha256 (str): sha256 hash to look up + sha256: sha256 hash to look up pkg (spack.package_base.PackageBase): Package object to get patch for. We build patch objects lazily because building them requires that - we have information about the package's location in its repo. - - """ + we have information about the package's location in its repo.""" sha_index = self.index.get(sha256) if not sha_index: - raise NoSuchPatchError( - "Couldn't find patch for package %s with sha256: %s" % (pkg.fullname, sha256) + raise PatchLookupError( + f"Couldn't find patch for package {pkg.fullname} with sha256: {sha256}" ) # Find patches for this class or any class it inherits from @@ -368,8 +336,8 @@ def patch_for_package(self, sha256, pkg): if patch_dict: break else: - raise NoSuchPatchError( - "Couldn't find patch for package %s with sha256: %s" % (pkg.fullname, sha256) + raise PatchLookupError( + f"Couldn't find patch for package {pkg.fullname} with sha256: {sha256}" ) # add the sha256 back (we take it out on write to save space, @@ -438,5 +406,9 @@ class NoSuchPatchError(spack.error.SpackError): """Raised when a patch file doesn't exist.""" +class PatchLookupError(NoSuchPatchError): + """Raised when a patch file cannot be located from sha256.""" + + class PatchDirectiveError(spack.error.SpackError): """Raised when the wrong arguments are suppled to the patch directive.""" diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 18656b76c26c41..4e66adf11a6a1f 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -10,11 +10,12 @@ dependencies. """ import os +from pathlib import PurePath import llnl.util.filesystem #: This file lives in $prefix/lib/spack/spack/__file__ -prefix = llnl.util.filesystem.ancestor(__file__, 4) +prefix = str(PurePath(llnl.util.filesystem.ancestor(__file__, 4))) #: synonym for prefix spack_root = prefix @@ -88,7 +89,7 @@ def _get_user_cache_path(): return os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or "~%s.spack" % os.sep) -user_cache_path = _get_user_cache_path() +user_cache_path = str(PurePath(_get_user_cache_path())) #: junit, cdash, etc. reports about builds reports_path = os.path.join(user_cache_path, "reports") diff --git a/lib/spack/spack/platforms/__init__.py b/lib/spack/spack/platforms/__init__.py index f83f5fb98e361c..1a41d26971ac15 100644 --- a/lib/spack/spack/platforms/__init__.py +++ b/lib/spack/spack/platforms/__init__.py @@ -64,7 +64,7 @@ def use_platform(new_platform): host = _PickleableCallable(new_platform) # Clear configuration and compiler caches - spack.config.config.clear_caches() + spack.config.CONFIG.clear_caches() spack.compilers._cache_config_files = [] yield new_platform @@ -73,5 +73,5 @@ def use_platform(new_platform): host = original_host_fn # Clear configuration and compiler caches - spack.config.config.clear_caches() + spack.config.CONFIG.clear_caches() spack.compilers._cache_config_files = [] diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index 7028b0db34d4ad..af40510b2c0469 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -139,6 +139,8 @@ def craype_type_and_version(cls): # If no default version, sort available versions and return latest versions_available = [spack.version.Version(v) for v in os.listdir(craype_dir)] versions_available.sort(reverse=True) + if not versions_available: + return (craype_type, None) return (craype_type, versions_available[0]) @classmethod diff --git a/lib/spack/spack/provider_index.py b/lib/spack/spack/provider_index.py index 2624de56acd88e..32ace00a1669e7 100644 --- a/lib/spack/spack/provider_index.py +++ b/lib/spack/spack/provider_index.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Classes and functions to manage providers of virtual dependencies""" -import itertools from typing import Dict, List, Optional, Set import spack.error @@ -11,33 +10,6 @@ import spack.util.spack_json as sjson -def _cross_provider_maps(lmap, rmap): - """Return a dictionary that combines constraint requests from both input. - - Args: - lmap: main provider map - rmap: provider map with additional constraints - """ - # TODO: this is pretty darned nasty, and inefficient, but there - # TODO: are not that many vdeps in most specs. - result = {} - for lspec, rspec in itertools.product(lmap, rmap): - try: - constrained = lspec.constrained(rspec) - except spack.error.UnsatisfiableSpecError: - continue - - # lp and rp are left and right provider specs. - for lp_spec, rp_spec in itertools.product(lmap[lspec], rmap[rspec]): - if lp_spec.name == rp_spec.name: - try: - const = lp_spec.constrained(rp_spec, deps=False) - result.setdefault(constrained, set()).add(const) - except spack.error.UnsatisfiableSpecError: - continue - return result - - class _IndexBase: #: This is a dict of dicts used for finding providers of particular #: virtual dependencies. The dict of dicts looks like: @@ -81,29 +53,6 @@ def providers_for(self, virtual_spec): def __contains__(self, name): return name in self.providers - def satisfies(self, other): - """Determine if the providers of virtual specs are compatible. - - Args: - other: another provider index - - Returns: - True if the providers are compatible, False otherwise. - """ - common = set(self.providers) & set(other.providers) - if not common: - return True - - # This ensures that some provider in other COULD satisfy the - # vpkg constraints on self. - result = {} - for name in common: - crossed = _cross_provider_maps(self.providers[name], other.providers[name]) - if crossed: - result[name] = crossed - - return all(c in result for c in common) - def __eq__(self, other): return self.providers == other.providers diff --git a/lib/spack/spack/relocate.py b/lib/spack/spack/relocate.py index 30cdb4bd064d4b..756e3fa9dc60d1 100644 --- a/lib/spack/spack/relocate.py +++ b/lib/spack/spack/relocate.py @@ -599,19 +599,6 @@ def make_elf_binaries_relative(new_binaries, orig_binaries, orig_layout_root): _set_elf_rpaths(new_binary, new_rpaths) -def ensure_binaries_are_relocatable(binaries): - """Raise an error if any binary in the list is not relocatable. - - Args: - binaries (list): list of binaries to check - - Raises: - InstallRootStringError: if the file is not relocatable - """ - for binary in binaries: - ensure_binary_is_relocatable(binary) - - def warn_if_link_cant_be_relocated(link, target): if not os.path.isabs(target): return @@ -663,83 +650,6 @@ def relocate_text_bin(binaries, prefixes): return BinaryFilePrefixReplacer.from_strings_or_bytes(prefixes).apply(binaries) -def is_relocatable(spec): - """Returns True if an installed spec is relocatable. - - Args: - spec (spack.spec.Spec): spec to be analyzed - - Returns: - True if the binaries of an installed spec - are relocatable and False otherwise. - - Raises: - ValueError: if the spec is not installed - """ - if not spec.installed: - raise ValueError("spec is not installed [{0}]".format(str(spec))) - - if spec.external or spec.virtual: - tty.warn("external or virtual package %s is not relocatable" % spec.name) - return False - - # Explore the installation prefix of the spec - for root, dirs, files in os.walk(spec.prefix, topdown=True): - dirs[:] = [d for d in dirs if d not in (".spack", "man")] - try: - abs_paths = (os.path.join(root, f) for f in files) - ensure_binaries_are_relocatable(filter(is_binary, abs_paths)) - except InstallRootStringError: - return False - - return True - - -def ensure_binary_is_relocatable(filename, paths_to_relocate=None): - """Raises if any given or default absolute path is found in the - binary (apart from rpaths / load commands). - - Args: - filename: absolute path of the file to be analyzed - - Raises: - InstallRootStringError: if the binary contains an absolute path - ValueError: if the filename does not exist or the path is not absolute - """ - paths_to_relocate = paths_to_relocate or [spack.store.layout.root, spack.paths.prefix] - - if not os.path.exists(filename): - raise ValueError("{0} does not exist".format(filename)) - - if not os.path.isabs(filename): - raise ValueError("{0} is not an absolute path".format(filename)) - - strings = executable.Executable("strings") - - # Remove the RPATHS from the strings in the executable - set_of_strings = set(strings(filename, output=str).split()) - - m_type, m_subtype = fs.mime_type(filename) - if m_type == "application": - tty.debug("{0},{1}".format(m_type, m_subtype), level=2) - - if not is_macos: - if m_subtype == "x-executable" or m_subtype == "x-sharedlib": - rpaths = ":".join(_elf_rpaths_for(filename)) - set_of_strings.discard(rpaths) - else: - if m_subtype == "x-mach-binary": - rpaths, deps, idpath = macholib_get_paths(filename) - set_of_strings.discard(set(rpaths)) - set_of_strings.discard(set(deps)) - if idpath is not None: - set_of_strings.discard(idpath) - - for path_to_relocate in paths_to_relocate: - if any(path_to_relocate in x for x in set_of_strings): - raise InstallRootStringError(filename, path_to_relocate) - - def is_binary(filename): """Returns true if a file is binary, False otherwise @@ -793,7 +703,7 @@ def fixup_macos_rpath(root, filename): args = [] # Check dependencies for non-rpath entries - spack_root = spack.store.layout.root + spack_root = spack.store.STORE.layout.root for name in deps: if name.startswith(spack_root): tty.debug("Spack-installed dependency for {0}: {1}".format(abspath, name)) diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index dba5b0c337aec9..5918454005df85 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -6,6 +6,7 @@ import abc import collections.abc import contextlib +import difflib import errno import functools import importlib @@ -24,8 +25,9 @@ import traceback import types import uuid -from typing import Dict, Union +from typing import Any, Dict, List, Tuple, Union +import llnl.path import llnl.util.filesystem as fs import llnl.util.lang import llnl.util.tty as tty @@ -149,7 +151,7 @@ def compute_loader(self, fullname): # If it's a module in some repo, or if it is the repo's # namespace, let the repo handle it. - for repo in path.repos: + for repo in PATH.repos: # We are using the namespace of the repo and the repo contains the package if namespace == repo.full_namespace: # With 2 nested conditionals we can call "repo.real_name" only once @@ -163,7 +165,7 @@ def compute_loader(self, fullname): # No repo provides the namespace, but it is a valid prefix of # something in the RepoPath. - if path.by_namespace.is_prefix(fullname): + if PATH.by_namespace.is_prefix(fullname): return SpackNamespaceLoader() return None @@ -184,9 +186,9 @@ def compute_loader(self, fullname): def packages_path(): """Get the test repo if it is active, otherwise the builtin repo.""" try: - return spack.repo.path.get_repo("builtin.mock").packages_path + return spack.repo.PATH.get_repo("builtin.mock").packages_path except spack.repo.UnknownNamespaceError: - return spack.repo.path.get_repo("builtin").packages_path + return spack.repo.PATH.get_repo("builtin").packages_path class GitExe: @@ -282,7 +284,7 @@ def add_package_to_git_stage(packages): git = GitExe() for pkg_name in packages: - filename = spack.repo.path.filename_for_package_name(pkg_name) + filename = spack.repo.PATH.filename_for_package_name(pkg_name) if not os.path.isfile(filename): tty.die("No such package: %s. Path does not exist:" % pkg_name, filename) @@ -387,7 +389,7 @@ def _create_new_cache(self) -> Dict[str, os.stat_result]: # Warn about invalid names that look like packages. if not nm.valid_module_name(pkg_name): - if not pkg_name.startswith("."): + if not pkg_name.startswith(".") and pkg_name != "repo.yaml": tty.warn( 'Skipping package at {0}. "{1}" is not ' "a valid Spack module name.".format(pkg_dir, pkg_name) @@ -422,7 +424,7 @@ def _create_new_cache(self) -> Dict[str, os.stat_result]: def last_mtime(self): return max(sinfo.st_mtime for sinfo in self._packages_to_stats.values()) - def modified_since(self, since): + def modified_since(self, since: float) -> List[str]: return [name for name, sinfo in self._packages_to_stats.items() if sinfo.st_mtime > since] def __getitem__(self, item): @@ -548,35 +550,34 @@ class RepoIndex: when they're needed. ``Indexers`` should be added to the ``RepoIndex`` using - ``add_index(name, indexer)``, and they should support the interface + ``add_indexer(name, indexer)``, and they should support the interface defined by ``Indexer``, so that the ``RepoIndex`` can read, generate, and update stored indices. - Generated indexes are accessed by name via ``__getitem__()``. + Generated indexes are accessed by name via ``__getitem__()``.""" - """ - - def __init__(self, package_checker, namespace, cache): + def __init__( + self, + package_checker: FastPackageChecker, + namespace: str, + cache: spack.util.file_cache.FileCache, + ): self.checker = package_checker self.packages_path = self.checker.packages_path if sys.platform == "win32": - self.packages_path = spack.util.path.convert_to_posix_path(self.packages_path) + self.packages_path = llnl.path.convert_to_posix_path(self.packages_path) self.namespace = namespace - self.indexers = {} - self.indexes = {} + self.indexers: Dict[str, Indexer] = {} + self.indexes: Dict[str, Any] = {} self.cache = cache - def add_indexer(self, name, indexer): + def add_indexer(self, name: str, indexer: Indexer): """Add an indexer to the repo index. Arguments: - name (str): name of this indexer - - indexer (object): an object that supports create(), read(), - write(), and get_index() operations - - """ + name: name of this indexer + indexer: object implementing the ``Indexer`` interface""" self.indexers[name] = indexer def __getitem__(self, name): @@ -597,17 +598,15 @@ def _build_all_indexes(self): because the main bottleneck here is loading all the packages. It can take tens of seconds to regenerate sequentially, and we'd rather only pay that cost once rather than on several - invocations. - - """ + invocations.""" for name, indexer in self.indexers.items(): self.indexes[name] = self._build_index(name, indexer) - def _build_index(self, name, indexer): + def _build_index(self, name: str, indexer: Indexer): """Determine which packages need an update, and update indexes.""" # Filename of the provider index cache (we assume they're all json) - cache_filename = "{0}/{1}-index.json".format(name, self.namespace) + cache_filename = f"{name}/{self.namespace}-index.json" # Compute which packages needs to be updated in the cache index_mtime = self.cache.mtime(cache_filename) @@ -631,8 +630,7 @@ def _build_index(self, name, indexer): needs_update = self.checker.modified_since(new_index_mtime) for pkg_name in needs_update: - namespaced_name = "%s.%s" % (self.namespace, pkg_name) - indexer.update(namespaced_name) + indexer.update(f"{self.namespace}.{pkg_name}") indexer.write(new) @@ -651,7 +649,7 @@ class RepoPath: """ def __init__(self, *repos, **kwargs): - cache = kwargs.get("cache", spack.caches.misc_cache) + cache = kwargs.get("cache", spack.caches.MISC_CACHE) self.repos = [] self.by_namespace = nm.NamespaceTrie() @@ -748,10 +746,18 @@ def all_package_paths(self): for name in self.all_package_names(): yield self.package_path(name) - def packages_with_tags(self, *tags): + def packages_with_tags(self, *tags, full=False): + """Returns a list of packages matching any of the tags in input. + + Args: + full: if True the package names in the output are fully-qualified + """ r = set() for repo in self.repos: - r |= set(repo.packages_with_tags(*tags)) + current = repo.packages_with_tags(*tags) + if full: + current = [f"{repo.namespace}.{x}" for x in current] + r |= set(current) return sorted(r) def all_package_classes(self): @@ -970,7 +976,7 @@ def check(condition, msg): # Indexes for this repository, computed lazily self._repo_index = None - self._cache = cache or spack.caches.misc_cache + self._cache = cache or spack.caches.MISC_CACHE def real_name(self, import_name): """Allow users to import Spack packages using Python identifiers. @@ -1127,7 +1133,8 @@ def extensions_for(self, extendee_spec): def dirname_for_package_name(self, pkg_name): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" - return os.path.join(self.packages_path, pkg_name) + _, unqualified_name = self.partition_package_name(pkg_name) + return os.path.join(self.packages_path, unqualified_name) def filename_for_package_name(self, pkg_name): """Get the filename for the module we should load for a particular @@ -1225,15 +1232,10 @@ def get_pkg_class(self, pkg_name): package. Then extracts the package class from the module according to Spack's naming convention. """ - namespace, _, pkg_name = pkg_name.rpartition(".") - if namespace and (namespace != self.namespace): - raise InvalidNamespaceError( - "Invalid namespace for %s repo: %s" % (self.namespace, namespace) - ) - + namespace, pkg_name = self.partition_package_name(pkg_name) class_name = nm.mod_to_class(pkg_name) + fullname = f"{self.full_namespace}.{pkg_name}" - fullname = "{0}.{1}".format(self.full_namespace, pkg_name) try: module = importlib.import_module(fullname) except ImportError: @@ -1244,7 +1246,7 @@ def get_pkg_class(self, pkg_name): cls = getattr(module, class_name) if not inspect.isclass(cls): - tty.die("%s.%s is not a class" % (pkg_name, class_name)) + tty.die(f"{pkg_name}.{class_name} is not a class") new_cfg_settings = ( spack.config.get("packages").get(pkg_name, {}).get("package_attributes", {}) @@ -1283,6 +1285,15 @@ def get_pkg_class(self, pkg_name): return cls + def partition_package_name(self, pkg_name: str) -> Tuple[str, str]: + namespace, pkg_name = partition_package_name(pkg_name) + if namespace and (namespace != self.namespace): + raise InvalidNamespaceError( + f"Invalid namespace for the '{self.namespace}' repo: {namespace}" + ) + + return namespace, pkg_name + def __str__(self): return "[Repo '%s' at '%s']" % (self.namespace, self.root) @@ -1296,6 +1307,20 @@ def __contains__(self, pkg_name): RepoType = Union[Repo, RepoPath] +def partition_package_name(pkg_name: str) -> Tuple[str, str]: + """Given a package name that might be fully-qualified, returns the namespace part, + if present and the unqualified package name. + + If the package name is unqualified, the namespace is an empty string. + + Args: + pkg_name: a package name, either unqualified like "llvl", or + fully-qualified, like "builtin.llvm" + """ + namespace, _, pkg_name = pkg_name.rpartition(".") + return namespace, pkg_name + + def create_repo(root, namespace=None, subdir=packages_dir_name): """Create a new repository in root with the specified namespace. @@ -1361,7 +1386,7 @@ def create_or_construct(path, namespace=None): def _path(configuration=None): """Get the singleton RepoPath instance for Spack.""" - configuration = configuration or spack.config.config + configuration = configuration or spack.config.CONFIG return create(configuration=configuration) @@ -1378,7 +1403,7 @@ def create(configuration): #: Singleton repo path instance -path: Union[RepoPath, llnl.util.lang.Singleton] = llnl.util.lang.Singleton(_path) +PATH: Union[RepoPath, llnl.util.lang.Singleton] = llnl.util.lang.Singleton(_path) # Add the finder to sys.meta_path REPOS_FINDER = ReposFinder() @@ -1387,7 +1412,7 @@ def create(configuration): def all_package_names(include_virtuals=False): """Convenience wrapper around ``spack.repo.all_package_names()``.""" - return path.all_package_names(include_virtuals) + return PATH.all_package_names(include_virtuals) @contextlib.contextmanager @@ -1402,21 +1427,21 @@ def use_repositories(*paths_and_repos, **kwargs): Returns: Corresponding RepoPath object """ - global path + global PATH # TODO (Python 2.7): remove this kwargs on deprecation of Python 2.7 support override = kwargs.get("override", True) paths = [getattr(x, "root", x) for x in paths_and_repos] scope_name = "use-repo-{}".format(uuid.uuid4()) repos_key = "repos:" if override else "repos" - spack.config.config.push_scope( + spack.config.CONFIG.push_scope( spack.config.InternalConfigScope(name=scope_name, data={repos_key: paths}) ) - path, saved = create(configuration=spack.config.config), path + PATH, saved = create(configuration=spack.config.CONFIG), PATH try: - yield path + yield PATH finally: - spack.config.config.remove_scope(scope_name=scope_name) - path = saved + spack.config.CONFIG.remove_scope(scope_name=scope_name) + PATH = saved class MockRepositoryBuilder: @@ -1472,10 +1497,6 @@ class UnknownEntityError(RepoError): """Raised when we encounter a package spack doesn't have.""" -class IndexError(RepoError): - """Raised when there's an error with an index.""" - - class UnknownPackageError(UnknownEntityError): """Raised when we encounter a package spack doesn't have.""" @@ -1496,7 +1517,18 @@ def __init__(self, name, repo=None): long_msg = "Did you mean to specify a filename with './{0}'?" long_msg = long_msg.format(name) else: - long_msg = "You may need to run 'spack clean -m'." + long_msg = "Use 'spack create' to create a new package." + + if not repo: + repo = spack.repo.PATH + + # We need to compare the base package name + pkg_name = name.rsplit(".", 1)[-1] + similar = difflib.get_close_matches(pkg_name, repo.all_package_names()) + + if 1 <= len(similar) <= 5: + long_msg += "\n\nDid you mean one of the following packages?\n " + long_msg += "\n ".join(similar) super().__init__(msg, long_msg) self.name = name diff --git a/lib/spack/spack/rewiring.py b/lib/spack/spack/rewiring.py index 75684f7a2a6ebe..2c18827e872294 100644 --- a/lib/spack/spack/rewiring.py +++ b/lib/spack/spack/rewiring.py @@ -80,8 +80,8 @@ def rewire_node(spec, explicit): if "macho" in platform.binary_formats: relocate.relocate_macho_binaries( bins_to_relocate, - str(spack.store.layout.root), - str(spack.store.layout.root), + str(spack.store.STORE.layout.root), + str(spack.store.STORE.layout.root), prefix_to_prefix, False, spec.build_spec.prefix, @@ -90,8 +90,8 @@ def rewire_node(spec, explicit): if "elf" in platform.binary_formats: relocate.relocate_elf_binaries( bins_to_relocate, - str(spack.store.layout.root), - str(spack.store.layout.root), + str(spack.store.STORE.layout.root), + str(spack.store.STORE.layout.root), prefix_to_prefix, False, spec.build_spec.prefix, @@ -114,9 +114,9 @@ def rewire_node(spec, explicit): # (spliced) spec into spec.json, without this, Database.add would fail on # the next line (because it checks the spec.json in the prefix against the # spec being added to look for mismatches) - spack.store.layout.write_spec(spec, spack.store.layout.spec_file_path(spec)) + spack.store.STORE.layout.write_spec(spec, spack.store.STORE.layout.spec_file_path(spec)) # add to database, not sure about explicit - spack.store.db.add(spec, spack.store.layout, explicit=explicit) + spack.store.STORE.db.add(spec, spack.store.STORE.layout, explicit=explicit) # run post install hooks spack.hooks.post_install(spec, explicit) diff --git a/lib/spack/spack/s3_handler.py b/lib/spack/spack/s3_handler.py deleted file mode 100644 index efab23a5ea9f62..00000000000000 --- a/lib/spack/spack/s3_handler.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import urllib.error -import urllib.parse -import urllib.request -import urllib.response -from io import BufferedReader, BytesIO, IOBase - -import spack.util.s3 as s3_util - - -# NOTE(opadron): Workaround issue in boto where its StreamingBody -# implementation is missing several APIs expected from IOBase. These missing -# APIs prevent the streams returned by boto from being passed as-are along to -# urllib. -# -# https://github.com/boto/botocore/issues/879 -# https://github.com/python/cpython/pull/3249 -class WrapStream(BufferedReader): - def __init__(self, raw): - # In botocore >=1.23.47, StreamingBody inherits from IOBase, so we - # only add missing attributes in older versions. - # https://github.com/boto/botocore/commit/a624815eabac50442ed7404f3c4f2664cd0aa784 - if not isinstance(raw, IOBase): - raw.readable = lambda: True - raw.writable = lambda: False - raw.seekable = lambda: False - raw.closed = False - raw.flush = lambda: None - super().__init__(raw) - - def detach(self): - self.raw = None - - def read(self, *args, **kwargs): - return self.raw.read(*args, **kwargs) - - def __getattr__(self, key): - return getattr(self.raw, key) - - -def _s3_open(url, method="GET"): - parsed = urllib.parse.urlparse(url) - s3 = s3_util.get_s3_session(url, method="fetch") - - bucket = parsed.netloc - key = parsed.path - - if key.startswith("/"): - key = key[1:] - - if method not in ("GET", "HEAD"): - raise urllib.error.URLError( - "Only GET and HEAD verbs are currently supported for the s3:// scheme" - ) - - try: - if method == "GET": - obj = s3.get_object(Bucket=bucket, Key=key) - # NOTE(opadron): Apply workaround here (see above) - stream = WrapStream(obj["Body"]) - elif method == "HEAD": - obj = s3.head_object(Bucket=bucket, Key=key) - stream = BytesIO() - except s3.ClientError as e: - raise urllib.error.URLError(e) from e - - headers = obj["ResponseMetadata"]["HTTPHeaders"] - - return url, headers, stream - - -class UrllibS3Handler(urllib.request.BaseHandler): - def s3_open(self, req): - orig_url = req.get_full_url() - url, headers, stream = _s3_open(orig_url, method=req.get_method()) - return urllib.response.addinfourl(stream, headers, url) diff --git a/lib/spack/spack/schema/__init__.py b/lib/spack/spack/schema/__init__.py index f99f47a455e42c..bdb1a272d03754 100644 --- a/lib/spack/spack/schema/__init__.py +++ b/lib/spack/spack/schema/__init__.py @@ -62,3 +62,25 @@ def _deprecated_properties(validator, deprecated, instance, schema): Validator = llnl.util.lang.Singleton(_make_validator) + +spec_list_schema = { + "type": "array", + "default": [], + "items": { + "anyOf": [ + { + "type": "object", + "additionalProperties": False, + "properties": { + "matrix": { + "type": "array", + "items": {"type": "array", "items": {"type": "string"}}, + }, + "exclude": {"type": "array", "items": {"type": "string"}}, + }, + }, + {"type": "string"}, + {"type": "null"}, + ] + }, +} diff --git a/lib/spack/spack/schema/buildcache_spec.py b/lib/spack/spack/schema/buildcache_spec.py index eeeb6dc08524c5..7c74c7f4eba012 100644 --- a/lib/spack/spack/schema/buildcache_spec.py +++ b/lib/spack/spack/schema/buildcache_spec.py @@ -6,7 +6,7 @@ """Schema for a buildcache spec.yaml file .. literalinclude:: _spack_root/lib/spack/spack/schema/buildcache_spec.py - :lines: 14- + :lines: 13- """ import spack.schema.spec @@ -16,15 +16,8 @@ "type": "object", "additionalProperties": False, "properties": { - "buildinfo": { - "type": "object", - "additionalProperties": False, - "required": ["relative_prefix"], - "properties": { - "relative_prefix": {"type": "string"}, - "relative_rpaths": {"type": "boolean"}, - }, - }, + # `buildinfo` is no longer needed as of Spack 0.21 + "buildinfo": {"type": "object"}, "spec": { "type": "object", "additionalProperties": True, diff --git a/lib/spack/spack/schema/ci.py b/lib/spack/spack/schema/ci.py index 92edf2f13968bc..9ba65b26820830 100644 --- a/lib/spack/spack/schema/ci.py +++ b/lib/spack/spack/schema/ci.py @@ -141,6 +141,7 @@ } ) +# TODO: Remove in Spack 0.23 ci_properties = { "anyOf": [ { @@ -166,6 +167,7 @@ properties = { "ci": { "oneOf": [ + # TODO: Replace with core-shared-properties in Spack 0.23 ci_properties, # Allow legacy format under `ci` for `config update ci` spack.schema.gitlab_ci.gitlab_ci_properties, diff --git a/lib/spack/spack/schema/compilers.py b/lib/spack/spack/schema/compilers.py index 6caaf9cc2385b6..924fee7a21ff76 100644 --- a/lib/spack/spack/schema/compilers.py +++ b/lib/spack/spack/schema/compilers.py @@ -14,63 +14,61 @@ properties = { "compilers": { "type": "array", - "items": [ - { - "type": "object", - "additionalProperties": False, - "properties": { - "compiler": { - "type": "object", - "additionalProperties": False, - "required": ["paths", "spec", "modules", "operating_system"], - "properties": { - "paths": { - "type": "object", - "required": ["cc", "cxx", "f77", "fc"], - "additionalProperties": False, - "properties": { - "cc": {"anyOf": [{"type": "string"}, {"type": "null"}]}, - "cxx": {"anyOf": [{"type": "string"}, {"type": "null"}]}, - "f77": {"anyOf": [{"type": "string"}, {"type": "null"}]}, - "fc": {"anyOf": [{"type": "string"}, {"type": "null"}]}, - }, + "items": { + "type": "object", + "additionalProperties": False, + "properties": { + "compiler": { + "type": "object", + "additionalProperties": False, + "required": ["paths", "spec", "modules", "operating_system"], + "properties": { + "paths": { + "type": "object", + "required": ["cc", "cxx", "f77", "fc"], + "additionalProperties": False, + "properties": { + "cc": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "cxx": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "f77": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "fc": {"anyOf": [{"type": "string"}, {"type": "null"}]}, }, - "flags": { - "type": "object", - "additionalProperties": False, - "properties": { - "cflags": {"anyOf": [{"type": "string"}, {"type": "null"}]}, - "cxxflags": {"anyOf": [{"type": "string"}, {"type": "null"}]}, - "fflags": {"anyOf": [{"type": "string"}, {"type": "null"}]}, - "cppflags": {"anyOf": [{"type": "string"}, {"type": "null"}]}, - "ldflags": {"anyOf": [{"type": "string"}, {"type": "null"}]}, - "ldlibs": {"anyOf": [{"type": "string"}, {"type": "null"}]}, - }, - }, - "spec": {"type": "string"}, - "operating_system": {"type": "string"}, - "target": {"type": "string"}, - "alias": {"anyOf": [{"type": "string"}, {"type": "null"}]}, - "modules": { - "anyOf": [{"type": "string"}, {"type": "null"}, {"type": "array"}] - }, - "implicit_rpaths": { - "anyOf": [ - {"type": "array", "items": {"type": "string"}}, - {"type": "boolean"}, - ] - }, - "environment": spack.schema.environment.definition, - "extra_rpaths": { - "type": "array", - "default": [], - "items": {"type": "string"}, + }, + "flags": { + "type": "object", + "additionalProperties": False, + "properties": { + "cflags": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "cxxflags": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "fflags": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "cppflags": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "ldflags": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "ldlibs": {"anyOf": [{"type": "string"}, {"type": "null"}]}, }, }, - } - }, - } - ], + "spec": {"type": "string"}, + "operating_system": {"type": "string"}, + "target": {"type": "string"}, + "alias": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "modules": { + "anyOf": [{"type": "string"}, {"type": "null"}, {"type": "array"}] + }, + "implicit_rpaths": { + "anyOf": [ + {"type": "array", "items": {"type": "string"}}, + {"type": "boolean"}, + ] + }, + "environment": spack.schema.environment.definition, + "extra_rpaths": { + "type": "array", + "default": [], + "items": {"type": "string"}, + }, + }, + } + }, + }, } } diff --git a/lib/spack/spack/schema/concretizer.py b/lib/spack/spack/schema/concretizer.py index a62786f4040fc0..a4bd82e267c190 100644 --- a/lib/spack/spack/schema/concretizer.py +++ b/lib/spack/spack/schema/concretizer.py @@ -28,6 +28,12 @@ "unify": { "oneOf": [{"type": "boolean"}, {"type": "string", "enum": ["when_possible"]}] }, + "duplicates": { + "type": "object", + "properties": { + "strategy": {"type": "string", "enum": ["none", "minimal", "full"]} + }, + }, }, } } diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 106e45ce7ab5c1..6818cd78f39079 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -87,15 +87,17 @@ "anyOf": [{"type": "integer", "minimum": 1}, {"type": "null"}] }, "allow_sgid": {"type": "boolean"}, + "install_status": {"type": "boolean"}, "binary_index_root": {"type": "string"}, "url_fetch_method": {"type": "string", "enum": ["urllib", "curl"]}, "additional_external_search_paths": {"type": "array", "items": {"type": "string"}}, "binary_index_ttl": {"type": "integer", "minimum": 0}, + "aliases": {"type": "object", "patternProperties": {r"\w[\w-]*": {"type": "string"}}}, }, "deprecatedProperties": { - "properties": ["module_roots"], - "message": "config:module_roots has been replaced by " - "modules:[module set]:roots and is ignored", + "properties": ["terminal_title"], + "message": "config:terminal_title has been replaced by " + "install_status and is ignored", "error": False, }, } diff --git a/lib/spack/spack/schema/container.py b/lib/spack/spack/schema/container.py index cf63dd58aa6fe2..ef409c3d56bf0e 100644 --- a/lib/spack/spack/schema/container.py +++ b/lib/spack/spack/schema/container.py @@ -94,15 +94,6 @@ "docker": {"type": "object", "additionalProperties": False, "default": {}}, "depfile": {"type": "boolean", "default": False}, }, - "deprecatedProperties": { - "properties": ["extra_instructions"], - "message": ( - "container:extra_instructions has been deprecated and will be removed " - "in Spack v0.21. Set container:template appropriately to use custom Jinja2 " - "templates instead." - ), - "error": False, - }, } properties = {"container": container_schema} diff --git a/lib/spack/spack/schema/definitions.py b/lib/spack/spack/schema/definitions.py new file mode 100644 index 00000000000000..470eb7e8989ce4 --- /dev/null +++ b/lib/spack/spack/schema/definitions.py @@ -0,0 +1,34 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Schema for definitions + +.. literalinclude:: _spack_root/lib/spack/spack/schema/definitions.py + :lines: 13- +""" + +import spack.schema + +#: Properties for inclusion in other schemas +properties = { + "definitions": { + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": {"when": {"type": "string"}}, + "patternProperties": {r"^(?!when$)\w*": spack.schema.spec_list_schema}, + }, + } +} + +#: Full schema with metadata +schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Spack definitions configuration file schema", + "type": "object", + "additionalProperties": False, + "properties": properties, +} diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index 6548ca4b2b400d..463c6680f0d47e 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -12,34 +12,11 @@ import spack.schema.gitlab_ci # DEPRECATED import spack.schema.merged -import spack.schema.packages import spack.schema.projections #: Top level key in a manifest file TOP_LEVEL_KEY = "spack" -spec_list_schema = { - "type": "array", - "default": [], - "items": { - "anyOf": [ - { - "type": "object", - "additionalProperties": False, - "properties": { - "matrix": { - "type": "array", - "items": {"type": "array", "items": {"type": "string"}}, - }, - "exclude": {"type": "array", "items": {"type": "string"}}, - }, - }, - {"type": "string"}, - {"type": "null"}, - ] - }, -} - projections_scheme = spack.schema.projections.properties["projections"] schema = { @@ -75,16 +52,7 @@ } }, }, - "definitions": { - "type": "array", - "default": [], - "items": { - "type": "object", - "properties": {"when": {"type": "string"}}, - "patternProperties": {r"^(?!when$)\w*": spec_list_schema}, - }, - }, - "specs": spec_list_schema, + "specs": spack.schema.spec_list_schema, "view": { "anyOf": [ {"type": "boolean"}, diff --git a/lib/spack/spack/schema/merged.py b/lib/spack/spack/schema/merged.py index b20700a03cebfb..7ceb6494108d0e 100644 --- a/lib/spack/spack/schema/merged.py +++ b/lib/spack/spack/schema/merged.py @@ -17,6 +17,7 @@ import spack.schema.concretizer import spack.schema.config import spack.schema.container +import spack.schema.definitions import spack.schema.mirrors import spack.schema.modules import spack.schema.packages @@ -32,6 +33,7 @@ spack.schema.config.properties, spack.schema.container.properties, spack.schema.ci.properties, + spack.schema.definitions.properties, spack.schema.mirrors.properties, spack.schema.modules.properties, spack.schema.packages.properties, diff --git a/lib/spack/spack/schema/mirrors.py b/lib/spack/spack/schema/mirrors.py index 41dbddb051ad93..8001172afd4b64 100644 --- a/lib/spack/spack/schema/mirrors.py +++ b/lib/spack/spack/schema/mirrors.py @@ -6,29 +6,55 @@ """Schema for mirrors.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/mirrors.py + :lines: 12-69 """ +#: Common properties for connection specification +connection = { + "url": {"type": "string"}, + # todo: replace this with named keys "username" / "password" or "id" / "secret" + "access_pair": { + "type": "array", + "items": {"type": ["string", "null"], "minItems": 2, "maxItems": 2}, + }, + "access_token": {"type": ["string", "null"]}, + "profile": {"type": ["string", "null"]}, + "endpoint_url": {"type": ["string", "null"]}, +} + +#: Mirror connection inside pull/push keys +fetch_and_push = { + "anyOf": [ + {"type": "string"}, + { + "type": "object", + "additionalProperties": False, + "properties": {**connection}, # type: ignore + }, + ] +} + +#: Mirror connection when no pull/push keys are set +mirror_entry = { + "type": "object", + "additionalProperties": False, + "anyOf": [{"required": ["url"]}, {"required": ["fetch"]}, {"required": ["pull"]}], + "properties": { + "source": {"type": "boolean"}, + "binary": {"type": "boolean"}, + "fetch": fetch_and_push, + "push": fetch_and_push, + **connection, # type: ignore + }, +} + #: Properties for inclusion in other schemas properties = { "mirrors": { "type": "object", "default": {}, "additionalProperties": False, - "patternProperties": { - r"\w[\w-]*": { - "anyOf": [ - {"type": "string"}, - { - "type": "object", - "required": ["fetch", "push"], - "properties": { - "fetch": {"type": ["string", "object"]}, - "push": {"type": ["string", "object"]}, - }, - }, - ] - } - }, + "patternProperties": {r"\w[\w-]*": {"anyOf": [{"type": "string"}, mirror_entry]}}, } } diff --git a/lib/spack/spack/schema/modules.py b/lib/spack/spack/schema/modules.py index 1d285f851bb85f..fb4130d345d02b 100644 --- a/lib/spack/spack/schema/modules.py +++ b/lib/spack/spack/schema/modules.py @@ -17,10 +17,8 @@ #: THIS NEEDS TO BE UPDATED FOR EVERY NEW KEYWORD THAT #: IS ADDED IMMEDIATELY BELOW THE MODULE TYPE ATTRIBUTE spec_regex = ( - r"(?!hierarchy|core_specs|verbose|hash_length|defaults|filter_hierarchy_specs|" - r"whitelist|blacklist|" # DEPRECATED: remove in 0.20. - r"include|exclude|" # use these more inclusive/consistent options - r"projections|naming_scheme|core_compilers|all)(^\w[\w-]*)" + r"(?!hierarchy|core_specs|verbose|hash_length|defaults|filter_hierarchy_specs|hide|" + r"include|exclude|projections|naming_scheme|core_compilers|all)(^\w[\w-]*)" ) #: Matches a valid name for a module set @@ -46,14 +44,7 @@ "default": {}, "additionalProperties": False, "properties": { - # DEPRECATED: remove in 0.20. - "environment_blacklist": { - "type": "array", - "default": [], - "items": {"type": "string"}, - }, - # use exclude_env_vars instead - "exclude_env_vars": {"type": "array", "default": [], "items": {"type": "string"}}, + "exclude_env_vars": {"type": "array", "default": [], "items": {"type": "string"}} }, }, "template": {"type": "string"}, @@ -80,15 +71,11 @@ "properties": { "verbose": {"type": "boolean", "default": False}, "hash_length": {"type": "integer", "minimum": 0, "default": 7}, - # DEPRECATED: remove in 0.20. - "whitelist": array_of_strings, - "blacklist": array_of_strings, - "blacklist_implicits": {"type": "boolean", "default": False}, - # whitelist/blacklist have been replaced with include/exclude "include": array_of_strings, "exclude": array_of_strings, "exclude_implicits": {"type": "boolean", "default": False}, "defaults": array_of_strings, + "hide_implicits": {"type": "boolean", "default": False}, "naming_scheme": {"type": "string"}, # Can we be more specific here? "projections": projections_scheme, "all": module_file_configuration, diff --git a/lib/spack/spack/schema/packages.py b/lib/spack/spack/schema/packages.py index 2cc4534d0711fa..2e651ec798355d 100644 --- a/lib/spack/spack/schema/packages.py +++ b/lib/spack/spack/schema/packages.py @@ -8,6 +8,66 @@ :lines: 13- """ +permissions = { + "type": "object", + "additionalProperties": False, + "properties": { + "read": {"type": "string", "enum": ["user", "group", "world"]}, + "write": {"type": "string", "enum": ["user", "group", "world"]}, + "group": {"type": "string"}, + }, +} + +variants = {"oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}]} + +requirements = { + "oneOf": [ + # 'require' can be a list of requirement_groups. + # each requirement group is a list of one or more + # specs. Either at least one or exactly one spec + # in the group must be satisfied (depending on + # whether you use "any_of" or "one_of", + # repectively) + { + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "additionalProperties": False, + "properties": { + "one_of": {"type": "array", "items": {"type": "string"}}, + "any_of": {"type": "array", "items": {"type": "string"}}, + "spec": {"type": "string"}, + "message": {"type": "string"}, + "when": {"type": "string"}, + }, + }, + {"type": "string"}, + ] + }, + }, + # Shorthand for a single requirement group with + # one member + {"type": "string"}, + ] +} + +permissions = { + "type": "object", + "additionalProperties": False, + "properties": { + "read": {"type": "string", "enum": ["user", "group", "world"]}, + "write": {"type": "string", "enum": ["user", "group", "world"]}, + "group": {"type": "string"}, + }, +} + +package_attributes = { + "type": "object", + "additionalProperties": False, + "patternProperties": {r"\w+": {}}, +} #: Properties for inclusion in other schemas properties = { @@ -15,57 +75,14 @@ "type": "object", "default": {}, "additionalProperties": False, - "patternProperties": { - r"\w[\w-]*": { # package name + "properties": { + "all": { # package name "type": "object", "default": {}, "additionalProperties": False, "properties": { - "require": { - "oneOf": [ - # 'require' can be a list of requirement_groups. - # each requirement group is a list of one or more - # specs. Either at least one or exactly one spec - # in the group must be satisfied (depending on - # whether you use "any_of" or "one_of", - # repectively) - { - "type": "array", - "items": { - "oneOf": [ - { - "type": "object", - "additionalProperties": False, - "properties": { - "one_of": { - "type": "array", - "items": {"type": "string"}, - }, - "any_of": { - "type": "array", - "items": {"type": "string"}, - }, - "spec": {"type": "string"}, - "message": {"type": "string"}, - "when": {"type": "string"}, - }, - }, - {"type": "string"}, - ] - }, - }, - # Shorthand for a single requirement group with - # one member - {"type": "string"}, - ] - }, - "version": { - "type": "array", - "default": [], - # version strings (type should be string, number is still possible - # but deprecated. this is to avoid issues with e.g. 3.10 -> 3.1) - "items": {"anyOf": [{"type": "string"}, {"type": "number"}]}, - }, + "require": requirements, + "version": {}, # Here only to warn users on ignored properties "target": { "type": "array", "default": [], @@ -78,22 +95,10 @@ "items": {"type": "string"}, }, # compiler specs "buildable": {"type": "boolean", "default": True}, - "permissions": { - "type": "object", - "additionalProperties": False, - "properties": { - "read": {"type": "string", "enum": ["user", "group", "world"]}, - "write": {"type": "string", "enum": ["user", "group", "world"]}, - "group": {"type": "string"}, - }, - }, + "permissions": permissions, # If 'get_full_repo' is promoted to a Package-level # attribute, it could be useful to set it here - "package_attributes": { - "type": "object", - "additionalProperties": False, - "patternProperties": {r"\w+": {}}, - }, + "package_attributes": package_attributes, "providers": { "type": "object", "default": {}, @@ -106,12 +111,40 @@ } }, }, - "variants": { - "oneOf": [ - {"type": "string"}, - {"type": "array", "items": {"type": "string"}}, - ] + "variants": variants, + }, + "deprecatedProperties": { + "properties": ["version"], + "message": "setting version preferences in the 'all' section of packages.yaml " + "is deprecated and will be removed in v0.22\n\n\tThese preferences " + "will be ignored by Spack. You can set them only in package specific sections " + "of the same file.\n", + "error": False, + }, + } + }, + "patternProperties": { + r"(?!^all$)(^\w[\w-]*)": { # package name + "type": "object", + "default": {}, + "additionalProperties": False, + "properties": { + "require": requirements, + "version": { + "type": "array", + "default": [], + # version strings + "items": {"anyOf": [{"type": "string"}, {"type": "number"}]}, }, + "target": {}, # Here only to warn users on ignored properties + "compiler": {}, # Here only to warn users on ignored properties + "buildable": {"type": "boolean", "default": True}, + "permissions": permissions, + # If 'get_full_repo' is promoted to a Package-level + # attribute, it could be useful to set it here + "package_attributes": package_attributes, + "providers": {}, # Here only to warn users on ignored properties + "variants": variants, "externals": { "type": "array", "items": { @@ -127,6 +160,14 @@ }, }, }, + "deprecatedProperties": { + "properties": ["target", "compiler", "providers"], + "message": "setting compiler, target or provider preferences in a package " + "specific section of packages.yaml is deprecated, and will be removed in " + "v0.22.\n\n\tThese preferences will be ignored by Spack. You " + "can set them only in the 'all' section of the same file.\n", + "error": False, + }, } }, } diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index 4ff4f402a3dc34..4514bd0e96bc0f 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -8,14 +8,17 @@ import enum import itertools import os +import pathlib import pprint import re import types import warnings -from typing import List +from typing import Callable, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, Union import archspec.cpu +import spack.deptypes as dt + try: import clingo # type: ignore[import] @@ -33,7 +36,6 @@ import spack.cmd import spack.compilers import spack.config -import spack.dependency import spack.directives import spack.environment as ev import spack.error @@ -43,11 +45,17 @@ import spack.repo import spack.spec import spack.store -import spack.traverse +import spack.util.crypto import spack.util.path import spack.util.timer import spack.variant import spack.version as vn +import spack.version.git_ref_lookup +from spack import traverse + +from .counter import FullDuplicatesCounter, MinimalDuplicatesCounter, NoDuplicatesCounter + +GitOrStandardVersion = Union[spack.version.GitVersion, spack.version.StandardVersion] # these are from clingo.ast and bootstrapped later ASTType = None @@ -76,9 +84,7 @@ def default_clingo_control(): """Return a control object with the default settings used in Spack""" control = clingo.Control() control.configuration.configuration = "tweety" - control.configuration.solve.models = 0 control.configuration.solver.heuristic = "Domain" - control.configuration.solve.parallel_mode = "1" control.configuration.solver.opt_strategy = "usc,one" return control @@ -132,7 +138,15 @@ class RequirementKind(enum.Enum): PACKAGE = enum.auto() -DeclaredVersion = collections.namedtuple("DeclaredVersion", ["version", "idx", "origin"]) +class DeclaredVersion(NamedTuple): + """Data class to contain information on declared versions used in the solve""" + + #: String representation of the version + version: str + #: Unique index assigned to this version + idx: int + #: Provenance of the version + origin: Provenance # Below numbers are used to map names of criteria to the order @@ -236,7 +250,7 @@ def listify(args): def packagize(pkg): if isinstance(pkg, str): - return spack.repo.path.get_pkg_class(pkg) + return spack.repo.PATH.get_pkg_class(pkg) else: return pkg @@ -265,12 +279,14 @@ def _id(thing): @llnl.util.lang.key_ordering class AspFunction(AspObject): + __slots__ = ["name", "args"] + def __init__(self, name, args=None): self.name = name self.args = () if args is None else tuple(args) def _cmp_key(self): - return (self.name, self.args) + return self.name, self.args def __call__(self, *args): """Return a new instance of this function with added arguments. @@ -301,6 +317,8 @@ def argify(arg): return clingo.String(str(arg)) elif isinstance(arg, int): return clingo.Number(arg) + elif isinstance(arg, AspFunction): + return clingo.Function(arg.name, [argify(x) for x in arg.args], positive=positive) else: return clingo.String(str(arg)) @@ -320,6 +338,22 @@ def __getattr__(self, name): fn = AspFunctionBuilder() +TransformFunction = Callable[[spack.spec.Spec, List[AspFunction]], List[AspFunction]] + + +def remove_node(spec: spack.spec.Spec, facts: List[AspFunction]) -> List[AspFunction]: + """Transformation that removes all "node" and "virtual_node" from the input list of facts.""" + return list(filter(lambda x: x.args[0] not in ("node", "virtual_node"), facts)) + + +def _create_counter(specs, tests): + strategy = spack.config.CONFIG.get("concretizer:duplicates:strategy", "none") + if strategy == "full": + return FullDuplicatesCounter(specs, tests=tests) + if strategy == "minimal": + return MinimalDuplicatesCounter(specs, tests=tests) + return NoDuplicatesCounter(specs, tests=tests) + def all_compilers_in_config(): return spack.compilers.all_compilers() @@ -341,7 +375,7 @@ def extend_flag_list(flag_list, new_flags): def check_packages_exist(specs): """Ensure all packages mentioned in specs exist.""" - repo = spack.repo.path + repo = spack.repo.PATH for spec in specs: for s in spec.traverse(): try: @@ -512,15 +546,17 @@ def _compute_specs_from_answer_set(self): best = min(self.answers) opt, _, answer = best for input_spec in self.abstract_specs: - key = input_spec.name + node = SpecBuilder.make_node(pkg=input_spec.name) if input_spec.virtual: - providers = [spec.name for spec in answer.values() if spec.package.provides(key)] - key = providers[0] - candidate = answer.get(key) + providers = [ + spec.name for spec in answer.values() if spec.package.provides(input_spec.name) + ] + node = SpecBuilder.make_node(pkg=providers[0]) + candidate = answer.get(node) if candidate and candidate.satisfies(input_spec): - self._concrete_specs.append(answer[key]) - self._concrete_specs_by_input[input_spec] = answer[key] + self._concrete_specs.append(answer[node]) + self._concrete_specs_by_input[input_spec] = answer[node] else: self._unsolved_specs.append(input_spec) @@ -528,7 +564,7 @@ def _compute_specs_from_answer_set(self): def _normalize_packages_yaml(packages_yaml): normalized_yaml = copy.copy(packages_yaml) for pkg_name in packages_yaml: - is_virtual = spack.repo.path.is_virtual(pkg_name) + is_virtual = spack.repo.PATH.is_virtual(pkg_name) if pkg_name == "all" or not is_virtual: continue @@ -536,7 +572,7 @@ def _normalize_packages_yaml(packages_yaml): data = normalized_yaml.pop(pkg_name) is_buildable = data.get("buildable", True) if not is_buildable: - for provider in spack.repo.path.providers_for(pkg_name): + for provider in spack.repo.PATH.providers_for(pkg_name): entry = normalized_yaml.setdefault(provider.name, {}) entry["buildable"] = False @@ -552,6 +588,41 @@ def keyfn(x): return normalized_yaml +def _is_checksummed_git_version(v): + return isinstance(v, vn.GitVersion) and v.is_commit + + +def _is_checksummed_version(version_info: Tuple[GitOrStandardVersion, dict]): + """Returns true iff the version is not a moving target""" + version, info = version_info + if isinstance(version, spack.version.StandardVersion): + if any(h in info for h in spack.util.crypto.hashes.keys()) or "checksum" in info: + return True + return "commit" in info and len(info["commit"]) == 40 + return _is_checksummed_git_version(version) + + +def _concretization_version_order(version_info: Tuple[GitOrStandardVersion, dict]): + """Version order key for concretization, where preferred > not preferred, + not deprecated > deprecated, finite > any infinite component; only if all are + the same, do we use default version ordering.""" + version, info = version_info + return ( + info.get("preferred", False), + not info.get("deprecated", False), + not version.isdevelop(), + version, + ) + + +def _spec_with_default_name(spec_str, name): + """Return a spec with a default name if none is provided, used for requirement specs""" + spec = spack.spec.Spec(spec_str) + if not spec.name: + spec.name = name + return spec + + def bootstrap_clingo(): global clingo, ASTType, parse_files @@ -571,16 +642,33 @@ def bootstrap_clingo(): from clingo import parse_files -def stringify(sym): - """Stringify symbols from clingo models. +class NodeArgument(NamedTuple): + id: str + pkg: str + - This will turn a ``clingo.Symbol`` into a string, or a sequence of ``clingo.Symbol`` - objects into a tuple of strings. +def intermediate_repr(sym): + """Returns an intermediate representation of clingo models for Spack's spec builder. + Currently, transforms symbols from clingo models either to strings or to NodeArgument objects. + + Returns: + This will turn a ``clingo.Symbol`` into a string or NodeArgument, or a sequence of + ``clingo.Symbol`` objects into a tuple of those objects. """ # TODO: simplify this when we no longer have to support older clingo versions. if isinstance(sym, (list, tuple)): - return tuple(stringify(a) for a in sym) + return tuple(intermediate_repr(a) for a in sym) + + try: + if sym.name == "node": + return NodeArgument( + id=intermediate_repr(sym.arguments[0]), pkg=intermediate_repr(sym.arguments[1]) + ) + except RuntimeError: + # This happens when using clingo w/ CFFI and trying to access ".name" for symbols + # that are not functions + pass if clingo_cffi: # Clingo w/ CFFI will throw an exception on failure @@ -595,16 +683,16 @@ def stringify(sym): def extract_args(model, predicate_name): """Extract the arguments to predicates with the provided name from a model. - Pull out all the predicates with name ``predicate_name`` from the model, and return - their stringified arguments as tuples. + Pull out all the predicates with name ``predicate_name`` from the model, and + return their intermediate representation. """ - return [stringify(sym.arguments) for sym in model if sym.name == predicate_name] + return [intermediate_repr(sym.arguments) for sym in model if sym.name == predicate_name] class ErrorHandler: def __init__(self, model): self.model = model - self.error_args = extract_args(model, "error") + self.full_model = None def multiple_values_error(self, attribute, pkg): return f'Cannot select a single "{attribute}" for package "{pkg}"' @@ -612,6 +700,48 @@ def multiple_values_error(self, attribute, pkg): def no_value_error(self, attribute, pkg): return f'Cannot select a single "{attribute}" for package "{pkg}"' + def _get_cause_tree( + self, + cause: Tuple[str, str], + conditions: Dict[str, str], + condition_causes: List[Tuple[Tuple[str, str], Tuple[str, str]]], + seen: Set, + indent: str = " ", + ) -> List[str]: + """ + Implementation of recursion for self.get_cause_tree. Much of this operates on tuples + (condition_id, set_id) in which the latter idea means that the condition represented by + the former held in the condition set represented by the latter. + """ + seen = set(seen) | set(cause) + parents = [c for e, c in condition_causes if e == cause and c not in seen] + local = "required because %s " % conditions[cause[0]] + + return [indent + local] + [ + c + for parent in parents + for c in self._get_cause_tree( + parent, conditions, condition_causes, seen, indent=indent + " " + ) + ] + + def get_cause_tree(self, cause: Tuple[str, str]) -> List[str]: + """ + Get the cause tree associated with the given cause. + + Arguments: + cause: The root cause of the tree (final condition) + + Returns: + A list of strings describing the causes, formatted to display tree structure. + """ + conditions: Dict[str, str] = dict(extract_args(self.full_model, "condition_reason")) + condition_causes: List[Tuple[Tuple[str, str], Tuple[str, str]]] = list( + ((Effect, EID), (Cause, CID)) + for Effect, EID, Cause, CID in extract_args(self.full_model, "condition_cause") + ) + return self._get_cause_tree(cause, conditions, condition_causes, set()) + def handle_error(self, msg, *args): """Handle an error state derived by the solver.""" if msg == "multiple_values_error": @@ -620,14 +750,31 @@ def handle_error(self, msg, *args): if msg == "no_value_error": return self.no_value_error(*args) + try: + idx = args.index("startcauses") + except ValueError: + msg_args = args + causes = [] + else: + msg_args = args[:idx] + cause_args = args[idx + 1 :] + cause_args_conditions = cause_args[::2] + cause_args_ids = cause_args[1::2] + causes = list(zip(cause_args_conditions, cause_args_ids)) + + msg = msg.format(*msg_args) + # For variant formatting, we sometimes have to construct specs # to format values properly. Find/replace all occurances of # Spec(...) with the string representation of the spec mentioned - msg = msg.format(*args) specs_to_construct = re.findall(r"Spec\(([^)]*)\)", msg) for spec_str in specs_to_construct: msg = msg.replace("Spec(%s)" % spec_str, str(spack.spec.Spec(spec_str))) + for cause in set(causes): + for c in self.get_cause_tree(cause): + msg += f"\n{c}" + return msg def message(self, errors) -> str: @@ -639,11 +786,31 @@ def message(self, errors) -> str: return "\n".join([header] + messages) def raise_if_errors(self): - if not self.error_args: + initial_error_args = extract_args(self.model, "error") + if not initial_error_args: return + error_causation = clingo.Control() + + parent_dir = pathlib.Path(__file__).parent + errors_lp = parent_dir / "error_messages.lp" + + def on_model(model): + self.full_model = model.symbols(shown=True, terms=True) + + with error_causation.backend() as backend: + for atom in self.model: + atom_id = backend.add_atom(atom) + backend.add_rule([atom_id], [], choice=False) + + error_causation.load(str(errors_lp)) + error_causation.ground([("base", []), ("error_messages", [])]) + _ = error_causation.solve(on_model=on_model) + + # No choices so there will be only one model + error_args = extract_args(self.full_model, "error") errors = sorted( - [(int(priority), msg, args) for priority, msg, *args in self.error_args], reverse=True + [(int(priority), msg, args) for priority, msg, *args in error_args], reverse=True ) msg = self.message(errors) raise UnsatisfiableSpecError(msg) @@ -699,7 +866,9 @@ def fact(self, head): """ symbol = head.symbol() if hasattr(head, "symbol") else head - self.out.write("%s.\n" % str(symbol)) + # This is commented out to avoid evaluating str(symbol) when we have no stream + if not isinstance(self.out, llnl.util.lang.Devnull): + self.out.write(f"{str(symbol)}.\n") atom = self.backend.add_atom(symbol) @@ -710,7 +879,7 @@ def fact(self, head): if choice: self.assumptions.append(atom) - def solve(self, setup, specs, reuse=None, output=None, control=None): + def solve(self, setup, specs, reuse=None, output=None, control=None, allow_deprecated=False): """Set up the input and solve for dependencies of ``specs``. Arguments: @@ -721,6 +890,7 @@ def solve(self, setup, specs, reuse=None, output=None, control=None): the output of this solve. control (clingo.Control): configuration for the solver. If None, default values will be used + allow_deprecated: if True, allow deprecated versions in the solve Return: A tuple of the solve result, the timer for the different phases of the @@ -740,7 +910,7 @@ def solve(self, setup, specs, reuse=None, output=None, control=None): timer.start("setup") with self.control.backend() as backend: self.backend = backend - setup.setup(self, specs, reuse=reuse) + setup.setup(self, specs, reuse=reuse, allow_deprecated=allow_deprecated) timer.stop("setup") timer.start("load") @@ -771,8 +941,13 @@ def visit(node): # Load the file itself self.control.load(os.path.join(parent_dir, "concretize.lp")) + self.control.load(os.path.join(parent_dir, "heuristic.lp")) + if spack.config.CONFIG.get("concretizer:duplicates:strategy", "none") != "none": + self.control.load(os.path.join(parent_dir, "heuristic_separate.lp")) self.control.load(os.path.join(parent_dir, "os_compatibility.lp")) self.control.load(os.path.join(parent_dir, "display.lp")) + if not setup.concretize_everything: + self.control.load(os.path.join(parent_dir, "when_possible.lp")) timer.stop("load") # Grounding is the first step in the solve -- it turns our facts @@ -835,7 +1010,8 @@ def on_model(model): for sym in best_model: if sym.name not in ("attr", "error", "opt_criterion"): tty.debug( - "UNKNOWN SYMBOL: %s(%s)" % (sym.name, ", ".join(stringify(sym.arguments))) + "UNKNOWN SYMBOL: %s(%s)" + % (sym.name, ", ".join([str(s) for s in intermediate_repr(sym.arguments)])) ) elif cores: @@ -853,6 +1029,70 @@ def on_model(model): return result, timer, self.control.statistics +class ConcreteSpecsByHash(collections.abc.Mapping): + """Mapping containing concrete specs keyed by DAG hash. + + The mapping is ensured to be consistent, i.e. if a spec in the mapping has a dependency with + hash X, it is ensured to be the same object in memory as the spec keyed by X. + """ + + def __init__(self) -> None: + self.data: Dict[str, spack.spec.Spec] = {} + + def __getitem__(self, dag_hash: str) -> spack.spec.Spec: + return self.data[dag_hash] + + def add(self, spec: spack.spec.Spec) -> bool: + """Adds a new concrete spec to the mapping. Returns True if the spec was just added, + False if the spec was already in the mapping. + + Args: + spec: spec to be added + + Raises: + ValueError: if the spec is not concrete + """ + if not spec.concrete: + msg = ( + f"trying to store the non-concrete spec '{spec}' in a container " + f"that only accepts concrete" + ) + raise ValueError(msg) + + dag_hash = spec.dag_hash() + if dag_hash in self.data: + return False + + # Here we need to iterate on the input and rewire the copy. + self.data[spec.dag_hash()] = spec.copy(deps=False) + nodes_to_reconstruct = [spec] + + while nodes_to_reconstruct: + input_parent = nodes_to_reconstruct.pop() + container_parent = self.data[input_parent.dag_hash()] + + for edge in input_parent.edges_to_dependencies(): + input_child = edge.spec + container_child = self.data.get(input_child.dag_hash()) + # Copy children that don't exist yet + if container_child is None: + container_child = input_child.copy(deps=False) + self.data[input_child.dag_hash()] = container_child + nodes_to_reconstruct.append(input_child) + + # Rewire edges + container_parent.add_dependency_edge( + dependency_spec=container_child, depflag=edge.depflag, virtuals=edge.virtuals + ) + return True + + def __len__(self) -> int: + return len(self.data) + + def __iter__(self): + return iter(self.data) + + class SpackSolverSetup: """Class to set up and run a Spack concretization solve.""" @@ -876,12 +1116,14 @@ def __init__(self, tests=False): # (ID, CompilerSpec) -> dictionary of attributes self.compiler_info = collections.defaultdict(dict) - # hashes we've already added facts for - self.seen_hashes = set() - self.reusable_and_possible = {} + self.reusable_and_possible = ConcreteSpecsByHash() # id for dummy variables self._condition_id_counter = itertools.count() + self._trigger_id_counter = itertools.count() + self._trigger_cache = collections.defaultdict(dict) + self._effect_id_counter = itertools.count() + self._effect_cache = collections.defaultdict(dict) # Caches to optimize the setup phase of the solver self.target_specs_cache = None @@ -918,15 +1160,18 @@ def key_fn(version): for weight, declared_version in enumerate(most_to_least_preferred): self.gen.fact( - fn.version_declared( - pkg.name, declared_version.version, weight, str(declared_version.origin) + fn.pkg_fact( + pkg.name, + fn.version_declared( + declared_version.version, weight, str(declared_version.origin) + ), ) ) # Declare deprecated versions for this package, if any deprecated = self.deprecated_versions[pkg.name] for v in sorted(deprecated): - self.gen.fact(fn.deprecated_version(pkg.name, v)) + self.gen.fact(fn.pkg_fact(pkg.name, fn.deprecated_version(v))) def spec_versions(self, spec): """Return list of clauses expressing spec's version constraints.""" @@ -955,11 +1200,14 @@ def target_ranges(self, spec, single_target_fn): return [fn.attr("node_target_satisfies", spec.name, target)] def conflict_rules(self, pkg): - default_msg = "{0} '{1}' conflicts with '{2}'" - no_constraint_msg = "{0} conflicts with '{1}'" + default_msg = "{0}: '{1}' conflicts with '{2}'" + no_constraint_msg = "{0}: conflicts with '{1}'" for trigger, constraints in pkg.conflicts.items(): - trigger_msg = "conflict trigger %s" % str(trigger) - trigger_id = self.condition(spack.spec.Spec(trigger), name=pkg.name, msg=trigger_msg) + trigger_msg = f"conflict is triggered when {str(trigger)}" + trigger_spec = spack.spec.Spec(trigger) + trigger_id = self.condition( + trigger_spec, name=trigger_spec.name or pkg.name, msg=trigger_msg + ) for constraint, conflict_msg in constraints: if conflict_msg is None: @@ -967,9 +1215,15 @@ def conflict_rules(self, pkg): conflict_msg = no_constraint_msg.format(pkg.name, trigger) else: conflict_msg = default_msg.format(pkg.name, trigger, constraint) - constraint_msg = "conflict constraint %s" % str(constraint) + + spec_for_msg = ( + spack.spec.Spec(pkg.name) if constraint == spack.spec.Spec() else constraint + ) + constraint_msg = f"conflict applies to spec {str(spec_for_msg)}" constraint_id = self.condition(constraint, name=pkg.name, msg=constraint_msg) - self.gen.fact(fn.conflict(pkg.name, trigger_id, constraint_id, conflict_msg)) + self.gen.fact( + fn.pkg_fact(pkg.name, fn.conflict(trigger_id, constraint_id, conflict_msg)) + ) self.gen.newline() def compiler_facts(self): @@ -1004,29 +1258,9 @@ def compiler_facts(self): matches = sorted(indexed_possible_compilers, key=lambda x: ppk(x[1].spec)) for weight, (compiler_id, cspec) in enumerate(matches): - f = fn.default_compiler_preference(compiler_id, weight) + f = fn.compiler_weight(compiler_id, weight) self.gen.fact(f) - def package_compiler_defaults(self, pkg): - """Facts about packages' compiler prefs.""" - - packages = spack.config.get("packages") - pkg_prefs = packages.get(pkg.name) - if not pkg_prefs or "compiler" not in pkg_prefs: - return - - compiler_list = self.possible_compilers - compiler_list = sorted(compiler_list, key=lambda x: (x.name, x.version), reverse=True) - ppk = spack.package_prefs.PackagePrefs(pkg.name, "compiler", all=False) - matches = sorted(compiler_list, key=lambda x: ppk(x.spec)) - - for i, compiler in enumerate(reversed(matches)): - self.gen.fact( - fn.node_compiler_preference( - pkg.name, compiler.spec.name, compiler.spec.version, -i * 100 - ) - ) - def package_requirement_rules(self, pkg): rules = self.requirement_rules_from_package_py(pkg) rules.extend(self.requirement_rules_from_packages_yaml(pkg)) @@ -1113,12 +1347,62 @@ def pkg_rules(self, pkg, tests): self.gen.newline() # variants + self.variant_rules(pkg) + + # conflicts + self.conflict_rules(pkg) + + # virtuals + self.package_provider_rules(pkg) + + # dependencies + self.package_dependencies_rules(pkg) + + # virtual preferences + self.virtual_preferences( + pkg.name, + lambda v, p, i: self.gen.fact(fn.pkg_fact(pkg.name, fn.provider_preference(v, p, i))), + ) + + self.package_requirement_rules(pkg) + + # trigger and effect tables + self.trigger_rules() + self.effect_rules() + + def trigger_rules(self): + """Flushes all the trigger rules collected so far, and clears the cache.""" + self.gen.h2("Trigger conditions") + for name in self._trigger_cache: + cache = self._trigger_cache[name] + for (spec_str, _), (trigger_id, requirements) in cache.items(): + self.gen.fact(fn.pkg_fact(name, fn.trigger_id(trigger_id))) + self.gen.fact(fn.pkg_fact(name, fn.trigger_msg(spec_str))) + for predicate in requirements: + self.gen.fact(fn.condition_requirement(trigger_id, *predicate.args)) + self.gen.newline() + self._trigger_cache.clear() + + def effect_rules(self): + """Flushes all the effect rules collected so far, and clears the cache.""" + self.gen.h2("Imposed requirements") + for name in self._effect_cache: + cache = self._effect_cache[name] + for (spec_str, _), (effect_id, requirements) in cache.items(): + self.gen.fact(fn.pkg_fact(name, fn.effect_id(effect_id))) + self.gen.fact(fn.pkg_fact(name, fn.effect_msg(spec_str))) + for predicate in requirements: + self.gen.fact(fn.imposed_constraint(effect_id, *predicate.args)) + self.gen.newline() + self._effect_cache.clear() + + def variant_rules(self, pkg): for name, entry in sorted(pkg.variants.items()): variant, when = entry if spack.spec.Spec() in when: # unconditional variant - self.gen.fact(fn.variant(pkg.name, name)) + self.gen.fact(fn.pkg_fact(pkg.name, fn.variant(name))) else: # conditional variant for w in when: @@ -1127,19 +1411,23 @@ def pkg_rules(self, pkg, tests): msg += " when %s" % w cond_id = self.condition(w, name=pkg.name, msg=msg) - self.gen.fact(fn.variant_condition(cond_id, pkg.name, name)) + self.gen.fact(fn.pkg_fact(pkg.name, fn.conditional_variant(cond_id, name))) single_value = not variant.multi if single_value: - self.gen.fact(fn.variant_single_value(pkg.name, name)) + self.gen.fact(fn.pkg_fact(pkg.name, fn.variant_single_value(name))) self.gen.fact( - fn.variant_default_value_from_package_py(pkg.name, name, variant.default) + fn.pkg_fact( + pkg.name, fn.variant_default_value_from_package_py(name, variant.default) + ) ) else: spec_variant = variant.make_default() defaults = spec_variant.value for val in sorted(defaults): - self.gen.fact(fn.variant_default_value_from_package_py(pkg.name, name, val)) + self.gen.fact( + fn.pkg_fact(pkg.name, fn.variant_default_value_from_package_py(name, val)) + ) values = variant.values if values is None: @@ -1150,7 +1438,9 @@ def pkg_rules(self, pkg, tests): for sid, s in enumerate(values.sets): for value in s: self.gen.fact( - fn.variant_value_from_disjoint_sets(pkg.name, name, value, sid) + fn.pkg_fact( + pkg.name, fn.variant_value_from_disjoint_sets(name, value, sid) + ) ) union.update(s) values = union @@ -1177,7 +1467,9 @@ def pkg_rules(self, pkg, tests): msg="empty (total) conflict constraint", ) msg = "variant {0}={1} is conditionally disabled".format(name, value) - self.gen.fact(fn.conflict(pkg.name, trigger_id, constraint_id, msg)) + self.gen.fact( + fn.pkg_fact(pkg.name, fn.conflict(trigger_id, constraint_id, msg)) + ) else: imposed = spack.spec.Spec(value.when) imposed.name = pkg.name @@ -1188,64 +1480,77 @@ def pkg_rules(self, pkg, tests): name=pkg.name, msg="%s variant %s value %s when %s" % (pkg.name, name, value, when), ) - self.gen.fact(fn.variant_possible_value(pkg.name, name, value)) + self.gen.fact(fn.pkg_fact(pkg.name, fn.variant_possible_value(name, value))) if variant.sticky: - self.gen.fact(fn.variant_sticky(pkg.name, name)) + self.gen.fact(fn.pkg_fact(pkg.name, fn.variant_sticky(name))) self.gen.newline() - # conflicts - self.conflict_rules(pkg) - - # default compilers for this package - self.package_compiler_defaults(pkg) - - # virtuals - self.package_provider_rules(pkg) - - # dependencies - self.package_dependencies_rules(pkg) - - # virtual preferences - self.virtual_preferences( - pkg.name, lambda v, p, i: self.gen.fact(fn.pkg_provider_preference(pkg.name, v, p, i)) - ) - - self.package_requirement_rules(pkg) - - def condition(self, required_spec, imposed_spec=None, name=None, msg=None, node=False): + def condition( + self, + required_spec: spack.spec.Spec, + imposed_spec: Optional[spack.spec.Spec] = None, + name: Optional[str] = None, + msg: Optional[str] = None, + transform_required: Optional[TransformFunction] = None, + transform_imposed: Optional[TransformFunction] = remove_node, + ): """Generate facts for a dependency or virtual provider condition. Arguments: - required_spec (spack.spec.Spec): the spec that triggers this condition - imposed_spec (spack.spec.Spec or None): the spec with constraints that - are imposed when this condition is triggered - name (str or None): name for `required_spec` (required if - required_spec is anonymous, ignored if not) - msg (str or None): description of the condition - node (bool): if False does not emit "node" or "virtual_node" requirements - from the imposed spec + required_spec: the constraints that triggers this condition + imposed_spec: the constraints that are imposed when this condition is triggered + name: name for `required_spec` (required if required_spec is anonymous, ignored if not) + msg: description of the condition + transform_required: transformation applied to facts from the required spec. Defaults + to leave facts as they are. + transform_imposed: transformation applied to facts from the imposed spec. Defaults + to removing "node" and "virtual_node" facts. Returns: int: id of the condition created by this function """ named_cond = required_spec.copy() named_cond.name = named_cond.name or name - assert named_cond.name, "must provide name for anonymous condtions!" + assert named_cond.name, "must provide name for anonymous conditions!" # Check if we can emit the requirements before updating the condition ID counter. # In this way, if a condition can't be emitted but the exception is handled in the caller, # we won't emit partial facts. - requirements = self.spec_clauses(named_cond, body=True, required_from=name) condition_id = next(self._condition_id_counter) - self.gen.fact(fn.condition(condition_id, msg)) - for pred in requirements: - self.gen.fact(fn.condition_requirement(condition_id, *pred.args)) + self.gen.fact(fn.pkg_fact(named_cond.name, fn.condition(condition_id))) + self.gen.fact(fn.condition_reason(condition_id, msg)) + + cache = self._trigger_cache[named_cond.name] + + named_cond_key = (str(named_cond), transform_required) + if named_cond_key not in cache: + trigger_id = next(self._trigger_id_counter) + requirements = self.spec_clauses(named_cond, body=True, required_from=name) + + if transform_required: + requirements = transform_required(named_cond, requirements) + + cache[named_cond_key] = (trigger_id, requirements) + trigger_id, requirements = cache[named_cond_key] + self.gen.fact(fn.pkg_fact(named_cond.name, fn.condition_trigger(condition_id, trigger_id))) + + if not imposed_spec: + return condition_id + + cache = self._effect_cache[named_cond.name] + imposed_spec_key = (str(imposed_spec), transform_imposed) + if imposed_spec_key not in cache: + effect_id = next(self._effect_id_counter) + requirements = self.spec_clauses(imposed_spec, body=False, required_from=name) - if imposed_spec: - self.impose(condition_id, imposed_spec, node=node, name=name) + if transform_imposed: + requirements = transform_imposed(imposed_spec, requirements) + cache[imposed_spec_key] = (effect_id, requirements) + effect_id, requirements = cache[imposed_spec_key] + self.gen.fact(fn.pkg_fact(named_cond.name, fn.condition_effect(condition_id, effect_id))) return condition_id def impose(self, condition_id, imposed_spec, node=True, name=None, body=False): @@ -1258,43 +1563,76 @@ def impose(self, condition_id, imposed_spec, node=True, name=None, body=False): def package_provider_rules(self, pkg): for provider_name in sorted(set(s.name for s in pkg.provided.keys())): - self.gen.fact(fn.possible_provider(pkg.name, provider_name)) + if provider_name not in self.possible_virtuals: + continue + self.gen.fact(fn.pkg_fact(pkg.name, fn.possible_provider(provider_name))) for provided, whens in pkg.provided.items(): + if provided.name not in self.possible_virtuals: + continue for when in whens: msg = "%s provides %s when %s" % (pkg.name, provided, when) condition_id = self.condition(when, provided, pkg.name, msg) - self.gen.fact(fn.provider_condition(condition_id, when.name, provided.name)) + self.gen.fact( + fn.pkg_fact(when.name, fn.provider_condition(condition_id, provided.name)) + ) + self.gen.newline() + + for when, sets_of_virtuals in pkg.provided_together.items(): + condition_id = self.condition( + when, name=pkg.name, msg="Virtuals are provided together" + ) + for set_id, virtuals_together in enumerate(sets_of_virtuals): + for name in virtuals_together: + self.gen.fact( + fn.pkg_fact(pkg.name, fn.provided_together(condition_id, set_id, name)) + ) self.gen.newline() def package_dependencies_rules(self, pkg): """Translate 'depends_on' directives into ASP logic.""" for _, conditions in sorted(pkg.dependencies.items()): for cond, dep in sorted(conditions.items()): - deptypes = dep.type.copy() + depflag = dep.depflag # Skip test dependencies if they're not requested if not self.tests: - deptypes.discard("test") + depflag &= ~dt.TEST # ... or if they are requested only for certain packages - if not isinstance(self.tests, bool) and pkg.name not in self.tests: - deptypes.discard("test") + elif not isinstance(self.tests, bool) and pkg.name not in self.tests: + depflag &= ~dt.TEST # if there are no dependency types to be considered # anymore, don't generate the dependency - if not deptypes: + if not depflag: continue - msg = "%s depends on %s" % (pkg.name, dep.spec.name) + msg = f"{pkg.name} depends on {dep.spec}" if cond != spack.spec.Spec(): - msg += " when %s" % cond + msg += f" when {cond}" + else: + pass - condition_id = self.condition(cond, dep.spec, pkg.name, msg) - self.gen.fact(fn.dependency_condition(condition_id, pkg.name, dep.spec.name)) + def track_dependencies(input_spec, requirements): + return requirements + [fn.attr("track_dependencies", input_spec.name)] - for t in sorted(deptypes): - # there is a declared dependency of type t - self.gen.fact(fn.dependency_type(condition_id, t)) + def dependency_holds(input_spec, requirements): + return remove_node(input_spec, requirements) + [ + fn.attr( + "dependency_holds", pkg.name, input_spec.name, dt.flag_to_string(t) + ) + for t in dt.ALL_FLAGS + if t & depflag + ] + + self.condition( + cond, + dep.spec, + name=pkg.name, + msg=msg, + transform_required=track_dependencies, + transform_imposed=dependency_holds, + ) self.gen.newline() @@ -1309,6 +1647,7 @@ def virtual_preferences(self, pkg_name, func): for i, provider in enumerate(providers): provider_name = spack.spec.Spec(provider).name func(vspec, provider_name, i) + self.gen.newline() def provider_defaults(self): self.gen.h2("Default virtual providers") @@ -1327,7 +1666,7 @@ def provider_requirements(self): "Internal Error: possible_virtuals is not populated. Please report to the spack" " maintainers" ) - packages_yaml = spack.config.config.get("packages") + packages_yaml = spack.config.CONFIG.get("packages") assert self.possible_virtuals is not None, msg for virtual_str in sorted(self.possible_virtuals): requirements = packages_yaml.get(virtual_str, {}).get("require", []) @@ -1335,6 +1674,8 @@ def provider_requirements(self): virtual_str, requirements, kind=RequirementKind.VIRTUAL ) self.emit_facts_from_requirement_rules(rules) + self.trigger_rules() + self.effect_rules() def emit_facts_from_requirement_rules(self, rules: List[RequirementRule]): """Generate facts to enforce requirements. @@ -1361,7 +1702,9 @@ def emit_facts_from_requirement_rules(self, rules: List[RequirementRule]): ) except Exception as e: if rule.kind != RequirementKind.DEFAULT: - raise RuntimeError("cannot emit requirements for the solver") from e + raise RuntimeError( + "cannot emit requirements for the solver: " + str(e) + ) from e continue self.gen.fact( @@ -1385,15 +1728,26 @@ def emit_facts_from_requirement_rules(self, rules: List[RequirementRule]): when_spec = spack.spec.Spec(pkg_name) try: + # With virtual we want to emit "node" and "virtual_node" in imposed specs + transform: Optional[TransformFunction] = remove_node + if virtual: + transform = None + member_id = self.condition( - required_spec=when_spec, imposed_spec=spec, name=pkg_name, node=virtual + required_spec=when_spec, + imposed_spec=spec, + name=pkg_name, + transform_imposed=transform, + msg=f"{spec_str} is a requirement for package {pkg_name}", ) except Exception as e: # Do not raise if the rule comes from the 'all' subsection, since usability # would be impaired. If a rule does not apply for a specific package, just # discard it. if rule.kind != RequirementKind.DEFAULT: - raise RuntimeError("cannot emit requirements for the solver") from e + raise RuntimeError( + "cannot emit requirements for the solver: " + str(e) + ) from e continue self.gen.fact(fn.requirement_group_member(member_id, pkg_name, requirement_grp_id)) @@ -1415,7 +1769,7 @@ def external_packages(self): continue # This package does not appear in any repository - if pkg_name not in spack.repo.path: + if pkg_name not in spack.repo.PATH: continue self.gen.h2("External package: {0}".format(pkg_name)) @@ -1447,11 +1801,21 @@ def external_packages(self): # Declare external conditions with a local index into packages.yaml for local_idx, spec in enumerate(external_specs): msg = "%s available as external when satisfying %s" % (spec.name, spec) - condition_id = self.condition(spec, msg=msg) - self.gen.fact(fn.possible_external(condition_id, pkg_name, local_idx)) + + def external_imposition(input_spec, _): + return [fn.attr("external_conditions_hold", input_spec.name, local_idx)] + + self.condition( + spec, + spack.spec.Spec(spec.name), + msg=msg, + transform_imposed=external_imposition, + ) self.possible_versions[spec.name].add(spec.version) self.gen.newline() + self.trigger_rules() + def preferred_variants(self, pkg_name): """Facts on concretization preferences, as read from packages.yaml""" preferences = spack.package_prefs.PackagePrefs @@ -1476,8 +1840,8 @@ def preferred_variants(self, pkg_name): fn.variant_default_value_from_packages_yaml(pkg_name, variant.name, value) ) - def target_preferences(self, pkg_name): - key_fn = spack.package_prefs.PackagePrefs(pkg_name, "target") + def target_preferences(self): + key_fn = spack.package_prefs.PackagePrefs("all", "target") if not self.target_specs_cache: self.target_specs_cache = [ @@ -1487,15 +1851,25 @@ def target_preferences(self, pkg_name): package_targets = self.target_specs_cache[:] package_targets.sort(key=key_fn) - - offset = 0 - best_default = self.default_targets[0][1] for i, preferred in enumerate(package_targets): - if str(preferred.architecture.target) == best_default and i != 0: - offset = 100 - self.gen.fact( - fn.target_weight(pkg_name, str(preferred.architecture.target), i + offset) - ) + self.gen.fact(fn.target_weight(str(preferred.architecture.target), i)) + + def flag_defaults(self): + self.gen.h2("Compiler flag defaults") + + # types of flags that can be on specs + for flag in spack.spec.FlagMap.valid_compiler_flags(): + self.gen.fact(fn.flag_type(flag)) + self.gen.newline() + + # flags from compilers.yaml + compilers = all_compilers_in_config() + for compiler in compilers: + for name, flags in compiler.flags.items(): + for flag in flags: + self.gen.fact( + fn.compiler_version_flag(compiler.name, compiler.version, name, flag) + ) def spec_clauses(self, *args, **kwargs): """Wrap a call to `_spec_clauses()` into a try/except block that @@ -1548,7 +1922,7 @@ class Head: node_flag = fn.attr("node_flag_set") node_flag_source = fn.attr("node_flag_source") node_flag_propagate = fn.attr("node_flag_propagate") - variant_propagate = fn.attr("variant_propagate") + variant_propagation_candidate = fn.attr("variant_propagation_candidate") class Body: node = fn.attr("node") @@ -1562,7 +1936,7 @@ class Body: node_flag = fn.attr("node_flag") node_flag_source = fn.attr("node_flag_source") node_flag_propagate = fn.attr("node_flag_propagate") - variant_propagate = fn.attr("variant_propagate") + variant_propagation_candidate = fn.attr("variant_propagation_candidate") f = Body if body else Head @@ -1597,7 +1971,7 @@ class Body: if not spec.concrete: reserved_names = spack.directives.reserved_names if not spec.virtual and vname not in reserved_names: - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) try: variant_def, _ = pkg_cls.variants[vname] except KeyError: @@ -1605,13 +1979,15 @@ class Body: raise RuntimeError(msg.format(vname, spec.name)) else: variant_def.validate_or_raise( - variant, spack.repo.path.get_pkg_class(spec.name) + variant, spack.repo.PATH.get_pkg_class(spec.name) ) clauses.append(f.variant_value(spec.name, vname, value)) if variant.propagate: - clauses.append(f.variant_propagate(spec.name, vname, value, spec.name)) + clauses.append( + f.variant_propagation_candidate(spec.name, vname, value, spec.name) + ) # Tell the concretizer that this is a possible value for the # variant, to account for things like int/str values where we @@ -1653,6 +2029,16 @@ class Body: clauses.append(fn.attr("package_hash", spec.name, spec._package_hash)) clauses.append(fn.attr("hash", spec.name, spec.dag_hash())) + edges = spec.edges_from_dependents() + virtuals = [x for x in itertools.chain.from_iterable([edge.virtuals for edge in edges])] + if not body: + for virtual in virtuals: + clauses.append(fn.attr("provider_set", spec.name, virtual)) + clauses.append(fn.attr("virtual_node", virtual)) + else: + for virtual in virtuals: + clauses.append(fn.attr("virtual_on_incoming_edges", spec.name, virtual)) + # add all clauses from dependencies if transitive: # TODO: Eventually distinguish 2 deps on the same pkg (build and link) @@ -1662,45 +2048,25 @@ class Body: if spec.concrete: # We know dependencies are real for concrete specs. For abstract # specs they just mean the dep is somehow in the DAG. - for dtype in dspec.deptypes: + for dtype in dt.ALL_FLAGS: + if not dspec.depflag & dtype: + continue # skip build dependencies of already-installed specs - if concrete_build_deps or dtype != "build": - clauses.append(fn.attr("depends_on", spec.name, dep.name, dtype)) - - # TODO: We have to look up info from package.py here, but we'd - # TODO: like to avoid this entirely. We should not need to look - # TODO: up potentially wrong info if we have virtual edge info. - try: - try: - pkg = dep.package - - except spack.repo.UnknownNamespaceError: - # Try to look up the package of the same name and use its - # providers. This is as good as we can do without edge info. - pkg_class = spack.repo.path.get_pkg_class(dep.name) - spec = spack.spec.Spec(f"{dep.name}@{dep.version}") - pkg = pkg_class(spec) - - virtuals = pkg.virtuals_provided - - except spack.repo.UnknownPackageError: - # Skip virtual node constriants for renamed/deleted packages, - # so their binaries can still be installed. - # NOTE: with current specs (which lack edge attributes) this - # can allow concretizations with two providers, but it's unlikely. - continue - - # Don't concretize with two providers of the same virtual. - # See above for exception for unknown packages. - # TODO: we will eventually record provider information on edges, - # TODO: which avoids the need for the package lookup above. - for virtual in virtuals: - clauses.append(fn.attr("virtual_node", virtual.name)) - clauses.append(fn.provider(dep.name, virtual.name)) + if concrete_build_deps or dtype != dt.BUILD: + clauses.append( + fn.attr( + "depends_on", spec.name, dep.name, dt.flag_to_string(dtype) + ) + ) + for virtual_name in dspec.virtuals: + clauses.append( + fn.attr("virtual_on_edge", spec.name, dep.name, virtual_name) + ) + clauses.append(fn.attr("virtual_node", virtual_name)) # imposing hash constraints for all but pure build deps of # already-installed concrete specs. - if concrete_build_deps or dspec.deptypes != ("build",): + if concrete_build_deps or dspec.depflag != dt.BUILD: clauses.append(fn.attr("hash", dep.name, dep.dag_hash())) # if the spec is abstract, descend into dependencies. @@ -1718,54 +2084,57 @@ class Body: return clauses - def build_version_dict(self, possible_pkgs): + def define_package_versions_and_validate_preferences( + self, possible_pkgs, *, require_checksum: bool, allow_deprecated: bool + ): """Declare any versions in specs not declared in packages.""" packages_yaml = spack.config.get("packages") - packages_yaml = _normalize_packages_yaml(packages_yaml) for pkg_name in possible_pkgs: - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) # All the versions from the corresponding package.py file. Since concepts # like being a "develop" version or being preferred exist only at a # package.py level, sort them in this partial list here - def key_fn(item): - version, info = item - # When COMPARING VERSIONS, the '@develop' version is always - # larger than other versions. BUT when CONCRETIZING, the largest - # NON-develop version is selected by default. - return ( - info.get("preferred", False), - not info.get("deprecated", False), - not version.isdevelop(), - version, - ) + package_py_versions = sorted( + pkg_cls.versions.items(), key=_concretization_version_order, reverse=True + ) + + if require_checksum and pkg_cls.has_code: + package_py_versions = [ + x for x in package_py_versions if _is_checksummed_version(x) + ] + + for idx, (v, version_info) in enumerate(package_py_versions): + if version_info.get("deprecated", False): + self.deprecated_versions[pkg_name].add(v) + if not allow_deprecated: + continue - for idx, item in enumerate(sorted(pkg_cls.versions.items(), key=key_fn, reverse=True)): - v, version_info = item self.possible_versions[pkg_name].add(v) self.declared_versions[pkg_name].append( DeclaredVersion(version=v, idx=idx, origin=Provenance.PACKAGE_PY) ) - deprecated = version_info.get("deprecated", False) - if deprecated: - self.deprecated_versions[pkg_name].add(v) - # All the preferred version from packages.yaml, versions in external - # specs will be computed later - version_preferences = packages_yaml.get(pkg_name, {}).get("version", []) + if pkg_name not in packages_yaml or "version" not in packages_yaml[pkg_name]: + continue + version_defs = [] - pkg_class = spack.repo.path.get_pkg_class(pkg_name) - for vstr in version_preferences: + + for vstr in packages_yaml[pkg_name]["version"]: v = vn.ver(vstr) + if isinstance(v, vn.GitVersion): - version_defs.append(v) + if not require_checksum or v.is_commit: + version_defs.append(v) else: - satisfying_versions = self._check_for_defined_matching_versions(pkg_class, v) - # Amongst all defined versions satisfying this specific - # preference, the highest-numbered version is the - # most-preferred: therefore sort satisfying versions - # from greatest to least - version_defs.extend(sorted(satisfying_versions, reverse=True)) + matches = [x for x in self.possible_versions[pkg_name] if x.satisfies(v)] + matches.sort(reverse=True) + if not matches: + raise spack.config.ConfigError( + f"Preference for version {v} does not match any known " + f"version of {pkg_name} (in its package.py or any external)" + ) + version_defs.extend(matches) for weight, vdef in enumerate(llnl.util.lang.dedupe(version_defs)): self.declared_versions[pkg_name].append( @@ -1773,31 +2142,11 @@ def key_fn(item): ) self.possible_versions[pkg_name].add(vdef) - def _check_for_defined_matching_versions(self, pkg_class, v): - """Given a version specification (which may be a concrete version, - range, etc.), determine if any package.py version declarations - or externals define a version which satisfies it. - - This is primarily for determining whether a version request (e.g. - version preferences, which should not themselves define versions) - refers to a defined version. - - This function raises an exception if no satisfying versions are - found. - """ - pkg_name = pkg_class.name - satisfying_versions = list(x for x in pkg_class.versions if x.satisfies(v)) - satisfying_versions.extend(x for x in self.possible_versions[pkg_name] if x.satisfies(v)) - if not satisfying_versions: - raise spack.config.ConfigError( - "Preference for version {0} does not match any version" - " defined for {1} (in its package.py or any external)".format(str(v), pkg_name) - ) - return satisfying_versions - - def add_concrete_versions_from_specs(self, specs, origin): + def define_ad_hoc_versions_from_specs( + self, specs, origin, *, allow_deprecated: bool, require_checksum: bool + ): """Add concrete versions to possible versions from lists of CLI/dev specs.""" - for s in spack.traverse.traverse_nodes(specs): + for s in traverse.traverse_nodes(specs): # If there is a concrete version on the CLI *that we know nothing # about*, add it to the known versions. Use idx=0, which is the # best possible, so they're guaranteed to be used preferentially. @@ -1806,9 +2155,16 @@ def add_concrete_versions_from_specs(self, specs, origin): if version is None or any(v == version for v in self.possible_versions[s.name]): continue - self.declared_versions[s.name].append( - DeclaredVersion(version=version, idx=0, origin=origin) - ) + if require_checksum and not _is_checksummed_git_version(version): + raise UnsatisfiableSpecError( + s.format("No matching version for constraint {name}{@versions}") + ) + + if not allow_deprecated and version in self.deprecated_versions[s.name]: + continue + + declared = DeclaredVersion(version=version, idx=0, origin=origin) + self.declared_versions[s.name].append(declared) self.possible_versions[s.name].add(version) def _supported_targets(self, compiler_name, compiler_version, targets): @@ -1969,6 +2325,8 @@ def target_defaults(self, specs): self.default_targets = list(sorted(set(self.default_targets))) + self.target_preferences() + def virtual_providers(self): self.gen.h2("Virtual providers") msg = ( @@ -2005,7 +2363,7 @@ def generate_possible_compilers(self, specs): # add compiler specs from the input line to possibilities if we # don't require compilers to exist. strict = spack.concretize.Concretizer().check_for_compiler_existence - for s in spack.traverse.traverse_nodes(specs): + for s in traverse.traverse_nodes(specs): # we don't need to validate compilers for already-built specs if s.concrete or not s.compiler: continue @@ -2040,11 +2398,11 @@ def define_version_constraints(self): # generate facts for each package constraint and the version # that satisfies it for v in sorted(v for v in self.possible_versions[pkg_name] if v.satisfies(versions)): - self.gen.fact(fn.version_satisfies(pkg_name, versions, v)) + self.gen.fact(fn.pkg_fact(pkg_name, fn.version_satisfies(versions, v))) self.gen.newline() - def define_virtual_constraints(self): + def collect_virtual_constraints(self): """Define versions for constraints on virtuals. Must be called before define_version_constraints(). @@ -2052,7 +2410,7 @@ def define_virtual_constraints(self): # aggregate constraints into per-virtual sets constraint_map = collections.defaultdict(lambda: set()) for pkg_name, versions in self.version_constraints: - if not spack.repo.path.is_virtual(pkg_name): + if not spack.repo.PATH.is_virtual(pkg_name): continue constraint_map[pkg_name].add(versions) @@ -2130,27 +2488,31 @@ def define_variant_values(self): # spec_clauses(). We might want to order these facts by pkg and name # if we are debugging. for pkg, variant, value in self.variant_values_from_specs: - self.gen.fact(fn.variant_possible_value(pkg, variant, value)) + self.gen.fact(fn.pkg_fact(pkg, fn.variant_possible_value(variant, value))) - def _facts_from_concrete_spec(self, spec, possible): + def register_concrete_spec(self, spec, possible): # tell the solver about any installed packages that could # be dependencies (don't tell it about the others) - h = spec.dag_hash() - if spec.name in possible and h not in self.seen_hashes: - self.reusable_and_possible[h] = spec - try: - # Only consider installed packages for repo we know - spack.repo.path.get(spec) - except (spack.repo.UnknownNamespaceError, spack.repo.UnknownPackageError): - return + if spec.name not in possible: + return + + try: + # Only consider installed packages for repo we know + spack.repo.PATH.get(spec) + except (spack.repo.UnknownNamespaceError, spack.repo.UnknownPackageError) as e: + tty.debug(f"[REUSE] Issues when trying to reuse {spec.short_spec}: {str(e)}") + return + self.reusable_and_possible.add(spec) + + def concrete_specs(self): + """Emit facts for reusable specs""" + for h, spec in self.reusable_and_possible.items(): # this indicates that there is a spec like this installed self.gen.fact(fn.installed_hash(spec.name, h)) - # this describes what constraints it imposes on the solve self.impose(h, spec, body=True) self.gen.newline() - # Declare as possible parts of specs that are not in package.py # - Add versions to possible versions # - Add OS to possible OS's @@ -2161,17 +2523,21 @@ def _facts_from_concrete_spec(self, spec, possible): ) self.possible_oses.add(dep.os) - # add the hash to the one seen so far - self.seen_hashes.add(h) - def define_concrete_input_specs(self, specs, possible): # any concrete specs in the input spec list for input_spec in specs: for spec in input_spec.traverse(): if spec.concrete: - self._facts_from_concrete_spec(spec, possible) - - def setup(self, driver, specs, reuse=None): + self.register_concrete_spec(spec, possible) + + def setup( + self, + driver: PyclingoDriver, + specs: Sequence[spack.spec.Spec], + *, + reuse: Optional[List[spack.spec.Spec]] = None, + allow_deprecated: bool = False, + ): """Generate an ASP program with relevant constraints for specs. This calls methods on the solve driver to set up the problem with @@ -2179,9 +2545,10 @@ def setup(self, driver, specs, reuse=None): specs, as well as constraints from the specs themselves. Arguments: - driver (PyclingoDriver): driver instance of this solve - specs (list): list of Specs to solve - reuse (None or list): list of concrete specs that can be reused + driver: driver instance of this solve + specs: list of Specs to solve + reuse: list of concrete specs that can be reused + allow_deprecated: if True adds deprecated versions into the solve """ self._condition_id_counter = itertools.count() @@ -2190,28 +2557,30 @@ def setup(self, driver, specs, reuse=None): # get list of all possible dependencies self.possible_virtuals = set(x.name for x in specs if x.virtual) - possible = spack.package_base.possible_dependencies( - *specs, virtuals=self.possible_virtuals, deptype=spack.dependency.all_deptypes - ) + + node_counter = _create_counter(specs, tests=self.tests) + self.possible_virtuals = node_counter.possible_virtuals() + self.pkgs = node_counter.possible_dependencies() # Fail if we already know an unreachable node is requested for spec in specs: missing_deps = [ - str(d) for d in spec.traverse() if d.name not in possible and not d.virtual + str(d) for d in spec.traverse() if d.name not in self.pkgs and not d.virtual ] if missing_deps: raise spack.spec.InvalidDependencyError(spec.name, missing_deps) - self.pkgs = set(possible) - # driver is used by all the functions below to add facts and # rules to generate an ASP program. self.gen = driver + if not allow_deprecated: + self.gen.fact(fn.deprecated_versions_not_allowed()) + # Calculate develop specs # they will be used in addition to command line specs # in determining known versions/targets/os - dev_specs = () + dev_specs: Tuple[spack.spec.Spec, ...] = () env = ev.active_environment() if env: dev_specs = tuple( @@ -2226,14 +2595,16 @@ def setup(self, driver, specs, reuse=None): # get possible compilers self.possible_compilers = self.generate_possible_compilers(specs) - self.gen.h1("Concrete input spec definitions") - self.define_concrete_input_specs(specs, possible) - + self.gen.h1("Reusable concrete specs") + self.define_concrete_input_specs(specs, self.pkgs) if reuse: - self.gen.h1("Reusable specs") self.gen.fact(fn.optimize_for_reuse()) for reusable_spec in reuse: - self._facts_from_concrete_spec(reusable_spec, possible) + self.register_concrete_spec(reusable_spec, self.pkgs) + self.concrete_specs() + + self.gen.h1("Generic statements on possible packages") + node_counter.possible_packages_facts(self.gen, fn) self.gen.h1("Possible flags on nodes") for flag in spack.spec.FlagMap.valid_compiler_flags(): @@ -2253,13 +2624,23 @@ def setup(self, driver, specs, reuse=None): self.provider_requirements() self.external_packages() - # traverse all specs and packages to build dict of possible versions - self.build_version_dict(possible) - self.add_concrete_versions_from_specs(specs, Provenance.SPEC) - self.add_concrete_versions_from_specs(dev_specs, Provenance.DEV_SPEC) - - req_version_specs = self._get_versioned_specs_from_pkg_requirements() - self.add_concrete_versions_from_specs(req_version_specs, Provenance.PACKAGE_REQUIREMENT) + # TODO: make a config option for this undocumented feature + checksummed = "SPACK_CONCRETIZER_REQUIRE_CHECKSUM" in os.environ + self.define_package_versions_and_validate_preferences( + self.pkgs, allow_deprecated=allow_deprecated, require_checksum=checksummed + ) + self.define_ad_hoc_versions_from_specs( + specs, Provenance.SPEC, allow_deprecated=allow_deprecated, require_checksum=checksummed + ) + self.define_ad_hoc_versions_from_specs( + dev_specs, + Provenance.DEV_SPEC, + allow_deprecated=allow_deprecated, + require_checksum=checksummed, + ) + self.validate_and_define_versions_from_requirements( + allow_deprecated=allow_deprecated, require_checksum=checksummed + ) self.gen.h1("Package Constraints") for pkg in sorted(self.pkgs): @@ -2267,11 +2648,13 @@ def setup(self, driver, specs, reuse=None): self.pkg_rules(pkg, tests=self.tests) self.gen.h2("Package preferences: %s" % pkg) self.preferred_variants(pkg) - self.target_preferences(pkg) + self.gen.h1("Develop specs") # Inject dev_path from environment for ds in dev_specs: self.condition(spack.spec.Spec(ds.name), ds, msg="%s is a develop spec" % ds.name) + self.trigger_rules() + self.effect_rules() self.gen.h1("Spec Constraints") self.literal_specs(specs) @@ -2279,10 +2662,8 @@ def setup(self, driver, specs, reuse=None): self.gen.h1("Variant Values defined in specs") self.define_variant_values() - self.gen.h1("Virtual Constraints") - self.define_virtual_constraints() - self.gen.h1("Version Constraints") + self.collect_virtual_constraints() self.define_version_constraints() self.gen.h1("Compiler Version Constraints") @@ -2292,93 +2673,113 @@ def setup(self, driver, specs, reuse=None): self.define_target_constraints() def literal_specs(self, specs): - for idx, spec in enumerate(specs): + for spec in specs: self.gen.h2("Spec: %s" % str(spec)) - self.gen.fact(fn.literal(idx)) + condition_id = next(self._condition_id_counter) + trigger_id = next(self._trigger_id_counter) - self.gen.fact(fn.literal(idx, "virtual_root" if spec.virtual else "root", spec.name)) - for clause in self.spec_clauses(spec): - self.gen.fact(fn.literal(idx, *clause.args)) - if clause.args[0] == "variant_set": - self.gen.fact( - fn.literal(idx, "variant_default_value_from_cli", *clause.args[1:]) + # Special condition triggered by "literal_solved" + self.gen.fact(fn.literal(trigger_id)) + self.gen.fact(fn.pkg_fact(spec.name, fn.condition_trigger(condition_id, trigger_id))) + self.gen.fact(fn.condition_reason(condition_id, f"{spec} requested from CLI")) + + # Effect imposes the spec + imposed_spec_key = str(spec), None + cache = self._effect_cache[spec.name] + msg = ( + "literal specs have different requirements. clear cache before computing literals" + ) + assert imposed_spec_key not in cache, msg + effect_id = next(self._effect_id_counter) + requirements = self.spec_clauses(spec) + root_name = spec.name + for clause in requirements: + clause_name = clause.args[0] + if clause_name == "variant_set": + requirements.append( + fn.attr("variant_default_value_from_cli", *clause.args[1:]) ) + elif clause_name in ("node", "virtual_node", "hash"): + # These facts are needed to compute the "condition_set" of the root + pkg_name = clause.args[1] + self.gen.fact(fn.mentioned_in_literal(trigger_id, root_name, pkg_name)) + + requirements.append(fn.attr("virtual_root" if spec.virtual else "root", spec.name)) + cache[imposed_spec_key] = (effect_id, requirements) + self.gen.fact(fn.pkg_fact(spec.name, fn.condition_effect(condition_id, effect_id))) - if self.concretize_everything: - self.gen.fact(fn.concretize_everything()) + if self.concretize_everything: + self.gen.fact(fn.solve_literal(trigger_id)) - def _get_versioned_specs_from_pkg_requirements(self): - """If package requirements mention versions that are not mentioned + self.effect_rules() + + def validate_and_define_versions_from_requirements( + self, *, allow_deprecated: bool, require_checksum: bool + ): + """If package requirements mention concrete versions that are not mentioned elsewhere, then we need to collect those to mark them as possible - versions. - """ - req_version_specs = list() - config = spack.config.get("packages") - for pkg_name, d in config.items(): - if pkg_name == "all": + versions. If they are abstract and statically have no match, then we + need to throw an error. This function assumes all possible versions are already + registered in self.possible_versions.""" + for pkg_name, d in spack.config.get("packages").items(): + if pkg_name == "all" or "require" not in d: continue - if "require" in d: - req_version_specs.extend(self._specs_from_requires(pkg_name, d["require"])) - return req_version_specs + + for s in traverse.traverse_nodes(self._specs_from_requires(pkg_name, d["require"])): + name, versions = s.name, s.versions + + if name not in self.pkgs or versions == spack.version.any_version: + continue + + s.attach_git_version_lookup() + v = versions.concrete + + if not v: + # If the version is not concrete, check it's statically concretizable. If + # not throw an error, which is just so that users know they need to change + # their config, instead of getting a hard to decipher concretization error. + if not any(x for x in self.possible_versions[name] if x.satisfies(versions)): + raise spack.config.ConfigError( + f"Version requirement {versions} on {pkg_name} for {name} " + f"cannot match any known version from package.py or externals" + ) + continue + + if v in self.possible_versions[name]: + continue + + if not allow_deprecated and v in self.deprecated_versions[name]: + continue + + # If concrete an not yet defined, conditionally define it, like we do for specs + # from the command line. + if not require_checksum or _is_checksummed_git_version(v): + self.declared_versions[name].append( + DeclaredVersion(version=v, idx=0, origin=Provenance.PACKAGE_REQUIREMENT) + ) + self.possible_versions[name].add(v) def _specs_from_requires(self, pkg_name, section): - """Collect specs from requirements which define versions (i.e. those that - have a concrete version). Requirements can define *new* versions if - they are included as part of an equivalence (hash=number) but not - otherwise. - """ + """Collect specs from a requirement rule""" if isinstance(section, str): - spec = spack.spec.Spec(section) - if not spec.name: - spec.name = pkg_name - extracted_specs = [spec] - else: - spec_strs = [] - for spec_group in section: - if isinstance(spec_group, str): - spec_strs.append(spec_group) - else: - # Otherwise it is an object. The object can contain a single - # "spec" constraint, or a list of them with "any_of" or - # "one_of" policy. - if "spec" in spec_group: - new_constraints = [spec_group["spec"]] - else: - key = "one_of" if "one_of" in spec_group else "any_of" - new_constraints = spec_group[key] - spec_strs.extend(new_constraints) - - extracted_specs = [] - for spec_str in spec_strs: - spec = spack.spec.Spec(spec_str) - if not spec.name: - spec.name = pkg_name - extracted_specs.append(spec) + yield _spec_with_default_name(section, pkg_name) + return - version_specs = [] - for spec in extracted_specs: - if spec.versions.concrete: - # Note: this includes git versions - version_specs.append(spec) + for spec_group in section: + if isinstance(spec_group, str): + yield _spec_with_default_name(spec_group, pkg_name) continue - # Prefer spec's name if it exists, in case the spec is - # requiring a specific implementation inside of a virtual section - # e.g. packages:mpi:require:openmpi@4.0.1 - pkg_class = spack.repo.path.get_pkg_class(spec.name or pkg_name) - satisfying_versions = self._check_for_defined_matching_versions( - pkg_class, spec.versions - ) - - # Version ranges ("@1.3" without the "=", "@1.2:1.4") and lists - # will end up here - ordered_satisfying_versions = sorted(satisfying_versions, reverse=True) - vspecs = list(spack.spec.Spec("@{0}".format(x)) for x in ordered_satisfying_versions) - version_specs.extend(vspecs) + # Otherwise it is an object. The object can contain a single + # "spec" constraint, or a list of them with "any_of" or + # "one_of" policy. + if "spec" in spec_group: + yield _spec_with_default_name(spec_group["spec"], pkg_name) + continue - for spec in version_specs: - spec.attach_git_version_lookup() - return version_specs + key = "one_of" if "one_of" in spec_group else "any_of" + for s in spec_group[key]: + yield _spec_with_default_name(s, pkg_name) class SpecBuilder: @@ -2394,17 +2795,27 @@ class SpecBuilder: r"^node_compiler$", r"^package_hash$", r"^root$", + r"^variant_default_value_from_cli$", r"^virtual_node$", r"^virtual_root$", ] ) ) + @staticmethod + def make_node(*, pkg: str) -> NodeArgument: + """Given a package name, returns the string representation of the "min_dupe_id" node in + the ASP encoding. + + Args: + pkg: name of a package + """ + return NodeArgument(id="0", pkg=pkg) + def __init__(self, specs, hash_lookup=None): self._specs = {} self._result = None self._command_line_specs = specs - self._hash_specs = [] self._flag_sources = collections.defaultdict(lambda: set()) self._flag_compiler_defaults = set() @@ -2412,101 +2823,105 @@ def __init__(self, specs, hash_lookup=None): # from this dictionary during reconstruction self._hash_lookup = hash_lookup or {} - def hash(self, pkg, h): - if pkg not in self._specs: - self._specs[pkg] = self._hash_lookup[h] - self._hash_specs.append(pkg) + def hash(self, node, h): + if node not in self._specs: + self._specs[node] = self._hash_lookup[h] - def node(self, pkg): - if pkg not in self._specs: - self._specs[pkg] = spack.spec.Spec(pkg) + def node(self, node): + if node not in self._specs: + self._specs[node] = spack.spec.Spec(node.pkg) - def _arch(self, pkg): - arch = self._specs[pkg].architecture + def _arch(self, node): + arch = self._specs[node].architecture if not arch: arch = spack.spec.ArchSpec() - self._specs[pkg].architecture = arch + self._specs[node].architecture = arch return arch - def node_platform(self, pkg, platform): - self._arch(pkg).platform = platform + def node_platform(self, node, platform): + self._arch(node).platform = platform - def node_os(self, pkg, os): - self._arch(pkg).os = os + def node_os(self, node, os): + self._arch(node).os = os - def node_target(self, pkg, target): - self._arch(pkg).target = target + def node_target(self, node, target): + self._arch(node).target = target - def variant_value(self, pkg, name, value): + def variant_value(self, node, name, value): # FIXME: is there a way not to special case 'dev_path' everywhere? if name == "dev_path": - self._specs[pkg].variants.setdefault( + self._specs[node].variants.setdefault( name, spack.variant.SingleValuedVariant(name, value) ) return if name == "patches": - self._specs[pkg].variants.setdefault( + self._specs[node].variants.setdefault( name, spack.variant.MultiValuedVariant(name, value) ) return - self._specs[pkg].update_variant_validate(name, value) + self._specs[node].update_variant_validate(name, value) - def version(self, pkg, version): - self._specs[pkg].versions = vn.VersionList([vn.Version(version)]) + def version(self, node, version): + self._specs[node].versions = vn.VersionList([vn.Version(version)]) - def node_compiler_version(self, pkg, compiler, version): - self._specs[pkg].compiler = spack.spec.CompilerSpec(compiler) - self._specs[pkg].compiler.versions = vn.VersionList([vn.Version(version)]) + def node_compiler_version(self, node, compiler, version): + self._specs[node].compiler = spack.spec.CompilerSpec(compiler) + self._specs[node].compiler.versions = vn.VersionList([vn.Version(version)]) - def node_flag_compiler_default(self, pkg): - self._flag_compiler_defaults.add(pkg) + def node_flag_compiler_default(self, node): + self._flag_compiler_defaults.add(node) - def node_flag(self, pkg, flag_type, flag): - self._specs[pkg].compiler_flags.add_flag(flag_type, flag, False) + def node_flag(self, node, flag_type, flag): + self._specs[node].compiler_flags.add_flag(flag_type, flag, False) - def node_flag_source(self, pkg, flag_type, source): - self._flag_sources[(pkg, flag_type)].add(source) + def node_flag_source(self, node, flag_type, source): + self._flag_sources[(node, flag_type)].add(source) - def no_flags(self, pkg, flag_type): - self._specs[pkg].compiler_flags[flag_type] = [] + def no_flags(self, node, flag_type): + self._specs[node].compiler_flags[flag_type] = [] - def external_spec_selected(self, pkg, idx): + def external_spec_selected(self, node, idx): """This means that the external spec and index idx has been selected for this package. """ + packages_yaml = spack.config.get("packages") packages_yaml = _normalize_packages_yaml(packages_yaml) - spec_info = packages_yaml[pkg]["externals"][int(idx)] - self._specs[pkg].external_path = spec_info.get("prefix", None) - self._specs[pkg].external_modules = spack.spec.Spec._format_module_list( + spec_info = packages_yaml[node.pkg]["externals"][int(idx)] + self._specs[node].external_path = spec_info.get("prefix", None) + self._specs[node].external_modules = spack.spec.Spec._format_module_list( spec_info.get("modules", None) ) - self._specs[pkg].extra_attributes = spec_info.get("extra_attributes", {}) + self._specs[node].extra_attributes = spec_info.get("extra_attributes", {}) # If this is an extension, update the dependencies to include the extendee - package = self._specs[pkg].package_class(self._specs[pkg]) + package = self._specs[node].package_class(self._specs[node]) extendee_spec = package.extendee_spec - if extendee_spec: - package.update_external_dependencies(self._specs.get(extendee_spec.name, None)) - - def depends_on(self, pkg, dep, type): - dependencies = self._specs[pkg].edges_to_dependencies(name=dep) - # TODO: assertion to be removed when cross-compilation is handled correctly - msg = "Current solver does not handle multiple dependency edges of the same name" - assert len(dependencies) < 2, msg - - if not dependencies: - self._specs[pkg].add_dependency_edge(self._specs[dep], deptypes=(type,), virtuals=()) + if extendee_spec: + extendee_node = SpecBuilder.make_node(pkg=extendee_spec.name) + package.update_external_dependencies(self._specs.get(extendee_node, None)) + + def depends_on(self, parent_node, dependency_node, type): + dependency_spec = self._specs[dependency_node] + edges = self._specs[parent_node].edges_to_dependencies(name=dependency_spec.name) + edges = [x for x in edges if id(x.spec) == id(dependency_spec)] + depflag = dt.flag_from_string(type) + + if not edges: + self._specs[parent_node].add_dependency_edge( + self._specs[dependency_node], depflag=depflag, virtuals=() + ) else: - # TODO: This assumes that each solve unifies dependencies - dependencies[0].update_deptypes(deptypes=(type,)) + edges[0].update_deptypes(depflag=depflag) - def virtual_on_edge(self, pkg, provider, virtual): - dependencies = self._specs[pkg].edges_to_dependencies(name=provider) - assert len(dependencies) == 1 + def virtual_on_edge(self, parent_node, provider_node, virtual): + dependencies = self._specs[parent_node].edges_to_dependencies(name=(provider_node.pkg)) + provider_spec = self._specs[provider_node] + dependencies = [x for x in dependencies if id(x.spec) == id(provider_spec)] + assert len(dependencies) == 1, f"{virtual}: {provider_node.pkg}" dependencies[0].update_virtuals((virtual,)) def reorder_flags(self): @@ -2536,19 +2951,23 @@ def reorder_flags(self): # order is determined by the DAG. A spec's flags come after any of its ancestors # on the compile line - source_key = (spec.name, flag_type) + node = SpecBuilder.make_node(pkg=spec.name) + source_key = (node, flag_type) if source_key in self._flag_sources: - order = [s.name for s in spec.traverse(order="post", direction="parents")] + order = [ + SpecBuilder.make_node(pkg=s.name) + for s in spec.traverse(order="post", direction="parents") + ] sorted_sources = sorted( self._flag_sources[source_key], key=lambda s: order.index(s) ) # add flags from each source, lowest to highest precedence - for name in sorted_sources: + for node in sorted_sources: all_src_flags = list() - per_pkg_sources = [self._specs[name]] - if name in cmd_specs: - per_pkg_sources.append(cmd_specs[name]) + per_pkg_sources = [self._specs[node]] + if node.pkg in cmd_specs: + per_pkg_sources.append(cmd_specs[node.pkg]) for source in per_pkg_sources: all_src_flags.extend(source.compiler_flags.get(flag_type, [])) extend_flag_list(from_sources, all_src_flags) @@ -2562,9 +2981,8 @@ def reorder_flags(self): spec.compiler_flags.update({flag_type: ordered_compiler_flags}) - def deprecated(self, pkg, version): - msg = 'using "{0}@{1}" which is a deprecated version' - tty.warn(msg.format(pkg, version)) + def deprecated(self, node: NodeArgument, version: str) -> None: + tty.warn(f'using "{node.pkg}@{version}" which is a deprecated version') @staticmethod def sort_fn(function_tuple): @@ -2619,14 +3037,15 @@ def build_specs(self, function_tuples): # solving but don't construct anything. Do not ignore error # predicates on virtual packages. if name != "error": - pkg = args[0] - if spack.repo.path.is_virtual(pkg): + node = args[0] + pkg = node.pkg + if spack.repo.PATH.is_virtual(pkg): continue # if we've already gotten a concrete spec for this pkg, # do not bother calling actions on it except for node_flag_source, # since node_flag_source is tracking information not in the spec itself - spec = self._specs.get(pkg) + spec = self._specs.get(args[0]) if spec and spec.concrete: if name != "node_flag_source": continue @@ -2638,7 +3057,7 @@ def build_specs(self, function_tuples): for spec in self._specs.values(): if spec.namespace: continue - repo = spack.repo.path.repo_for_pkg(spec) + repo = spack.repo.PATH.repo_for_pkg(spec) spec.namespace = repo.namespace # fix flags after all specs are constructed @@ -2672,7 +3091,9 @@ def build_specs(self, function_tuples): for root in self._specs.values(): for spec in root.traverse(): if isinstance(spec.version, vn.GitVersion): - spec.version.attach_git_lookup_from_package(spec.fullname) + spec.version.attach_lookup( + spack.version.git_ref_lookup.GitRefLookup(spec.fullname) + ) return self._specs @@ -2733,11 +3154,11 @@ def _reusable_specs(self, specs): reusable_specs = [] if self.reuse: # Specs from the local Database - with spack.store.db.read_transaction(): + with spack.store.STORE.db.read_transaction(): reusable_specs.extend( [ s - for s in spack.store.db.query(installed=True) + for s in spack.store.STORE.db.query(installed=True) if not s.satisfies("dev_path=*") ] ) @@ -2760,27 +3181,42 @@ def _reusable_specs(self, specs): return reusable_specs - def solve(self, specs, out=None, timers=False, stats=False, tests=False, setup_only=False): + def solve( + self, + specs, + out=None, + timers=False, + stats=False, + tests=False, + setup_only=False, + allow_deprecated=False, + ): """ Arguments: specs (list): List of ``Spec`` objects to solve for. out: Optionally write the generate ASP program to a file-like object. - timers (bool): Print out coarse fimers for different solve phases. + timers (bool): Print out coarse timers for different solve phases. stats (bool): Print out detailed stats from clingo. tests (bool or tuple): If True, concretize test dependencies for all packages. If a tuple of package names, concretize test dependencies for named packages (defaults to False: do not concretize test dependencies). setup_only (bool): if True, stop after setup and don't solve (default False). + allow_deprecated (bool): allow deprecated version in the solve """ # Check upfront that the variants are admissible + specs = [s.lookup_hash() for s in specs] reusable_specs = self._check_input_and_extract_concrete_specs(specs) reusable_specs.extend(self._reusable_specs(specs)) setup = SpackSolverSetup(tests=tests) output = OutputConfiguration(timers=timers, stats=stats, out=out, setup_only=setup_only) - result, _, _ = self.driver.solve(setup, specs, reuse=reusable_specs, output=output) + result, _, _ = self.driver.solve( + setup, specs, reuse=reusable_specs, output=output, allow_deprecated=allow_deprecated + ) return result - def solve_in_rounds(self, specs, out=None, timers=False, stats=False, tests=False): + def solve_in_rounds( + self, specs, out=None, timers=False, stats=False, tests=False, allow_deprecated=False + ): """Solve for a stable model of specs in multiple rounds. This relaxes the assumption of solve that everything must be consistent and @@ -2795,7 +3231,9 @@ def solve_in_rounds(self, specs, out=None, timers=False, stats=False, tests=Fals timers (bool): print timing if set to True stats (bool): print internal statistics if set to True tests (bool): add test dependencies to the solve + allow_deprecated (bool): allow deprecated version in the solve """ + specs = [s.lookup_hash() for s in specs] reusable_specs = self._check_input_and_extract_concrete_specs(specs) reusable_specs.extend(self._reusable_specs(specs)) setup = SpackSolverSetup(tests=tests) @@ -2807,7 +3245,11 @@ def solve_in_rounds(self, specs, out=None, timers=False, stats=False, tests=Fals output = OutputConfiguration(timers=timers, stats=stats, out=out, setup_only=False) while True: result, _, _ = self.driver.solve( - setup, input_specs, reuse=reusable_specs, output=output + setup, + input_specs, + reuse=reusable_specs, + output=output, + allow_deprecated=allow_deprecated, ) yield result @@ -2845,10 +3287,11 @@ def __init__(self, provided, conflicts): msg = ( "Spack concretizer internal error. Please submit a bug report and include the " "command, environment if applicable and the following error message." - f"\n {provided} is unsatisfiable, errors are:" + f"\n {provided} is unsatisfiable" ) - msg += "".join([f"\n {conflict}" for conflict in conflicts]) + if conflicts: + msg += ", errors are:" + "".join([f"\n {conflict}" for conflict in conflicts]) super(spack.error.UnsatisfiableSpecError, self).__init__(msg) diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp index f90aa872f8bb16..d5f24ddc3b33f0 100644 --- a/lib/spack/spack/solver/concretize.lp +++ b/lib/spack/spack/solver/concretize.lp @@ -7,38 +7,135 @@ % This logic program implements Spack's concretizer %============================================================================= +% ID of the nodes in the "root" link-run sub-DAG +#const min_dupe_id = 0. + +#const direct_link_run = 0. +#const direct_build = 1. + +% Allow clingo to create nodes +{ attr("node", node(0..X-1, Package)) } :- max_dupes(Package, X), not virtual(Package). +{ attr("virtual_node", node(0..X-1, Package)) } :- max_dupes(Package, X), virtual(Package). + +% Integrity constraints on DAG nodes +:- attr("root", PackageNode), not attr("node", PackageNode). +:- attr("version", PackageNode, _), not attr("node", PackageNode), not attr("virtual_node", PackageNode). +:- attr("node_version_satisfies", PackageNode), not attr("node", PackageNode). +:- attr("hash", PackageNode, _), not attr("node", PackageNode). +:- attr("node_platform", PackageNode, _), not attr("node", PackageNode). +:- attr("node_os", PackageNode, _), not attr("node", PackageNode). +:- attr("node_target", PackageNode, _), not attr("node", PackageNode). +:- attr("node_compiler_version", PackageNode, _, _), not attr("node", PackageNode). +:- attr("variant_value", PackageNode, _, _), not attr("node", PackageNode). +:- attr("node_flag_compiler_default", PackageNode), not attr("node", PackageNode). +:- attr("node_flag", PackageNode, _, _), not attr("node", PackageNode). +:- attr("no_flags", PackageNode, _), not attr("node", PackageNode). +:- attr("external_spec_selected", PackageNode, _), not attr("node", PackageNode). +:- attr("depends_on", ParentNode, _, _), not attr("node", ParentNode). +:- attr("depends_on", _, ChildNode, _), not attr("node", ChildNode). +:- attr("node_flag_source", ParentNode, _, _), not attr("node", ParentNode). +:- attr("node_flag_source", _, _, ChildNode), not attr("node", ChildNode). +:- attr("virtual_node", VirtualNode), not provider(_, VirtualNode), internal_error("virtual node with no provider"). +:- provider(_, VirtualNode), not attr("virtual_node", VirtualNode), internal_error("provider with no virtual node"). +:- provider(PackageNode, _), not attr("node", PackageNode), internal_error("provider with no real node"). + +:- attr("root", node(ID, PackageNode)), ID > min_dupe_id, internal_error("root with a non-minimal duplicate ID"). + +% Nodes in the "root" unification set cannot depend on non-root nodes if the dependency is "link" or "run" +:- attr("depends_on", node(min_dupe_id, Package), node(ID, _), "link"), ID != min_dupe_id, unification_set("root", node(min_dupe_id, Package)), internal_error("link dependency out of the root unification set"). +:- attr("depends_on", node(min_dupe_id, Package), node(ID, _), "run"), ID != min_dupe_id, unification_set("root", node(min_dupe_id, Package)), internal_error("run dependency out of the root unification set"). + +% Rules on "unification sets", i.e. on sets of nodes allowing a single configuration of any given package +unify(SetID, PackageName) :- unification_set(SetID, node(_, PackageName)). +:- 2 { unification_set(SetID, node(_, PackageName)) }, unify(SetID, PackageName). + +unification_set("root", PackageNode) :- attr("root", PackageNode). +unification_set(SetID, ChildNode) :- attr("depends_on", ParentNode, ChildNode, Type), Type != "build", unification_set(SetID, ParentNode). + +unification_set(("build", node(X, Child)), node(X, Child)) + :- attr("depends_on", ParentNode, node(X, Child), Type), + Type == "build", + multiple_unification_sets(Child), + unification_set(SetID, ParentNode). + +unification_set("generic_build", node(X, Child)) + :- attr("depends_on", ParentNode, node(X, Child), Type), + Type == "build", + not multiple_unification_sets(Child), + unification_set(_, ParentNode). + +unification_set(SetID, VirtualNode) + :- provider(PackageNode, VirtualNode), + unification_set(SetID, PackageNode). + +% Do not allow split dependencies, for now. This ensures that we don't construct graphs where e.g. +% a python extension depends on setuptools@63.4 as a run dependency, but uses e.g. setuptools@68 +% as a build dependency. +% +% We'll need to relax the rule before we get to actual cross-compilation +:- depends_on(ParentNode, node(X, Dependency)), depends_on(ParentNode, node(Y, Dependency)), X < Y. + + +#defined multiple_unification_sets/1. + +%---- +% Rules to break symmetry and speed-up searches +%---- + +% In the "root" unification set only ID = 0 are allowed +:- unification_set("root", node(ID, _)), ID != 0, internal_error("root unification set has node with non-zero unification set ID"). + +% In the "root" unification set we allow only packages from the link-run possible subDAG +:- unification_set("root", node(_, Package)), not possible_in_link_run(Package), not virtual(Package), internal_error("package outside possible link/run graph in root unification set"). + +% Each node must belong to at least one unification set +:- attr("node", PackageNode), not unification_set(_, PackageNode), internal_error("node belongs to no unification set"). + +% Cannot have a node with an ID, if lower ID of the same package are not used +:- attr("node", node(ID1, Package)), + not attr("node", node(ID2, Package)), + max_dupes(Package, X), ID1=0..X-1, ID2=0..X-1, ID2 < ID1, + internal_error("node skipped id number"). + +:- attr("virtual_node", node(ID1, Package)), + not attr("virtual_node", node(ID2, Package)), + max_dupes(Package, X), ID1=0..X-1, ID2=0..X-1, ID2 < ID1, + internal_error("virtual node skipped id number"). + %----------------------------------------------------------------------------- % Map literal input specs to facts that drive the solve %----------------------------------------------------------------------------- -% Give clingo the choice to solve an input spec or not -{ literal_solved(ID) } :- literal(ID). -literal_not_solved(ID) :- not literal_solved(ID), literal(ID). +% TODO: literals, at the moment, can only influence the "root" unification set. This needs to be extended later. + +% Node attributes that have multiple node arguments (usually, only the first argument is a node) +multiple_nodes_attribute("node_flag_source"). +multiple_nodes_attribute("depends_on"). +multiple_nodes_attribute("virtual_on_edge"). +multiple_nodes_attribute("provider_set"). -% If concretize_everything() is a fact, then we cannot have unsolved specs -:- literal_not_solved(ID), concretize_everything. +trigger_condition_holds(TriggerID, node(min_dupe_id, Package)) :- + solve_literal(TriggerID), + pkg_fact(Package, condition_trigger(_, TriggerID)), + literal(TriggerID). -% Make a problem with "zero literals solved" unsat. This is to trigger -% looking for solutions to the ASP problem with "errors", which results -% in better reporting for users. See #30669 for details. -1 { literal_solved(ID) : literal(ID) }. +trigger_node(TriggerID, Node, Node) :- + trigger_condition_holds(TriggerID, Node), + literal(TriggerID). -opt_criterion(300, "number of input specs not concretized"). -#minimize{ 0@300: #true }. -#minimize { 1@300,ID : literal_not_solved(ID) }. +% Since we trigger the existence of literal nodes from a condition, we need to construct +% the condition_set/2 manually below +mentioned_in_literal(Root, Mentioned) :- mentioned_in_literal(TriggerID, Root, Mentioned), solve_literal(TriggerID). +condition_set(node(min_dupe_id, Root), node(min_dupe_id, Mentioned)) :- mentioned_in_literal(Root, Mentioned). -% Map constraint on the literal ID to the correct PSID -attr(Name, A1) :- literal(LiteralID, Name, A1), literal_solved(LiteralID). -attr(Name, A1, A2) :- literal(LiteralID, Name, A1, A2), literal_solved(LiteralID). -attr(Name, A1, A2, A3) :- literal(LiteralID, Name, A1, A2, A3), literal_solved(LiteralID). -attr(Name, A1, A2, A3, A4) :- literal(LiteralID, Name, A1, A2, A3, A4), literal_solved(LiteralID). +% Discriminate between "roots" that have been explicitly requested, and roots that are deduced from "virtual roots" +explicitly_requested_root(node(min_dupe_id, Package)) :- + solve_literal(TriggerID), + trigger_and_effect(Package, TriggerID, EffectID), + imposed_constraint(EffectID, "root", Package). #defined concretize_everything/0. #defined literal/1. -#defined literal/3. -#defined literal/4. -#defined literal/5. -#defined literal/6. % Attributes for node packages which must have a single value attr_single_value("version"). @@ -48,15 +145,15 @@ attr_single_value("node_target"). % Error when no attribute is selected error(100, no_value_error, Attribute, Package) - :- attr("node", Package), + :- attr("node", node(ID, Package)), attr_single_value(Attribute), - not attr(Attribute, Package, _). + not attr(Attribute, node(ID, Package), _). % Error when multiple attr need to be selected error(100, multiple_values_error, Attribute, Package) - :- attr("node", Package), + :- attr("node", node(ID, Package)), attr_single_value(Attribute), - 2 { attr(Attribute, Package, Version) }. + 2 { attr(Attribute, node(ID, Package), Value) }. %----------------------------------------------------------------------------- % Version semantics @@ -64,28 +161,23 @@ error(100, multiple_values_error, Attribute, Package) % Versions are declared with a weight and an origin, which indicates where the % version was declared (e.g. "package_py" or "external"). -version_declared(Package, Version, Weight) :- version_declared(Package, Version, Weight, _). +pkg_fact(Package, version_declared(Version, Weight)) :- pkg_fact(Package, version_declared(Version, Weight, _)). % We can't emit the same version **with the same weight** from two different sources -:- version_declared(Package, Version, Weight, Origin1), - version_declared(Package, Version, Weight, Origin2), +:- pkg_fact(Package, version_declared(Version, Weight, Origin1)), + pkg_fact(Package, version_declared(Version, Weight, Origin2)), Origin1 < Origin2, internal_error("Two versions with identical weights"). % We cannot use a version declared for an installed package if we end up building it -:- version_declared(Package, Version, Weight, "installed"), - attr("version", Package, Version), - version_weight(Package, Weight), - not attr("hash", Package, _), +:- pkg_fact(Package, version_declared(Version, Weight, "installed")), + attr("version", node(ID, Package), Version), + version_weight(node(ID, Package), Weight), + not attr("hash", node(ID, Package), _), internal_error("Reuse version weight used for built package"). % versions are declared w/priority -- declared with priority implies declared -version_declared(Package, Version) :- version_declared(Package, Version, _). - -% a spec with a git hash version is equivalent to one with the same matched version -version_satisfies(Package, Constraint, HashVersion) :- version_satisfies(Package, Constraint, EquivalentVersion), - version_equivalent(Package, HashVersion, EquivalentVersion). -#defined version_equivalent/3. +pkg_fact(Package, version_declared(Version)) :- pkg_fact(Package, version_declared(Version, _)). % If something is a package, it has only one version and that must be a % declared version. @@ -93,74 +185,85 @@ version_satisfies(Package, Constraint, HashVersion) :- version_satisfies(Package % is not precisely one version chosen. Error facts are heavily optimized % against to ensure they cannot be inferred when a non-error solution is % possible -{ attr("version", Package, Version) : version_declared(Package, Version) } - :- attr("node", Package). +{ attr("version", node(ID, Package), Version) : pkg_fact(Package, version_declared(Version)) } + :- attr("node", node(ID, Package)). % A virtual package may or may not have a version, but never has more than one error(100, "Cannot select a single version for virtual '{0}'", Virtual) - :- attr("virtual_node", Virtual), - 2 { attr("version", Virtual, Version) }. + :- attr("virtual_node", node(ID, Virtual)), + 2 { attr("version", node(ID, Virtual), Version) }. % If we select a deprecated version, mark the package as deprecated -attr("deprecated", Package, Version) :- - attr("version", Package, Version), - deprecated_version(Package, Version). +attr("deprecated", node(ID, Package), Version) :- + attr("version", node(ID, Package), Version), + pkg_fact(Package, deprecated_version(Version)). + +error(100, "Package '{0}' needs the deprecated version '{1}', and this is not allowed", Package, Version) + :- deprecated_versions_not_allowed(), + attr("version", node(ID, Package), Version), + not external(node(ID, Package)), + not concrete(node(ID, Package)), + pkg_fact(Package, deprecated_version(Version)). -possible_version_weight(Package, Weight) - :- attr("version", Package, Version), - version_declared(Package, Version, Weight). +possible_version_weight(node(ID, Package), Weight) + :- attr("version", node(ID, Package), Version), + pkg_fact(Package, version_declared(Version, Weight)). % we can't use the weight for an external version if we don't use the % corresponding external spec. -:- attr("version", Package, Version), - version_weight(Package, Weight), - version_declared(Package, Version, Weight, "external"), - not external(Package), +:- attr("version", node(ID, Package), Version), + version_weight(node(ID, Package), Weight), + pkg_fact(Package, version_declared(Version, Weight, "external")), + not external(node(ID, Package)), internal_error("External weight used for built package"). % we can't use a weight from an installed spec if we are building it % and vice-versa -:- attr("version", Package, Version), - version_weight(Package, Weight), - version_declared(Package, Version, Weight, "installed"), - build(Package), +:- attr("version", node(ID, Package), Version), + version_weight(node(ID, Package), Weight), + pkg_fact(Package, version_declared(Version, Weight, "installed")), + build(node(ID, Package)), internal_error("Reuse version weight used for build package"). -:- attr("version", Package, Version), - version_weight(Package, Weight), - not version_declared(Package, Version, Weight, "installed"), - not build(Package), +:- attr("version", node(ID, Package), Version), + version_weight(node(ID, Package), Weight), + not pkg_fact(Package, version_declared(Version, Weight, "installed")), + not build(node(ID, Package)), internal_error("Build version weight used for reused package"). -1 { version_weight(Package, Weight) : version_declared(Package, Version, Weight) } 1 - :- attr("version", Package, Version), - attr("node", Package). +1 { version_weight(node(ID, Package), Weight) : pkg_fact(Package, version_declared(Version, Weight)) } 1 + :- attr("version", node(ID, Package), Version), + attr("node", node(ID, Package)), + internal_error("version weights must exist and be unique"). % node_version_satisfies implies that exactly one of the satisfying versions % is the package's version, and vice versa. % While this choice rule appears redundant with the initial choice rule for % versions, virtual nodes with version constraints require this rule to be % able to choose versions -{ attr("version", Package, Version) : version_satisfies(Package, Constraint, Version) } - :- attr("node_version_satisfies", Package, Constraint). +{ attr("version", node(ID, Package), Version) : pkg_fact(Package, version_satisfies(Constraint, Version)) } + :- attr("node_version_satisfies", node(ID, Package), Constraint). % If there is at least a version that satisfy the constraint, impose a lower % bound on the choice rule to avoid false positives with the error below -1 { attr("version", Package, Version) : version_satisfies(Package, Constraint, Version) } - :- attr("node_version_satisfies", Package, Constraint), - version_satisfies(Package, Constraint, _). +1 { attr("version", node(ID, Package), Version) : pkg_fact(Package, version_satisfies(Constraint, Version)) } + :- attr("node_version_satisfies", node(ID, Package), Constraint), + pkg_fact(Package, version_satisfies(Constraint, _)), + internal_error("must choose a single version to satisfy version constraints"). % More specific error message if the version cannot satisfy some constraint % Otherwise covered by `no_version_error` and `versions_conflict_error`. error(10, "Cannot satisfy '{0}@{1}'", Package, Constraint) - :- attr("node_version_satisfies", Package, Constraint), - attr("version", Package, Version), - not version_satisfies(Package, Constraint, Version). + :- attr("node_version_satisfies", node(ID, Package), Constraint), + attr("version", node(ID, Package), Version), + not pkg_fact(Package, version_satisfies(Constraint, Version)). -attr("node_version_satisfies", Package, Constraint) - :- attr("version", Package, Version), version_satisfies(Package, Constraint, Version). +attr("node_version_satisfies", node(ID, Package), Constraint) + :- attr("version", node(ID, Package), Version), + pkg_fact(Package, version_satisfies(Constraint, Version)). #defined version_satisfies/3. +#defined deprecated_versions_not_allowed/0. #defined deprecated_version/2. %----------------------------------------------------------------------------- @@ -175,32 +278,141 @@ attr("node_version_satisfies", Package, Constraint) %----------------------------------------------------------------------------- % conditions are specified with `condition_requirement` and hold when % corresponding spec attributes hold. -condition_holds(ID) :- - condition(ID, _); - attr(Name, A1) : condition_requirement(ID, Name, A1); - attr(Name, A1, A2) : condition_requirement(ID, Name, A1, A2); - attr(Name, A1, A2, A3) : condition_requirement(ID, Name, A1, A2, A3); - attr(Name, A1, A2, A3, A4) : condition_requirement(ID, Name, A1, A2, A3, A4). - -% condition_holds(ID) implies all imposed_constraints, unless do_not_impose(ID) -% is derived. This allows imposed constraints to be canceled in special cases. -impose(ID) :- condition_holds(ID), not do_not_impose(ID). -% conditions that hold impose constraints on other specs -attr(Name, A1) :- impose(ID), imposed_constraint(ID, Name, A1). -attr(Name, A1, A2) :- impose(ID), imposed_constraint(ID, Name, A1, A2). -attr(Name, A1, A2, A3) :- impose(ID), imposed_constraint(ID, Name, A1, A2, A3). -attr(Name, A1, A2, A3, A4) :- impose(ID), imposed_constraint(ID, Name, A1, A2, A3, A4). +% A "condition_set(PackageNode, _)" is the set of nodes on which PackageNode can require / impose conditions +% Currently, for a given node, this is the link-run sub-DAG of PackageNode and its direct build dependencies +condition_set(PackageNode, PackageNode, direct_link_run) :- attr("node", PackageNode). + +condition_set(PackageNode, PackageNode, direct_link_run) :- provider(PackageNode, VirtualNode). +condition_set(PackageNode, VirtualNode, direct_link_run) :- provider(PackageNode, VirtualNode). + +condition_set(PackageNode, DependencyNode, direct_build) :- condition_set(PackageNode, PackageNode, direct_link_run), attr("depends_on", PackageNode, DependencyNode, "build"). +condition_set(PackageNode, DependencyNode, direct_link_run) :- condition_set(PackageNode, PackageNode, direct_link_run), attr("depends_on", PackageNode, DependencyNode, Type), Type != "build". + +condition_set(ID, VirtualNode, Type) :- condition_set(ID, PackageNode, Type), provider(PackageNode, VirtualNode). + +condition_set(ID, PackageNode) :- condition_set(ID, PackageNode, _). + +condition_set(VirtualNode, X) :- provider(PackageNode, VirtualNode), condition_set(PackageNode, X). + +condition_packages(ID, A1) :- condition_requirement(ID, _, A1). +condition_packages(ID, A1) :- condition_requirement(ID, _, A1, _). +condition_packages(ID, A1) :- condition_requirement(ID, _, A1, _, _). +condition_packages(ID, A1) :- condition_requirement(ID, _, A1, _, _, _). + +trigger_node(ID, node(PackageID, Package), node(PackageID, Package)) :- pkg_fact(Package, trigger_id(ID)), attr("node", node(PackageID, Package)). +trigger_node(ID, node(PackageID, Package), node(VirtualID, Virtual)) :- pkg_fact(Virtual, trigger_id(ID)), provider(node(PackageID, Package), node(VirtualID, Virtual)). + +condition_nodes(TriggerID, PackageNode, node(X, A1)) + :- condition_packages(TriggerID, A1), + condition_set(PackageNode, node(X, A1)), + trigger_node(TriggerID, PackageNode, _). + +cannot_hold(TriggerID, PackageNode) + :- condition_packages(TriggerID, A1), + not condition_set(PackageNode, node(_, A1)), + trigger_node(TriggerID, PackageNode, _). + +trigger_condition_holds(ID, RequestorNode) :- + trigger_node(ID, PackageNode, RequestorNode); + attr(Name, node(X, A1)) : condition_requirement(ID, Name, A1), condition_nodes(ID, PackageNode, node(X, A1)); + attr(Name, node(X, A1), A2) : condition_requirement(ID, Name, A1, A2), condition_nodes(ID, PackageNode, node(X, A1)); + attr(Name, node(X, A1), A2, A3) : condition_requirement(ID, Name, A1, A2, A3), condition_nodes(ID, PackageNode, node(X, A1)), not multiple_nodes_attribute(Name); + attr(Name, node(X, A1), A2, A3, A4) : condition_requirement(ID, Name, A1, A2, A3, A4), condition_nodes(ID, PackageNode, node(X, A1)); + % Special cases + attr("node_flag_source", node(X, A1), A2, node(Y, A3)) : condition_requirement(ID, "node_flag_source", A1, A2, A3), condition_nodes(ID, PackageNode, node(X, A1)), condition_nodes(ID, PackageNode, node(Y, A3)); + not cannot_hold(ID, PackageNode). + +condition_holds(ConditionID, node(X, Package)) + :- pkg_fact(Package, condition_trigger(ConditionID, TriggerID)), + trigger_condition_holds(TriggerID, node(X, Package)). + +trigger_and_effect(Package, TriggerID, EffectID) + :- pkg_fact(Package, condition_trigger(ID, TriggerID)), + pkg_fact(Package, condition_effect(ID, EffectID)). + +% condition_holds(ID, node(ID, Package)) implies all imposed_constraints, unless do_not_impose(ID, node(ID, Package)) +% is derived. This allows imposed constraints to be canceled in special cases. +impose(EffectID, node(X, Package)) + :- trigger_and_effect(Package, TriggerID, EffectID), + trigger_node(TriggerID, _, node(X, Package)), + trigger_condition_holds(TriggerID, node(X, Package)), + not do_not_impose(EffectID, node(X, Package)). + +imposed_packages(ID, A1) :- imposed_constraint(ID, _, A1). +imposed_packages(ID, A1) :- imposed_constraint(ID, _, A1, _). +imposed_packages(ID, A1) :- imposed_constraint(ID, _, A1, _, _). +imposed_packages(ID, A1) :- imposed_constraint(ID, _, A1, _, _, _). +imposed_packages(ID, A1) :- imposed_constraint(ID, "depends_on", _, A1, _). + +imposed_nodes(EffectID, node(NodeID, Package), node(X, A1)) + :- pkg_fact(Package, condition_trigger(ID, TriggerID)), + pkg_fact(Package, condition_effect(ID, EffectID)), + imposed_packages(EffectID, A1), + condition_set(node(NodeID, Package), node(X, A1)), + trigger_node(TriggerID, _, node(NodeID, Package)). + +imposed_nodes(ConditionID, PackageNode, node(X, A1)) + :- imposed_packages(ConditionID, A1), + condition_set(PackageNode, node(X, A1)), + attr("hash", PackageNode, ConditionID). + +:- imposed_packages(ID, A1), impose(ID, PackageNode), not condition_set(PackageNode, node(_, A1)). +:- imposed_packages(ID, A1), impose(ID, PackageNode), not imposed_nodes(ID, PackageNode, node(_, A1)). + +% Conditions that hold impose may impose constraints on other specs +attr(Name, node(X, A1)) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1), imposed_nodes(ID, PackageNode, node(X, A1)). +attr(Name, node(X, A1), A2) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1, A2), imposed_nodes(ID, PackageNode, node(X, A1)), not multiple_nodes_attribute(Name). +attr(Name, node(X, A1), A2, A3) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1, A2, A3), imposed_nodes(ID, PackageNode, node(X, A1)), not multiple_nodes_attribute(Name). +attr(Name, node(X, A1), A2, A3, A4) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1, A2, A3, A4), imposed_nodes(ID, PackageNode, node(X, A1)). + +% For node flag sources we need to look at the condition_set of the source, since it is the dependent +% of the package on which I want to impose the constraint +attr("node_flag_source", node(X, A1), A2, node(Y, A3)) + :- impose(ID, node(X, A1)), + imposed_constraint(ID, "node_flag_source", A1, A2, A3), + condition_set(node(Y, A3), node(X, A1)). + +% Provider set is relevant only for literals, since it's the only place where `^[virtuals=foo] bar` +% might appear in the HEAD of a rule +attr("provider_set", node(min_dupe_id, Provider), node(min_dupe_id, Virtual)) + :- solve_literal(TriggerID), + trigger_and_effect(_, TriggerID, EffectID), + impose(EffectID, _), + imposed_constraint(EffectID, "provider_set", Provider, Virtual). + +provider(ProviderNode, VirtualNode) :- attr("provider_set", ProviderNode, VirtualNode). + +% Here we can't use the condition set because it's a recursive definition, that doesn't define the +% node index, and leads to unsatisfiability. Hence we say that one and only one node index must +% satisfy the dependency. +1 { attr("depends_on", node(X, A1), node(0..Y-1, A2), A3) : max_dupes(A2, Y) } 1 + :- impose(ID, node(X, A1)), + imposed_constraint(ID, "depends_on", A1, A2, A3). + +% Reconstruct virtual dependencies for reused specs +attr("virtual_on_edge", node(X, A1), node(Y, A2), Virtual) + :- impose(ID, node(X, A1)), + depends_on(node(X, A1), node(Y, A2)), + imposed_constraint(ID, "virtual_on_edge", A1, A2, Virtual), + not build(node(X, A1)). + +virtual_condition_holds(node(Y, A2), Virtual) + :- impose(ID, node(X, A1)), + attr("virtual_on_edge", node(X, A1), node(Y, A2), Virtual), + not build(node(X, A1)). % we cannot have additional variant values when we are working with concrete specs -:- attr("node", Package), attr("hash", Package, Hash), - attr("variant_value", Package, Variant, Value), +:- attr("node", node(ID, Package)), + attr("hash", node(ID, Package), Hash), + attr("variant_value", node(ID, Package), Variant, Value), not imposed_constraint(Hash, "variant_value", Package, Variant, Value), internal_error("imposed hash without imposing all variant values"). % we cannot have additional flag values when we are working with concrete specs -:- attr("node", Package), attr("hash", Package, Hash), - attr("node_flag", Package, FlagType, Flag), +:- attr("node", node(ID, Package)), + attr("hash", node(ID, Package), Hash), + attr("node_flag", node(ID, Package), FlagType, Flag), not imposed_constraint(Hash, "node_flag", Package, FlagType, Flag), internal_error("imposed hash without imposing all flag values"). @@ -218,137 +430,148 @@ attr(Name, A1, A2, A3, A4) :- impose(ID), imposed_constraint(ID, Name, A1, A2, A % Concrete specs %----------------------------------------------------------------------------- % if a package is assigned a hash, it's concrete. -concrete(Package) :- attr("hash", Package, _), attr("node", Package). +concrete(PackageNode) :- attr("hash", PackageNode, _), attr("node", PackageNode). %----------------------------------------------------------------------------- % Dependency semantics %----------------------------------------------------------------------------- % Dependencies of any type imply that one package "depends on" another -depends_on(Package, Dependency) :- attr("depends_on", Package, Dependency, _). +depends_on(PackageNode, DependencyNode) :- attr("depends_on", PackageNode, DependencyNode, _). % a dependency holds if its condition holds and if it is not external or % concrete. We chop off dependencies for externals, and dependencies of % concrete specs don't need to be resolved -- they arise from the concrete % specs themselves. -dependency_holds(Package, Dependency, Type) :- - dependency_condition(ID, Package, Dependency), - dependency_type(ID, Type), - build(Package), - not external(Package), - condition_holds(ID). - -% We cut off dependencies of externals (as we don't really know them). -% Don't impose constraints on dependencies that don't exist. -do_not_impose(ID) :- - not dependency_holds(Package, Dependency, _), - dependency_condition(ID, Package, Dependency). - -% declared dependencies are real if they're not virtual AND -% the package is not an external. -% They're only triggered if the associated dependnecy condition holds. -attr("depends_on", Package, Dependency, Type) - :- dependency_holds(Package, Dependency, Type), - not virtual(Dependency). - -% every root must be a node -attr("node", Package) :- attr("root", Package). - -% dependencies imply new nodes -attr("node", Dependency) :- attr("node", Package), depends_on(Package, Dependency). +attr("track_dependencies", Node) :- build(Node), not external(Node). + +% If a dependency holds on a package node, there must be one and only one dependency node satisfying it +1 { attr("depends_on", PackageNode, node(0..Y-1, Dependency), Type) : max_dupes(Dependency, Y) } 1 + :- attr("dependency_holds", PackageNode, Dependency, Type), + not virtual(Dependency). % all nodes in the graph must be reachable from some root % this ensures a user can't say `zlib ^libiconv` (neither of which have any % dependencies) and get a two-node unconnected graph -needed(Package) :- attr("root", Package). -needed(Dependency) :- needed(Package), depends_on(Package, Dependency). +needed(PackageNode) :- attr("root", PackageNode). +needed(DependencyNode) :- needed(PackageNode), depends_on(PackageNode, DependencyNode). error(10, "'{0}' is not a valid dependency for any package in the DAG", Package) - :- attr("node", Package), - not needed(Package). - -% Avoid cycles in the DAG -% some combinations of conditional dependencies can result in cycles; -% this ensures that we solve around them -path(Parent, Child) :- depends_on(Parent, Child). -path(Parent, Descendant) :- path(Parent, A), depends_on(A, Descendant). -error(100, "Cyclic dependency detected between '{0}' and '{1}' (consider changing variants to avoid the cycle)", A, B) - :- path(A, B), - path(B, A). + :- attr("node", node(ID, Package)), + not needed(node(ID, Package)). #defined dependency_type/2. -#defined dependency_condition/3. %----------------------------------------------------------------------------- % Conflicts %----------------------------------------------------------------------------- -error(1, Msg) :- attr("node", Package), - conflict(Package, TriggerID, ConstraintID, Msg), - condition_holds(TriggerID), - condition_holds(ConstraintID), - not external(Package), % ignore conflicts for externals - not attr("hash", Package, _). % ignore conflicts for installed packages - -#defined conflict/4. +error(1, Msg) + :- attr("node", node(ID, Package)), + pkg_fact(Package, conflict(TriggerID, ConstraintID, Msg)), + % node(ID1, TriggerPackage) is node(ID2, Package) in most, but not all, cases + condition_holds(TriggerID, node(ID1, TriggerPackage)), + condition_holds(ConstraintID, node(ID2, Package)), + unification_set(X, node(ID2, Package)), + unification_set(X, node(ID1, TriggerPackage)), + not external(node(ID, Package)), % ignore conflicts for externals + not attr("hash", node(ID, Package), _). % ignore conflicts for installed packages %----------------------------------------------------------------------------- % Virtual dependencies %----------------------------------------------------------------------------- +% If the provider is set from the command line, its weight is 0 +possible_provider_weight(ProviderNode, VirtualNode, 0, "Set on the command line") + :- attr("provider_set", ProviderNode, VirtualNode). + +% Enforces all virtuals to be provided, if multiple of them are provided together +error(100, "Package '{0}' needs to provide both '{1}' and '{2}' together, but provides only '{1}'", Package, Virtual1, Virtual2) +:- condition_holds(ID, node(X, Package)), + pkg_fact(Package, provided_together(ID, SetID, Virtual1)), + pkg_fact(Package, provided_together(ID, SetID, Virtual2)), + Virtual1 != Virtual2, + attr("virtual_on_incoming_edges", node(X, Package), Virtual1), + not attr("virtual_on_incoming_edges", node(X, Package), Virtual2), + attr("virtual_node", node(_, Virtual1)), + attr("virtual_node", node(_, Virtual2)). + % if a package depends on a virtual, it's not external and we have a % provider for that virtual then it depends on the provider -attr("depends_on", Package, Provider, Type) - :- dependency_holds(Package, Virtual, Type), - provider(Provider, Virtual), - not external(Package). +node_depends_on_virtual(PackageNode, Virtual, Type) + :- attr("dependency_holds", PackageNode, Virtual, Type), + virtual(Virtual), + not external(PackageNode). + +node_depends_on_virtual(PackageNode, Virtual) :- node_depends_on_virtual(PackageNode, Virtual, Type). + +1 { attr("depends_on", PackageNode, ProviderNode, Type) : provider(ProviderNode, node(VirtualID, Virtual)) } 1 + :- node_depends_on_virtual(PackageNode, Virtual, Type). + +attr("virtual_on_edge", PackageNode, ProviderNode, Virtual) + :- attr("dependency_holds", PackageNode, Virtual, Type), + attr("depends_on", PackageNode, ProviderNode, Type), + provider(ProviderNode, node(_, Virtual)), + not external(PackageNode). -attr("virtual_on_edge", Package, Provider, Virtual) - :- dependency_holds(Package, Virtual, Type), - provider(Provider, Virtual), - not external(Package). +attr("virtual_on_incoming_edges", ProviderNode, Virtual) + :- attr("virtual_on_edge", _, ProviderNode, Virtual). % dependencies on virtuals also imply that the virtual is a virtual node -attr("virtual_node", Virtual) - :- dependency_holds(Package, Virtual, Type), - virtual(Virtual), not external(Package). +1 { attr("virtual_node", node(0..X-1, Virtual)) : max_dupes(Virtual, X) } + :- node_depends_on_virtual(PackageNode, Virtual). % If there's a virtual node, we must select one and only one provider. % The provider must be selected among the possible providers. -{ provider(Package, Virtual) : possible_provider(Package, Virtual) } - :- attr("virtual_node", Virtual). + +error(100, "'{0}' cannot be a provider for the '{1}' virtual", Package, Virtual) + :- attr("provider_set", node(min_dupe_id, Package), node(min_dupe_id, Virtual)), + not virtual_condition_holds( node(min_dupe_id, Package), Virtual). error(100, "Cannot find valid provider for virtual {0}", Virtual) - :- attr("virtual_node", Virtual), - not provider(_, Virtual). + :- attr("virtual_node", node(X, Virtual)), + not provider(_, node(X, Virtual)). error(100, "Cannot select a single provider for virtual '{0}'", Virtual) - :- attr("virtual_node", Virtual), - 2 { provider(P, Virtual) }. + :- attr("virtual_node", node(X, Virtual)), + 2 { provider(P, node(X, Virtual)) }. % virtual roots imply virtual nodes, and that one provider is a root -attr("virtual_node", Virtual) :- attr("virtual_root", Virtual). +attr("virtual_node", VirtualNode) :- attr("virtual_root", VirtualNode). % If we asked for a virtual root and we have a provider for that, % then the provider is the root package. -attr("root", Package) :- attr("virtual_root", Virtual), provider(Package, Virtual). +attr("root", PackageNode) :- attr("virtual_root", VirtualNode), provider(PackageNode, VirtualNode). -% If we asked for a root package and that root provides a virtual, -% the root is a provider for that virtual. This rule is mostly relevant -% for environments that are concretized together (e.g. where we -% asks to install "mpich" and "hdf5+mpi" and we want "mpich" to -% be the mpi provider) -provider(Package, Virtual) :- attr("node", Package), virtual_condition_holds(Package, Virtual). +% The provider is selected among the nodes for which the virtual condition holds +1 { provider(PackageNode, node(X, Virtual)) : + attr("node", PackageNode), virtual_condition_holds(PackageNode, Virtual) } 1 + :- attr("virtual_node", node(X, Virtual)). % The provider provides the virtual if some provider condition holds. -virtual_condition_holds(Provider, Virtual) :- - provider_condition(ID, Provider, Virtual), - condition_holds(ID), +virtual_condition_holds(node(ProviderID, Provider), Virtual) :- virtual_condition_holds(ID, node(ProviderID, Provider), Virtual). +virtual_condition_holds(ID, node(ProviderID, Provider), Virtual) :- + pkg_fact(Provider, provider_condition(ID, Virtual)), + condition_holds(ID, node(ProviderID, Provider)), virtual(Virtual). +% If a "provider" condition holds, but this package is not a provider, do not impose the "provider" condition +do_not_impose(EffectID, node(X, Package)) + :- virtual_condition_holds(ID, node(X, Package), Virtual), + pkg_fact(Package, condition_effect(ID, EffectID)), + not provider(node(X, Package), node(_, Virtual)). + +% Choose the provider among root specs, if possible +:- provider(ProviderNode, node(min_dupe_id, Virtual)), + virtual_condition_holds(_, PossibleProvider, Virtual), + PossibleProvider != ProviderNode, + explicitly_requested_root(PossibleProvider), + not explicitly_requested_root(ProviderNode). + % A package cannot be the actual provider for a virtual if it does not % fulfill the conditions to provide that virtual -:- provider(Package, Virtual), not virtual_condition_holds(Package, Virtual), +:- provider(PackageNode, node(VirtualID, Virtual)), + not virtual_condition_holds(PackageNode, Virtual), internal_error("Virtual when provides not respected"). -#defined possible_provider/2. +#defined provided_together/4. %----------------------------------------------------------------------------- % Virtual dependency weights @@ -357,49 +580,30 @@ virtual_condition_holds(Provider, Virtual) :- % A provider may have different possible weights depending on whether it's an external % or not, or on preferences expressed in packages.yaml etc. This rule ensures that % we select the weight, among the possible ones, that minimizes the overall objective function. -1 { provider_weight(Dependency, Virtual, Weight, Reason) : - possible_provider_weight(Dependency, Virtual, Weight, Reason) } 1 - :- provider(Dependency, Virtual), internal_error("Package provider weights must be unique"). - -% Get rid or the reason for enabling the possible weight (useful for debugging) -provider_weight(Dependency, Virtual, Weight) :- provider_weight(Dependency, Virtual, Weight, _). +1 { provider_weight(DependencyNode, VirtualNode, Weight) : + possible_provider_weight(DependencyNode, VirtualNode, Weight, _) } 1 + :- provider(DependencyNode, VirtualNode), internal_error("Package provider weights must be unique"). % A provider that is an external can use a weight of 0 -possible_provider_weight(Dependency, Virtual, 0, "external") - :- provider(Dependency, Virtual), - external(Dependency). - -% A provider mentioned in packages.yaml can use a weight -% according to its priority in the list of providers -possible_provider_weight(Dependency, Virtual, Weight, "packages_yaml") - :- provider(Dependency, Virtual), - depends_on(Package, Dependency), - pkg_provider_preference(Package, Virtual, Dependency, Weight). +possible_provider_weight(DependencyNode, VirtualNode, 0, "external") + :- provider(DependencyNode, VirtualNode), + external(DependencyNode). % A provider mentioned in the default configuration can use a weight % according to its priority in the list of providers -possible_provider_weight(Dependency, Virtual, Weight, "default") - :- provider(Dependency, Virtual), - default_provider_preference(Virtual, Dependency, Weight). +possible_provider_weight(node(ProviderID, Provider), node(VirtualID, Virtual), Weight, "default") + :- provider(node(ProviderID, Provider), node(VirtualID, Virtual)), + default_provider_preference(Virtual, Provider, Weight). % Any provider can use 100 as a weight, which is very high and discourage its use -possible_provider_weight(Dependency, Virtual, 100, "fallback") :- provider(Dependency, Virtual). +possible_provider_weight(node(ProviderID, Provider), VirtualNode, 100, "fallback") + :- provider(node(ProviderID, Provider), VirtualNode). % do not warn if generated program contains none of these. -#defined possible_provider/2. -#defined provider_condition/3. -#defined required_provider_condition/3. -#defined required_provider_condition/4. -#defined required_provider_condition/5. -#defined required_provider_condition/6. -#defined declared_dependency/3. #defined virtual/1. #defined virtual_condition_holds/2. #defined external/1. -#defined external_spec/2. -#defined external_version_declared/4. #defined buildable_false/1. -#defined pkg_provider_preference/4. #defined default_provider_preference/3. %----------------------------------------------------------------------------- @@ -407,134 +611,130 @@ possible_provider_weight(Dependency, Virtual, 100, "fallback") :- provider(Depen %----------------------------------------------------------------------------- % if a package is external its version must be one of the external versions -{ external_version(Package, Version, Weight): - version_declared(Package, Version, Weight, "external") } - :- external(Package). -error(100, "Attempted to use external for '{0}' which does not satisfy any configured external spec", Package) - :- external(Package), - not external_version(Package, _, _). +{ external_version(node(ID, Package), Version, Weight): + pkg_fact(Package, version_declared(Version, Weight, "external")) } + :- external(node(ID, Package)). -error(100, "Attempted to use external for '{0}' which does not satisfy any configured external spec", Package) - :- external(Package), - 2 { external_version(Package, Version, Weight) }. +error(100, "Attempted to use external for '{0}' which does not satisfy any configured external spec version", Package) + :- external(node(ID, Package)), + not external_version(node(ID, Package), _, _). + +error(100, "Attempted to use external for '{0}' which does not satisfy a unique configured external spec version", Package) + :- external(node(ID, Package)), + 2 { external_version(node(ID, Package), Version, Weight) }. -version_weight(Package, Weight) :- external_version(Package, Version, Weight). -attr("version", Package, Version) :- external_version(Package, Version, Weight). +version_weight(PackageNode, Weight) :- external_version(PackageNode, Version, Weight). +attr("version", PackageNode, Version) :- external_version(PackageNode, Version, Weight). % if a package is not buildable, only externals or hashed specs are allowed -external(Package) :- buildable_false(Package), - attr("node", Package), - not attr("hash", Package, _). +external(node(ID, Package)) + :- buildable_false(Package), + attr("node", node(ID, Package)), + not attr("hash", node(ID, Package), _). % a package is a real_node if it is not external -real_node(Package) :- attr("node", Package), not external(Package). +real_node(PackageNode) :- attr("node", PackageNode), not external(PackageNode). % a package is external if we are using an external spec for it -external(Package) :- attr("external_spec_selected", Package, _). +external(PackageNode) :- attr("external_spec_selected", PackageNode, _). % we can't use the weight for an external version if we don't use the % corresponding external spec. -:- attr("version", Package, Version), - version_weight(Package, Weight), - version_declared(Package, Version, Weight, "external"), - not external(Package), +:- attr("version", node(ID, Package), Version), + version_weight(node(ID, Package), Weight), + pkg_fact(Package, version_declared(Version, Weight, "external")), + not external(node(ID, Package)), internal_error("External weight used for internal spec"). % determine if an external spec has been selected -attr("external_spec_selected", Package, LocalIndex) :- - external_conditions_hold(Package, LocalIndex), - attr("node", Package), - not attr("hash", Package, _). - -external_conditions_hold(Package, LocalIndex) :- - possible_external(ID, Package, LocalIndex), condition_holds(ID). +attr("external_spec_selected", node(ID, Package), LocalIndex) :- + attr("external_conditions_hold", node(ID, Package), LocalIndex), + attr("node", node(ID, Package)), + not attr("hash", node(ID, Package), _). % it cannot happen that a spec is external, but none of the external specs % conditions hold. error(100, "Attempted to use external for '{0}' which does not satisfy any configured external spec", Package) - :- external(Package), - not external_conditions_hold(Package, _). - -#defined possible_external/3. -#defined external_spec_index/3. -#defined external_spec_condition/3. -#defined external_spec_condition/4. -#defined external_spec_condition/5. -#defined external_spec_condition/6. + :- external(node(ID, Package)), + not attr("external_conditions_hold", node(ID, Package), _). %----------------------------------------------------------------------------- % Config required semantics %----------------------------------------------------------------------------- -package_in_dag(Package) :- attr("node", Package). -package_in_dag(Package) :- attr("virtual_node", Package). +package_in_dag(Node) :- attr("node", Node). +package_in_dag(Node) :- attr("virtual_node", Node). -activate_requirement(Package, X) :- - package_in_dag(Package), +activate_requirement(node(ID, Package), X) :- + package_in_dag(node(ID, Package)), requirement_group(Package, X), not requirement_conditional(Package, X, _). -activate_requirement(Package, X) :- - package_in_dag(Package), +activate_requirement(node(ID, Package), X) :- + package_in_dag(node(ID, Package)), requirement_group(Package, X), - condition_holds(Y), + condition_holds(Y, node(ID, Package)), requirement_conditional(Package, X, Y). -requirement_group_satisfied(Package, X) :- - 1 { condition_holds(Y) : requirement_group_member(Y, Package, X) } 1, +requirement_group_satisfied(node(ID, Package), X) :- + 1 { condition_holds(Y, node(ID, Package)) : requirement_group_member(Y, Package, X) } 1, requirement_policy(Package, X, "one_of"), - activate_requirement(Package, X), + activate_requirement(node(ID, Package), X), requirement_group(Package, X). -requirement_weight(Package, Group, W) :- - condition_holds(Y), +requirement_weight(node(ID, Package), Group, W) :- + condition_holds(Y, node(ID, Package)), requirement_has_weight(Y, W), requirement_group_member(Y, Package, Group), requirement_policy(Package, Group, "one_of"), - requirement_group_satisfied(Package, Group). + requirement_group_satisfied(node(ID, Package), Group). -requirement_group_satisfied(Package, X) :- - 1 { condition_holds(Y) : requirement_group_member(Y, Package, X) } , +requirement_group_satisfied(node(ID, Package), X) :- + 1 { condition_holds(Y, node(ID, Package)) : requirement_group_member(Y, Package, X) } , requirement_policy(Package, X, "any_of"), - activate_requirement(Package, X), + activate_requirement(node(ID, Package), X), requirement_group(Package, X). % TODO: the following two choice rules allow the solver to add compiler % flags if their only source is from a requirement. This is overly-specific % and should use a more-generic approach like in https://github.com/spack/spack/pull/37180 -{ attr("node_flag", A1, A2, A3) } :- - requirement_group_member(Y, Package, X), - activate_requirement(Package, X), - imposed_constraint(Y,"node_flag_set", A1, A2, A3). -{ attr("node_flag_source", A1, A2, A3) } :- - requirement_group_member(Y, Package, X), - activate_requirement(Package, X), - imposed_constraint(Y,"node_flag_source", A1, A2, A3). +{ attr("node_flag", node(ID, Package), FlagType, FlagValue) } :- + requirement_group_member(ConditionID, Package, RequirementID), + activate_requirement(node(ID, Package), RequirementID), + pkg_fact(Package, condition_effect(ConditionID, EffectID)), + imposed_constraint(EffectID, "node_flag_set", Package, FlagType, FlagValue). -requirement_weight(Package, Group, W) :- +{ attr("node_flag_source", node(NodeID1, Package1), FlagType, node(NodeID2, Package2)) } :- + requirement_group_member(ConditionID, Package1, RequirementID), + activate_requirement(node(NodeID1, Package1), RequirementID), + pkg_fact(Package1, condition_effect(ConditionID, EffectID)), + imposed_constraint(EffectID, "node_flag_source", Package1, FlagType, Package2), + imposed_nodes(EffectID, node(NodeID2, Package2), node(NodeID1, Package1)). + +requirement_weight(node(ID, Package), Group, W) :- W = #min { - Z : requirement_has_weight(Y, Z), condition_holds(Y), requirement_group_member(Y, Package, Group); + Z : requirement_has_weight(Y, Z), condition_holds(Y, node(ID, Package)), requirement_group_member(Y, Package, Group); % We need this to avoid an annoying warning during the solve % concretize.lp:1151:5-11: info: tuple ignored: % #sup@73 10000 }, requirement_policy(Package, Group, "any_of"), - requirement_group_satisfied(Package, Group). + requirement_group_satisfied(node(ID, Package), Group). error(100, "cannot satisfy a requirement for package '{0}'.", Package) :- - activate_requirement(Package, X), + activate_requirement(node(ID, Package), X), requirement_group(Package, X), not requirement_message(Package, X, _), - not requirement_group_satisfied(Package, X). + not requirement_group_satisfied(node(ID, Package), X). error(10, Message) :- - activate_requirement(Package, X), + activate_requirement(node(ID, Package), X), requirement_group(Package, X), requirement_message(Package, X, Message), - not requirement_group_satisfied(Package, X). + not requirement_group_satisfied(node(ID, Package), X). #defined requirement_group/2. @@ -549,130 +749,150 @@ error(10, Message) :- %----------------------------------------------------------------------------- % a variant is a variant of a package if it is a variant under some condition % and that condition holds -variant(Package, Variant) :- variant_condition(ID, Package, Variant), - condition_holds(ID). - -attr("variant_propagate", Package, Variant, Value, Source) :- - attr("node", Package), - depends_on(Parent, Package), - attr("variant_propagate", Parent, Variant, Value, Source), - not attr("variant_set", Package, Variant). - -attr("variant_value", Package, Variant, Value) :- - attr("node", Package), - variant(Package, Variant), - attr("variant_propagate", Package, Variant, Value, _), - variant_possible_value(Package, Variant, Value). - +node_has_variant(node(NodeID, Package), Variant) :- + pkg_fact(Package, conditional_variant(ID, Variant)), + condition_holds(ID, node(NodeID, Package)). + +node_has_variant(node(ID, Package), Variant) :- + pkg_fact(Package, variant(Variant)), + attr("node", node(ID, Package)). + +% Variant propagation is forwarded to dependencies +attr("variant_propagation_candidate", PackageNode, Variant, Value, Source) :- + attr("node", PackageNode), + depends_on(ParentNode, PackageNode), + attr("variant_value", node(_, Source), Variant, Value), + attr("variant_propagation_candidate", ParentNode, Variant, _, Source). + +% If the node is a candidate, and it has the variant and value, +% then those variant and value should be propagated +attr("variant_propagate", node(ID, Package), Variant, Value, Source) :- + attr("variant_propagation_candidate", node(ID, Package), Variant, Value, Source), + node_has_variant(node(ID, Package), Variant), + pkg_fact(Package, variant_possible_value(Variant, Value)), + not attr("variant_set", node(ID, Package), Variant). + +% Propagate the value, if there is the corresponding attribute +attr("variant_value", PackageNode, Variant, Value) :- attr("variant_propagate", PackageNode, Variant, Value, _). + +% If a variant is propagated, we cannot have extraneous values (this is for multi valued variants) +variant_is_propagated(PackageNode, Variant) :- attr("variant_propagate", PackageNode, Variant, _, _). +:- variant_is_propagated(PackageNode, Variant), + attr("variant_value", PackageNode, Variant, Value), + not attr("variant_propagate", PackageNode, Variant, Value, _). + +% Cannot receive different values from different sources on the same variant error(100, "{0} and {1} cannot both propagate variant '{2}' to package {3} with values '{4}' and '{5}'", Source1, Source2, Variant, Package, Value1, Value2) :- - attr("variant_propagate", Package, Variant, Value1, Source1), - attr("variant_propagate", Package, Variant, Value2, Source2), - variant(Package, Variant), - Value1 < Value2. + attr("variant_propagate", node(X, Package), Variant, Value1, Source1), + attr("variant_propagate", node(X, Package), Variant, Value2, Source2), + node_has_variant(node(X, Package), Variant), + Value1 < Value2, Source1 < Source2. % a variant cannot be set if it is not a variant on the package error(100, "Cannot set variant '{0}' for package '{1}' because the variant condition cannot be satisfied for the given spec", Variant, Package) - :- attr("variant_set", Package, Variant), - not variant(Package, Variant), - build(Package). + :- attr("variant_set", node(X, Package), Variant), + not node_has_variant(node(X, Package), Variant), + build(node(X, Package)). % a variant cannot take on a value if it is not a variant of the package error(100, "Cannot set variant '{0}' for package '{1}' because the variant condition cannot be satisfied for the given spec", Variant, Package) - :- attr("variant_value", Package, Variant, _), - not variant(Package, Variant), - build(Package). + :- attr("variant_value", node(X, Package), Variant, _), + not node_has_variant(node(X, Package), Variant), + build(node(X, Package)). % if a variant is sticky and not set its value is the default value -attr("variant_value", Package, Variant, Value) :- - variant(Package, Variant), - not attr("variant_set", Package, Variant), - variant_sticky(Package, Variant), +attr("variant_value", node(ID, Package), Variant, Value) :- + node_has_variant(node(ID, Package), Variant), + not attr("variant_set", node(ID, Package), Variant), + pkg_fact(Package, variant_sticky(Variant)), variant_default_value(Package, Variant, Value), - build(Package). + build(node(ID, Package)). % at most one variant value for single-valued variants. { - attr("variant_value", Package, Variant, Value) - : variant_possible_value(Package, Variant, Value) + attr("variant_value", node(ID, Package), Variant, Value) + : pkg_fact(Package, variant_possible_value(Variant, Value)) } - :- attr("node", Package), - variant(Package, Variant), - build(Package). + :- attr("node", node(ID, Package)), + node_has_variant(node(ID, Package), Variant), + build(node(ID, Package)). error(100, "'{0}' required multiple values for single-valued variant '{1}'", Package, Variant) - :- attr("node", Package), - variant(Package, Variant), - variant_single_value(Package, Variant), - build(Package), - 2 { attr("variant_value", Package, Variant, Value) }. + :- attr("node", node(ID, Package)), + node_has_variant(node(ID, Package), Variant), + pkg_fact(Package, variant_single_value(Variant)), + build(node(ID, Package)), + 2 { attr("variant_value", node(ID, Package), Variant, Value) }. error(100, "No valid value for variant '{1}' of package '{0}'", Package, Variant) - :- attr("node", Package), - variant(Package, Variant), - build(Package), - not attr("variant_value", Package, Variant, _). + :- attr("node", node(X, Package)), + node_has_variant(node(X, Package), Variant), + build(node(X, Package)), + not attr("variant_value", node(X, Package), Variant, _). % if a variant is set to anything, it is considered 'set'. -attr("variant_set", Package, Variant) :- attr("variant_set", Package, Variant, _). +attr("variant_set", PackageNode, Variant) :- attr("variant_set", PackageNode, Variant, _). % A variant cannot have a value that is not also a possible value % This only applies to packages we need to build -- concrete packages may % have been built w/different variants from older/different package versions. error(10, "'Spec({1}={2})' is not a valid value for '{0}' variant '{1}'", Package, Variant, Value) - :- attr("variant_value", Package, Variant, Value), - not variant_possible_value(Package, Variant, Value), - build(Package). + :- attr("variant_value", node(ID, Package), Variant, Value), + not pkg_fact(Package, variant_possible_value(Variant, Value)), + build(node(ID, Package)). % Some multi valued variants accept multiple values from disjoint sets. % Ensure that we respect that constraint and we don't pick values from more % than one set at once error(100, "{0} variant '{1}' cannot have values '{2}' and '{3}' as they come from disjoint value sets", Package, Variant, Value1, Value2) - :- attr("variant_value", Package, Variant, Value1), - attr("variant_value", Package, Variant, Value2), - variant_value_from_disjoint_sets(Package, Variant, Value1, Set1), - variant_value_from_disjoint_sets(Package, Variant, Value2, Set2), + :- attr("variant_value", node(ID, Package), Variant, Value1), + attr("variant_value", node(ID, Package), Variant, Value2), + pkg_fact(Package, variant_value_from_disjoint_sets(Variant, Value1, Set1)), + pkg_fact(Package, variant_value_from_disjoint_sets(Variant, Value2, Set2)), Set1 < Set2, % see[1] - build(Package). + build(node(ID, Package)). % variant_set is an explicitly set variant value. If it's not 'set', % we revert to the default value. If it is set, we force the set value -attr("variant_value", Package, Variant, Value) - :- attr("node", Package), - variant(Package, Variant), - attr("variant_set", Package, Variant, Value). +attr("variant_value", PackageNode, Variant, Value) + :- attr("node", PackageNode), + node_has_variant(PackageNode, Variant), + attr("variant_set", PackageNode, Variant, Value). % The rules below allow us to prefer default values for variants % whenever possible. If a variant is set in a spec, or if it is % specified in an external, we score it as if it was a default value. -variant_not_default(Package, Variant, Value) - :- attr("variant_value", Package, Variant, Value), +variant_not_default(node(ID, Package), Variant, Value) + :- attr("variant_value", node(ID, Package), Variant, Value), not variant_default_value(Package, Variant, Value), % variants set explicitly on the CLI don't count as non-default - not attr("variant_set", Package, Variant, Value), + not attr("variant_set", node(ID, Package), Variant, Value), % variant values forced by propagation don't count as non-default - not attr("variant_propagate", Package, Variant, Value, _), + not attr("variant_propagate", node(ID, Package), Variant, Value, _), % variants set on externals that we could use don't count as non-default % this makes spack prefer to use an external over rebuilding with the % default configuration - not external_with_variant_set(Package, Variant, Value), - attr("node", Package). + not external_with_variant_set(node(ID, Package), Variant, Value), + attr("node", node(ID, Package)). % A default variant value that is not used -variant_default_not_used(Package, Variant, Value) +variant_default_not_used(node(ID, Package), Variant, Value) :- variant_default_value(Package, Variant, Value), - variant(Package, Variant), - not attr("variant_value", Package, Variant, Value), - attr("node", Package). + node_has_variant(node(ID, Package), Variant), + not attr("variant_value", node(ID, Package), Variant, Value), + not attr("variant_propagate", node(ID, Package), Variant, _, _), + attr("node", node(ID, Package)). % The variant is set in an external spec -external_with_variant_set(Package, Variant, Value) - :- attr("variant_value", Package, Variant, Value), - condition_requirement(ID, "variant_value", Package, Variant, Value), - possible_external(ID, Package, _), - external(Package), - attr("node", Package). +external_with_variant_set(node(NodeID, Package), Variant, Value) + :- attr("variant_value", node(NodeID, Package), Variant, Value), + condition_requirement(TriggerID, "variant_value", Package, Variant, Value), + trigger_and_effect(Package, TriggerID, EffectID), + imposed_constraint(EffectID, "external_conditions_hold", Package, _), + external(node(NodeID, Package)), + attr("node", node(NodeID, Package)). % The default value for a variant in a package is what is prescribed: % @@ -682,68 +902,62 @@ external_with_variant_set(Package, Variant, Value) % packages.yaml and the command line) % variant_default_value(Package, Variant, Value) - :- variant_default_value_from_package_py(Package, Variant, Value), + :- pkg_fact(Package, variant_default_value_from_package_py(Variant, Value)), not variant_default_value_from_packages_yaml(Package, Variant, _), - not attr("variant_default_value_from_cli", Package, Variant, _). + not attr("variant_default_value_from_cli", node(min_dupe_id, Package), Variant, _). variant_default_value(Package, Variant, Value) :- variant_default_value_from_packages_yaml(Package, Variant, Value), - not attr("variant_default_value_from_cli", Package, Variant, _). + not attr("variant_default_value_from_cli", node(min_dupe_id, Package), Variant, _). variant_default_value(Package, Variant, Value) :- - attr("variant_default_value_from_cli", Package, Variant, Value). + attr("variant_default_value_from_cli", node(min_dupe_id, Package), Variant, Value). % Treat 'none' in a special way - it cannot be combined with other % values even if the variant is multi-valued error(100, "{0} variant '{1}' cannot have values '{2}' and 'none'", Package, Variant, Value) - :- attr("variant_value", Package, Variant, Value), - attr("variant_value", Package, Variant, "none"), + :- attr("variant_value", node(X, Package), Variant, Value), + attr("variant_value", node(X, Package), Variant, "none"), Value != "none", - build(Package). + build(node(X, Package)). % patches and dev_path are special variants -- they don't have to be % declared in the package, so we just allow them to spring into existence % when assigned a value. auto_variant("dev_path"). auto_variant("patches"). -variant(Package, Variant) - :- attr("variant_set", Package, Variant, _), auto_variant(Variant). -variant_single_value(Package, "dev_path") - :- attr("variant_set", Package, "dev_path", _). + +node_has_variant(PackageNode, Variant) + :- attr("variant_set", PackageNode, Variant, _), auto_variant(Variant). + +pkg_fact(Package, variant_single_value("dev_path")) + :- attr("variant_set", node(ID, Package), "dev_path", _). % suppress warnings about this atom being unset. It's only set if some % spec or some package sets it, and without this, clingo will give % warnings like 'info: atom does not occur in any rule head'. -#defined variant/2. -#defined variant_sticky/2. -#defined variant_set/3. -#defined variant_condition/3. -#defined variant_single_value/2. #defined variant_default_value/3. -#defined variant_possible_value/3. -#defined variant_default_value_from_cli/3. #defined variant_default_value_from_packages_yaml/3. -#defined variant_default_value_from_package_py/3. -#defined variant_value_from_disjoint_sets/4. %----------------------------------------------------------------------------- % Platform semantics %----------------------------------------------------------------------------- % if no platform is set, fall back to the default -:- attr("node_platform", _, Platform), not allowed_platform(Platform). +error(100, "platform '{0}' is not allowed on the current host", Platform) + :- attr("node_platform", _, Platform), not allowed_platform(Platform). -attr("node_platform", Package, Platform) - :- attr("node", Package), - not attr("node_platform_set", Package), +attr("node_platform", PackageNode, Platform) + :- attr("node", PackageNode), + not attr("node_platform_set", PackageNode), node_platform_default(Platform). % setting platform on a node is a hard constraint -attr("node_platform", Package, Platform) - :- attr("node", Package), attr("node_platform_set", Package, Platform). +attr("node_platform", PackageNode, Platform) + :- attr("node", PackageNode), attr("node_platform_set", PackageNode, Platform). % platform is set if set to anything -attr("node_platform_set", Package) :- attr("node_platform_set", Package, _). +attr("node_platform_set", PackageNode) :- attr("node_platform_set", PackageNode, _). %----------------------------------------------------------------------------- % OS semantics @@ -752,33 +966,37 @@ attr("node_platform_set", Package) :- attr("node_platform_set", Package, _). os(OS) :- os(OS, _). % one os per node -{ attr("node_os", Package, OS) : os(OS) } :- attr("node", Package). +{ attr("node_os", PackageNode, OS) : os(OS) } :- attr("node", PackageNode). % can't have a non-buildable OS on a node we need to build error(100, "Cannot select '{0} os={1}' (operating system '{1}' is not buildable)", Package, OS) - :- build(Package), - attr("node_os", Package, OS), + :- build(node(X, Package)), + attr("node_os", node(X, Package), OS), not buildable_os(OS). % can't have dependencies on incompatible OS's -error(100, "{0} and dependency {1} have incompatible operating systems 'os={2}' and 'os={3}'", Package, Dependency, PackageOS, DependencyOS) - :- depends_on(Package, Dependency), - attr("node_os", Package, PackageOS), - attr("node_os", Dependency, DependencyOS), - not os_compatible(PackageOS, DependencyOS), - build(Package). +error(100, "{0} and dependency {1} have incompatible operating systems 'os={2}' and 'os={3}'", Package, Dependency, PackageNodeOS, DependencyOS) + :- depends_on(node(X, Package), node(Y, Dependency)), + attr("node_os", node(X, Package), PackageNodeOS), + attr("node_os", node(Y, Dependency), DependencyOS), + not os_compatible(PackageNodeOS, DependencyOS), + build(node(X, Package)). % give OS choice weights according to os declarations -node_os_weight(Package, Weight) - :- attr("node", Package), - attr("node_os", Package, OS), +node_os_weight(PackageNode, Weight) + :- attr("node", PackageNode), + attr("node_os", PackageNode, OS), os(OS, Weight). % match semantics for OS's -node_os_match(Package, Dependency) :- - depends_on(Package, Dependency), attr("node_os", Package, OS), attr("node_os", Dependency, OS). -node_os_mismatch(Package, Dependency) :- - depends_on(Package, Dependency), not node_os_match(Package, Dependency). +node_os_match(PackageNode, DependencyNode) :- + depends_on(PackageNode, DependencyNode), + attr("node_os", PackageNode, OS), + attr("node_os", DependencyNode, OS). + +node_os_mismatch(PackageNode, DependencyNode) :- + depends_on(PackageNode, DependencyNode), + not node_os_match(PackageNode, DependencyNode). % every OS is compatible with itself. We can use `os_compatible` to declare os_compatible(OS, OS) :- os(OS). @@ -794,7 +1012,7 @@ os_compatible(OS1, OS3) :- os_compatible(OS1, OS2), os_compatible(OS2, OS3). internal_error("Reused OS incompatible with build OS"). % If an OS is set explicitly respect the value -attr("node_os", Package, OS) :- attr("node_os_set", Package, OS), attr("node", Package). +attr("node_os", PackageNode, OS) :- attr("node_os_set", PackageNode, OS), attr("node", PackageNode). #defined os_compatible/2. @@ -803,134 +1021,127 @@ attr("node_os", Package, OS) :- attr("node_os_set", Package, OS), attr("node", P %----------------------------------------------------------------------------- % Each node has only one target chosen among the known targets -{ attr("node_target", Package, Target) : target(Target) } :- attr("node", Package). +{ attr("node_target", PackageNode, Target) : target(Target) } :- attr("node", PackageNode). % If a node must satisfy a target constraint, enforce it error(10, "'{0} target={1}' cannot satisfy constraint 'target={2}'", Package, Target, Constraint) - :- attr("node_target", Package, Target), - attr("node_target_satisfies", Package, Constraint), + :- attr("node_target", node(X, Package), Target), + attr("node_target_satisfies", node(X, Package), Constraint), not target_satisfies(Constraint, Target). % If a node has a target and the target satisfies a constraint, then the target % associated with the node satisfies the same constraint -attr("node_target_satisfies", Package, Constraint) - :- attr("node_target", Package, Target), target_satisfies(Constraint, Target). +attr("node_target_satisfies", PackageNode, Constraint) + :- attr("node_target", PackageNode, Target), target_satisfies(Constraint, Target). % If a node has a target, all of its dependencies must be compatible with that target error(100, "Cannot find compatible targets for {0} and {1}", Package, Dependency) - :- depends_on(Package, Dependency), - attr("node_target", Package, Target), - not node_target_compatible(Dependency, Target). + :- depends_on(node(X, Package), node(Y, Dependency)), + attr("node_target", node(X, Package), Target), + not node_target_compatible(node(Y, Dependency), Target). % Intermediate step for performance reasons % When the integrity constraint above was formulated including this logic % we suffered a substantial performance penalty -node_target_compatible(Package, Target) - :- attr("node_target", Package, MyTarget), +node_target_compatible(PackageNode, Target) + :- attr("node_target", PackageNode, MyTarget), target_compatible(Target, MyTarget). #defined target_satisfies/2. % can't use targets on node if the compiler for the node doesn't support them error(100, "{0} compiler '{2}@{3}' incompatible with 'target={1}'", Package, Target, Compiler, Version) - :- attr("node_target", Package, Target), - node_compiler(Package, CompilerID), + :- attr("node_target", node(X, Package), Target), + node_compiler(node(X, Package), CompilerID), not compiler_supports_target(CompilerID, Target), compiler_name(CompilerID, Compiler), compiler_version(CompilerID, Version), - build(Package). + build(node(X, Package)). % if a target is set explicitly, respect it -attr("node_target", Package, Target) - :- attr("node", Package), attr("node_target_set", Package, Target). +attr("node_target", PackageNode, Target) + :- attr("node", PackageNode), attr("node_target_set", PackageNode, Target). % each node has the weight of its assigned target -node_target_weight(Package, Weight) - :- attr("node", Package), - attr("node_target", Package, Target), - target_weight(Package, Target, Weight). +node_target_weight(node(ID, Package), Weight) + :- attr("node", node(ID, Package)), + attr("node_target", node(ID, Package), Target), + target_weight(Target, Weight). % compatibility rules for targets among nodes -node_target_match(Parent, Dependency) - :- depends_on(Parent, Dependency), - attr("node_target", Parent, Target), - attr("node_target", Dependency, Target). +node_target_match(ParentNode, DependencyNode) + :- attr("depends_on", ParentNode, DependencyNode, Type), Type != "build", + attr("node_target", ParentNode, Target), + attr("node_target", DependencyNode, Target). -node_target_mismatch(Parent, Dependency) - :- depends_on(Parent, Dependency), - not node_target_match(Parent, Dependency). +node_target_mismatch(ParentNode, DependencyNode) + :- attr("depends_on", ParentNode, DependencyNode, Type), Type != "build", + not node_target_match(ParentNode, DependencyNode). % disallow reusing concrete specs that don't have a compatible target error(100, "'{0} target={1}' is not compatible with this machine", Package, Target) - :- attr("node", Package), - attr("node_target", Package, Target), + :- attr("node", node(X, Package)), + attr("node_target", node(X, Package), Target), not target(Target). -#defined package_target_weight/3. - %----------------------------------------------------------------------------- % Compiler semantics %----------------------------------------------------------------------------- % There must be only one compiler set per built node. -{ node_compiler(Package, CompilerID) : compiler_id(CompilerID) } :- - attr("node", Package), - build(Package). +{ node_compiler(PackageNode, CompilerID) : compiler_id(CompilerID) } :- + attr("node", PackageNode), + build(PackageNode). % Infer the compiler that matches a reused node -node_compiler(Package, CompilerID) - :- attr("node_compiler_version", Package, CompilerName, CompilerVersion), - attr("node", Package), +node_compiler(PackageNode, CompilerID) + :- attr("node_compiler_version", PackageNode, CompilerName, CompilerVersion), + attr("node", PackageNode), compiler_name(CompilerID, CompilerName), compiler_version(CompilerID, CompilerVersion), - concrete(Package). + concrete(PackageNode). % Expand the internal attribute into "attr("node_compiler_version") -attr("node_compiler_version", Package, CompilerName, CompilerVersion) - :- node_compiler(Package, CompilerID), +attr("node_compiler_version", PackageNode, CompilerName, CompilerVersion) + :- node_compiler(PackageNode, CompilerID), compiler_name(CompilerID, CompilerName), compiler_version(CompilerID, CompilerVersion), - build(Package). + build(PackageNode). -attr("node_compiler", Package, CompilerName) - :- attr("node_compiler_version", Package, CompilerName, CompilerVersion). +attr("node_compiler", PackageNode, CompilerName) + :- attr("node_compiler_version", PackageNode, CompilerName, CompilerVersion). error(100, "No valid compiler version found for '{0}'", Package) - :- attr("node", Package), - not node_compiler(Package, _). + :- attr("node", node(X, Package)), + not node_compiler(node(X, Package), _). % We can't have a compiler be enforced and select the version from another compiler error(100, "Cannot select a single compiler for package {0}", Package) - :- attr("node", Package), - 2 { attr("node_compiler_version", Package, C, V) }. - -error(100, "Cannot concretize {0} with two compilers {1} and {2}@{3}", Package, Compiler1, Compiler2, Version) - :- attr("node_compiler", Package, Compiler1), - attr("node_compiler_version", Package, Compiler2, Version), - Compiler1 != Compiler2. + :- attr("node", node(X, Package)), + 2 { attr("node_compiler_version", node(X, Package), C, V) }. % If the compiler of a node cannot be satisfied, raise error(10, "No valid compiler for {0} satisfies '%{1}'", Package, Compiler) - :- attr("node", Package), - attr("node_compiler_version_satisfies", Package, Compiler, ":"), + :- attr("node", node(X, Package)), + attr("node_compiler_version_satisfies", node(X, Package), Compiler, ":"), not compiler_version_satisfies(Compiler, ":", _). % If the compiler of a node must satisfy a constraint, then its version % must be chosen among the ones that satisfy said constraint error(100, "No valid version for '{0}' compiler '{1}' satisfies '@{2}'", Package, Compiler, Constraint) - :- attr("node", Package), - attr("node_compiler_version_satisfies", Package, Compiler, Constraint), + :- attr("node", node(X, Package)), + attr("node_compiler_version_satisfies", node(X, Package), Compiler, Constraint), not compiler_version_satisfies(Compiler, Constraint, _). error(100, "No valid version for '{0}' compiler '{1}' satisfies '@{2}'", Package, Compiler, Constraint) - :- attr("node", Package), - attr("node_compiler_version_satisfies", Package, Compiler, Constraint), + :- attr("node", node(X, Package)), + attr("node_compiler_version_satisfies", node(X, Package), Compiler, Constraint), not compiler_version_satisfies(Compiler, Constraint, ID), - node_compiler(Package, ID). + node_compiler(node(X, Package), ID). % If the node is associated with a compiler and the compiler satisfy a constraint, then % the compiler associated with the node satisfy the same constraint -attr("node_compiler_version_satisfies", Package, Compiler, Constraint) - :- node_compiler(Package, CompilerID), +attr("node_compiler_version_satisfies", PackageNode, Compiler, Constraint) + :- node_compiler(PackageNode, CompilerID), compiler_name(CompilerID, Compiler), compiler_version_satisfies(Compiler, Constraint, CompilerID). @@ -938,91 +1149,90 @@ attr("node_compiler_version_satisfies", Package, Compiler, Constraint) % If the compiler version was set from the command line, % respect it verbatim -:- attr("node_compiler_version_set", Package, Compiler, Version), - not attr("node_compiler_version", Package, Compiler, Version). +:- attr("node_compiler_version_set", PackageNode, Compiler, Version), + not attr("node_compiler_version", PackageNode, Compiler, Version). -:- attr("node_compiler_set", Package, Compiler), - not attr("node_compiler_version", Package, Compiler, _). +:- attr("node_compiler_set", PackageNode, Compiler), + not attr("node_compiler_version", PackageNode, Compiler, _). % Cannot select a compiler if it is not supported on the OS % Compilers that are explicitly marked as allowed % are excluded from this check error(100, "{0} compiler '%{1}@{2}' incompatible with 'os={3}'", Package, Compiler, Version, OS) - :- attr("node_os", Package, OS), - node_compiler(Package, CompilerID), + :- attr("node_os", node(X, Package), OS), + node_compiler(node(X, Package), CompilerID), compiler_name(CompilerID, Compiler), compiler_version(CompilerID, Version), not compiler_os(CompilerID, OS), not allow_compiler(Compiler, Version), - build(Package). + build(node(X, Package)). % If a package and one of its dependencies don't have the % same compiler there's a mismatch. -compiler_match(Package, Dependency) - :- depends_on(Package, Dependency), - node_compiler(Package, CompilerID), - node_compiler(Dependency, CompilerID). +compiler_match(PackageNode, DependencyNode) + :- depends_on(PackageNode, DependencyNode), + node_compiler(PackageNode, CompilerID), + node_compiler(DependencyNode, CompilerID). -compiler_mismatch(Package, Dependency) - :- depends_on(Package, Dependency), - not attr("node_compiler_set", Dependency, _), - not compiler_match(Package, Dependency). +compiler_mismatch(PackageNode, DependencyNode) + :- depends_on(PackageNode, DependencyNode), + not attr("node_compiler_set", DependencyNode, _), + not compiler_match(PackageNode, DependencyNode). -compiler_mismatch_required(Package, Dependency) - :- depends_on(Package, Dependency), - attr("node_compiler_set", Dependency, _), - not compiler_match(Package, Dependency). +compiler_mismatch_required(PackageNode, DependencyNode) + :- depends_on(PackageNode, DependencyNode), + attr("node_compiler_set", DependencyNode, _), + not compiler_match(PackageNode, DependencyNode). #defined compiler_os/3. #defined allow_compiler/2. % compilers weighted by preference according to packages.yaml -compiler_weight(Package, Weight) - :- node_compiler(Package, CompilerID), - compiler_name(CompilerID, Compiler), - compiler_version(CompilerID, V), - node_compiler_preference(Package, Compiler, V, Weight). -compiler_weight(Package, Weight) - :- node_compiler(Package, CompilerID), +node_compiler_weight(node(ID, Package), Weight) + :- node_compiler(node(ID, Package), CompilerID), compiler_name(CompilerID, Compiler), compiler_version(CompilerID, V), - not node_compiler_preference(Package, Compiler, V, _), - default_compiler_preference(CompilerID, Weight). -compiler_weight(Package, 100) - :- node_compiler(Package, CompilerID), + compiler_weight(CompilerID, Weight). + +node_compiler_weight(node(ID, Package), 100) + :- node_compiler(node(ID, Package), CompilerID), compiler_name(CompilerID, Compiler), compiler_version(CompilerID, V), - not node_compiler_preference(Package, Compiler, V, _), - not default_compiler_preference(CompilerID, _). + not compiler_weight(CompilerID, _). % For the time being, be strict and reuse only if the compiler match one we have on the system error(100, "Compiler {1}@{2} requested for {0} cannot be found. Set install_missing_compilers:true if intended.", Package, Compiler, Version) - :- attr("node_compiler_version", Package, Compiler, Version), - not node_compiler(Package, _). + :- attr("node_compiler_version", node(ID, Package), Compiler, Version), + not node_compiler(node(ID, Package), _). #defined node_compiler_preference/4. -#defined default_compiler_preference/3. +#defined compiler_weight/3. %----------------------------------------------------------------------------- % Compiler flags %----------------------------------------------------------------------------- % propagate flags when compiler match -can_inherit_flags(Package, Dependency, FlagType) - :- depends_on(Package, Dependency), - node_compiler(Package, CompilerID), - node_compiler(Dependency, CompilerID), - not attr("node_flag_set", Dependency, FlagType, _), - compiler_id(CompilerID), - flag_type(FlagType). - -node_flag_inherited(Dependency, FlagType, Flag) - :- attr("node_flag_set", Package, FlagType, Flag), can_inherit_flags(Package, Dependency, FlagType), - attr("node_flag_propagate", Package, FlagType). +can_inherit_flags(PackageNode, DependencyNode, FlagType) + :- same_compiler(PackageNode, DependencyNode), + not attr("node_flag_set", DependencyNode, FlagType, _), + flag_type(FlagType). + +same_compiler(PackageNode, DependencyNode) + :- depends_on(PackageNode, DependencyNode), + node_compiler(PackageNode, CompilerID), + node_compiler(DependencyNode, CompilerID), + compiler_id(CompilerID). + +node_flag_inherited(DependencyNode, FlagType, Flag) + :- attr("node_flag_set", PackageNode, FlagType, Flag), + can_inherit_flags(PackageNode, DependencyNode, FlagType), + attr("node_flag_propagate", PackageNode, FlagType). + % Ensure propagation -:- node_flag_inherited(Package, FlagType, Flag), - can_inherit_flags(Package, Dependency, FlagType), - attr("node_flag_propagate", Package, FlagType). +:- node_flag_inherited(PackageNode, FlagType, Flag), + can_inherit_flags(PackageNode, DependencyNode, FlagType), + attr("node_flag_propagate", PackageNode, FlagType). error(100, "{0} and {1} cannot both propagate compiler flags '{2}' to {3}", Source1, Source2, Package, FlagType) :- depends_on(Source1, Package), @@ -1034,37 +1244,41 @@ error(100, "{0} and {1} cannot both propagate compiler flags '{2}' to {3}", Sour Source1 < Source2. % remember where flags came from -attr("node_flag_source", Package, FlagType, Package) :- attr("node_flag_set", Package, FlagType, _). -attr("node_flag_source", Dependency, FlagType, Q) - :- attr("node_flag_source", Package, FlagType, Q), node_flag_inherited(Dependency, FlagType, _), - attr("node_flag_propagate", Package, FlagType). +attr("node_flag_source", PackageNode, FlagType, PackageNode) + :- attr("node_flag_set", PackageNode, FlagType, _). + +attr("node_flag_source", DependencyNode, FlagType, Q) + :- attr("node_flag_source", PackageNode, FlagType, Q), + node_flag_inherited(DependencyNode, FlagType, _), + attr("node_flag_propagate", PackageNode, FlagType). % compiler flags from compilers.yaml are put on nodes if compiler matches -attr("node_flag", Package, FlagType, Flag) - :- compiler_flag(CompilerID, FlagType, Flag), - node_compiler(Package, CompilerID), - flag_type(FlagType), - compiler_id(CompilerID), - compiler_name(CompilerID, CompilerName), - compiler_version(CompilerID, Version). - -attr("node_flag_compiler_default", Package) - :- not attr("node_flag_set", Package, FlagType, _), - compiler_flag(CompilerID, FlagType, Flag), - node_compiler(Package, CompilerID), - flag_type(FlagType), - compiler_id(CompilerID), - compiler_name(CompilerID, CompilerName), - compiler_version(CompilerID, Version). +attr("node_flag", PackageNode, FlagType, Flag) + :- compiler_flag(CompilerID, FlagType, Flag), + node_compiler(PackageNode, CompilerID), + flag_type(FlagType), + compiler_id(CompilerID), + compiler_name(CompilerID, CompilerName), + compiler_version(CompilerID, Version). + +attr("node_flag_compiler_default", PackageNode) + :- not attr("node_flag_set", PackageNode, FlagType, _), + compiler_flag(CompilerID, FlagType, Flag), + node_compiler(PackageNode, CompilerID), + flag_type(FlagType), + compiler_id(CompilerID), + compiler_name(CompilerID, CompilerName), + compiler_version(CompilerID, Version). % if a flag is set to something or inherited, it's included -attr("node_flag", Package, FlagType, Flag) :- attr("node_flag_set", Package, FlagType, Flag). -attr("node_flag", Package, FlagType, Flag) - :- node_flag_inherited(Package, FlagType, Flag). +attr("node_flag", PackageNode, FlagType, Flag) :- attr("node_flag_set", PackageNode, FlagType, Flag). +attr("node_flag", PackageNode, FlagType, Flag) :- node_flag_inherited(PackageNode, FlagType, Flag). % if no node flags are set for a type, there are no flags. -attr("no_flags", Package, FlagType) - :- not attr("node_flag", Package, FlagType, _), attr("node", Package), flag_type(FlagType). +attr("no_flags", PackageNode, FlagType) + :- not attr("node_flag", PackageNode, FlagType, _), + attr("node", PackageNode), + flag_type(FlagType). #defined compiler_flag/3. @@ -1073,22 +1287,22 @@ attr("no_flags", Package, FlagType) % Installed packages %----------------------------------------------------------------------------- % the solver is free to choose at most one installed hash for each package -{ attr("hash", Package, Hash) : installed_hash(Package, Hash) } 1 - :- attr("node", Package), internal_error("Package must resolve to at most one hash"). +{ attr("hash", node(ID, Package), Hash) : installed_hash(Package, Hash) } 1 + :- attr("node", node(ID, Package)), internal_error("Package must resolve to at most one hash"). % you can't choose an installed hash for a dev spec -:- attr("hash", Package, Hash), attr("variant_value", Package, "dev_path", _). +:- attr("hash", PackageNode, Hash), attr("variant_value", PackageNode, "dev_path", _). % You can't install a hash, if it is not installed -:- attr("hash", Package, Hash), not installed_hash(Package, Hash). +:- attr("hash", node(ID, Package), Hash), not installed_hash(Package, Hash). % This should be redundant given the constraint above -:- attr("node", Package), 2 { attr("hash", Package, Hash) }. +:- attr("node", PackageNode), 2 { attr("hash", PackageNode, Hash) }. % if a hash is selected, we impose all the constraints that implies -impose(Hash) :- attr("hash", Package, Hash). +impose(Hash, PackageNode) :- attr("hash", PackageNode, Hash). % if we haven't selected a hash for a package, we'll be building it -build(Package) :- not attr("hash", Package, _), attr("node", Package). +build(PackageNode) :- not attr("hash", PackageNode, _), attr("node", PackageNode). % Minimizing builds is tricky. We want a minimizing criterion @@ -1105,11 +1319,8 @@ build(Package) :- not attr("hash", Package, _), attr("node", Package). % 200+ Shifted priorities for build nodes; correspond to priorities 0 - 99. % 100 - 199 Unshifted priorities. Currently only includes minimizing #builds. % 0 - 99 Priorities for non-built nodes. -build_priority(Package, 200) :- build(Package), attr("node", Package), optimize_for_reuse(). -build_priority(Package, 0) :- not build(Package), attr("node", Package), optimize_for_reuse(). - -% don't adjust build priorities if reuse is not enabled -build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse(). +build_priority(PackageNode, 200) :- build(PackageNode), attr("node", PackageNode). +build_priority(PackageNode, 0) :- not build(PackageNode), attr("node", PackageNode). % don't assign versions from installed packages unless reuse is enabled % NOTE: that "installed" means the declared version was only included because @@ -1122,13 +1333,17 @@ build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse(). % currently *won't* force versions for `bar`'s build dependencies -- `--fresh` % will instead build the latest bar. When we actually include transitive % build deps in the solve, consider using them as a preference to resolve this. -:- attr("version", Package, Version), - version_weight(Package, Weight), - version_declared(Package, Version, Weight, "installed"), +:- attr("version", node(ID, Package), Version), + version_weight(node(ID, Package), Weight), + pkg_fact(Package, version_declared(Version, Weight, "installed")), not optimize_for_reuse(). #defined installed_hash/2. +% This statement, which is a hidden feature of clingo, let us avoid cycles in the DAG +#edge (A, B) : depends_on(A, B). + + %----------------------------------------------------------------- % Optimization to avoid errors %----------------------------------------------------------------- @@ -1151,9 +1366,14 @@ build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse(). % is displayed (clingo doesn't display sums over empty sets by default) % Try hard to reuse installed packages (i.e., minimize the number built) -opt_criterion(100, "number of packages to build (vs. reuse)"). +opt_criterion(110, "number of packages to build (vs. reuse)"). +#minimize { 0@110: #true }. +#minimize { 1@110,PackageNode : build(PackageNode), optimize_for_reuse() }. + +opt_criterion(100, "number of nodes from the same package"). #minimize { 0@100: #true }. -#minimize { 1@100,Package : build(Package), optimize_for_reuse() }. +#minimize { ID@100,Package : attr("node", node(ID, Package)) }. +#minimize { ID@100,Package : attr("virtual_node", node(ID, Package)) }. #defined optimize_for_reuse/0. % A condition group specifies one or more specs that must be satisfied. @@ -1163,9 +1383,9 @@ opt_criterion(75, "requirement weight"). #minimize{ 0@275: #true }. #minimize{ 0@75: #true }. #minimize { - Weight@75+Priority,Package,Group - : requirement_weight(Package, Group, Weight), - build_priority(Package, Priority) + Weight@75+Priority,PackageNode,Group + : requirement_weight(PackageNode, Group, Weight), + build_priority(PackageNode, Priority) }. % Minimize the number of deprecated versions being used @@ -1173,9 +1393,9 @@ opt_criterion(73, "deprecated versions used"). #minimize{ 0@273: #true }. #minimize{ 0@73: #true }. #minimize{ - 1@73+Priority,Package - : attr("deprecated", Package, _), - build_priority(Package, Priority) + 1@73+Priority,PackageNode + : attr("deprecated", PackageNode, _), + build_priority(PackageNode, Priority) }. % Minimize the: @@ -1187,38 +1407,39 @@ opt_criterion(70, "version weight"). #minimize{ 0@70: #true }. #minimize { Weight@70+Priority - : attr("root", Package), version_weight(Package, Weight), - build_priority(Package, Priority) + : attr("root", PackageNode), + version_weight(PackageNode, Weight), + build_priority(PackageNode, Priority) }. opt_criterion(65, "number of non-default variants (roots)"). #minimize{ 0@265: #true }. #minimize{ 0@65: #true }. #minimize { - 1@65+Priority,Package,Variant,Value - : variant_not_default(Package, Variant, Value), - attr("root", Package), - build_priority(Package, Priority) + 1@65+Priority,PackageNode,Variant,Value + : variant_not_default(PackageNode, Variant, Value), + attr("root", PackageNode), + build_priority(PackageNode, Priority) }. opt_criterion(60, "preferred providers for roots"). #minimize{ 0@260: #true }. #minimize{ 0@60: #true }. #minimize{ - Weight@60+Priority,Provider,Virtual - : provider_weight(Provider, Virtual, Weight), - attr("root", Provider), - build_priority(Provider, Priority) + Weight@60+Priority,ProviderNode,Virtual + : provider_weight(ProviderNode, Virtual, Weight), + attr("root", ProviderNode), + build_priority(ProviderNode, Priority) }. opt_criterion(55, "default values of variants not being used (roots)"). #minimize{ 0@255: #true }. #minimize{ 0@55: #true }. #minimize{ - 1@55+Priority,Package,Variant,Value - : variant_default_not_used(Package, Variant, Value), - attr("root", Package), - build_priority(Package, Priority) + 1@55+Priority,PackageNode,Variant,Value + : variant_default_not_used(PackageNode, Variant, Value), + attr("root", PackageNode), + build_priority(PackageNode, Priority) }. % Try to use default variants or variants that have been set @@ -1226,10 +1447,10 @@ opt_criterion(50, "number of non-default variants (non-roots)"). #minimize{ 0@250: #true }. #minimize{ 0@50: #true }. #minimize { - 1@50+Priority,Package,Variant,Value - : variant_not_default(Package, Variant, Value), - not attr("root", Package), - build_priority(Package, Priority) + 1@50+Priority,PackageNode,Variant,Value + : variant_not_default(PackageNode, Variant, Value), + not attr("root", PackageNode), + build_priority(PackageNode, Priority) }. % Minimize the weights of the providers, i.e. use as much as @@ -1238,9 +1459,10 @@ opt_criterion(45, "preferred providers (non-roots)"). #minimize{ 0@245: #true }. #minimize{ 0@45: #true }. #minimize{ - Weight@45+Priority,Provider,Virtual - : provider_weight(Provider, Virtual, Weight), not attr("root", Provider), - build_priority(Provider, Priority) + Weight@45+Priority,ProviderNode,Virtual + : provider_weight(ProviderNode, Virtual, Weight), + not attr("root", ProviderNode), + build_priority(ProviderNode, Priority) }. % Try to minimize the number of compiler mismatches in the DAG. @@ -1248,18 +1470,18 @@ opt_criterion(40, "compiler mismatches that are not from CLI"). #minimize{ 0@240: #true }. #minimize{ 0@40: #true }. #minimize{ - 1@40+Priority,Package,Dependency - : compiler_mismatch(Package, Dependency), - build_priority(Package, Priority) + 1@40+Priority,PackageNode,DependencyNode + : compiler_mismatch(PackageNode, DependencyNode), + build_priority(PackageNode, Priority) }. opt_criterion(39, "compiler mismatches that are not from CLI"). #minimize{ 0@239: #true }. #minimize{ 0@39: #true }. #minimize{ - 1@39+Priority,Package,Dependency - : compiler_mismatch_required(Package, Dependency), - build_priority(Package, Priority) + 1@39+Priority,PackageNode,DependencyNode + : compiler_mismatch_required(PackageNode, DependencyNode), + build_priority(PackageNode, Priority) }. % Try to minimize the number of compiler mismatches in the DAG. @@ -1267,18 +1489,18 @@ opt_criterion(35, "OS mismatches"). #minimize{ 0@235: #true }. #minimize{ 0@35: #true }. #minimize{ - 1@35+Priority,Package,Dependency - : node_os_mismatch(Package, Dependency), - build_priority(Package, Priority) + 1@35+Priority,PackageNode,DependencyNode + : node_os_mismatch(PackageNode, DependencyNode), + build_priority(PackageNode, Priority) }. opt_criterion(30, "non-preferred OS's"). #minimize{ 0@230: #true }. #minimize{ 0@30: #true }. #minimize{ - Weight@30+Priority,Package - : node_os_weight(Package, Weight), - build_priority(Package, Priority) + Weight@30+Priority,PackageNode + : node_os_weight(PackageNode, Weight), + build_priority(PackageNode, Priority) }. % Choose more recent versions for nodes @@ -1286,9 +1508,9 @@ opt_criterion(25, "version badness"). #minimize{ 0@225: #true }. #minimize{ 0@25: #true }. #minimize{ - Weight@25+Priority,Package - : version_weight(Package, Weight), - build_priority(Package, Priority) + Weight@25+Priority,PackageNode + : version_weight(PackageNode, Weight), + build_priority(PackageNode, Priority) }. % Try to use all the default values of variants @@ -1296,10 +1518,10 @@ opt_criterion(20, "default values of variants not being used (non-roots)"). #minimize{ 0@220: #true }. #minimize{ 0@20: #true }. #minimize{ - 1@20+Priority,Package,Variant,Value - : variant_default_not_used(Package, Variant, Value), - not attr("root", Package), - build_priority(Package, Priority) + 1@20+Priority,PackageNode,Variant,Value + : variant_default_not_used(PackageNode, Variant, Value), + not attr("root", PackageNode), + build_priority(PackageNode, Priority) }. % Try to use preferred compilers @@ -1307,9 +1529,9 @@ opt_criterion(15, "non-preferred compilers"). #minimize{ 0@215: #true }. #minimize{ 0@15: #true }. #minimize{ - Weight@15+Priority,Package - : compiler_weight(Package, Weight), - build_priority(Package, Priority) + Weight@15+Priority,PackageNode + : node_compiler_weight(PackageNode, Weight), + build_priority(PackageNode, Priority) }. % Minimize the number of mismatches for targets in the DAG, try @@ -1318,47 +1540,30 @@ opt_criterion(10, "target mismatches"). #minimize{ 0@210: #true }. #minimize{ 0@10: #true }. #minimize{ - 1@10+Priority,Package,Dependency - : node_target_mismatch(Package, Dependency), - build_priority(Package, Priority) + 1@10+Priority,PackageNode,Dependency + : node_target_mismatch(PackageNode, Dependency), + build_priority(PackageNode, Priority) }. opt_criterion(5, "non-preferred targets"). #minimize{ 0@205: #true }. #minimize{ 0@5: #true }. #minimize{ - Weight@5+Priority,Package - : node_target_weight(Package, Weight), - build_priority(Package, Priority) + Weight@5+Priority,PackageNode + : node_target_weight(PackageNode, Weight), + build_priority(PackageNode, Priority) }. -%----------------- -% Domain heuristic -%----------------- -#heuristic literal_solved(ID) : literal(ID). [1, sign] -#heuristic literal_solved(ID) : literal(ID). [50, init] -#heuristic attr("hash", Package, Hash) : attr("root", Package). [45, init] - -#heuristic attr("version", Package, Version) : version_declared(Package, Version, 0), attr("root", Package). [40, true] -#heuristic version_weight(Package, 0) : version_declared(Package, Version, 0), attr("root", Package). [40, true] -#heuristic attr("variant_value", Package, Variant, Value) : variant_default_value(Package, Variant, Value), attr("root", Package). [40, true] -#heuristic attr("node_target", Package, Target) : package_target_weight(Target, Package, 0), attr("root", Package). [40, true] -#heuristic node_target_weight(Package, 0) : attr("root", Package). [40, true] -#heuristic node_compiler(Package, CompilerID) : default_compiler_preference(ID, 0), compiler_id(ID), attr("root", Package). [40, true] - -#heuristic provider(Package, Virtual) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [30, true] -#heuristic provider_weight(Package, Virtual, 0, R) : possible_provider_weight(Package, Virtual, 0, R), attr("virtual_node", Virtual). [30, true] -#heuristic attr("node", Package) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [30, true] - -#heuristic attr("version", Package, Version) : version_declared(Package, Version, 0), attr("node", Package). [20, true] -#heuristic version_weight(Package, 0) : version_declared(Package, Version, 0), attr("node", Package). [20, true] - -#heuristic attr("node_target", Package, Target) : package_target_weight(Target, Package, 0), attr("node", Package). [20, true] -#heuristic node_target_weight(Package, 0) : attr("node", Package). [20, true] -#heuristic node_compiler(Package, CompilerID) : default_compiler_preference(ID, 0), compiler_id(ID), attr("node", Package). [15, true] - -#heuristic attr("variant_value", Package, Variant, Value) : variant_default_value(Package, Variant, Value), attr("node", Package). [10, true] -#heuristic attr("node_os", Package, OS) : buildable_os(OS). [10, true] +% Choose more recent versions for nodes +opt_criterion(1, "edge wiring"). +#minimize{ 0@201: #true }. +#minimize{ 0@1: #true }. +#minimize{ + Weight@1,ParentNode,PackageNode + : version_weight(PackageNode, Weight), + not attr("root", PackageNode), + depends_on(ParentNode, PackageNode) +}. %----------- % Notes diff --git a/lib/spack/spack/solver/counter.py b/lib/spack/spack/solver/counter.py new file mode 100644 index 00000000000000..28883817dfe564 --- /dev/null +++ b/lib/spack/spack/solver/counter.py @@ -0,0 +1,171 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import collections +from typing import List, Set + +from llnl.util import lang + +import spack.deptypes as dt +import spack.package_base +import spack.repo + +PossibleDependencies = Set[str] + + +class Counter: + """Computes the possible packages and the maximum number of duplicates + allowed for each of them. + + Args: + specs: abstract specs to concretize + tests: if True, add test dependencies to the list of possible packages + """ + + def __init__(self, specs: List["spack.spec.Spec"], tests: bool) -> None: + self.specs = specs + + self.link_run_types: dt.DepFlag = dt.LINK | dt.RUN | dt.TEST + self.all_types: dt.DepFlag = dt.ALL + if not tests: + self.link_run_types = dt.LINK | dt.RUN + self.all_types = dt.LINK | dt.RUN | dt.BUILD + + self._possible_dependencies: PossibleDependencies = set() + self._possible_virtuals: Set[str] = set(x.name for x in specs if x.virtual) + + def possible_dependencies(self) -> PossibleDependencies: + """Returns the list of possible dependencies""" + self.ensure_cache_values() + return self._possible_dependencies + + def possible_virtuals(self) -> Set[str]: + """Returns the list of possible virtuals""" + self.ensure_cache_values() + return self._possible_virtuals + + def ensure_cache_values(self) -> None: + """Ensure the cache values have been computed""" + if self._possible_dependencies: + return + self._compute_cache_values() + + def possible_packages_facts(self, gen: "spack.solver.asp.PyclingoDriver", fn) -> None: + """Emit facts associated with the possible packages""" + raise NotImplementedError("must be implemented by derived classes") + + def _compute_cache_values(self): + raise NotImplementedError("must be implemented by derived classes") + + +class NoDuplicatesCounter(Counter): + def _compute_cache_values(self): + result = spack.package_base.possible_dependencies( + *self.specs, virtuals=self._possible_virtuals, depflag=self.all_types + ) + self._possible_dependencies = set(result) + + def possible_packages_facts(self, gen, fn): + gen.h2("Maximum number of nodes (packages)") + for package_name in sorted(self.possible_dependencies()): + gen.fact(fn.max_dupes(package_name, 1)) + gen.newline() + gen.h2("Maximum number of nodes (virtual packages)") + for package_name in sorted(self.possible_virtuals()): + gen.fact(fn.max_dupes(package_name, 1)) + gen.newline() + gen.h2("Possible package in link-run subDAG") + for name in sorted(self.possible_dependencies()): + gen.fact(fn.possible_in_link_run(name)) + gen.newline() + + +class MinimalDuplicatesCounter(NoDuplicatesCounter): + def __init__(self, specs, tests): + super().__init__(specs, tests) + self._link_run: PossibleDependencies = set() + self._direct_build: PossibleDependencies = set() + self._total_build: PossibleDependencies = set() + self._link_run_virtuals: Set[str] = set() + + def _compute_cache_values(self): + self._link_run = set( + spack.package_base.possible_dependencies( + *self.specs, virtuals=self._possible_virtuals, depflag=self.link_run_types + ) + ) + self._link_run_virtuals.update(self._possible_virtuals) + for x in self._link_run: + build_dependencies = spack.repo.PATH.get_pkg_class(x).dependencies_of_type(dt.BUILD) + virtuals, reals = lang.stable_partition( + build_dependencies, spack.repo.PATH.is_virtual_safe + ) + + self._possible_virtuals.update(virtuals) + for virtual_dep in virtuals: + providers = spack.repo.PATH.providers_for(virtual_dep) + self._direct_build.update(str(x) for x in providers) + + self._direct_build.update(reals) + + self._total_build = set( + spack.package_base.possible_dependencies( + *self._direct_build, virtuals=self._possible_virtuals, depflag=self.all_types + ) + ) + self._possible_dependencies = set(self._link_run) | set(self._total_build) + + def possible_packages_facts(self, gen, fn): + build_tools = set(spack.repo.PATH.packages_with_tags("build-tools")) + gen.h2("Packages with at most a single node") + for package_name in sorted(self.possible_dependencies() - build_tools): + gen.fact(fn.max_dupes(package_name, 1)) + gen.newline() + + gen.h2("Packages with at multiple possible nodes (build-tools)") + for package_name in sorted(self.possible_dependencies() & build_tools): + gen.fact(fn.max_dupes(package_name, 2)) + gen.fact(fn.multiple_unification_sets(package_name)) + gen.newline() + + gen.h2("Maximum number of nodes (virtual packages)") + for package_name in sorted(self.possible_virtuals()): + gen.fact(fn.max_dupes(package_name, 1)) + gen.newline() + + gen.h2("Possible package in link-run subDAG") + for name in sorted(self._link_run): + gen.fact(fn.possible_in_link_run(name)) + gen.newline() + + +class FullDuplicatesCounter(MinimalDuplicatesCounter): + def possible_packages_facts(self, gen, fn): + build_tools = set(spack.repo.PATH.packages_with_tags("build-tools")) + counter = collections.Counter( + list(self._link_run) + list(self._total_build) + list(self._direct_build) + ) + gen.h2("Maximum number of nodes") + for pkg, count in sorted(counter.items(), key=lambda x: (x[1], x[0])): + count = min(count, 2) + gen.fact(fn.max_dupes(pkg, count)) + gen.newline() + + gen.h2("Build unification sets ") + for name in sorted(self.possible_dependencies() & build_tools): + gen.fact(fn.multiple_unification_sets(name)) + gen.newline() + + gen.h2("Possible package in link-run subDAG") + for name in sorted(self._link_run): + gen.fact(fn.possible_in_link_run(name)) + gen.newline() + + counter = collections.Counter( + list(self._link_run_virtuals) + list(self._possible_virtuals) + ) + gen.h2("Maximum number of virtual nodes") + for pkg, count in sorted(counter.items(), key=lambda x: (x[1], x[0])): + gen.fact(fn.max_dupes(pkg, count)) + gen.newline() diff --git a/lib/spack/spack/solver/display.lp b/lib/spack/spack/solver/display.lp index fffffb2c0430bd..58d04d42ea3002 100644 --- a/lib/spack/spack/solver/display.lp +++ b/lib/spack/spack/solver/display.lp @@ -24,4 +24,29 @@ #show error/5. #show error/6. +% for error causation +#show condition_reason/2. + +% For error messages to use later +#show pkg_fact/2. +#show condition_holds/2. +#show imposed_constraint/3. +#show imposed_constraint/4. +#show imposed_constraint/5. +#show imposed_constraint/6. +#show condition_requirement/3. +#show condition_requirement/4. +#show condition_requirement/5. +#show condition_requirement/6. +#show node_has_variant/2. +#show build/1. +#show external/1. +#show external_version/3. +#show trigger_and_effect/3. +#show unification_set/2. +#show provider/2. +#show condition_nodes/3. +#show trigger_node/3. +#show imposed_nodes/3. + % debug diff --git a/lib/spack/spack/solver/error_messages.lp b/lib/spack/spack/solver/error_messages.lp new file mode 100644 index 00000000000000..7eb383860d8c75 --- /dev/null +++ b/lib/spack/spack/solver/error_messages.lp @@ -0,0 +1,239 @@ +% Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +% Spack Project Developers. See the top-level COPYRIGHT file for details. +% +% SPDX-License-Identifier: (Apache-2.0 OR MIT) + +%============================================================================= +% This logic program adds detailed error messages to Spack's concretizer +%============================================================================= + +#program error_messages. + +% Create a causal tree between trigger conditions by locating the effect conditions +% that are triggers for another condition. Condition2 is caused by Condition1 +condition_cause(Condition2, ID2, Condition1, ID1) :- + condition_holds(Condition2, node(ID2, Package2)), + pkg_fact(Package2, condition_trigger(Condition2, Trigger)), + condition_requirement(Trigger, Name, Package), + condition_nodes(Trigger, TriggerNode, node(ID, Package)), + trigger_node(Trigger, TriggerNode, node(ID2, Package2)), + attr(Name, node(ID, Package)), + condition_holds(Condition1, node(ID1, Package1)), + pkg_fact(Package1, condition_effect(Condition1, Effect)), + imposed_constraint(Effect, Name, Package), + imposed_nodes(Effect, node(ID1, Package1), node(ID, Package)). + +condition_cause(Condition2, ID2, Condition1, ID1) :- + condition_holds(Condition2, node(ID2, Package2)), + pkg_fact(Package2, condition_trigger(Condition2, Trigger)), + condition_requirement(Trigger, Name, Package, A1), + condition_nodes(Trigger, TriggerNode, node(ID, Package)), + trigger_node(Trigger, TriggerNode, node(ID2, Package2)), + attr(Name, node(ID, Package), A1), + condition_holds(Condition1, node(ID1, Package1)), + pkg_fact(Package1, condition_effect(Condition1, Effect)), + imposed_constraint(Effect, Name, Package, A1), + imposed_nodes(Effect, node(ID1, Package1), node(ID, Package)). + +condition_cause(Condition2, ID2, Condition1, ID1) :- + condition_holds(Condition2, node(ID2, Package2)), + pkg_fact(Package2, condition_trigger(Condition2, Trigger)), + condition_requirement(Trigger, Name, Package, A1, A2), + condition_nodes(Trigger, TriggerNode, node(ID, Package)), + trigger_node(Trigger, TriggerNode, node(ID2, Package2)), + attr(Name, node(ID, Package), A1, A2), + condition_holds(Condition1, node(ID1, Package1)), + pkg_fact(Package1, condition_effect(Condition1, Effect)), + imposed_constraint(Effect, Name, Package, A1, A2), + imposed_nodes(Effect, node(ID1, Package1), node(ID, Package)). + +condition_cause(Condition2, ID2, Condition1, ID1) :- + condition_holds(Condition2, node(ID2, Package2)), + pkg_fact(Package2, condition_trigger(Condition2, Trigger)), + condition_requirement(Trigger, Name, Package, A1, A2, A3), + condition_nodes(Trigger, TriggerNode, node(ID, Package)), + trigger_node(Trigger, TriggerNode, node(ID2, Package2)), + attr(Name, node(ID, Package), A1, A2, A3), + condition_holds(Condition1, node(ID1, Package1)), + pkg_fact(Package1, condition_effect(Condition1, Effect)), + imposed_constraint(Effect, Name, Package, A1, A2, A3), + imposed_nodes(Effect, node(ID1, Package1), node(ID, Package)). + +% special condition cause for dependency conditions +% we can't simply impose the existence of the node for dependency conditions +% because we need to allow for the choice of which dupe ID the node gets +condition_cause(Condition2, ID2, Condition1, ID1) :- + condition_holds(Condition2, node(ID2, Package2)), + pkg_fact(Package2, condition_trigger(Condition2, Trigger)), + condition_requirement(Trigger, "node", Package), + condition_nodes(Trigger, TriggerNode, node(ID, Package)), + trigger_node(Trigger, TriggerNode, node(ID2, Package2)), + attr("node", node(ID, Package)), + condition_holds(Condition1, node(ID1, Package1)), + pkg_fact(Package1, condition_effect(Condition1, Effect)), + imposed_constraint(Effect, "dependency_holds", Parent, Package, Type), + imposed_nodes(Effect, node(ID1, Package1), node(ID, Package)), + attr("depends_on", node(X, Parent), node(ID, Package), Type). + +% The literal startcauses is used to separate the variables that are part of the error from the +% ones describing the causal tree of the error. After startcauses, each successive pair must be +% a condition and a condition_set id for which it holds. + +% More specific error message if the version cannot satisfy some constraint +% Otherwise covered by `no_version_error` and `versions_conflict_error`. +error(1, "Cannot satisfy '{0}@{1}'", Package, Constraint, startcauses, ConstraintCause, CauseID) + :- attr("node_version_satisfies", node(ID, Package), Constraint), + pkg_fact(TriggerPkg, condition_effect(ConstraintCause, EffectID)), + imposed_constraint(EffectID, "node_version_satisfies", Package, Constraint), + condition_holds(ConstraintCause, node(CauseID, TriggerPkg)), + attr("version", node(ID, Package), Version), + not pkg_fact(Package, version_satisfies(Constraint, Version)). + +error(0, "Cannot satisfy '{0}@{1}' and '{0}@{2}", Package, Constraint1, Constraint2, startcauses, Cause1, C1ID, Cause2, C2ID) + :- attr("node_version_satisfies", node(ID, Package), Constraint1), + pkg_fact(TriggerPkg1, condition_effect(Cause1, EffectID1)), + imposed_constraint(EffectID1, "node_version_satisfies", Package, Constraint1), + condition_holds(Cause1, node(C1ID, TriggerPkg1)), + % two constraints + attr("node_version_satisfies", node(ID, Package), Constraint2), + pkg_fact(TriggerPkg2, condition_effect(Cause2, EffectID2)), + imposed_constraint(EffectID2, "node_version_satisfies", Package, Constraint2), + condition_holds(Cause2, node(C2ID, TriggerPkg2)), + % version chosen + attr("version", node(ID, Package), Version), + % version satisfies one but not the other + pkg_fact(Package, version_satisfies(Constraint1, Version)), + not pkg_fact(Package, version_satisfies(Constraint2, Version)). + +% causation tracking error for no or multiple virtual providers +error(0, "Cannot find a valid provider for virtual {0}", Virtual, startcauses, Cause, CID) + :- attr("virtual_node", node(X, Virtual)), + not provider(_, node(X, Virtual)), + imposed_constraint(EID, "dependency_holds", Parent, Virtual, Type), + pkg_fact(TriggerPkg, condition_effect(Cause, EID)), + condition_holds(Cause, node(CID, TriggerPkg)). + + +% At most one variant value for single-valued variants +error(0, "'{0}' required multiple values for single-valued variant '{1}'\n Requested 'Spec({1}={2})' and 'Spec({1}={3})'", Package, Variant, Value1, Value2, startcauses, Cause1, X, Cause2, X) + :- attr("node", node(X, Package)), + node_has_variant(node(X, Package), Variant), + pkg_fact(Package, variant_single_value(Variant)), + build(node(X, Package)), + attr("variant_value", node(X, Package), Variant, Value1), + imposed_constraint(EID1, "variant_set", Package, Variant, Value1), + pkg_fact(TriggerPkg1, condition_effect(Cause1, EID1)), + condition_holds(Cause1, node(X, TriggerPkg1)), + attr("variant_value", node(X, Package), Variant, Value2), + imposed_constraint(EID2, "variant_set", Package, Variant, Value2), + pkg_fact(TriggerPkg2, condition_effect(Cause2, EID2)), + condition_holds(Cause2, node(X, TriggerPkg2)), + Value1 < Value2. % see[1] in concretize.lp + +% Externals have to specify external conditions +error(0, "Attempted to use external for {0} which does not satisfy any configured external spec version", Package, startcauses, ExternalCause, CID) + :- external(node(ID, Package)), + attr("external_spec_selected", node(ID, Package), Index), + imposed_constraint(EID, "external_conditions_hold", Package, Index), + pkg_fact(TriggerPkg, condition_effect(ExternalCause, EID)), + condition_holds(ExternalCause, node(CID, TriggerPkg)), + not external_version(node(ID, Package), _, _). + +error(0, "Attempted to build package {0} which is not buildable and does not have a satisfying external\n attr('{1}', '{2}') is an external constraint for {0} which was not satisfied", Package, Name, A1) + :- external(node(ID, Package)), + not attr("external_conditions_hold", node(ID, Package), _), + imposed_constraint(EID, "external_conditions_hold", Package, _), + trigger_and_effect(Package, TID, EID), + condition_requirement(TID, Name, A1), + not attr(Name, node(_, A1)). + +error(0, "Attempted to build package {0} which is not buildable and does not have a satisfying external\n attr('{1}', '{2}', '{3}') is an external constraint for {0} which was not satisfied", Package, Name, A1, A2) + :- external(node(ID, Package)), + not attr("external_conditions_hold", node(ID, Package), _), + imposed_constraint(EID, "external_conditions_hold", Package, _), + trigger_and_effect(Package, TID, EID), + condition_requirement(TID, Name, A1, A2), + not attr(Name, node(_, A1), A2). + +error(0, "Attempted to build package {0} which is not buildable and does not have a satisfying external\n attr('{1}', '{2}', '{3}', '{4}') is an external constraint for {0} which was not satisfied", Package, Name, A1, A2, A3) + :- external(node(ID, Package)), + not attr("external_conditions_hold", node(ID, Package), _), + imposed_constraint(EID, "external_conditions_hold", Package, _), + trigger_and_effect(Package, TID, EID), + condition_requirement(TID, Name, A1, A2, A3), + not attr(Name, node(_, A1), A2, A3). + +error(0, "Attempted to build package {0} which is not buildable and does not have a satisfying external\n 'Spec({0} {1}={2})' is an external constraint for {0} which was not satisfied\n 'Spec({0} {1}={3})' required", Package, Variant, Value, OtherValue, startcauses, OtherValueCause, CID) + :- external(node(ID, Package)), + not attr("external_conditions_hold", node(ID, Package), _), + imposed_constraint(EID, "external_conditions_hold", Package, _), + trigger_and_effect(Package, TID, EID), + condition_requirement(TID, "variant_value", Package, Variant, Value), + not attr("variant_value", node(ID, Package), Variant, Value), + attr("variant_value", node(ID, Package), Variant, OtherValue), + imposed_constraint(EID2, "variant_set", Package, Variant, OtherValue), + pkg_fact(TriggerPkg, condition_effect(OtherValueCause, EID2)), + condition_holds(OtherValueCause, node(CID, TriggerPkg)). + +error(0, "Attempted to build package {0} which is not buildable and does not have a satisfying external\n attr('{1}', '{2}', '{3}', '{4}', '{5}') is an external constraint for {0} which was not satisfied", Package, Name, A1, A2, A3, A4) + :- external(node(ID, Package)), + not attr("external_conditions_hold", node(ID, Package), _), + imposed_constraint(EID, "external_conditions_hold", Package, _), + trigger_and_effect(Package, TID, EID), + condition_requirement(TID, Name, A1, A2, A3, A4), + not attr(Name, node(_, A1), A2, A3, A4). + +% error message with causes for conflicts +error(0, Msg, startcauses, TriggerID, ID1, ConstraintID, ID2) + :- attr("node", node(ID, Package)), + pkg_fact(Package, conflict(TriggerID, ConstraintID, Msg)), + % node(ID1, TriggerPackage) is node(ID2, Package) in most, but not all, cases + condition_holds(TriggerID, node(ID1, TriggerPackage)), + condition_holds(ConstraintID, node(ID2, Package)), + unification_set(X, node(ID2, Package)), + unification_set(X, node(ID1, TriggerPackage)), + not external(node(ID, Package)), % ignore conflicts for externals + not attr("hash", node(ID, Package), _). % ignore conflicts for installed packages + +% variables to show +#show error/2. +#show error/3. +#show error/4. +#show error/5. +#show error/6. +#show error/7. +#show error/8. +#show error/9. +#show error/10. +#show error/11. + +#show condition_cause/4. +#show condition_reason/2. + +% Define all variables used to avoid warnings at runtime when the model doesn't happen to have one +#defined error/2. +#defined error/3. +#defined error/4. +#defined error/5. +#defined error/6. +#defined attr/2. +#defined attr/3. +#defined attr/4. +#defined attr/5. +#defined pkg_fact/2. +#defined imposed_constraint/3. +#defined imposed_constraint/4. +#defined imposed_constraint/5. +#defined imposed_constraint/6. +#defined condition_requirement/3. +#defined condition_requirement/4. +#defined condition_requirement/5. +#defined condition_requirement/6. +#defined condition_holds/2. +#defined unification_set/2. +#defined external/1. +#defined trigger_and_effect/3. +#defined build/1. +#defined node_has_variant/2. +#defined provider/2. +#defined external_version/3. diff --git a/lib/spack/spack/solver/heuristic.lp b/lib/spack/spack/solver/heuristic.lp new file mode 100644 index 00000000000000..cc87207047d438 --- /dev/null +++ b/lib/spack/spack/solver/heuristic.lp @@ -0,0 +1,24 @@ +% Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +% Spack Project Developers. See the top-level COPYRIGHT file for details. +% +% SPDX-License-Identifier: (Apache-2.0 OR MIT) + +%============================================================================= +% Heuristic to speed-up solves (node with ID 0) +%============================================================================= + + +%----------------- +% Domain heuristic +%----------------- + +% Root node +#heuristic attr("version", node(0, Package), Version) : pkg_fact(Package, version_declared(Version, 0)), attr("root", node(0, Package)). [35, true] +#heuristic version_weight(node(0, Package), 0) : pkg_fact(Package, version_declared(Version, 0)), attr("root", node(0, Package)). [35, true] +#heuristic attr("variant_value", node(0, Package), Variant, Value) : variant_default_value(Package, Variant, Value), attr("root", node(0, Package)). [35, true] +#heuristic attr("node_target", node(0, Package), Target) : target_weight(Target, 0), attr("root", node(0, Package)). [35, true] +#heuristic node_target_weight(node(0, Package), 0) : attr("root", node(0, Package)). [35, true] +#heuristic node_compiler(node(0, Package), CompilerID) : compiler_weight(ID, 0), compiler_id(ID), attr("root", node(0, Package)). [35, true] + +% Providers +#heuristic attr("node", node(0, Package)) : default_provider_preference(Virtual, Package, 0), possible_in_link_run(Package). [30, true] diff --git a/lib/spack/spack/solver/heuristic_separate.lp b/lib/spack/spack/solver/heuristic_separate.lp new file mode 100644 index 00000000000000..caa47aa09d84d1 --- /dev/null +++ b/lib/spack/spack/solver/heuristic_separate.lp @@ -0,0 +1,24 @@ +% Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +% Spack Project Developers. See the top-level COPYRIGHT file for details. +% +% SPDX-License-Identifier: (Apache-2.0 OR MIT) + +%============================================================================= +% Heuristic to speed-up solves (node with ID > 0) +%============================================================================= + +% node(ID, _) +#heuristic attr("version", node(ID, Package), Version) : pkg_fact(Package, version_declared(Version, 0)), attr("node", node(ID, Package)), ID > 0. [25-5*ID, true] +#heuristic version_weight(node(ID, Package), 0) : pkg_fact(Package, version_declared(Version, 0)), attr("node", node(ID, Package)), ID > 0. [25-5*ID, true] +#heuristic attr("variant_value", node(ID, Package), Variant, Value) : variant_default_value(Package, Variant, Value), attr("node", node(ID, Package)), ID > 0. [25-5*ID, true] +#heuristic attr("node_target", node(ID, Package), Target) : pkg_fact(Package, target_weight(Target, 0)), attr("node", node(ID, Package)), ID > 0. [25-5*ID, true] +#heuristic node_target_weight(node(ID, Package), 0) : attr("node", node(ID, Package)), ID > 0. [25-5*ID, true] +#heuristic node_compiler(node(ID, Package), CompilerID) : compiler_weight(CompilerID, 0), compiler_id(CompilerID), attr("node", node(ID, Package)), ID > 0. [25-5*ID, true] + +% node(ID, _), split build dependencies +#heuristic attr("version", node(ID, Package), Version) : pkg_fact(Package, version_declared(Version, 0)), attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true] +#heuristic version_weight(node(ID, Package), 0) : pkg_fact(Package, version_declared(Version, 0)), attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true] +#heuristic attr("variant_value", node(ID, Package), Variant, Value) : variant_default_value(Package, Variant, Value), attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true] +#heuristic attr("node_target", node(ID, Package), Target) : pkg_fact(Package, target_weight(Target, 0)), attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true] +#heuristic node_target_weight(node(ID, Package), 0) : attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true] +#heuristic node_compiler(node(ID, Package), CompilerID) : compiler_weight(CompilerID, 0), compiler_id(CompilerID), attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true] diff --git a/lib/spack/spack/solver/os_compatibility.lp b/lib/spack/spack/solver/os_compatibility.lp index a484a90a3d31b6..3c0c223a868f84 100644 --- a/lib/spack/spack/solver/os_compatibility.lp +++ b/lib/spack/spack/solver/os_compatibility.lp @@ -3,9 +3,11 @@ % % SPDX-License-Identifier: (Apache-2.0 OR MIT) +%============================================================================= % OS compatibility rules for reusing solves. % os_compatible(RecentOS, OlderOS) % OlderOS binaries can be used on RecentOS +%============================================================================= % macOS os_compatible("monterey", "bigsur"). diff --git a/lib/spack/spack/solver/when_possible.lp b/lib/spack/spack/solver/when_possible.lp new file mode 100644 index 00000000000000..b887c9175f7df6 --- /dev/null +++ b/lib/spack/spack/solver/when_possible.lp @@ -0,0 +1,27 @@ +% Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +% Spack Project Developers. See the top-level COPYRIGHT file for details. +% +% SPDX-License-Identifier: (Apache-2.0 OR MIT) + +%============================================================================= +% Minimize the number of literals that are not solved +% +% This minimization is used for the "when_possible" concretization mode, +% otherwise we assume that all literals must be solved. +%============================================================================= + +% Give clingo the choice to solve an input spec or not +{ solve_literal(ID) } :- literal(ID). +literal_not_solved(ID) :- not solve_literal(ID), literal(ID). + +% Make a problem with "zero literals solved" unsat. This is to trigger +% looking for solutions to the ASP problem with "errors", which results +% in better reporting for users. See #30669 for details. +1 { solve_literal(ID) : literal(ID) }. + +opt_criterion(300, "number of input specs not concretized"). +#minimize{ 0@300: #true }. +#minimize { 1@300,ID : literal_not_solved(ID) }. + +#heuristic literal_solved(ID) : literal(ID). [1, sign] +#heuristic literal_solved(ID) : literal(ID). [50, init] diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 95ee9deac03183..20e5c3ffa33e05 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -54,10 +54,15 @@ import io import itertools import os +import pathlib +import platform import re +import socket import warnings -from typing import Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union +import llnl.path +import llnl.string import llnl.util.filesystem as fs import llnl.util.lang as lang import llnl.util.tty as tty @@ -67,8 +72,11 @@ import spack.compilers import spack.config import spack.dependency as dp +import spack.deptypes as dt import spack.error import spack.hash_types as ht +import spack.parser +import spack.patch import spack.paths import spack.platforms import spack.provider_index @@ -81,13 +89,12 @@ import spack.util.executable import spack.util.hash import spack.util.module_cmd as md -import spack.util.path as pth import spack.util.prefix import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml -import spack.util.string import spack.variant as vt import spack.version as vn +import spack.version.git_ref_lookup __all__ = [ "CompilerSpec", @@ -111,50 +118,49 @@ "UnsatisfiableDependencySpecError", "AmbiguousHashError", "InvalidHashError", - "RedundantSpecError", "SpecDeprecatedError", ] #: Valid pattern for an identifier in Spack -identifier_re = r"\w[\w-]*" +IDENTIFIER_RE = r"\w[\w-]*" -compiler_color = "@g" #: color for highlighting compilers -version_color = "@c" #: color for highlighting versions -architecture_color = "@m" #: color for highlighting architectures -enabled_variant_color = "@B" #: color for highlighting enabled variants -disabled_variant_color = "r" #: color for highlighting disabled varaints -dependency_color = "@." #: color for highlighting dependencies -hash_color = "@K" #: color for highlighting package hashes +COMPILER_COLOR = "@g" #: color for highlighting compilers +VERSION_COLOR = "@c" #: color for highlighting versions +ARCHITECTURE_COLOR = "@m" #: color for highlighting architectures +ENABLED_VARIANT_COLOR = "@B" #: color for highlighting enabled variants +DISABLED_VARIANT_COLOR = "r" #: color for highlighting disabled varaints +DEPENDENCY_COLOR = "@." #: color for highlighting dependencies +HASH_COLOR = "@K" #: color for highlighting package hashes #: This map determines the coloring of specs when using color output. #: We make the fields different colors to enhance readability. #: See llnl.util.tty.color for descriptions of the color codes. -color_formats = { - "%": compiler_color, - "@": version_color, - "=": architecture_color, - "+": enabled_variant_color, - "~": disabled_variant_color, - "^": dependency_color, - "#": hash_color, +COLOR_FORMATS = { + "%": COMPILER_COLOR, + "@": VERSION_COLOR, + "=": ARCHITECTURE_COLOR, + "+": ENABLED_VARIANT_COLOR, + "~": DISABLED_VARIANT_COLOR, + "^": DEPENDENCY_COLOR, + "#": HASH_COLOR, } #: Regex used for splitting by spec field separators. #: These need to be escaped to avoid metacharacters in -#: ``color_formats.keys()``. -_separators = "[\\%s]" % "\\".join(color_formats.keys()) +#: ``COLOR_FORMATS.keys()``. +_SEPARATORS = "[\\%s]" % "\\".join(COLOR_FORMATS.keys()) #: Default format for Spec.format(). This format can be round-tripped, so that: #: Spec(Spec("string").format()) == Spec("string)" -default_format = ( +DEFAULT_FORMAT = ( "{name}{@versions}" "{%compiler.name}{@compiler.versions}{compiler_flags}" "{variants}{arch=architecture}{/abstract_hash}" ) #: Display format, which eliminates extra `@=` in the output, for readability. -display_format = ( +DISPLAY_FORMAT = ( "{name}{@version}" "{%compiler.name}{@compiler.version}{compiler_flags}" "{variants}{arch=architecture}{/abstract_hash}" @@ -174,9 +180,12 @@ SPECFILE_FORMAT_VERSION = 4 -# InstallStatus is used to map install statuses to symbols for display -# Options are artificially disjoint for dispay purposes class InstallStatus(enum.Enum): + """Maps install statuses to symbols for display. + + Options are artificially disjoint for display purposes + """ + installed = "@g{[+]} " upstream = "@g{[^]} " external = "@g{[e]} " @@ -186,7 +195,7 @@ class InstallStatus(enum.Enum): def colorize_spec(spec): """Returns a spec colorized according to the colors specified in - color_formats.""" + COLOR_FORMATS.""" class insert_color: def __init__(self): @@ -199,9 +208,9 @@ def __call__(self, match): return clr.cescape(sep) self.last = sep - return "%s%s" % (color_formats[sep], clr.cescape(sep)) + return "%s%s" % (COLOR_FORMATS[sep], clr.cescape(sep)) - return clr.colorize(re.sub(_separators, insert_color(), str(spec)) + "@.") + return clr.colorize(re.sub(_SEPARATORS, insert_color(), str(spec)) + "@.") @lang.lazy_lexicographic_ordering @@ -724,81 +733,54 @@ class DependencySpec: Args: parent: starting node of the edge spec: ending node of the edge. - deptypes: list of strings, representing dependency relationships. + depflag: represents dependency relationships. virtuals: virtual packages provided from child to parent node. """ - __slots__ = "parent", "spec", "parameters" + __slots__ = "parent", "spec", "depflag", "virtuals" def __init__( - self, - parent: "Spec", - spec: "Spec", - *, - deptypes: dp.DependencyArgument, - virtuals: Tuple[str, ...], + self, parent: "Spec", spec: "Spec", *, depflag: dt.DepFlag, virtuals: Tuple[str, ...] ): self.parent = parent self.spec = spec - self.parameters = { - "deptypes": dp.canonical_deptype(deptypes), - "virtuals": tuple(sorted(set(virtuals))), - } - - @property - def deptypes(self) -> Tuple[str, ...]: - return self.parameters["deptypes"] + self.depflag = depflag + self.virtuals = tuple(sorted(set(virtuals))) - @property - def virtuals(self) -> Tuple[str, ...]: - return self.parameters["virtuals"] - - def _update_edge_multivalued_property( - self, property_name: str, value: Tuple[str, ...] - ) -> bool: - current = self.parameters[property_name] - update = set(current) | set(value) - update = tuple(sorted(update)) - changed = current != update - - if not changed: + def update_deptypes(self, depflag: dt.DepFlag) -> bool: + """Update the current dependency types""" + old = self.depflag + new = depflag | old + if new == old: return False - - self.parameters[property_name] = update + self.depflag = new return True - def update_deptypes(self, deptypes: Tuple[str, ...]) -> bool: - """Update the current dependency types""" - return self._update_edge_multivalued_property("deptypes", deptypes) - def update_virtuals(self, virtuals: Tuple[str, ...]) -> bool: """Update the list of provided virtuals""" - return self._update_edge_multivalued_property("virtuals", virtuals) + old = self.virtuals + self.virtuals = tuple(sorted(set(virtuals).union(self.virtuals))) + return old != self.virtuals def copy(self) -> "DependencySpec": """Return a copy of this edge""" - return DependencySpec( - self.parent, self.spec, deptypes=self.deptypes, virtuals=self.virtuals - ) + return DependencySpec(self.parent, self.spec, depflag=self.depflag, virtuals=self.virtuals) def _cmp_iter(self): yield self.parent.name if self.parent else None yield self.spec.name if self.spec else None - yield self.deptypes + yield self.depflag yield self.virtuals def __str__(self) -> str: parent = self.parent.name if self.parent else None child = self.spec.name if self.spec else None - return f"{parent} {self.deptypes}[virtuals={','.join(self.virtuals)}] --> {child}" - - def canonical(self) -> Tuple[str, str, Tuple[str, ...], Tuple[str, ...]]: - return self.parent.dag_hash(), self.spec.dag_hash(), self.deptypes, self.virtuals + return f"{parent} {self.depflag}[virtuals={','.join(self.virtuals)}] --> {child}" def flip(self) -> "DependencySpec": """Flip the dependency, and drop virtual information""" return DependencySpec( - parent=self.spec, spec=self.parent, deptypes=self.deptypes, virtuals=() + parent=self.spec, spec=self.parent, depflag=self.depflag, virtuals=() ) @@ -943,9 +925,8 @@ def __str__(self): ) -def _sort_by_dep_types(dspec): - # Use negation since False < True for sorting - return tuple(t not in dspec.deptypes for t in ("link", "run", "build", "test")) +def _sort_by_dep_types(dspec: DependencySpec): + return dspec.depflag #: Enum for edge directions @@ -984,16 +965,14 @@ def __iter__(self): def __len__(self): return len(self.edges) - def add(self, edge): - """Adds a new edge to this object. - - Args: - edge (DependencySpec): edge to be added - """ + def add(self, edge: DependencySpec): key = edge.spec.name if self.store_by_child else edge.parent.name - current_list = self.edges.setdefault(key, []) - current_list.append(edge) - current_list.sort(key=_sort_by_dep_types) + if key in self.edges: + lst = self.edges[key] + lst.append(edge) + lst.sort(key=_sort_by_dep_types) + else: + self.edges[key] = [edge] def __str__(self): return "{deps: %s}" % ", ".join(str(d) for d in sorted(self.values())) @@ -1013,7 +992,7 @@ def copy(self): return clone - def select(self, parent=None, child=None, deptypes=dp.all_deptypes): + def select(self, parent=None, child=None, depflag: dt.DepFlag = dt.ALL): """Select a list of edges and return them. If an edge: @@ -1021,18 +1000,18 @@ def select(self, parent=None, child=None, deptypes=dp.all_deptypes): - Matches the parent and/or child name, if passed then it is selected. - The deptypes argument needs to be canonical, since the method won't + The deptypes argument needs to be a flag, since the method won't convert it for performance reason. Args: parent (str): name of the parent package child (str): name of the child package - deptypes (tuple): allowed dependency types in canonical form + depflag: allowed dependency types in flag form Returns: List of DependencySpec objects """ - if not deptypes: + if not depflag: return [] # Start from all the edges we store @@ -1047,12 +1026,7 @@ def select(self, parent=None, child=None, deptypes=dp.all_deptypes): selected = (d for d in selected if d.spec.name == child) # Filter by allowed dependency types - if deptypes: - selected = ( - dep - for dep in selected - if not dep.deptypes or any(d in deptypes for d in dep.deptypes) - ) + selected = (dep for dep in selected if not dep.depflag or (depflag & dep.depflag)) return list(selected) @@ -1298,7 +1272,7 @@ def __init__(self, spec, name, query_parameters): original_spec = getattr(spec, "wrapped_obj", spec) self.wrapped_obj = original_spec self.token = original_spec, name, query_parameters - is_virtual = spack.repo.path.is_virtual(name) + is_virtual = spack.repo.PATH.is_virtual(name) self.last_query = QueryState( name=name, extra_parameters=query_parameters, isvirtual=is_virtual ) @@ -1345,8 +1319,6 @@ def __init__( self.external_path = external_path self.external_module = external_module """ - import spack.parser - # Copy if spec_like is a Spec. if isinstance(spec_like, Spec): self._dup(spec_like) @@ -1421,7 +1393,7 @@ def _format_module_list(modules): @property def external_path(self): - return pth.path_to_os_path(self._external_path)[0] + return llnl.path.path_to_os_path(self._external_path)[0] @external_path.setter def external_path(self, ext_path): @@ -1472,47 +1444,69 @@ def _get_dependency(self, name): raise spack.error.SpecError(err_msg.format(name, len(deps))) return deps[0] - def edges_from_dependents(self, name=None, deptype="all"): + def edges_from_dependents(self, name=None, depflag: dt.DepFlag = dt.ALL): """Return a list of edges connecting this node in the DAG to parents. Args: name (str): filter dependents by package name - deptype (str or tuple): allowed dependency types + depflag: allowed dependency types """ - deptype = dp.canonical_deptype(deptype) - return [d for d in self._dependents.select(parent=name, deptypes=deptype)] + return [d for d in self._dependents.select(parent=name, depflag=depflag)] - def edges_to_dependencies(self, name=None, deptype="all"): + def edges_to_dependencies(self, name=None, depflag: dt.DepFlag = dt.ALL): """Return a list of edges connecting this node in the DAG to children. Args: name (str): filter dependencies by package name - deptype (str or tuple): allowed dependency types + depflag: allowed dependency types """ - deptype = dp.canonical_deptype(deptype) - return [d for d in self._dependencies.select(child=name, deptypes=deptype)] + return [d for d in self._dependencies.select(child=name, depflag=depflag)] - def dependencies(self, name=None, deptype="all"): + @property + def edge_attributes(self) -> str: + """Helper method to print edge attributes in spec literals""" + edges = self.edges_from_dependents() + if not edges: + return "" + + union = DependencySpec(parent=Spec(), spec=self, depflag=0, virtuals=()) + for edge in edges: + union.update_deptypes(edge.depflag) + union.update_virtuals(edge.virtuals) + deptypes_str = ( + f"deptypes={','.join(dt.flag_to_tuple(union.depflag))}" if union.depflag else "" + ) + virtuals_str = f"virtuals={','.join(union.virtuals)}" if union.virtuals else "" + if not deptypes_str and not virtuals_str: + return "" + result = f"{deptypes_str} {virtuals_str}".strip() + return f"[{result}]" + + def dependencies(self, name=None, deptype: Union[dt.DepTypes, dt.DepFlag] = dt.ALL): """Return a list of direct dependencies (nodes in the DAG). Args: name (str): filter dependencies by package name - deptype (str or tuple): allowed dependency types + deptype: allowed dependency types """ - return [d.spec for d in self.edges_to_dependencies(name, deptype=deptype)] + if not isinstance(deptype, dt.DepFlag): + deptype = dt.canonicalize(deptype) + return [d.spec for d in self.edges_to_dependencies(name, depflag=deptype)] - def dependents(self, name=None, deptype="all"): + def dependents(self, name=None, deptype: Union[dt.DepTypes, dt.DepFlag] = dt.ALL): """Return a list of direct dependents (nodes in the DAG). Args: name (str): filter dependents by package name - deptype (str or tuple): allowed dependency types + deptype: allowed dependency types """ - return [d.parent for d in self.edges_from_dependents(name, deptype=deptype)] + if not isinstance(deptype, dt.DepFlag): + deptype = dt.canonicalize(deptype) + return [d.parent for d in self.edges_from_dependents(name, depflag=deptype)] - def _dependencies_dict(self, deptype="all"): + def _dependencies_dict(self, depflag: dt.DepFlag = dt.ALL): """Return a dictionary, keyed by package name, of the direct dependencies. @@ -1521,10 +1515,9 @@ def _dependencies_dict(self, deptype="all"): Args: deptype: allowed dependency types """ - _sort_fn = lambda x: (x.spec.name,) + _sort_by_dep_types(x) + _sort_fn = lambda x: (x.spec.name, _sort_by_dep_types(x)) _group_fn = lambda x: x.spec.name - deptype = dp.canonical_deptype(deptype) - selected_edges = self._dependencies.select(deptypes=deptype) + selected_edges = self._dependencies.select(depflag=depflag) result = {} for key, group in itertools.groupby(sorted(selected_edges, key=_sort_fn), key=_group_fn): result[key] = list(group) @@ -1620,35 +1613,36 @@ def _set_compiler(self, compiler): ) self.compiler = compiler - def _add_dependency( - self, spec: "Spec", *, deptypes: dp.DependencyArgument, virtuals: Tuple[str, ...] - ): + def _add_dependency(self, spec: "Spec", *, depflag: dt.DepFlag, virtuals: Tuple[str, ...]): """Called by the parser to add another spec as a dependency.""" if spec.name not in self._dependencies or not spec.name: - self.add_dependency_edge(spec, deptypes=deptypes, virtuals=virtuals) + self.add_dependency_edge(spec, depflag=depflag, virtuals=virtuals) return # Keep the intersection of constraints when a dependency is added # multiple times. Currently, we only allow identical edge types. orig = self._dependencies[spec.name] try: - dspec = next(dspec for dspec in orig if deptypes == dspec.deptypes) + dspec = next(dspec for dspec in orig if depflag == dspec.depflag) except StopIteration: - raise DuplicateDependencyError("Cannot depend on '%s' twice" % spec) + current_deps = ", ".join( + dt.flag_to_chars(x.depflag) + " " + x.spec.short_spec for x in orig + ) + raise DuplicateDependencyError( + f"{self.short_spec} cannot depend on '{spec.short_spec}' multiple times.\n" + f"\tRequired: {dt.flag_to_chars(depflag)}\n" + f"\tDependency: {current_deps}" + ) try: dspec.spec.constrain(spec) except spack.error.UnsatisfiableSpecError: raise DuplicateDependencyError( - "Cannot depend on incompatible specs '%s' and '%s'" % (dspec.spec, spec) + f"Cannot depend on incompatible specs '{dspec.spec}' and '{spec}'" ) def add_dependency_edge( - self, - dependency_spec: "Spec", - *, - deptypes: dp.DependencyArgument, - virtuals: Tuple[str, ...], + self, dependency_spec: "Spec", *, depflag: dt.DepFlag, virtuals: Tuple[str, ...] ): """Add a dependency edge to this spec. @@ -1657,19 +1651,17 @@ def add_dependency_edge( deptypes: dependency types for this edge virtuals: virtuals provided by this edge """ - deptypes = dp.canonical_deptype(deptypes) - # Check if we need to update edges that are already present selected = self._dependencies.select(child=dependency_spec.name) for edge in selected: has_errors, details = False, [] msg = f"cannot update the edge from {edge.parent.name} to {edge.spec.name}" - if any(d in edge.deptypes for d in deptypes): + if edge.depflag & depflag: has_errors = True details.append( ( f"{edge.parent.name} has already an edge matching any" - f" of these types {str(deptypes)}" + f" of these types {depflag}" ) ) @@ -1678,7 +1670,7 @@ def add_dependency_edge( details.append( ( f"{edge.parent.name} has already an edge matching any" - f" of these virtuals {str(virtuals)}" + f" of these virtuals {virtuals}" ) ) @@ -1690,11 +1682,11 @@ def add_dependency_edge( # If we are here, it means the edge object was previously added to # both the parent and the child. When we update this object they'll # both see the deptype modification. - edge.update_deptypes(deptypes=deptypes) + edge.update_deptypes(depflag=depflag) edge.update_virtuals(virtuals=virtuals) return - edge = DependencySpec(self, dependency_spec, deptypes=deptypes, virtuals=virtuals) + edge = DependencySpec(self, dependency_spec, depflag=depflag, virtuals=virtuals) self._dependencies.add(edge) dependency_spec._dependents.add(edge) @@ -1732,7 +1724,7 @@ def package(self): self.name ) if not self._package: - self._package = spack.repo.path.get(self) + self._package = spack.repo.PATH.get(self) return self._package @property @@ -1740,11 +1732,11 @@ def package_class(self): """Internal package call gets only the class object for a package. Use this to just get package metadata. """ - return spack.repo.path.get_pkg_class(self.fullname) + return spack.repo.PATH.get_pkg_class(self.fullname) @property def virtual(self): - return spack.repo.path.is_virtual(self.name) + return spack.repo.PATH.is_virtual(self.name) @property def concrete(self): @@ -1780,7 +1772,7 @@ def installed(self): try: # If the spec is in the DB, check the installed # attribute of the record - return spack.store.db.get_record(self).installed + return spack.store.STORE.db.get_record(self).installed except KeyError: # If the spec is not in the DB, the method # above raises a Key error @@ -1796,7 +1788,7 @@ def installed_upstream(self): if not self.concrete: return False - upstream, _ = spack.store.db.query_by_spec_hash(self.dag_hash()) + upstream, _ = spack.store.STORE.db.query_by_spec_hash(self.dag_hash()) return upstream def traverse(self, **kwargs): @@ -1828,16 +1820,16 @@ def prefix(self): raise spack.error.SpecError("Spec is not concrete: " + str(self)) if self._prefix is None: - upstream, record = spack.store.db.query_by_spec_hash(self.dag_hash()) + upstream, record = spack.store.STORE.db.query_by_spec_hash(self.dag_hash()) if record and record.path: self.prefix = record.path else: - self.prefix = spack.store.layout.path_for_spec(self) + self.prefix = spack.store.STORE.layout.path_for_spec(self) return self._prefix @prefix.setter def prefix(self, value): - self._prefix = spack.util.prefix.Prefix(pth.convert_to_platform_path(value)) + self._prefix = spack.util.prefix.Prefix(llnl.path.convert_to_platform_path(value)) def spec_hash(self, hash): """Utility method for computing different types of Spec hashes. @@ -1926,19 +1918,15 @@ def _lookup_hash(self): store, or finally, binary caches.""" import spack.environment - matches = [] active_env = spack.environment.active_environment() - if active_env: - env_matches = active_env.get_by_hash(self.abstract_hash) or [] - matches = [m for m in env_matches if m._satisfies(self)] - if not matches: - db_matches = spack.store.db.get_by_hash(self.abstract_hash) or [] - matches = [m for m in db_matches if m._satisfies(self)] - if not matches: - query = spack.binary_distribution.BinaryCacheQuery(True) - remote_matches = query("/" + self.abstract_hash) or [] - matches = [m for m in remote_matches if m._satisfies(self)] + # First env, then store, then binary cache + matches = ( + (active_env.all_matching_specs(self) if active_env else []) + or spack.store.STORE.db.query(self, installed=any) + or spack.binary_distribution.BinaryCacheQuery(True)(self) + ) + if not matches: raise InvalidHashError(self, self.abstract_hash) @@ -1959,20 +1947,18 @@ def lookup_hash(self): spec = self.copy(deps=False) # root spec is replaced if spec.abstract_hash: - new = self._lookup_hash() - spec._dup(new) + spec._dup(self._lookup_hash()) return spec # Get dependencies that need to be replaced for node in self.traverse(root=False): if node.abstract_hash: - new = node._lookup_hash() - spec._add_dependency(new, deptypes=(), virtuals=()) + spec._add_dependency(node._lookup_hash(), depflag=0, virtuals=()) # reattach nodes that were not otherwise satisfied by new dependencies for node in self.traverse(root=False): - if not any(n._satisfies(node) for n in spec.traverse()): - spec._add_dependency(node.copy(), deptypes=(), virtuals=()) + if not any(n.satisfies(node) for n in spec.traverse()): + spec._add_dependency(node.copy(), depflag=0, virtuals=()) return spec @@ -1984,9 +1970,7 @@ def replace_hash(self): if not any(node for node in self.traverse(order="post") if node.abstract_hash): return - spec_by_hash = self.lookup_hash() - - self._dup(spec_by_hash) + self._dup(self.lookup_hash()) def to_node_dict(self, hash=ht.dag_hash): """Create a dictionary representing the state of this Spec. @@ -2100,7 +2084,7 @@ def to_node_dict(self, hash=ht.dag_hash): d["package_hash"] = package_hash # Note: Relies on sorting dict by keys later in algorithm. - deps = self._dependencies_dict(deptype=hash.deptype) + deps = self._dependencies_dict(depflag=hash.depflag) if deps: deps_list = [] for name, edges_for_name in sorted(deps.items()): @@ -2110,7 +2094,10 @@ def to_node_dict(self, hash=ht.dag_hash): parameters_tuple = ( "parameters", syaml.syaml_dict( - (key, dspec.parameters[key]) for key in sorted(dspec.parameters) + ( + ("deptypes", dt.flag_to_tuple(dspec.depflag)), + ("virtuals", dspec.virtuals), + ) ), ) ordered_entries = [name_tuple, hash_tuple, parameters_tuple] @@ -2208,7 +2195,7 @@ def to_dict(self, hash=ht.dag_hash): """ node_list = [] # Using a list to preserve preorder traversal for hash. hash_set = set() - for s in self.traverse(order="pre", deptype=hash.deptype): + for s in self.traverse(order="pre", deptype=hash.depflag): spec_hash = s._cached_hash(hash) if spec_hash not in hash_set: @@ -2271,7 +2258,7 @@ def override(init_spec, change_spec): # TODO: this doesn't account for the case where the changed spec # (and the user spec) have dependencies new_spec = init_spec.copy() - package_cls = spack.repo.path.get_pkg_class(new_spec.name) + package_cls = spack.repo.PATH.get_pkg_class(new_spec.name) if change_spec.versions and not change_spec.versions == vn.any_version: new_spec.versions = change_spec.versions for variant, value in change_spec.variants.items(): @@ -2392,13 +2379,12 @@ def spec_builder(d): if dep_like is None: return spec - def name_and_dependency_types(s): + def name_and_dependency_types(s: str) -> Tuple[str, dt.DepFlag]: """Given a key in the dictionary containing the literal, extracts the name of the spec and its dependency types. Args: - s (str): key in the dictionary containing the literal - + s: key in the dictionary containing the literal """ t = s.split(":") @@ -2406,39 +2392,37 @@ def name_and_dependency_types(s): msg = 'more than one ":" separator in key "{0}"' raise KeyError(msg.format(s)) - n = t[0] + name = t[0] if len(t) == 2: - dtypes = tuple(dt.strip() for dt in t[1].split(",")) + depflag = dt.flag_from_strings(dep_str.strip() for dep_str in t[1].split(",")) else: - dtypes = () - - return n, dtypes + depflag = 0 + return name, depflag - def spec_and_dependency_types(s): + def spec_and_dependency_types( + s: Union[Spec, Tuple[Spec, str]] + ) -> Tuple[Spec, dt.DepFlag]: """Given a non-string key in the literal, extracts the spec and its dependency types. Args: - s (spec or tuple): either a Spec object or a tuple - composed of a Spec object and a string with the - dependency types - + s: either a Spec object, or a tuple of Spec and string of dependency types """ if isinstance(s, Spec): - return s, () + return s, 0 spec_obj, dtypes = s - return spec_obj, tuple(dt.strip() for dt in dtypes.split(",")) + return spec_obj, dt.flag_from_strings(dt.strip() for dt in dtypes.split(",")) # Recurse on dependencies for s, s_dependencies in dep_like.items(): if isinstance(s, str): - dag_node, dependency_types = name_and_dependency_types(s) + dag_node, dep_flag = name_and_dependency_types(s) else: - dag_node, dependency_types = spec_and_dependency_types(s) + dag_node, dep_flag = spec_and_dependency_types(s) dependency_spec = spec_builder({dag_node: s_dependencies}) - spec._add_dependency(dependency_spec, deptypes=dependency_types, virtuals=()) + spec._add_dependency(dependency_spec, depflag=dep_flag, virtuals=()) return spec @@ -2545,7 +2529,7 @@ def validate_detection(self): assert isinstance(self.extra_attributes, collections.abc.Mapping), msg # Validate the spec calling a package specific method - pkg_cls = spack.repo.path.get_pkg_class(self.name) + pkg_cls = spack.repo.PATH.get_pkg_class(self.name) validate_fn = getattr(pkg_cls, "validate_detected_spec", lambda x, y: None) validate_fn(self, self.extra_attributes) @@ -2611,7 +2595,7 @@ def _replace_with(self, concrete): virtuals = (self.name,) for dep_spec in itertools.chain.from_iterable(self._dependents.values()): dependent = dep_spec.parent - deptypes = dep_spec.deptypes + depflag = dep_spec.depflag # remove self from all dependents, unless it is already removed if self.name in dependent._dependencies: @@ -2619,7 +2603,7 @@ def _replace_with(self, concrete): # add the replacement, unless it is already a dep of dependent. if concrete.name not in dependent._dependencies: - dependent._add_dependency(concrete, deptypes=deptypes, virtuals=virtuals) + dependent._add_dependency(concrete, depflag=depflag, virtuals=virtuals) else: dependent.edges_to_dependencies(name=concrete.name)[0].update_virtuals( virtuals=virtuals @@ -2644,7 +2628,7 @@ def _expand_virtual_packages(self, concretizer): """ # Make an index of stuff this spec already provides self_index = spack.provider_index.ProviderIndex( - repository=spack.repo.path, specs=self.traverse(), restrict=True + repository=spack.repo.PATH, specs=self.traverse(), restrict=True ) changed = False done = False @@ -2784,7 +2768,7 @@ def _old_concretize(self, tests=False, deprecation_warning=True): visited_user_specs = set() for dep in self.traverse(): visited_user_specs.add(dep.name) - pkg_cls = spack.repo.path.get_pkg_class(dep.name) + pkg_cls = spack.repo.PATH.get_pkg_class(dep.name) visited_user_specs.update(x.name for x in pkg_cls(dep).provided) extra = set(user_spec_deps.keys()).difference(visited_user_specs) @@ -2867,7 +2851,7 @@ def inject_patches_variant(root): # we can do it as late as possible to allow as much # compatibility across repositories as possible. if s.namespace is None: - s.namespace = spack.repo.path.repo_for_pkg(s.name).namespace + s.namespace = spack.repo.PATH.repo_for_pkg(s.name).namespace if s.concrete: continue @@ -2925,7 +2909,7 @@ def ensure_external_path_if_external(external_spec): # Get the path from the module the package can override the default # (this is mostly needed for Cray) - pkg_cls = spack.repo.path.get_pkg_class(external_spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(external_spec.name) package = pkg_cls(external_spec) external_spec.external_path = getattr( package, "external_prefix", md.path_from_modules(external_spec.external_modules) @@ -2942,9 +2926,9 @@ def ensure_no_deprecated(root): SpecDeprecatedError: if any deprecated spec is found """ deprecated = [] - with spack.store.db.read_transaction(): + with spack.store.STORE.db.read_transaction(): for x in root.traverse(): - _, rec = spack.store.db.query_by_spec_hash(x.dag_hash()) + _, rec = spack.store.STORE.db.query_by_spec_hash(x.dag_hash()) if rec and rec.deprecated_for: deprecated.append(rec) if deprecated: @@ -2970,8 +2954,9 @@ def _new_concretize(self, tests=False): if self._concrete: return + allow_deprecated = spack.config.get("config:deprecated", False) solver = spack.solver.asp.Solver() - result = solver.solve([self], tests=tests) + result = solver.solve([self], tests=tests, allow_deprecated=allow_deprecated) result.raise_if_unsat() # take the best answer @@ -2982,9 +2967,12 @@ def _new_concretize(self, tests=False): providers = [spec.name for spec in answer.values() if spec.package.provides(name)] name = providers[0] - assert name in answer + node = spack.solver.asp.SpecBuilder.make_node(pkg=name) + assert ( + node in answer + ), f"cannot find {name} in the list of specs {','.join([n.pkg for n in answer.keys()])}" - concretized = answer[name] + concretized = answer[node] self._dup(concretized) def concretize(self, tests=False): @@ -3178,7 +3166,7 @@ def _evaluate_dependency_conditions(self, name): for when_spec, dependency in conditions.items(): if self.satisfies(when_spec): if dep is None: - dep = dp.Dependency(self.name, Spec(name), type=()) + dep = dp.Dependency(self.name, Spec(name), depflag=0) try: dep.merge(dependency) except spack.error.UnsatisfiableSpecError as e: @@ -3199,7 +3187,7 @@ def _find_provider(self, vdep, provider_index): Raise an exception if there is a conflicting virtual dependency already in this spec. """ - assert spack.repo.path.is_virtual_safe(vdep.name), vdep + assert spack.repo.PATH.is_virtual_safe(vdep.name), vdep # note that this defensively copies. providers = provider_index.providers_for(vdep) @@ -3265,7 +3253,7 @@ def _merge_dependency(self, dependency, visited, spec_deps, provider_index, test # If it's a virtual dependency, try to find an existing # provider in the spec, and merge that. virtuals = () - if spack.repo.path.is_virtual_safe(dep.name): + if spack.repo.PATH.is_virtual_safe(dep.name): virtuals = (dep.name,) visited.add(dep.name) provider = self._find_provider(dep, provider_index) @@ -3273,11 +3261,11 @@ def _merge_dependency(self, dependency, visited, spec_deps, provider_index, test dep = provider else: index = spack.provider_index.ProviderIndex( - repository=spack.repo.path, specs=[dep], restrict=True + repository=spack.repo.PATH, specs=[dep], restrict=True ) items = list(spec_deps.items()) for name, vspec in items: - if not spack.repo.path.is_virtual_safe(vspec.name): + if not spack.repo.PATH.is_virtual_safe(vspec.name): continue if index.providers_for(vspec): @@ -3322,7 +3310,7 @@ def _merge_dependency(self, dependency, visited, spec_deps, provider_index, test # Add merged spec to my deps and recurse spec_dependency = spec_deps[dep.name] if dep.name not in self._dependencies: - self._add_dependency(spec_dependency, deptypes=dependency.type, virtuals=virtuals) + self._add_dependency(spec_dependency, depflag=dependency.depflag, virtuals=virtuals) changed |= spec_dependency._normalize_helper(visited, spec_deps, provider_index, tests) return changed @@ -3363,7 +3351,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index, tests): or (tests and self.name in tests) or # this is not a test-only dependency - dep.type - set(["test"]) + (dep.depflag & ~dt.TEST) ) if merge: @@ -3427,7 +3415,7 @@ def normalize(self, force=False, tests=False, user_spec_deps=None): # Initialize index of virtual dependency providers if # concretize didn't pass us one already provider_index = spack.provider_index.ProviderIndex( - repository=spack.repo.path, specs=[s for s in all_spec_deps.values()], restrict=True + repository=spack.repo.PATH, specs=[s for s in all_spec_deps.values()], restrict=True ) # traverse the package DAG and fill out dependencies according @@ -3458,7 +3446,7 @@ def validate_or_raise(self): for spec in self.traverse(): # raise an UnknownPackageError if the spec's package isn't real. if (not spec.virtual) and spec.name: - spack.repo.path.get_pkg_class(spec.fullname) + spack.repo.PATH.get_pkg_class(spec.fullname) # validate compiler in addition to the package name. if spec.compiler: @@ -3518,7 +3506,8 @@ def update_variant_validate(self, variant_name, values): for value in values: if self.variants.get(variant_name): msg = ( - "Cannot append a value to a single-valued " "variant with an already set value" + f"cannot append the new value '{value}' to the single-valued " + f"variant '{self.variants[variant_name]}'" ) assert pkg_variant.multi, msg self.variants[variant_name].append(value) @@ -3526,7 +3515,7 @@ def update_variant_validate(self, variant_name, values): variant = pkg_variant.make_variant(value) self.variants[variant_name] = variant - pkg_cls = spack.repo.path.get_pkg_class(self.name) + pkg_cls = spack.repo.PATH.get_pkg_class(self.name) pkg_variant.validate_or_raise(self.variants[variant_name], pkg_cls) def constrain(self, other, deps=True): @@ -3656,9 +3645,7 @@ def _constrain_dependencies(self, other): # WARNING: using index 0 i.e. we assume that we have only # WARNING: one edge from package "name" edges_from_name = self._dependencies[name] - changed |= edges_from_name[0].update_deptypes( - other._dependencies[name][0].deptypes - ) + changed |= edges_from_name[0].update_deptypes(other._dependencies[name][0].depflag) changed |= edges_from_name[0].update_virtuals( other._dependencies[name][0].virtuals ) @@ -3670,7 +3657,7 @@ def _constrain_dependencies(self, other): dep_spec_copy = other._get_dependency(name) self._add_dependency( dep_spec_copy.spec.copy(), - deptypes=dep_spec_copy.deptypes, + depflag=dep_spec_copy.depflag, virtuals=dep_spec_copy.virtuals, ) changed = True @@ -3705,7 +3692,7 @@ def _autospec(self, spec_like): return spec_like return Spec(spec_like) - def intersects(self, other: "Spec", deps: bool = True) -> bool: + def intersects(self, other: Union[str, "Spec"], deps: bool = True) -> bool: """Return True if there exists at least one concrete spec that matches both self and other, otherwise False. @@ -3718,21 +3705,32 @@ def intersects(self, other: "Spec", deps: bool = True) -> bool: """ other = self._autospec(other) - lhs = self.lookup_hash() or self - rhs = other.lookup_hash() or other - - return lhs._intersects(rhs, deps) - - def _intersects(self, other: "Spec", deps: bool = True) -> bool: if other.concrete and self.concrete: return self.dag_hash() == other.dag_hash() + elif self.concrete: + return self.satisfies(other) + + elif other.concrete: + return other.satisfies(self) + + # From here we know both self and other are not concrete + self_hash = self.abstract_hash + other_hash = other.abstract_hash + + if ( + self_hash + and other_hash + and not (self_hash.startswith(other_hash) or other_hash.startswith(self_hash)) + ): + return False + # If the names are different, we need to consider virtuals if self.name != other.name and self.name and other.name: if self.virtual and other.virtual: # Two virtual specs intersect only if there are providers for both - lhs = spack.repo.path.providers_for(str(self)) - rhs = spack.repo.path.providers_for(str(other)) + lhs = spack.repo.PATH.providers_for(str(self)) + rhs = spack.repo.PATH.providers_for(str(other)) intersection = [s for s in lhs if any(s.intersects(z) for z in rhs)] return bool(intersection) @@ -3741,7 +3739,7 @@ def _intersects(self, other: "Spec", deps: bool = True) -> bool: virtual_spec, non_virtual_spec = (self, other) if self.virtual else (other, self) try: # Here we might get an abstract spec - pkg_cls = spack.repo.path.get_pkg_class(non_virtual_spec.fullname) + pkg_cls = spack.repo.PATH.get_pkg_class(non_virtual_spec.fullname) pkg = pkg_cls(non_virtual_spec) except spack.repo.UnknownEntityError: # If we can't get package info on this spec, don't treat @@ -3786,19 +3784,8 @@ def _intersects(self, other: "Spec", deps: bool = True) -> bool: # If we need to descend into dependencies, do it, otherwise we're done. if deps: return self._intersects_dependencies(other) - else: - return True - def satisfies(self, other, deps=True): - """ - This checks constraints on common dependencies against each other. - """ - other = self._autospec(other) - - lhs = self.lookup_hash() or self - rhs = other.lookup_hash() or other - - return lhs._satisfies(rhs, deps=deps) + return True def _intersects_dependencies(self, other): if not other._dependencies or not self._dependencies: @@ -3812,16 +3799,12 @@ def _intersects_dependencies(self, other): # For virtual dependencies, we need to dig a little deeper. self_index = spack.provider_index.ProviderIndex( - repository=spack.repo.path, specs=self.traverse(), restrict=True + repository=spack.repo.PATH, specs=self.traverse(), restrict=True ) other_index = spack.provider_index.ProviderIndex( - repository=spack.repo.path, specs=other.traverse(), restrict=True + repository=spack.repo.PATH, specs=other.traverse(), restrict=True ) - # This handles cases where there are already providers for both vpkgs - if not self_index.satisfies(other_index): - return False - # These two loops handle cases where there is an overly restrictive # vpkg in one spec for a provider in the other (e.g., mpi@3: is not # compatible with mpich2) @@ -3835,7 +3818,7 @@ def _intersects_dependencies(self, other): return True - def _satisfies(self, other: "Spec", deps: bool = True) -> bool: + def satisfies(self, other: Union[str, "Spec"], deps: bool = True) -> bool: """Return True if all concrete specs matching self also match other, otherwise False. Args: @@ -3850,13 +3833,20 @@ def _satisfies(self, other: "Spec", deps: bool = True) -> bool: # objects. return self.concrete and self.dag_hash() == other.dag_hash() + # If the right-hand side has an abstract hash, make sure it's a prefix of the + # left-hand side's (abstract) hash. + if other.abstract_hash: + compare_hash = self.dag_hash() if self.concrete else self.abstract_hash + if not compare_hash or not compare_hash.startswith(other.abstract_hash): + return False + # If the names are different, we need to consider virtuals if self.name != other.name and self.name and other.name: # A concrete provider can satisfy a virtual dependency. if not self.virtual and other.virtual: try: # Here we might get an abstract spec - pkg_cls = spack.repo.path.get_pkg_class(self.fullname) + pkg_cls = spack.repo.PATH.get_pkg_class(self.fullname) pkg = pkg_cls(self) except spack.repo.UnknownEntityError: # If we can't get package info on this spec, don't treat @@ -3912,7 +3902,46 @@ def _satisfies(self, other: "Spec", deps: bool = True) -> bool: return False # If we arrived here, then rhs is abstract. At the moment we don't care about the edge - # structure of an abstract DAG - hence the deps=False parameter. + # structure of an abstract DAG, so we check if any edge could satisfy the properties + # we ask for. + lhs_edges: Dict[str, Set[DependencySpec]] = collections.defaultdict(set) + for rhs_edge in other.traverse_edges(root=False, cover="edges"): + # If we are checking for ^mpi we need to verify if there is any edge + if rhs_edge.spec.virtual: + rhs_edge.update_virtuals(virtuals=(rhs_edge.spec.name,)) + + if not rhs_edge.virtuals: + continue + + if not lhs_edges: + # Construct a map of the link/run subDAG + direct "build" edges, + # keyed by dependency name + for lhs_edge in self.traverse_edges( + root=False, cover="edges", deptype=("link", "run") + ): + lhs_edges[lhs_edge.spec.name].add(lhs_edge) + for virtual_name in lhs_edge.virtuals: + lhs_edges[virtual_name].add(lhs_edge) + + build_edges = self.edges_to_dependencies(depflag=dt.BUILD) + for lhs_edge in build_edges: + lhs_edges[lhs_edge.spec.name].add(lhs_edge) + for virtual_name in lhs_edge.virtuals: + lhs_edges[virtual_name].add(lhs_edge) + + # We don't have edges to this dependency + current_dependency_name = rhs_edge.spec.name + if current_dependency_name not in lhs_edges: + return False + + for virtual in rhs_edge.virtuals: + has_virtual = any( + virtual in edge.virtuals for edge in lhs_edges[current_dependency_name] + ) + if not has_virtual: + return False + + # Edges have been checked above already, hence deps=False return all( any(lhs.satisfies(rhs, deps=False) for lhs in self.traverse(root=False)) for rhs in other.traverse(root=False) @@ -3938,14 +3967,22 @@ def patches(self): # translate patch sha256sums to patch objects by consulting the index if self._patches_assigned(): for sha256 in self.variants["patches"]._patches_in_order_of_appearance: - index = spack.repo.path.patch_index - pkg_cls = spack.repo.path.get_pkg_class(self.name) - patch = index.patch_for_package(sha256, pkg_cls) + index = spack.repo.PATH.patch_index + pkg_cls = spack.repo.PATH.get_pkg_class(self.name) + try: + patch = index.patch_for_package(sha256, pkg_cls) + except spack.patch.PatchLookupError as e: + raise spack.error.SpecError( + f"{e}. This usually means the patch was modified or removed. " + "To fix this, either reconcretize or use the original package " + "repository" + ) from e + self._patches.append(patch) return self._patches - def _dup(self, other, deps=True, cleardeps=True): + def _dup(self, other, deps: Union[bool, dt.DepTypes, dt.DepFlag] = True, cleardeps=True): """Copy the spec other into self. This is an overwriting copy. It does not copy any dependents (parents), but by default copies dependencies. @@ -3954,9 +3991,8 @@ def _dup(self, other, deps=True, cleardeps=True): Args: other (Spec): spec to be copied onto ``self`` - deps (bool or Sequence): if True copies all the dependencies. If - False copies None. If a sequence of dependency types copy - only those types. + deps: if True copies all the dependencies. If + False copies None. If deptype/depflag, copy matching types. cleardeps (bool): if True clears the dependencies of ``self``, before possibly copying the dependencies of ``other`` onto ``self`` @@ -4016,10 +4052,10 @@ def _dup(self, other, deps=True, cleardeps=True): if deps: # If caller restricted deptypes to be copied, adjust that here. # By default, just copy all deptypes - deptypes = dp.all_deptypes - if isinstance(deps, (tuple, list)): - deptypes = deps - self._dup_deps(other, deptypes) + depflag = dt.ALL + if isinstance(deps, (tuple, list, str)): + depflag = dt.canonicalize(deps) + self._dup_deps(other, depflag) self._concrete = other._concrete @@ -4040,13 +4076,13 @@ def _dup(self, other, deps=True, cleardeps=True): return changed - def _dup_deps(self, other, deptypes): + def _dup_deps(self, other, depflag: dt.DepFlag): def spid(spec): return id(spec) new_specs = {spid(other): self} for edge in other.traverse_edges(cover="edges", root=False): - if edge.deptypes and not any(d in deptypes for d in edge.deptypes): + if edge.depflag and not depflag & edge.depflag: continue if spid(edge.parent) not in new_specs: @@ -4056,17 +4092,16 @@ def spid(spec): new_specs[spid(edge.spec)] = edge.spec.copy(deps=False) new_specs[spid(edge.parent)].add_dependency_edge( - new_specs[spid(edge.spec)], deptypes=edge.deptypes, virtuals=edge.virtuals + new_specs[spid(edge.spec)], depflag=edge.depflag, virtuals=edge.virtuals ) - def copy(self, deps=True, **kwargs): + def copy(self, deps: Union[bool, dt.DepTypes, dt.DepFlag] = True, **kwargs): """Make a copy of this spec. Args: - deps (bool or tuple): Defaults to True. If boolean, controls + deps: Defaults to True. If boolean, controls whether dependencies are copied (copied if True). If a - tuple is provided, *only* dependencies of types matching - those in the tuple are copied. + DepTypes or DepFlag is provided, *only* matching dependencies are copied. kwargs: additional arguments for internal use (passed to ``_dup``). Returns: @@ -4108,9 +4143,7 @@ def __getitem__(self, name): """ query_parameters = name.split(":") if len(query_parameters) > 2: - msg = "key has more than one ':' symbol." - msg += " At most one is admitted." - raise KeyError(msg) + raise KeyError("key has more than one ':' symbol. At most one is admitted.") name, query_parameters = query_parameters[0], query_parameters[1:] if query_parameters: @@ -4126,7 +4159,7 @@ def __getitem__(self, name): # only when we don't find the package do we consider the full DAG. order = lambda: itertools.chain( self.traverse(deptype="link"), - self.dependencies(deptype=("build", "run", "test")), + self.dependencies(deptype=dt.BUILD | dt.RUN | dt.TEST), self.traverse(), # fall back to a full search ) @@ -4135,11 +4168,17 @@ def __getitem__(self, name): itertools.chain( # Regular specs (x for x in order() if x.name == name), + ( + x + for x in order() + if (not x.virtual) + and any(name in edge.virtuals for edge in x.edges_from_dependents()) + ), (x for x in order() if (not x.virtual) and x.package.provides(name)), ) ) except StopIteration: - raise KeyError("No spec with name %s in %s" % (name, self)) + raise KeyError(f"No spec with name {name} in {self}") if self._concrete: return SpecBuildInterface(value, name, query_parameters) @@ -4184,7 +4223,7 @@ def eq_dag(self, other, deptypes=True, vs=None, vo=None): for s_dspec, o_dspec in zip( itertools.chain.from_iterable(ssorted), itertools.chain.from_iterable(osorted) ): - if deptypes and s_dspec.deptypes != o_dspec.deptypes: + if deptypes and s_dspec.depflag != o_dspec.depflag: return False s, o = s_dspec.spec, o_dspec.spec @@ -4226,9 +4265,7 @@ def eq_node(self, other): def _cmp_iter(self): """Lazily yield components of self for comparison.""" - cmp_spec = self.lookup_hash() or self - - for item in cmp_spec._cmp_node(): + for item in self._cmp_node(): yield item # This needs to be in _cmp_iter so that no specs with different process hashes @@ -4239,12 +4276,12 @@ def _cmp_iter(self): # TODO: they exist for speed. We should benchmark whether it's really worth # TODO: having two types of hashing now that we use `json` instead of `yaml` for # TODO: spec hashing. - yield cmp_spec.process_hash() if cmp_spec.concrete else None + yield self.process_hash() if self.concrete else None def deps(): - for dep in sorted(itertools.chain.from_iterable(cmp_spec._dependencies.values())): + for dep in sorted(itertools.chain.from_iterable(self._dependencies.values())): yield dep.spec.name - yield tuple(sorted(dep.deptypes)) + yield dep.depflag yield hash(dep.spec) yield deps @@ -4252,7 +4289,7 @@ def deps(): def colorized(self): return colorize_spec(self) - def format(self, format_string=default_format, **kwargs): + def format(self, format_string=DEFAULT_FORMAT, **kwargs): r"""Prints out particular pieces of a spec, depending on what is in the format string. @@ -4331,7 +4368,7 @@ def format(self, format_string=default_format, **kwargs): def write(s, c=None): f = clr.cescape(s) if c is not None: - f = color_formats[c] + f + "@." + f = COLOR_FORMATS[c] + f + "@." clr.cwrite(f, stream=out, color=color) def write_attribute(spec, attribute, color): @@ -4377,7 +4414,7 @@ def write_attribute(spec, attribute, color): write(morph(spec, spack.paths.spack_root)) return elif attribute == "spack_install": - write(morph(spec, spack.store.layout.root)) + write(morph(spec, spack.store.STORE.layout.root)) return elif re.match(r"hash(:\d)?", attribute): col = "#" @@ -4482,11 +4519,63 @@ def cformat(self, *args, **kwargs): kwargs.setdefault("color", None) return self.format(*args, **kwargs) + def format_path( + # self, format_string: str, _path_ctor: Optional[pathlib.PurePath] = None + self, + format_string: str, + _path_ctor: Optional[Callable[[Any], pathlib.PurePath]] = None, + ) -> str: + """Given a `format_string` that is intended as a path, generate a string + like from `Spec.format`, but eliminate extra path separators introduced by + formatting of Spec properties. + + Path separators explicitly added to the string are preserved, so for example + "{name}/{version}" would generate a directory based on the Spec's name, and + a subdirectory based on its version; this function guarantees though that + the resulting string would only have two directories (i.e. that if under + normal circumstances that `str(Spec.version)` would contain a path + separator, it would not in this case). + """ + format_component_with_sep = r"\{[^}]*[/\\][^}]*}" + if re.search(format_component_with_sep, format_string): + raise SpecFormatPathError( + f"Invalid path format string: cannot contain {{/...}}\n\t{format_string}" + ) + + path_ctor = _path_ctor or pathlib.PurePath + format_string_as_path = path_ctor(format_string) + if format_string_as_path.is_absolute(): + output_path_components = [format_string_as_path.parts[0]] + input_path_components = list(format_string_as_path.parts[1:]) + else: + output_path_components = [] + input_path_components = list(format_string_as_path.parts) + output_path_components += [ + fs.polite_filename(self.format(x)) for x in input_path_components + ] + return str(path_ctor(*output_path_components)) + def __str__(self): - sorted_nodes = [self] + sorted( - self.traverse(root=False), key=lambda x: x.name or x.abstract_hash + root_str = [self.format()] + sorted_dependencies = sorted( + self.traverse(root=False), key=lambda x: (x.name, x.abstract_hash) ) - spec_str = " ^".join(d.format() for d in sorted_nodes) + sorted_dependencies = [ + d.format("{edge_attributes} " + DEFAULT_FORMAT) for d in sorted_dependencies + ] + spec_str = " ^".join(root_str + sorted_dependencies) + return spec_str.strip() + + @property + def colored_str(self): + root_str = [self.cformat()] + sorted_dependencies = sorted( + self.traverse(root=False), key=lambda x: (x.name, x.abstract_hash) + ) + sorted_dependencies = [ + d.cformat("{edge_attributes} " + DISPLAY_FORMAT) for d in sorted_dependencies + ] + spec_str = " ^".join(root_str + sorted_dependencies) return spec_str.strip() def install_status(self): @@ -4497,7 +4586,7 @@ def install_status(self): if self.external: return InstallStatus.external - upstream, record = spack.store.db.query_by_spec_hash(self.dag_hash()) + upstream, record = spack.store.STORE.db.query_by_spec_hash(self.dag_hash()) if not record: return InstallStatus.absent elif upstream and record.installed: @@ -4512,34 +4601,56 @@ def _installed_explicitly(self): if not self.concrete: return None try: - record = spack.store.db.get_record(self) + record = spack.store.STORE.db.get_record(self) return record.explicit except KeyError: return None - def tree(self, **kwargs): + def tree( + self, + *, + color: Optional[bool] = None, + depth: bool = False, + hashes: bool = False, + hashlen: Optional[int] = None, + cover: str = "nodes", + indent: int = 0, + format: str = DEFAULT_FORMAT, + deptypes: Union[Tuple[str, ...], str] = "all", + show_types: bool = False, + depth_first: bool = False, + recurse_dependencies: bool = True, + status_fn: Optional[Callable[["Spec"], InstallStatus]] = None, + prefix: Optional[Callable[["Spec"], str]] = None, + ) -> str: """Prints out this spec and its dependencies, tree-formatted with indentation. Status function may either output a boolean or an InstallStatus - """ - color = kwargs.pop("color", clr.get_color_when()) - depth = kwargs.pop("depth", False) - hashes = kwargs.pop("hashes", False) - hlen = kwargs.pop("hashlen", None) - status_fn = kwargs.pop("status_fn", False) - cover = kwargs.pop("cover", "nodes") - indent = kwargs.pop("indent", 0) - fmt = kwargs.pop("format", default_format) - prefix = kwargs.pop("prefix", None) - show_types = kwargs.pop("show_types", False) - deptypes = kwargs.pop("deptypes", "all") - recurse_dependencies = kwargs.pop("recurse_dependencies", True) - depth_first = kwargs.pop("depth_first", False) - lang.check_kwargs(kwargs, self.tree) + Args: + color: if True, always colorize the tree. If False, don't colorize the tree. If None, + use the default from llnl.tty.color + depth: print the depth from the root + hashes: if True, print the hash of each node + hashlen: length of the hash to be printed + cover: either "nodes" or "edges" + indent: extra indentation for the tree being printed + format: format to be used to print each node + deptypes: dependency types to be represented in the tree + show_types: if True, show the (merged) dependency type of a node + depth_first: if True, traverse the DAG depth first when representing it as a tree + recurse_dependencies: if True, recurse on dependencies + status_fn: optional callable that takes a node as an argument and return its + installation status + prefix: optional callable that takes a node as an argument and return its + installation prefix + """ out = "" + if color is None: + color = clr.get_color_when() + for d, dep_spec in traverse.traverse_tree( [self], cover=cover, deptype=deptypes, depth_first=depth_first ): @@ -4562,25 +4673,27 @@ def tree(self, **kwargs): out += clr.colorize("@r{[-]} ", color=color) if hashes: - out += clr.colorize("@K{%s} ", color=color) % node.dag_hash(hlen) + out += clr.colorize("@K{%s} ", color=color) % node.dag_hash(hashlen) if show_types: if cover == "nodes": # when only covering nodes, we merge dependency types # from all dependents before showing them. - types = [ds.deptypes for ds in node.edges_from_dependents()] + depflag = 0 + for ds in node.edges_from_dependents(): + depflag |= ds.depflag else: # when covering edges or paths, we show dependency # types only for the edge through which we visited - types = [dep_spec.deptypes] + depflag = dep_spec.depflag - type_chars = dp.deptype_chars(*types) + type_chars = dt.flag_to_chars(depflag) out += "[%s] " % type_chars out += " " * d if d > 0: out += "^" - out += node.format(fmt, color=color) + "\n" + out += node.format(format, color=color) + "\n" # Check if we wanted just the first line if not recurse_dependencies: @@ -4736,14 +4849,14 @@ def from_self(name, transitive): for edge in self[name].edges_to_dependencies(): dep_name = deps_to_replace.get(edge.spec, edge.spec).name nodes[name].add_dependency_edge( - nodes[dep_name], deptypes=edge.deptypes, virtuals=edge.virtuals + nodes[dep_name], depflag=edge.depflag, virtuals=edge.virtuals ) if any(dep not in self_nodes for dep in self[name]._dependencies): nodes[name].build_spec = self[name].build_spec else: for edge in other[name].edges_to_dependencies(): nodes[name].add_dependency_edge( - nodes[edge.spec.name], deptypes=edge.deptypes, virtuals=edge.virtuals + nodes[edge.spec.name], depflag=edge.depflag, virtuals=edge.virtuals ) if any(dep not in other_nodes for dep in other[name]._dependencies): nodes[name].build_spec = other[name].build_spec @@ -4801,7 +4914,7 @@ def attach_git_version_lookup(self): return for v in self.versions: if isinstance(v, vn.GitVersion) and v._ref_version is None: - v.attach_git_lookup_from_package(self.fullname) + v.attach_lookup(spack.version.git_ref_lookup.GitRefLookup(self.fullname)) def parse_with_version_concrete(string: str, compiler: bool = False): @@ -4834,8 +4947,9 @@ def merge_abstract_anonymous_specs(*abstract_specs: Spec): # Update with additional constraints from other spec for name in current_spec_constraint.direct_dep_difference(merged_spec): edge = next(iter(current_spec_constraint.edges_to_dependencies(name))) + merged_spec._add_dependency( - edge.spec.copy(), deptypes=edge.deptypes, virtuals=edge.virtuals + edge.spec.copy(), depflag=edge.depflag, virtuals=edge.virtuals ) return merged_spec @@ -4982,9 +5096,11 @@ def _load(cls, data): # Pass 2: Finish construction of all DAG edges (including build specs) for node_hash, node in hash_dict.items(): node_spec = node["node_spec"] - for _, dhash, dtypes, _, virtuals in cls.dependencies_from_node_dict(node): + for _, dhash, dtype, _, virtuals in cls.dependencies_from_node_dict(node): node_spec._add_dependency( - hash_dict[dhash]["node_spec"], deptypes=dtypes, virtuals=virtuals + hash_dict[dhash]["node_spec"], + depflag=dt.canonicalize(dtype), + virtuals=virtuals, ) if "build_spec" in node.keys(): _, bhash, _ = cls.build_spec_from_node_dict(node, hash_type=hash_type) @@ -5020,7 +5136,9 @@ def load(cls, data): # get dependency dict from the node. name, data = cls.name_and_data(node) for dname, _, dtypes, _, virtuals in cls.dependencies_from_node_dict(data): - deps[name]._add_dependency(deps[dname], deptypes=dtypes, virtuals=virtuals) + deps[name]._add_dependency( + deps[dname], depflag=dt.canonicalize(dtypes), virtuals=virtuals + ) reconstruct_virtuals_on_edges(result) return result @@ -5146,9 +5264,7 @@ def __missing__(self, key): return value -def save_dependency_specfiles( - root_spec_info, output_directory, dependencies=None, spec_format="json" -): +def save_dependency_specfiles(root: Spec, output_directory: str, dependencies: List[Spec]): """Given a root spec (represented as a yaml object), index it with a subset of its dependencies, and write each dependency to a separate yaml file in the output directory. By default, all dependencies will be written @@ -5157,26 +5273,52 @@ def save_dependency_specfiles( incoming spec is not json, that can be specified with the spec_format parameter. This can be used to convert from yaml specfiles to the json format.""" - if spec_format == "json": - root_spec = Spec.from_json(root_spec_info) - elif spec_format == "yaml": - root_spec = Spec.from_yaml(root_spec_info) - else: - raise SpecParseError("Unrecognized spec format {0}.".format(spec_format)) - dep_list = dependencies - if not dep_list: - dep_list = [dep.name for dep in root_spec.traverse()] + for spec in root.traverse(): + if not any(spec.satisfies(dep) for dep in dependencies): + continue - for dep_name in dep_list: - if dep_name not in root_spec: - msg = "Dependency {0} does not exist in root spec {1}".format(dep_name, root_spec.name) - raise SpecDependencyNotFoundError(msg) - dep_spec = root_spec[dep_name] - json_path = os.path.join(output_directory, "{0}.json".format(dep_name)) + json_path = os.path.join(output_directory, f"{spec.name}.json") with open(json_path, "w") as fd: - fd.write(dep_spec.to_json(hash=ht.dag_hash)) + fd.write(spec.to_json(hash=ht.dag_hash)) + + +def get_host_environment_metadata() -> Dict[str, str]: + """Get the host environment, reduce to a subset that we can store in + the install directory, and add the spack version. + """ + import spack.main + + environ = get_host_environment() + return { + "host_os": environ["os"], + "platform": environ["platform"], + "host_target": environ["target"], + "hostname": environ["hostname"], + "spack_version": spack.main.get_version(), + "kernel_version": platform.version(), + } + + +def get_host_environment() -> Dict[str, Any]: + """Return a dictionary (lookup) with host information (not including the + os.environ). + """ + host_platform = spack.platforms.host() + host_target = host_platform.target("default_target") + host_os = host_platform.operating_system("default_os") + arch_fmt = "platform={0} os={1} target={2}" + arch_spec = Spec(arch_fmt.format(host_platform, host_os, host_target)) + return { + "target": str(host_target), + "os": str(host_os), + "platform": str(host_platform), + "arch": arch_spec, + "architecture": arch_spec, + "arch_str": str(arch_spec), + "hostname": socket.gethostname(), + } class SpecParseError(spack.error.SpecError): @@ -5239,7 +5381,7 @@ class InvalidDependencyError(spack.error.SpecError): def __init__(self, pkg, deps): self.invalid_deps = deps super().__init__( - "Package {0} does not depend on {1}".format(pkg, spack.util.string.comma_or(deps)) + "Package {0} does not depend on {1}".format(pkg, llnl.string.comma_or(deps)) ) @@ -5351,18 +5493,14 @@ class NoSuchSpecFileError(SpecFilenameError): """Raised when a spec file doesn't exist.""" -class RedundantSpecError(spack.error.SpecError): - def __init__(self, spec, addition): - super().__init__( - "Attempting to add %s to spec %s which is already concrete." - " This is likely the result of adding to a spec specified by hash." % (addition, spec) - ) - - class SpecFormatStringError(spack.error.SpecError): """Called for errors in Spec format strings.""" +class SpecFormatPathError(spack.error.SpecError): + """Called for errors in Spec path-format strings.""" + + class SpecFormatSigilError(SpecFormatStringError): """Called for mismatched sigils and attributes in format strings""" @@ -5397,11 +5535,6 @@ def __init__(self, spec, matches): super().__init__(message, long_message) -class SpecDependencyNotFoundError(spack.error.SpecError): - """Raised when a failure is encountered writing the dependencies of - a spec.""" - - class SpecDeprecatedError(spack.error.SpecError): """Raised when a spec concretizes to a deprecated spec or dependency.""" diff --git a/lib/spack/spack/spec_list.py b/lib/spack/spack/spec_list.py index 23e3a7f056c252..6bb1ba8d047e9a 100644 --- a/lib/spack/spack/spec_list.py +++ b/lib/spack/spack/spec_list.py @@ -93,12 +93,14 @@ def remove(self, spec): if (isinstance(s, str) and not s.startswith("$")) and Spec(s) == Spec(spec) ] if not remove: - msg = "Cannot remove %s from SpecList %s\n" % (spec, self.name) - msg += "Either %s is not in %s or %s is " % (spec, self.name, spec) + msg = f"Cannot remove {spec} from SpecList {self.name}.\n" + msg += f"Either {spec} is not in {self.name} or {spec} is " msg += "expanded from a matrix and cannot be removed directly." raise SpecListError(msg) - assert len(remove) == 1 - self.yaml_list.remove(remove[0]) + + # Remove may contain more than one string representation of the same spec + for item in remove: + self.yaml_list.remove(item) # invalidate cache variables when we change the list self._expanded_list = None @@ -131,9 +133,8 @@ def _parse_reference(self, name): # Make sure the reference is valid if name not in self._reference: - msg = "SpecList %s refers to " % self.name - msg += "named list %s " % name - msg += "which does not appear in its reference dict" + msg = f"SpecList '{self.name}' refers to named list '{name}'" + msg += " which does not appear in its reference dict." raise UndefinedReferenceError(msg) return (name, sigil) @@ -197,7 +198,9 @@ def _expand_matrix_constraints(matrix_config): for combo in itertools.product(*expanded_rows): # Construct a combined spec to test against excludes flat_combo = [constraint for constraint_list in combo for constraint in constraint_list] - flat_combo = [Spec(x) for x in flat_combo] + + # Resolve abstract hashes so we can exclude by their concrete properties + flat_combo = [Spec(x).lookup_hash() for x in flat_combo] test_spec = flat_combo[0].copy() for constraint in flat_combo[1:]: diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 09ea1f88a457cc..7418b5a44ee694 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -2,18 +2,20 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - +import concurrent.futures import errno import getpass import glob import hashlib +import io import os import shutil import stat import sys import tempfile -from typing import Dict +from typing import Callable, Dict, Iterable, Optional, Set +import llnl.string import llnl.util.lang import llnl.util.tty as tty from llnl.util.filesystem import ( @@ -26,6 +28,8 @@ partition_path, remove_linked_tree, ) +from llnl.util.tty.colify import colify +from llnl.util.tty.color import colorize import spack.caches import spack.config @@ -33,13 +37,16 @@ import spack.fetch_strategy as fs import spack.mirror import spack.paths +import spack.resource import spack.spec +import spack.stage import spack.util.lock import spack.util.path as sup import spack.util.pattern as pattern import spack.util.url as url_util from spack.util.crypto import bit_length, prefix_bits -from spack.util.web import FetchError +from spack.util.editor import editor, executable +from spack.version import StandardVersion, VersionList # The well-known stage source subdirectory name. _source_path_subdir = "spack-src" @@ -50,9 +57,9 @@ def compute_stage_name(spec): """Determine stage name given a spec""" - default_stage_structure = "spack-stage-{name}-{version}-{hash}" + default_stage_structure = stage_prefix + "{name}-{version}-{hash}" stage_name_structure = spack.config.get("config:stage_name", default=default_stage_structure) - return spec.format(format_string=stage_name_structure) + return spec.format_path(format_string=stage_name_structure) def create_stage_root(path: str) -> None: @@ -241,10 +248,7 @@ class Stage: similar, and are intended to persist for only one run of spack. """ - """Shared dict of all stage locks.""" - stage_locks: Dict[str, spack.util.lock.Lock] = {} - - """Most staging is managed by Spack. DIYStage is one exception.""" + #: Most staging is managed by Spack. DIYStage is one exception. managed_by_spack = True def __init__( @@ -326,21 +330,16 @@ def __init__( self.keep = keep # File lock for the stage directory. We use one file for all - # stage locks. See spack.database.Database.prefix_lock for + # stage locks. See spack.database.Database.prefix_locker.lock for # details on this approach. self._lock = None if lock: - if self.name not in Stage.stage_locks: - sha1 = hashlib.sha1(self.name.encode("utf-8")).digest() - lock_id = prefix_bits(sha1, bit_length(sys.maxsize)) - stage_lock_path = os.path.join(get_stage_root(), ".lock") - - tty.debug("Creating stage lock {0}".format(self.name)) - Stage.stage_locks[self.name] = spack.util.lock.Lock( - stage_lock_path, lock_id, 1, desc=self.name - ) - - self._lock = Stage.stage_locks[self.name] + sha1 = hashlib.sha1(self.name.encode("utf-8")).digest() + lock_id = prefix_bits(sha1, bit_length(sys.maxsize)) + stage_lock_path = os.path.join(get_stage_root(), ".lock") + self._lock = spack.util.lock.Lock( + stage_lock_path, start=lock_id, length=1, desc=self.name + ) # When stages are reused, we need to know whether to re-create # it. This marks whether it has been created/destroyed. @@ -387,8 +386,7 @@ def expected_archive_files(self): expanded = True if isinstance(self.default_fetcher, fs.URLFetchStrategy): expanded = self.default_fetcher.expand_archive - clean_url = os.path.basename(sup.sanitize_file_path(self.default_fetcher.url)) - fnames.append(clean_url) + fnames.append(url_util.default_download_filename(self.default_fetcher.url)) if self.mirror_paths: fnames.extend(os.path.basename(x) for x in self.mirror_paths) @@ -429,6 +427,12 @@ def source_path(self): """Returns the well-known source directory path.""" return os.path.join(self.path, _source_path_subdir) + def disable_mirrors(self): + """The Stage will not attempt to look for the associated fetcher + target in any of Spack's mirrors (including the local download cache). + """ + self.mirror_paths = [] + def fetch(self, mirror_only=False, err_msg=None): """Retrieves the code or archive @@ -449,22 +453,12 @@ def fetch(self, mirror_only=False, err_msg=None): # Join URLs of mirror roots with mirror paths. Because # urljoin() will strip everything past the final '/' in # the root, so we add a '/' if it is not present. - mirror_urls = {} - for mirror in spack.mirror.MirrorCollection().values(): - for rel_path in self.mirror_paths: - mirror_url = url_util.join(mirror.fetch_url, rel_path) - mirror_urls[mirror_url] = {} - if ( - mirror.get_access_pair("fetch") - or mirror.get_access_token("fetch") - or mirror.get_profile("fetch") - ): - mirror_urls[mirror_url] = { - "access_token": mirror.get_access_token("fetch"), - "access_pair": mirror.get_access_pair("fetch"), - "access_profile": mirror.get_profile("fetch"), - "endpoint_url": mirror.get_endpoint_url("fetch"), - } + mirror_urls = [ + url_util.join(mirror.fetch_url, rel_path) + for mirror in spack.mirror.MirrorCollection(source=True).values() + if not mirror.fetch_url.startswith("oci://") + for rel_path in self.mirror_paths + ] # If this archive is normally fetched from a tarball URL, # then use the same digest. `spack mirror` ensures that @@ -483,21 +477,14 @@ def fetch(self, mirror_only=False, err_msg=None): # Add URL strategies for all the mirrors with the digest # Insert fetchers in the order that the URLs are provided. - for url in reversed(list(mirror_urls.keys())): + for url in reversed(mirror_urls): fetchers.insert( - 0, - fs.from_url_scheme( - url, - digest, - expand=expand, - extension=extension, - connection=mirror_urls[url], - ), + 0, fs.from_url_scheme(url, digest, expand=expand, extension=extension) ) if self.default_fetcher.cachable: for rel_path in reversed(list(self.mirror_paths)): - cache_fetcher = spack.caches.fetch_cache.fetcher( + cache_fetcher = spack.caches.FETCH_CACHE.fetcher( rel_path, digest, expand=expand, extension=extension ) fetchers.insert(0, cache_fetcher) @@ -535,7 +522,7 @@ def print_errors(errors): self.fetcher = self.default_fetcher default_msg = "All fetchers failed for {0}".format(self.name) - raise FetchError(err_msg or default_msg, None) + raise spack.error.FetchError(err_msg or default_msg, None) print_errors(errors) @@ -590,7 +577,7 @@ def check(self): self.fetcher.check() def cache_local(self): - spack.caches.fetch_cache.store(self.fetcher, self.mirror_paths.storage_path) + spack.caches.FETCH_CACHE.store(self.fetcher, self.mirror_paths.storage_path) def cache_mirror(self, mirror, stats): """Perform a fetch if the resource is not already cached @@ -673,8 +660,14 @@ def destroy(self): class ResourceStage(Stage): - def __init__(self, url_or_fetch_strategy, root, resource, **kwargs): - super().__init__(url_or_fetch_strategy, **kwargs) + def __init__( + self, + fetch_strategy: fs.FetchStrategy, + root: Stage, + resource: spack.resource.Resource, + **kwargs, + ): + super().__init__(fetch_strategy, **kwargs) self.root_stage = root self.resource = resource @@ -756,10 +749,18 @@ def __init__(self): "cache_local", "cache_mirror", "steal_source", + "disable_mirrors", "managed_by_spack", ] ) + @classmethod + def from_iterable(cls, iterable: Iterable[Stage]) -> "StageComposite": + """Create a new composite from an iterable of stages.""" + composite = cls() + composite.extend(iterable) + return composite + def __enter__(self): for item in self: item.__enter__() @@ -767,7 +768,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): - item.keep = getattr(self, "keep", False) item.__exit__(exc_type, exc_val, exc_tb) # @@ -789,6 +789,15 @@ def path(self): def archive_file(self): return self[0].archive_file + @property + def keep(self): + return self[0].keep + + @keep.setter + def keep(self, value): + for item in self: + item.keep = value + class DIYStage: """ @@ -865,115 +874,295 @@ def purge(): os.remove(stage_path) -def get_checksums_for_versions(url_dict, name, **kwargs): - """Fetches and checksums archives from URLs. - - This function is called by both ``spack checksum`` and ``spack - create``. The ``first_stage_function`` argument allows the caller to - inspect the first downloaded archive, e.g., to determine the build - system. +def interactive_version_filter( + url_dict: Dict[StandardVersion, str], + known_versions: Iterable[StandardVersion] = (), + *, + initial_verion_filter: Optional[VersionList] = None, + url_changes: Set[StandardVersion] = set(), + input: Callable[..., str] = input, +) -> Optional[Dict[StandardVersion, str]]: + """Interactively filter the list of spidered versions. Args: - url_dict (dict): A dictionary of the form: version -> URL - name (str): The name of the package - first_stage_function (typing.Callable): function that takes a Stage and a URL; - this is run on the stage of the first URL downloaded - keep_stage (bool): whether to keep staging area when command completes - batch (bool): whether to ask user how many versions to fetch (false) - or fetch all versions (true) - latest (bool): whether to take the latest version (true) or all (false) - fetch_options (dict): Options used for the fetcher (such as timeout - or cookies) + url_dict: Dictionary of versions to URLs + known_versions: Versions that can be skipped because they are already known Returns: - (str): A multi-line string containing versions and corresponding hashes - + Filtered dictionary of versions to URLs or None if the user wants to quit """ - batch = kwargs.get("batch", False) - fetch_options = kwargs.get("fetch_options", None) - first_stage_function = kwargs.get("first_stage_function", None) - keep_stage = kwargs.get("keep_stage", False) - latest = kwargs.get("latest", False) - - sorted_versions = sorted(url_dict.keys(), reverse=True) - if latest: - sorted_versions = sorted_versions[:1] - # Find length of longest string in the list for padding - max_len = max(len(str(v)) for v in sorted_versions) - num_ver = len(sorted_versions) - - tty.msg( - "Found {0} version{1} of {2}:".format(num_ver, "" if num_ver == 1 else "s", name), - "", - *llnl.util.lang.elide_list( - ["{0:{1}} {2}".format(str(v), max_len, url_dict[v]) for v in sorted_versions] - ), - ) - print() - - if batch or latest: - archives_to_fetch = len(sorted_versions) - else: - archives_to_fetch = tty.get_number( - "How many would you like to checksum?", default=1, abort="q" + version_filter = initial_verion_filter or VersionList([":"]) + max_len = max(len(str(v)) for v in url_dict) if url_dict else 0 + sorted_and_filtered = [v for v in url_dict if v.satisfies(version_filter)] + sorted_and_filtered.sort(reverse=True) + orig_url_dict = url_dict # only copy when using editor to modify + print_header = True + VERSION_COLOR = spack.spec.VERSION_COLOR + while True: + if print_header: + has_filter = version_filter != VersionList([":"]) + header = [] + if len(orig_url_dict) > 0 and len(sorted_and_filtered) == len(orig_url_dict): + header.append( + f"Selected {llnl.string.plural(len(sorted_and_filtered), 'version')}" + ) + else: + header.append( + f"Selected {len(sorted_and_filtered)} of " + f"{llnl.string.plural(len(orig_url_dict), 'version')}" + ) + if sorted_and_filtered and known_versions: + num_new = sum(1 for v in sorted_and_filtered if v not in known_versions) + header.append(f"{llnl.string.plural(num_new, 'new version')}") + if has_filter: + header.append(colorize(f"Filtered by {VERSION_COLOR}@@{version_filter}@.")) + + version_with_url = [ + colorize( + f"{VERSION_COLOR}{str(v):{max_len}}@. {url_dict[v]}" + f"{' @K{# NOTE: change of URL}' if v in url_changes else ''}" + ) + for v in sorted_and_filtered + ] + tty.msg(". ".join(header), *llnl.util.lang.elide_list(version_with_url)) + print() + + print_header = True + + tty.info(colorize("Enter @*{number} of versions to take, or use a @*{command}:")) + commands = ( + "@*b{[c]}hecksum", + "@*b{[e]}dit", + "@*b{[f]}ilter", + "@*b{[a]}sk each", + "@*b{[n]}ew only", + "@*b{[r]}estart", + "@*b{[q]}uit", ) + colify(list(map(colorize, commands)), indent=4) - if not archives_to_fetch: - tty.die("Aborted.") - - versions = sorted_versions[:archives_to_fetch] - urls = [url_dict[v] for v in versions] - - tty.debug("Downloading...") - version_hashes = [] - i = 0 - errors = [] - for url, version in zip(urls, versions): - # Wheels should not be expanded during staging - expand_arg = "" - if url.endswith(".whl") or ".whl#" in url: - expand_arg = ", expand=False" try: - if fetch_options: - url_or_fs = fs.URLFetchStrategy(url, fetch_options=fetch_options) - else: - url_or_fs = url - with Stage(url_or_fs, keep=keep_stage) as stage: - # Fetch the archive - stage.fetch() - if i == 0 and first_stage_function: - # Only run first_stage_function the first time, - # no need to run it every time - first_stage_function(stage, url) - - # Checksum the archive and add it to the list - version_hashes.append( - (version, spack.util.crypto.checksum(hashlib.sha256, stage.archive_file)) + command = input(colorize("@*g{action>} ")).strip().lower() + except EOFError: + print() + command = "q" + + if command == "c": + break + elif command == "e": + # Create a temporary file in the stage dir with lines of the form + # + # which the user can modify. Once the editor is closed, the file is + # read back in and the versions to url dict is updated. + + # Create a temporary file by hashing its contents. + buffer = io.StringIO() + buffer.write("# Edit this file to change the versions and urls to fetch\n") + for v in sorted_and_filtered: + buffer.write(f"{str(v):{max_len}} {url_dict[v]}\n") + data = buffer.getvalue().encode("utf-8") + + short_hash = hashlib.sha1(data).hexdigest()[:7] + filename = f"{spack.stage.stage_prefix}versions-{short_hash}.txt" + filepath = os.path.join(spack.stage.get_stage_root(), filename) + + # Write contents + with open(filepath, "wb") as f: + f.write(data) + + # Open editor + editor(filepath, exec_fn=executable) + + # Read back in + with open(filepath, "r") as f: + orig_url_dict, url_dict = url_dict, {} + for line in f: + line = line.strip() + # Skip empty lines and comments + if not line or line.startswith("#"): + continue + try: + version, url = line.split(None, 1) + except ValueError: + tty.warn(f"Couldn't parse: {line}") + continue + try: + url_dict[StandardVersion.from_string(version)] = url + except ValueError: + tty.warn(f"Invalid version: {version}") + continue + sorted_and_filtered = sorted(url_dict.keys(), reverse=True) + + os.unlink(filepath) + elif command == "f": + tty.msg( + colorize( + f"Examples filters: {VERSION_COLOR}1.2@. " + f"or {VERSION_COLOR}1.1:1.3@. " + f"or {VERSION_COLOR}=1.2, 1.2.2:@." ) - i += 1 - except FailedDownloadError: - errors.append("Failed to fetch {0}".format(url)) - except Exception as e: - tty.msg("Something failed on {0}, skipping. ({1})".format(url, e)) + ) + try: + # Allow a leading @ version specifier + filter_spec = input(colorize("@*g{filter>} ")).strip().lstrip("@") + except EOFError: + print() + continue + try: + version_filter.intersect(VersionList([filter_spec])) + except ValueError: + tty.warn(f"Invalid version specifier: {filter_spec}") + continue + # Apply filter + sorted_and_filtered = [v for v in sorted_and_filtered if v.satisfies(version_filter)] + elif command == "a": + i = 0 + while i < len(sorted_and_filtered): + v = sorted_and_filtered[i] + try: + answer = input(f" {str(v):{max_len}} {url_dict[v]} [Y/n]? ").strip().lower() + except EOFError: + # If ^D, don't fully exit, but go back to the command prompt, now with possibly + # fewer versions + print() + break + if answer in ("n", "no"): + del sorted_and_filtered[i] + elif answer in ("y", "yes", ""): + i += 1 + else: + # Went over each version, so go to checksumming + break + elif command == "n": + sorted_and_filtered = [v for v in sorted_and_filtered if v not in known_versions] + elif command == "r": + url_dict = orig_url_dict + sorted_and_filtered = sorted(url_dict.keys(), reverse=True) + version_filter = VersionList([":"]) + elif command == "q": + try: + if input("Really quit [y/N]? ").strip().lower() in ("y", "yes"): + return None + except EOFError: + print() + return None + else: + # Last restort: filter the top N versions + try: + n = int(command) + invalid_command = n < 1 + except ValueError: + invalid_command = True + + if invalid_command: + tty.warn(f"Ignoring invalid command: {command}") + print_header = False + continue - for msg in errors: - tty.debug(msg) + sorted_and_filtered = sorted_and_filtered[:n] - if not version_hashes: - tty.die("Could not fetch any versions for {0}".format(name)) + return {v: url_dict[v] for v in sorted_and_filtered} - # Generate the version directives to put in a package.py - version_lines = "\n".join( - [' version("{0}", sha256="{1}"{2})'.format(v, h, expand_arg) for v, h in version_hashes] - ) - num_hash = len(version_hashes) - tty.debug( - "Checksummed {0} version{1} of {2}:".format(num_hash, "" if num_hash == 1 else "s", name) - ) +def get_checksums_for_versions( + url_by_version: Dict[str, str], + package_name: str, + *, + first_stage_function: Optional[Callable[[Stage, str], None]] = None, + keep_stage: bool = False, + concurrency: Optional[int] = None, + fetch_options: Optional[Dict[str, str]] = None, +) -> Dict[str, str]: + """Computes the checksums for each version passed in input, and returns the results. + + Archives are fetched according to the usl dictionary passed as input. + + The ``first_stage_function`` argument allows the caller to inspect the first downloaded + archive, e.g., to determine the build system. - return version_lines + Args: + url_by_version: URL keyed by version + package_name: name of the package + first_stage_function: function that takes a Stage and a URL; this is run on the stage + of the first URL downloaded + keep_stage: whether to keep staging area when command completes + batch: whether to ask user how many versions to fetch (false) or fetch all versions (true) + fetch_options: options used for the fetcher (such as timeout or cookies) + concurrency: maximum number of workers to use for retrieving archives + + Returns: + A dictionary mapping each version to the corresponding checksum + """ + versions = sorted(url_by_version.keys(), reverse=True) + search_arguments = [(url_by_version[v], v) for v in versions] + + version_hashes, errors = {}, [] + + # Don't spawn 16 processes when we need to fetch 2 urls + if concurrency is not None: + concurrency = min(concurrency, len(search_arguments)) + else: + concurrency = min(os.cpu_count() or 1, len(search_arguments)) + + # The function might have side effects in memory, that would not be reflected in the + # parent process, if run in a child process. If this pattern happens frequently, we + # can move this function call *after* having distributed the work to executors. + if first_stage_function is not None: + (url, version), search_arguments = search_arguments[0], search_arguments[1:] + checksum, error = _fetch_and_checksum(url, fetch_options, keep_stage, first_stage_function) + if error is not None: + errors.append(error) + + if checksum is not None: + version_hashes[version] = checksum + + with concurrent.futures.ProcessPoolExecutor(max_workers=concurrency) as executor: + results = [] + for url, version in search_arguments: + future = executor.submit(_fetch_and_checksum, url, fetch_options, keep_stage) + results.append((version, future)) + + for version, future in results: + checksum, error = future.result() + if error is not None: + errors.append(error) + continue + version_hashes[version] = checksum + + for msg in errors: + tty.debug(msg) + + if not version_hashes: + tty.die(f"Could not fetch any versions for {package_name}") + + num_hash = len(version_hashes) + tty.debug(f"Checksummed {num_hash} version{'' if num_hash == 1 else 's'} of {package_name}:") + + return version_hashes + + +def _fetch_and_checksum(url, options, keep_stage, action_fn=None): + try: + url_or_fs = url + if options: + url_or_fs = fs.URLFetchStrategy(url, fetch_options=options) + + with Stage(url_or_fs, keep=keep_stage) as stage: + # Fetch the archive + stage.fetch() + if action_fn is not None: + # Only run first_stage_function the first time, + # no need to run it every time + action_fn(stage, url) + + # Checksum the archive and add it to the list + checksum = spack.util.crypto.checksum(hashlib.sha256, stage.archive_file) + return checksum, None + except FailedDownloadError: + return None, f"[WORKER] Failed to fetch {url}" + except Exception as e: + return None, f"[WORKER] Something failed on {url}, skipping. ({e})" class StageError(spack.error.SpackError): diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 1c6129d70e3847..709074dea344eb 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -19,21 +19,27 @@ """ import contextlib import os +import pathlib import re -from typing import Union +import uuid +from typing import Any, Callable, Dict, Generator, List, Optional, Union import llnl.util.lang -import llnl.util.tty as tty +from llnl.util import tty import spack.config import spack.database import spack.directory_layout import spack.error import spack.paths +import spack.spec import spack.util.path #: default installation root, relative to the Spack install path -default_install_tree_root = os.path.join(spack.paths.opt_path, "spack") +DEFAULT_INSTALL_TREE_ROOT = os.path.join(spack.paths.opt_path, "spack") + + +ConfigurationType = Union["spack.config.Configuration", "llnl.util.lang.Singleton"] def parse_install_tree(config_dict): @@ -79,7 +85,7 @@ def parse_install_tree(config_dict): projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", default_install_tree_root) + unpadded_root = install_tree.get("root", DEFAULT_INSTALL_TREE_ROOT) unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) @@ -123,143 +129,141 @@ def parse_install_tree(config_dict): else: root = unpadded_root - return (root, unpadded_root, projections) + return root, unpadded_root, projections class Store: """A store is a path full of installed Spack packages. - Stores consist of packages installed according to a - ``DirectoryLayout``, along with an index, or _database_ of their - contents. The directory layout controls what paths look like and how - Spack ensures that each uniqe spec gets its own unique directory (or - not, though we don't recommend that). The database is a signle file - that caches metadata for the entire Spack installation. It prevents - us from having to spider the install tree to figure out what's there. + Stores consist of packages installed according to a ``DirectoryLayout``, along with a database + of their contents. + + The directory layout controls what paths look like and how Spack ensures that each unique spec + gets its own unique directory (or not, though we don't recommend that). + + The database is a single file that caches metadata for the entire Spack installation. It + prevents us from having to spider the install tree to figure out what's there. + + The store is also able to lock installation prefixes, and to mark installation failures. Args: - root (str): path to the root of the install tree - unpadded_root (str): path to the root of the install tree without - padding; the sbang script has to be installed here to work with - padded roots - path_scheme (str): expression according to guidelines in - ``spack.util.path`` that describes how to construct a path to + root: path to the root of the install tree + unpadded_root: path to the root of the install tree without padding. The sbang script has + to be installed here to work with padded roots + projections: expression according to guidelines that describes how to construct a path to a package prefix in this store - hash_length (int): length of the hashes used in the directory - layout; spec hash suffixes will be truncated to this length + hash_length: length of the hashes used in the directory layout. Spec hash suffixes will be + truncated to this length + upstreams: optional list of upstream databases + lock_cfg: lock configuration for the database """ - def __init__(self, root, unpadded_root=None, projections=None, hash_length=None): + def __init__( + self, + root: str, + unpadded_root: Optional[str] = None, + projections: Optional[Dict[str, str]] = None, + hash_length: Optional[int] = None, + upstreams: Optional[List[spack.database.Database]] = None, + lock_cfg: spack.database.LockConfiguration = spack.database.NO_LOCK, + ) -> None: self.root = root self.unpadded_root = unpadded_root or root self.projections = projections self.hash_length = hash_length - self.db = spack.database.Database(root, upstream_dbs=retrieve_upstream_dbs()) + self.upstreams = upstreams + self.lock_cfg = lock_cfg + self.db = spack.database.Database(root, upstream_dbs=upstreams, lock_cfg=lock_cfg) + + timeout_format_str = ( + f"{str(lock_cfg.package_timeout)}s" if lock_cfg.package_timeout else "No timeout" + ) + tty.debug("PACKAGE LOCK TIMEOUT: {0}".format(str(timeout_format_str))) + + self.prefix_locker = spack.database.SpecLocker( + spack.database.prefix_lock_path(root), default_timeout=lock_cfg.package_timeout + ) + self.failure_tracker = spack.database.FailureTracker( + self.root, default_timeout=lock_cfg.package_timeout + ) + self.layout = spack.directory_layout.DirectoryLayout( root, projections=projections, hash_length=hash_length ) - def reindex(self): + def reindex(self) -> None: """Convenience function to reindex the store DB with its own layout.""" return self.db.reindex(self.layout) - def serialize(self): - """Return a pickle-able object that can be used to reconstruct - a store. - """ - return (self.root, self.unpadded_root, self.projections, self.hash_length) - - @staticmethod - def deserialize(token): - """Return a store reconstructed from a token created by - the serialize method. - - Args: - token: return value of the serialize method - - Returns: - Store object reconstructed from the token - """ - return Store(*token) + def __reduce__(self): + return Store, ( + self.root, + self.unpadded_root, + self.projections, + self.hash_length, + self.upstreams, + self.lock_cfg, + ) -def _store(): - """Get the singleton store instance.""" - import spack.bootstrap +def create(configuration: ConfigurationType) -> Store: + """Create a store from the configuration passed as input. - config_dict = spack.config.get("config") + Args: + configuration: configuration to create a store. + """ + configuration = configuration or spack.config.CONFIG + config_dict = configuration.get("config") root, unpadded_root, projections = parse_install_tree(config_dict) - hash_length = spack.config.get("config:install_hash_length") + hash_length = configuration.get("config:install_hash_length") + + install_roots = [ + install_properties["install_tree"] + for install_properties in configuration.get("upstreams", {}).values() + ] + upstreams = _construct_upstream_dbs_from_install_roots(install_roots) return Store( - root=root, unpadded_root=unpadded_root, projections=projections, hash_length=hash_length + root=root, + unpadded_root=unpadded_root, + projections=projections, + hash_length=hash_length, + upstreams=upstreams, + lock_cfg=spack.database.lock_configuration(configuration), ) -#: Singleton store instance -store: Union[Store, llnl.util.lang.Singleton] = llnl.util.lang.Singleton(_store) - +def _create_global() -> Store: + result = create(configuration=spack.config.CONFIG) + return result -def _store_root(): - return store.root - -def _store_unpadded_root(): - return store.unpadded_root - - -def _store_db(): - return store.db - - -def _store_layout(): - return store.layout - - -# convenience accessors for parts of the singleton store -root = llnl.util.lang.LazyReference(_store_root) -unpadded_root = llnl.util.lang.LazyReference(_store_unpadded_root) -db = llnl.util.lang.LazyReference(_store_db) -layout = llnl.util.lang.LazyReference(_store_layout) +#: Singleton store instance +STORE: Union[Store, llnl.util.lang.Singleton] = llnl.util.lang.Singleton(_create_global) def reinitialize(): """Restore globals to the same state they would have at start-up. Return a token containing the state of the store before reinitialization. """ - global store - global root, unpadded_root, db, layout + global STORE - token = store, root, unpadded_root, db, layout - - store = llnl.util.lang.Singleton(_store) - root = llnl.util.lang.LazyReference(_store_root) - unpadded_root = llnl.util.lang.LazyReference(_store_unpadded_root) - db = llnl.util.lang.LazyReference(_store_db) - layout = llnl.util.lang.LazyReference(_store_layout) + token = STORE + STORE = llnl.util.lang.Singleton(_create_global) return token def restore(token): """Restore the environment from a token returned by reinitialize""" - global store - global root, unpadded_root, db, layout - store, root, unpadded_root, db, layout = token - + global STORE + STORE = token -def retrieve_upstream_dbs(): - other_spack_instances = spack.config.get("upstreams", {}) - install_roots = [] - for install_properties in other_spack_instances.values(): - install_roots.append(install_properties["install_tree"]) - - return _construct_upstream_dbs_from_install_roots(install_roots) - - -def _construct_upstream_dbs_from_install_roots(install_roots, _test=False): - accumulated_upstream_dbs = [] +def _construct_upstream_dbs_from_install_roots( + install_roots: List[str], _test: bool = False +) -> List[spack.database.Database]: + accumulated_upstream_dbs: List[spack.database.Database] = [] for install_root in reversed(install_roots): upstream_dbs = list(accumulated_upstream_dbs) next_db = spack.database.Database( @@ -274,8 +278,13 @@ def _construct_upstream_dbs_from_install_roots(install_roots, _test=False): return accumulated_upstream_dbs -def find(constraints, multiple=False, query_fn=None, **kwargs): - """Return a list of specs matching the constraints passed as inputs. +def find( + constraints: Union[str, List[str], List["spack.spec.Spec"]], + multiple: bool = False, + query_fn: Optional[Callable[[Any], List["spack.spec.Spec"]]] = None, + **kwargs, +) -> List["spack.spec.Spec"]: + """Returns a list of specs matching the constraints passed as inputs. At least one spec per constraint must match, otherwise the function will error with an appropriate message. @@ -287,22 +296,18 @@ def find(constraints, multiple=False, query_fn=None, **kwargs): The query function must accept a spec as its first argument. Args: - constraints (List[spack.spec.Spec]): specs to be matched against - installed packages - multiple (bool): if True multiple matches per constraint are admitted + constraints: spec(s) to be matched against installed packages + multiple: if True multiple matches per constraint are admitted query_fn (Callable): query function to get matching specs. By default, - ``spack.store.db.query`` + ``spack.store.STORE.db.query`` **kwargs: keyword arguments forwarded to the query function - - Return: - List of matching specs """ - # Normalize input to list of specs if isinstance(constraints, str): constraints = [spack.spec.Spec(constraints)] - matching_specs, errors = [], [] - query_fn = query_fn or spack.store.db.query + matching_specs: List[spack.spec.Spec] = [] + errors = [] + query_fn = query_fn or spack.store.STORE.db.query for spec in constraints: current_matches = query_fn(spec, **kwargs) @@ -327,50 +332,58 @@ def find(constraints, multiple=False, query_fn=None, **kwargs): return matching_specs -def specfile_matches(filename, **kwargs): +def specfile_matches(filename: str, **kwargs) -> List["spack.spec.Spec"]: """Same as find but reads the query from a spec file. Args: - filename (str): YAML or JSON file from which to read the query. + filename: YAML or JSON file from which to read the query. **kwargs: keyword arguments forwarded to "find" - - Return: - List of matches """ query = [spack.spec.Spec.from_specfile(filename)] return spack.store.find(query, **kwargs) +def ensure_singleton_created() -> None: + """Ensures the lazily evaluated singleton is created""" + _ = STORE.db + + @contextlib.contextmanager -def use_store(store_or_path): +def use_store( + path: Union[str, pathlib.Path], extra_data: Optional[Dict[str, Any]] = None +) -> Generator[Store, None, None]: """Use the store passed as argument within the context manager. Args: - store_or_path: either a Store object ot a path to where the store resides + path: path to the store. + extra_data: extra configuration under "config:install_tree" to be + taken into account. - Returns: + Yields: Store object associated with the context manager's store """ - global store, db, layout, root, unpadded_root + global STORE - # Normalize input arguments - temporary_store = store_or_path - if not isinstance(store_or_path, Store): - temporary_store = Store(store_or_path) + assert not isinstance(path, Store), "cannot pass a store anymore" + scope_name = "use-store-{}".format(uuid.uuid4()) + data = {"root": str(path)} + if extra_data: + data.update(extra_data) # Swap the store with the one just constructed and return it - _ = store.db - original_store, store = store, temporary_store - db, layout = store.db, store.layout - root, unpadded_root = store.root, store.unpadded_root + ensure_singleton_created() + spack.config.CONFIG.push_scope( + spack.config.InternalConfigScope(name=scope_name, data={"config": {"install_tree": data}}) + ) + temporary_store = create(configuration=spack.config.CONFIG) + original_store, STORE = STORE, temporary_store try: yield temporary_store finally: # Restore the original store - store = original_store - db, layout = original_store.db, original_store.layout - root, unpadded_root = original_store.root, original_store.unpadded_root + STORE = original_store + spack.config.CONFIG.remove_scope(scope_name=scope_name) class MatchError(spack.error.SpackError): diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index 8283a3d3ba96e4..f0e004e4726e6b 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -27,7 +27,7 @@ import spack.repo import spack.store -_serialize = sys.platform == "win32" or (sys.version_info >= (3, 8) and sys.platform == "darwin") +_SERIALIZE = sys.platform == "win32" or (sys.version_info >= (3, 8) and sys.platform == "darwin") patches = None @@ -66,7 +66,7 @@ class PackageInstallContext: """ def __init__(self, pkg): - if _serialize: + if _SERIALIZE: self.serialized_pkg = serialize(pkg) self.serialized_env = serialize(spack.environment.active_environment()) else: @@ -78,8 +78,8 @@ def __init__(self, pkg): def restore(self): self.test_state.restore() spack.main.spack_working_dir = self.spack_working_dir - env = pickle.load(self.serialized_env) if _serialize else self.env - pkg = pickle.load(self.serialized_pkg) if _serialize else self.pkg + env = pickle.load(self.serialized_env) if _SERIALIZE else self.env + pkg = pickle.load(self.serialized_pkg) if _SERIALIZE else self.pkg if env: spack.environment.activate(env) return pkg @@ -93,26 +93,18 @@ class TestState: """ def __init__(self): - if _serialize: - self.repo_dirs = list(r.root for r in spack.repo.path.repos) - self.config = spack.config.config + if _SERIALIZE: + self.config = spack.config.CONFIG self.platform = spack.platforms.host self.test_patches = store_patches() - self.store_token = spack.store.store.serialize() + self.store = spack.store.STORE def restore(self): - if _serialize: - spack.config.config = self.config - spack.repo.path = spack.repo._path(self.config) + if _SERIALIZE: + spack.config.CONFIG = self.config + spack.repo.PATH = spack.repo.create(self.config) spack.platforms.host = self.platform - - new_store = spack.store.Store.deserialize(self.store_token) - spack.store.store = new_store - spack.store.root = new_store.root - spack.store.unpadded_root = new_store.unpadded_root - spack.store.db = new_store.db - spack.store.layout = new_store.layout - + spack.store.STORE = self.store self.test_patches.restore() diff --git a/lib/spack/spack/tag.py b/lib/spack/spack/tag.py index 987f27f3619f3d..1b32f617c70ff5 100644 --- a/lib/spack/spack/tag.py +++ b/lib/spack/spack/tag.py @@ -32,10 +32,10 @@ def packages_with_tags(tags, installed, skip_empty): """ tag_pkgs = collections.defaultdict(lambda: list) spec_names = _get_installed_package_names() if installed else [] - keys = spack.repo.path.tag_index if tags is None else tags + keys = spack.repo.PATH.tag_index if tags is None else tags for tag in keys: packages = [ - name for name in spack.repo.path.tag_index[tag] if not installed or name in spec_names + name for name in spack.repo.PATH.tag_index[tag] if not installed or name in spec_names ] if packages or not skip_empty: tag_pkgs[tag] = packages diff --git a/lib/spack/spack/tengine.py b/lib/spack/spack/tengine.py index 6a77e6becf88ed..6f2e3fb69902f8 100644 --- a/lib/spack/spack/tengine.py +++ b/lib/spack/spack/tengine.py @@ -100,9 +100,15 @@ def quote(text): return ['"{0}"'.format(line) for line in text] +def curly_quote(text): + """Encloses each line of text in curly braces""" + return ["{{{0}}}".format(line) for line in text] + + def _set_filters(env): """Sets custom filters to the template engine environment""" env.filters["textwrap"] = textwrap.wrap env.filters["prepend_to_line"] = prepend_to_line env.filters["join"] = "\n".join env.filters["quote"] = quote + env.filters["curly_quote"] = curly_quote diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index f00d66ad3295c3..7bea7a1d86a47f 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -13,8 +13,8 @@ import spack.concretize import spack.operating_systems import spack.platforms -import spack.spec import spack.target +from spack.spec import ArchSpec, CompilerSpec, Spec @pytest.fixture(scope="module") @@ -59,13 +59,12 @@ def test_platform(current_host_platform): assert str(detected_platform) == str(current_host_platform) -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") def test_user_input_combination(config, target_str, os_str): """Test for all the valid user input combinations that both the target and the operating system match. """ spec_str = "libelf os={} target={}".format(os_str, target_str) - spec = spack.spec.Spec(spec_str) + spec = Spec(spec_str) assert spec.architecture.os == str(TEST_PLATFORM.operating_system(os_str)) assert spec.architecture.target == TEST_PLATFORM.target(target_str) @@ -115,7 +114,7 @@ def test_target_container_semantic(cpu_flag, target_name): ], ) def test_arch_spec_container_semantic(item, architecture_str): - architecture = spack.spec.ArchSpec(architecture_str) + architecture = ArchSpec(architecture_str) assert item in architecture @@ -142,24 +141,24 @@ def test_optimization_flags(compiler_spec, target_name, expected_flags, config): @pytest.mark.parametrize( "compiler,real_version,target_str,expected_flags", [ - (spack.spec.CompilerSpec("gcc@=9.2.0"), None, "haswell", "-march=haswell -mtune=haswell"), + (CompilerSpec("gcc@=9.2.0"), None, "haswell", "-march=haswell -mtune=haswell"), # Check that custom string versions are accepted ( - spack.spec.CompilerSpec("gcc@=10foo"), + CompilerSpec("gcc@=10foo"), "9.2.0", "icelake", "-march=icelake-client -mtune=icelake-client", ), # Check that we run version detection (4.4.0 doesn't support icelake) ( - spack.spec.CompilerSpec("gcc@=4.4.0-special"), + CompilerSpec("gcc@=4.4.0-special"), "9.2.0", "icelake", "-march=icelake-client -mtune=icelake-client", ), # Check that the special case for Apple's clang is treated correctly # i.e. it won't try to detect the version again - (spack.spec.CompilerSpec("apple-clang@=9.1.0"), None, "x86_64", "-march=x86-64"), + (CompilerSpec("apple-clang@=9.1.0"), None, "x86_64", "-march=x86-64"), ], ) def test_optimization_flags_with_custom_versions( @@ -181,8 +180,8 @@ def test_optimization_flags_with_custom_versions( ], ) def test_satisfy_strict_constraint_when_not_concrete(architecture_tuple, constraint_tuple): - architecture = spack.spec.ArchSpec(architecture_tuple) - constraint = spack.spec.ArchSpec(constraint_tuple) + architecture = ArchSpec(architecture_tuple) + constraint = ArchSpec(constraint_tuple) assert not architecture.satisfies(constraint) @@ -201,24 +200,14 @@ def test_satisfy_strict_constraint_when_not_concrete(architecture_tuple, constra ], ) @pytest.mark.usefixtures("mock_packages", "config") +@pytest.mark.only_clingo("Fixing the parser broke this test for the original concretizer.") def test_concretize_target_ranges(root_target_range, dep_target_range, result, monkeypatch): # Monkeypatch so that all concretization is done as if the machine is core2 monkeypatch.setattr(spack.platforms.test.Test, "default", "core2") - - # use foobar=bar to make the problem simpler for the old concretizer - # the new concretizer should not need that help - if spack.config.get("config:concretizer") == "original": - pytest.skip("Fixing the parser broke this test for the original concretizer.") - - spec_str = "a %%gcc@10 foobar=bar target=%s ^b target=%s" % ( - root_target_range, - dep_target_range, - ) - spec = spack.spec.Spec(spec_str) + spec = Spec(f"a %gcc@10 foobar=bar target={root_target_range} ^b target={dep_target_range}") with spack.concretize.disable_compiler_existence_check(): spec.concretize() - - assert str(spec).count("arch=test-debian6-%s" % result) == 2 + assert spec.target == spec["b"].target == result @pytest.mark.parametrize( @@ -229,6 +218,7 @@ def test_concretize_target_ranges(root_target_range, dep_target_range, result, m (["21.11", "21.9"], None, False), ], ) +@pytest.mark.skipif(sys.platform == "win32", reason="Cray does not use windows") def test_cray_platform_detection(versions, default, expected, tmpdir, monkeypatch, working_env): ex_path = str(tmpdir.join("fake_craype_dir")) fs.mkdirp(ex_path) diff --git a/lib/spack/spack/test/audit.py b/lib/spack/spack/test/audit.py index 2efc2bbd88913a..a3d4bb8e3fbaf1 100644 --- a/lib/spack/spack/test/audit.py +++ b/lib/spack/spack/test/audit.py @@ -21,6 +21,10 @@ (["wrong-variant-in-depends-on"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), # This package has a GitHub patch URL without full_index=1 (["invalid-github-patch-url"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), + # This package has invalid GitLab patch URLs + (["invalid-gitlab-patch-url"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), + # This package has invalid GitLab patch URLs + (["invalid-selfhosted-gitlab-patch-url"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), # This package has a stand-alone 'test*' method in build-time callbacks (["fail-test-audit"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), # This package has no issues diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py index ed60a1056a0d57..ea9caf7fc031ee 100644 --- a/lib/spack/spack/test/bindist.py +++ b/lib/spack/spack/test/bindist.py @@ -4,7 +4,9 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import filecmp import glob +import gzip import io +import json import os import platform import sys @@ -12,6 +14,7 @@ import urllib.error import urllib.request import urllib.response +from pathlib import PurePath import py import pytest @@ -28,6 +31,7 @@ import spack.repo import spack.store import spack.util.gpg +import spack.util.spack_yaml as syaml import spack.util.url as url_util import spack.util.web as web_util from spack.binary_distribution import get_buildfile_manifest @@ -35,7 +39,7 @@ from spack.paths import test_path from spack.spec import Spec -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") mirror_cmd = spack.main.SpackCommand("mirror") install_cmd = spack.main.SpackCommand("install") @@ -49,7 +53,7 @@ def cache_directory(tmpdir): fetch_cache_dir = tmpdir.ensure("fetch_cache", dir=True) fsc = spack.fetch_strategy.FsCache(str(fetch_cache_dir)) - spack.config.caches, old_cache_path = fsc, spack.caches.fetch_cache + spack.config.caches, old_cache_path = fsc, spack.caches.FETCH_CACHE yield spack.config.caches @@ -113,8 +117,8 @@ def default_config(tmpdir, config_directory, monkeypatch, install_mockery_mutabl ] ) - spack.config.config, old_config = cfg, spack.config.config - spack.config.config.set("repos", [spack.paths.mock_packages_path]) + spack.config.CONFIG, old_config = cfg, spack.config.CONFIG + spack.config.CONFIG.set("repos", [spack.paths.mock_packages_path]) njobs = spack.config.get("config:build_jobs") if not njobs: spack.config.set("config:build_jobs", 4, scope="user") @@ -136,9 +140,9 @@ def default_config(tmpdir, config_directory, monkeypatch, install_mockery_mutabl if not timeout: spack.config.set("config:connect_timeout", 10, scope="user") - yield spack.config.config + yield spack.config.CONFIG - spack.config.config = old_config + spack.config.CONFIG = old_config mutable_dir.remove() @@ -148,15 +152,15 @@ def install_dir_default_layout(tmpdir): scheme = os.path.join( "${architecture}", "${compiler.name}-${compiler.version}", "${name}-${version}-${hash}" ) - real_store, real_layout = spack.store.store, spack.store.layout + real_store, real_layout = spack.store.STORE, spack.store.STORE.layout opt_dir = tmpdir.join("opt") - spack.store.store = spack.store.Store(str(opt_dir)) - spack.store.layout = DirectoryLayout(str(opt_dir), path_scheme=scheme) + spack.store.STORE = spack.store.Store(str(opt_dir)) + spack.store.STORE.layout = DirectoryLayout(str(opt_dir), path_scheme=scheme) try: yield spack.store finally: - spack.store.store = real_store - spack.store.layout = real_layout + spack.store.STORE = real_store + spack.store.STORE.layout = real_layout @pytest.fixture(scope="function") @@ -165,18 +169,45 @@ def install_dir_non_default_layout(tmpdir): scheme = os.path.join( "${name}", "${version}", "${architecture}-${compiler.name}-${compiler.version}-${hash}" ) - real_store, real_layout = spack.store.store, spack.store.layout + real_store, real_layout = spack.store.STORE, spack.store.STORE.layout opt_dir = tmpdir.join("opt") - spack.store.store = spack.store.Store(str(opt_dir)) - spack.store.layout = DirectoryLayout(str(opt_dir), path_scheme=scheme) + spack.store.STORE = spack.store.Store(str(opt_dir)) + spack.store.STORE.layout = DirectoryLayout(str(opt_dir), path_scheme=scheme) try: yield spack.store finally: - spack.store.store = real_store - spack.store.layout = real_layout + spack.store.STORE = real_store + spack.store.STORE.layout = real_layout -args = ["strings", "file"] +@pytest.fixture(scope="function") +def dummy_prefix(tmpdir): + """Dummy prefix used for testing tarball creation, validation, extraction""" + p = tmpdir.mkdir("prefix") + assert os.path.isabs(p) + + p.mkdir("bin") + p.mkdir("share") + p.mkdir(".spack") + + app = p.join("bin", "app") + relative_app_link = p.join("bin", "relative_app_link") + absolute_app_link = p.join("bin", "absolute_app_link") + data = p.join("share", "file") + + with open(app, "w") as f: + f.write("hello world") + + with open(data, "w") as f: + f.write("hello world") + + os.symlink("app", relative_app_link) + os.symlink(app, absolute_app_link) + + return str(p) + + +args = ["file"] if sys.platform == "darwin": args.extend(["/usr/bin/clang++", "install_name_tool"]) else: @@ -201,12 +232,14 @@ def test_default_rpaths_create_install_default_layout(mirror_dir): install_cmd("--no-cache", sy_spec.name) # Create a buildache - buildcache_cmd("push", "-au", mirror_dir, cspec.name, sy_spec.name) + buildcache_cmd("push", "-u", mirror_dir, cspec.name, sy_spec.name) + # Test force overwrite create buildcache (-f option) - buildcache_cmd("push", "-auf", mirror_dir, cspec.name) + buildcache_cmd("push", "-uf", mirror_dir, cspec.name) # Create mirror index buildcache_cmd("update-index", mirror_dir) + # List the buildcaches in the mirror buildcache_cmd("list", "-alv") @@ -376,7 +409,7 @@ def test_spec_needs_rebuild(monkeypatch, tmpdir): install_cmd(s.name) # Put installed package in the buildcache - buildcache_cmd("push", "-u", "-a", mirror_dir.strpath, s.name) + buildcache_cmd("push", "-u", mirror_dir.strpath, s.name) rebuild = bindist.needs_rebuild(s, mirror_url) @@ -405,7 +438,7 @@ def test_generate_index_missing(monkeypatch, tmpdir, mutable_config): install_cmd("--no-cache", s.name) # Create a buildcache and update index - buildcache_cmd("push", "-ua", mirror_dir.strpath, s.name) + buildcache_cmd("push", "-u", mirror_dir.strpath, s.name) buildcache_cmd("update-index", mirror_dir.strpath) # Check package and dependency in buildcache @@ -479,9 +512,6 @@ def test_update_sbang(tmpdir, test_mirror): into the non-default directory layout scheme, triggering an update of the sbang. """ - scheme = os.path.join( - "${name}", "${version}", "${architecture}-${compiler.name}-${compiler.version}-${hash}" - ) spec_str = "old-sbang" # Concretize a package with some old-fashioned sbang lines. old_spec = Spec(spec_str).concretized() @@ -494,7 +524,7 @@ def test_update_sbang(tmpdir, test_mirror): install_cmd("--no-cache", old_spec.name) # Create a buildcache with the installed spec. - buildcache_cmd("push", "-u", "-a", mirror_dir, old_spec_hash_str) + buildcache_cmd("push", "-u", mirror_dir, old_spec_hash_str) # Need to force an update of the buildcache index buildcache_cmd("update-index", mirror_dir) @@ -504,12 +534,8 @@ def test_update_sbang(tmpdir, test_mirror): # Switch the store to the new install tree locations newtree_dir = tmpdir.join("newtree") - s = spack.store.Store(str(newtree_dir)) - s.layout = DirectoryLayout(str(newtree_dir), path_scheme=scheme) - - with spack.store.use_store(s): - new_spec = Spec("old-sbang") - new_spec.concretize() + with spack.store.use_store(str(newtree_dir)): + new_spec = Spec("old-sbang").concretized() assert new_spec.dag_hash() == old_spec.dag_hash() # Install package from buildcache @@ -861,6 +887,38 @@ def urlopen(request: urllib.request.Request): fetcher.conditional_fetch() +def test_tarball_doesnt_include_buildinfo_twice(tmpdir): + """When tarballing a package that was installed from a buildcache, make + sure that the buildinfo file is not included twice in the tarball.""" + p = tmpdir.mkdir("prefix") + p.mkdir(".spack") + + # Create a binary_distribution file in the .spack folder + with open(p.join(".spack", "binary_distribution"), "w") as f: + f.write(syaml.dump({"metadata", "old"})) + + # Now create a tarball, which should include a new binary_distribution file + tarball = str(tmpdir.join("prefix.tar.gz")) + + bindist._do_create_tarball( + tarfile_path=tarball, binaries_dir=p.strpath, buildinfo={"metadata": "new"} + ) + + expected_prefix = p.strpath.lstrip("/") + + # Verify we don't have a repeated binary_distribution file, + # and that the tarball contains the new one, not the old one. + with tarfile.open(tarball) as tar: + assert syaml.load(tar.extractfile(f"{expected_prefix}/.spack/binary_distribution")) == { + "metadata": "new" + } + assert tar.getnames() == [ + f"{expected_prefix}", + f"{expected_prefix}/.spack", + f"{expected_prefix}/.spack/binary_distribution", + ] + + def test_reproducible_tarball_is_reproducible(tmpdir): p = tmpdir.mkdir("prefix") p.mkdir("bin") @@ -878,15 +936,17 @@ def test_reproducible_tarball_is_reproducible(tmpdir): # Create a tarball with a certain mtime of bin/app os.utime(app, times=(0, 0)) - bindist._do_create_tarball(tarball_1, binaries_dir=p, pkg_dir="pkg", buildinfo=buildinfo) + bindist._do_create_tarball(tarball_1, binaries_dir=p.strpath, buildinfo=buildinfo) # Do it another time with different mtime of bin/app os.utime(app, times=(10, 10)) - bindist._do_create_tarball(tarball_2, binaries_dir=p, pkg_dir="pkg", buildinfo=buildinfo) + bindist._do_create_tarball(tarball_2, binaries_dir=p.strpath, buildinfo=buildinfo) # They should be bitwise identical: assert filecmp.cmp(tarball_1, tarball_2, shallow=False) + expected_prefix = p.strpath.lstrip("/") + # Sanity check for contents: with tarfile.open(tarball_1, mode="r") as f: for m in f.getmembers(): @@ -894,11 +954,11 @@ def test_reproducible_tarball_is_reproducible(tmpdir): assert m.uname == m.gname == "" assert set(f.getnames()) == { - "pkg", - "pkg/bin", - "pkg/bin/app", - "pkg/.spack", - "pkg/.spack/binary_distribution", + f"{expected_prefix}", + f"{expected_prefix}/bin", + f"{expected_prefix}/bin/app", + f"{expected_prefix}/.spack", + f"{expected_prefix}/.spack/binary_distribution", } @@ -922,18 +982,209 @@ def test_tarball_normalized_permissions(tmpdir): with open(data, "w", opener=lambda path, flags: os.open(path, flags, 0o477)) as f: f.write("hello world") - bindist._do_create_tarball(tarball, binaries_dir=p, pkg_dir="pkg", buildinfo={}) + bindist._do_create_tarball(tarball, binaries_dir=p.strpath, buildinfo={}) + + expected_prefix = p.strpath.lstrip("/") with tarfile.open(tarball) as tar: path_to_member = {member.name: member for member in tar.getmembers()} # directories should have 0o755 - assert path_to_member["pkg"].mode == 0o755 - assert path_to_member["pkg/bin"].mode == 0o755 - assert path_to_member["pkg/.spack"].mode == 0o755 + assert path_to_member[f"{expected_prefix}"].mode == 0o755 + assert path_to_member[f"{expected_prefix}/bin"].mode == 0o755 + assert path_to_member[f"{expected_prefix}/.spack"].mode == 0o755 # executable-by-user files should be 0o755 - assert path_to_member["pkg/bin/app"].mode == 0o755 + assert path_to_member[f"{expected_prefix}/bin/app"].mode == 0o755 # not-executable-by-user files should be 0o644 - assert path_to_member["pkg/share/file"].mode == 0o644 + assert path_to_member[f"{expected_prefix}/share/file"].mode == 0o644 + + +def test_tarball_common_prefix(dummy_prefix, tmpdir): + """Tests whether Spack can figure out the package directory + from the tarball contents, and strip them when extracting.""" + + # When creating a tarball, Python (and tar) use relative paths, + # Absolute paths become relative to `/`, so drop the leading `/`. + assert os.path.isabs(dummy_prefix) + expected_prefix = PurePath(dummy_prefix).as_posix().lstrip("/") + + with tmpdir.as_cwd(): + # Create a tarball (using absolute path for prefix dir) + with tarfile.open("example.tar", mode="w") as tar: + tar.add(name=dummy_prefix) + + # Open, verify common prefix, and extract it. + with tarfile.open("example.tar", mode="r") as tar: + common_prefix = bindist._ensure_common_prefix(tar) + assert common_prefix == expected_prefix + + # Strip the prefix from the tar entries + bindist._tar_strip_component(tar, common_prefix) + + # Extract into prefix2 + tar.extractall(path="prefix2") + + # Verify files are all there at the correct level. + assert set(os.listdir("prefix2")) == {"bin", "share", ".spack"} + assert set(os.listdir(os.path.join("prefix2", "bin"))) == { + "app", + "relative_app_link", + "absolute_app_link", + } + assert set(os.listdir(os.path.join("prefix2", "share"))) == {"file"} + + # Relative symlink should still be correct + assert os.readlink(os.path.join("prefix2", "bin", "relative_app_link")) == "app" + + # Absolute symlink should remain absolute -- this is for relocation to fix up. + assert os.readlink(os.path.join("prefix2", "bin", "absolute_app_link")) == os.path.join( + dummy_prefix, "bin", "app" + ) + + +def test_tarfile_without_common_directory_prefix_fails(tmpdir): + """A tarfile that only contains files without a common package directory + should fail to extract, as we won't know where to put the files.""" + with tmpdir.as_cwd(): + # Create a broken tarball with just a file, no directories. + with tarfile.open("empty.tar", mode="w") as tar: + tar.addfile(tarfile.TarInfo(name="example/file"), fileobj=io.BytesIO(b"hello")) + + with pytest.raises(ValueError, match="Tarball does not contain a common prefix"): + bindist._ensure_common_prefix(tarfile.open("empty.tar", mode="r")) + + +def test_tarfile_with_files_outside_common_prefix(tmpdir, dummy_prefix): + """If a file is outside of the common prefix, we should fail.""" + with tmpdir.as_cwd(): + with tarfile.open("broken.tar", mode="w") as tar: + tar.add(name=dummy_prefix) + tar.addfile(tarfile.TarInfo(name="/etc/config_file"), fileobj=io.BytesIO(b"hello")) + + with pytest.raises( + ValueError, match="Tarball contains file /etc/config_file outside of prefix" + ): + bindist._ensure_common_prefix(tarfile.open("broken.tar", mode="r")) + + +def test_tarfile_of_spec_prefix(tmpdir): + """Tests whether hardlinks, symlinks, files and dirs are added correctly, + and that the order of entries is correct.""" + prefix = tmpdir.mkdir("prefix") + prefix.ensure("a_directory", dir=True).join("file").write("hello") + prefix.ensure("c_directory", dir=True).join("file").write("hello") + prefix.ensure("b_directory", dir=True).join("file").write("hello") + prefix.join("file").write("hello") + os.symlink(prefix.join("file"), prefix.join("symlink")) + os.link(prefix.join("file"), prefix.join("hardlink")) + + file = tmpdir.join("example.tar") + + with tarfile.open(file, mode="w") as tar: + bindist.tarfile_of_spec_prefix(tar, prefix.strpath) + + expected_prefix = prefix.strpath.lstrip("/") + + with tarfile.open(file, mode="r") as tar: + # Verify that entries are added in depth-first pre-order, files preceding dirs, + # entries ordered alphabetically + assert tar.getnames() == [ + f"{expected_prefix}", + f"{expected_prefix}/file", + f"{expected_prefix}/hardlink", + f"{expected_prefix}/symlink", + f"{expected_prefix}/a_directory", + f"{expected_prefix}/a_directory/file", + f"{expected_prefix}/b_directory", + f"{expected_prefix}/b_directory/file", + f"{expected_prefix}/c_directory", + f"{expected_prefix}/c_directory/file", + ] + + # Check that the types are all correct + assert tar.getmember(f"{expected_prefix}").isdir() + assert tar.getmember(f"{expected_prefix}/file").isreg() + assert tar.getmember(f"{expected_prefix}/hardlink").islnk() + assert tar.getmember(f"{expected_prefix}/symlink").issym() + assert tar.getmember(f"{expected_prefix}/a_directory").isdir() + assert tar.getmember(f"{expected_prefix}/a_directory/file").isreg() + assert tar.getmember(f"{expected_prefix}/b_directory").isdir() + assert tar.getmember(f"{expected_prefix}/b_directory/file").isreg() + assert tar.getmember(f"{expected_prefix}/c_directory").isdir() + assert tar.getmember(f"{expected_prefix}/c_directory/file").isreg() + + +@pytest.mark.parametrize("layout,expect_success", [(None, True), (1, True), (2, False)]) +def test_get_valid_spec_file(tmp_path, layout, expect_success): + # Test reading a spec.json file that does not specify a layout version. + spec_dict = Spec("example").to_dict() + path = tmp_path / "spec.json" + effective_layout = layout or 0 # If not specified it should be 0 + + # Add a layout version + if layout is not None: + spec_dict["buildcache_layout_version"] = layout + + # Save to file + with open(path, "w") as f: + json.dump(spec_dict, f) + + try: + spec_dict_disk, layout_disk = bindist._get_valid_spec_file( + str(path), max_supported_layout=1 + ) + assert expect_success + assert spec_dict_disk == spec_dict + assert layout_disk == effective_layout + except bindist.InvalidMetadataFile: + assert not expect_success + + +def test_get_valid_spec_file_doesnt_exist(tmp_path): + with pytest.raises(bindist.InvalidMetadataFile, match="No such file"): + bindist._get_valid_spec_file(str(tmp_path / "no-such-file"), max_supported_layout=1) + + +def test_get_valid_spec_file_gzipped(tmp_path): + # Create a gzipped file, contents don't matter + path = tmp_path / "spec.json.gz" + with gzip.open(path, "wb") as f: + f.write(b"hello") + with pytest.raises( + bindist.InvalidMetadataFile, match="Compressed spec files are not supported" + ): + bindist._get_valid_spec_file(str(path), max_supported_layout=1) + + +@pytest.mark.parametrize("filename", ["spec.json", "spec.json.sig"]) +def test_get_valid_spec_file_no_json(tmp_path, filename): + tmp_path.joinpath(filename).write_text("not json") + with pytest.raises(bindist.InvalidMetadataFile): + bindist._get_valid_spec_file(str(tmp_path / filename), max_supported_layout=1) + + +def test_download_tarball_with_unsupported_layout_fails(tmp_path, mutable_config, capsys): + layout_version = bindist.CURRENT_BUILD_CACHE_LAYOUT_VERSION + 1 + spec = Spec("gmake@4.4.1%gcc@13.1.0 arch=linux-ubuntu23.04-zen2") + spec._mark_concrete() + spec_dict = spec.to_dict() + spec_dict["buildcache_layout_version"] = layout_version + + # Setup a basic local build cache structure + path = ( + tmp_path / bindist.build_cache_relative_path() / bindist.tarball_name(spec, ".spec.json") + ) + path.parent.mkdir(parents=True) + with open(path, "w") as f: + json.dump(spec_dict, f) + + # Configure as a mirror. + mirror_cmd("add", "test-mirror", str(tmp_path)) + + # Shouldn't be able "download" this. + assert bindist.download_tarball(spec, unsigned=True) is None + + # And there should be a warning about an unsupported layout version. + assert f"Layout version {layout_version} is too new" in capsys.readouterr().err diff --git a/lib/spack/spack/test/bootstrap.py b/lib/spack/spack/test/bootstrap.py index 7eee7a21e3f5f6..dcc8bf6495e6bd 100644 --- a/lib/spack/spack/test/bootstrap.py +++ b/lib/spack/spack/test/bootstrap.py @@ -2,7 +2,6 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys import pytest @@ -23,20 +22,48 @@ def active_mock_environment(mutable_config, mutable_mock_env_path): @pytest.mark.regression("22294") def test_store_is_restored_correctly_after_bootstrap(mutable_config, tmpdir): - # Prepare a custom store path. This should be in a writeable location - # since Spack needs to initialize the DB. + """Tests that the store is correctly swapped during bootstrapping, and restored afterward.""" user_path = str(tmpdir.join("store")) - # Reassign global variables in spack.store to the value - # they would have at Spack startup. - spack.store.reinitialize() - # Set the custom user path - spack.config.set("config:install_tree:root", user_path) + with spack.store.use_store(user_path): + assert spack.store.STORE.root == user_path + assert spack.config.CONFIG.get("config:install_tree:root") == user_path + with spack.bootstrap.ensure_bootstrap_configuration(): + assert spack.store.STORE.root == spack.bootstrap.config.store_path() + assert spack.store.STORE.root == user_path + assert spack.config.CONFIG.get("config:install_tree:root") == user_path + - # Test that within the context manager we use the bootstrap store - # and that outside we restore the correct location +@pytest.mark.regression("38963") +def test_store_padding_length_is_zero_during_bootstrapping(mutable_config, tmpdir): + """Tests that, even though padded length is set in user config, the bootstrap store maintains + a padded length of zero. + """ + user_path = str(tmpdir.join("store")) + with spack.store.use_store(user_path, extra_data={"padded_length": 512}): + assert spack.config.CONFIG.get("config:install_tree:padded_length") == 512 + with spack.bootstrap.ensure_bootstrap_configuration(): + assert spack.store.STORE.root == spack.bootstrap.config.store_path() + assert spack.config.CONFIG.get("config:install_tree:padded_length") == 0 + assert spack.config.CONFIG.get("config:install_tree:padded_length") == 512 + + +@pytest.mark.regression("38963") +def test_install_tree_customization_is_respected(mutable_config, tmp_path): + """Tests that a custom user store is respected when we exit the bootstrapping + environment. + """ + spack.store.reinitialize() + store_dir = tmp_path / "store" + spack.config.CONFIG.set("config:install_tree:root", str(store_dir)) with spack.bootstrap.ensure_bootstrap_configuration(): - assert spack.store.root == spack.bootstrap.config.store_path() - assert spack.store.root == user_path + assert spack.store.STORE.root == spack.bootstrap.config.store_path() + assert ( + spack.config.CONFIG.get("config:install_tree:root") + == spack.bootstrap.config.store_path() + ) + assert spack.config.CONFIG.get("config:install_tree:padded_length") == 0 + assert spack.config.CONFIG.get("config:install_tree:root") == str(store_dir) + assert spack.store.STORE.root == str(store_dir) @pytest.mark.parametrize( @@ -153,20 +180,19 @@ def test_bootstrap_custom_store_in_environment(mutable_config, tmpdir): # Don't trigger evaluation here with spack.bootstrap.ensure_bootstrap_configuration(): pass - assert str(spack.store.root) == install_root + assert str(spack.store.STORE.root) == install_root def test_nested_use_of_context_manager(mutable_config): """Test nested use of the context manager""" - user_config = spack.config.config + user_config = spack.config.CONFIG with spack.bootstrap.ensure_bootstrap_configuration(): - assert spack.config.config != user_config + assert spack.config.CONFIG != user_config with spack.bootstrap.ensure_bootstrap_configuration(): - assert spack.config.config != user_config - assert spack.config.config == user_config + assert spack.config.CONFIG != user_config + assert spack.config.CONFIG == user_config -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") @pytest.mark.parametrize("expected_missing", [False, True]) def test_status_function_find_files( mutable_config, mock_executable, tmpdir, monkeypatch, expected_missing diff --git a/lib/spack/spack/test/build_distribution.py b/lib/spack/spack/test/build_distribution.py index 3277f0e4fbef5d..2f68ba9caeba98 100644 --- a/lib/spack/spack/test/build_distribution.py +++ b/lib/spack/spack/test/build_distribution.py @@ -5,7 +5,6 @@ import os import os.path -import sys import pytest @@ -16,7 +15,7 @@ install = spack.main.SpackCommand("install") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") def test_build_tarball_overwrite(install_mockery, mock_fetch, monkeypatch, tmpdir): diff --git a/lib/spack/spack/test/build_environment.py b/lib/spack/spack/test/build_environment.py index 99229aa6fe1e67..cbccbc429eac6f 100644 --- a/lib/spack/spack/test/build_environment.py +++ b/lib/spack/spack/test/build_environment.py @@ -6,10 +6,10 @@ import os import platform import posixpath -import sys import pytest +from llnl.path import Path, convert_to_platform_path from llnl.util.filesystem import HeaderList, LibraryList import spack.build_environment @@ -17,11 +17,12 @@ import spack.package_base import spack.spec import spack.util.spack_yaml as syaml -from spack.build_environment import _static_to_shared_library, determine_number_of_jobs, dso_suffix +from spack.build_environment import UseMode, _static_to_shared_library, dso_suffix +from spack.context import Context from spack.paths import build_env_path +from spack.util.cpus import determine_number_of_jobs from spack.util.environment import EnvironmentModifications from spack.util.executable import Executable -from spack.util.path import Path, convert_to_platform_path def os_pathsep_join(path, *pths): @@ -119,7 +120,7 @@ def __call__(self, *args, **kwargs): return mock_module_cmd -@pytest.mark.skipif(sys.platform == "win32", reason="Static to Shared not supported on Win (yet)") +@pytest.mark.not_on_windows("Static to Shared not supported on Win (yet)") def test_static_to_shared_library(build_environment): os.environ["SPACK_TEST_COMMAND"] = "dump-args" @@ -438,12 +439,12 @@ def test_parallel_false_is_not_propagating(default_mock_concretization): # b (parallel =True) s = default_mock_concretization("a foobar=bar") - spack.build_environment.set_module_variables_for_package(s.package) + spack.build_environment.set_package_py_globals(s.package) assert s["a"].package.module.make_jobs == 1 - spack.build_environment.set_module_variables_for_package(s["b"].package) + spack.build_environment.set_package_py_globals(s["b"].package) assert s["b"].package.module.make_jobs == spack.build_environment.determine_number_of_jobs( - s["b"].package.parallel + parallel=s["b"].package.parallel ) @@ -475,28 +476,62 @@ def test_setting_dtags_based_on_config(config_setting, expected_flag, config, mo def test_build_jobs_sequential_is_sequential(): assert ( - determine_number_of_jobs(parallel=False, command_line=8, config_default=8, max_cpus=8) == 1 + determine_number_of_jobs( + parallel=False, + max_cpus=8, + config=spack.config.Configuration( + spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 8}}), + spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 8}}), + ), + ) + == 1 ) def test_build_jobs_command_line_overrides(): assert ( - determine_number_of_jobs(parallel=True, command_line=10, config_default=1, max_cpus=1) + determine_number_of_jobs( + parallel=True, + max_cpus=1, + config=spack.config.Configuration( + spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 10}}), + spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 1}}), + ), + ) == 10 ) assert ( - determine_number_of_jobs(parallel=True, command_line=10, config_default=100, max_cpus=100) + determine_number_of_jobs( + parallel=True, + max_cpus=100, + config=spack.config.Configuration( + spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 10}}), + spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 100}}), + ), + ) == 10 ) def test_build_jobs_defaults(): assert ( - determine_number_of_jobs(parallel=True, command_line=None, config_default=1, max_cpus=10) + determine_number_of_jobs( + parallel=True, + max_cpus=10, + config=spack.config.Configuration( + spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 1}}) + ), + ) == 1 ) assert ( - determine_number_of_jobs(parallel=True, command_line=None, config_default=100, max_cpus=10) + determine_number_of_jobs( + parallel=True, + max_cpus=10, + config=spack.config.Configuration( + spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 100}}) + ), + ) == 10 ) @@ -541,3 +576,94 @@ def test_setting_attributes(self, default_mock_concretization): if current_module == spack.package_base: break assert current_module.SOME_ATTRIBUTE == 1 + + +def test_effective_deptype_build_environment(default_mock_concretization): + s = default_mock_concretization("dttop") + + # [ ] dttop@1.0 # + # [b ] ^dtbuild1@1.0 # <- direct build dep + # [b ] ^dtbuild2@1.0 # <- indirect build-only dep is dropped + # [bl ] ^dtlink2@1.0 # <- linkable, and runtime dep of build dep + # [ r ] ^dtrun2@1.0 # <- non-linkable, exectuable runtime dep of build dep + # [bl ] ^dtlink1@1.0 # <- direct build dep + # [bl ] ^dtlink3@1.0 # <- linkable, and runtime dep of build dep + # [b ] ^dtbuild2@1.0 # <- indirect build-only dep is dropped + # [bl ] ^dtlink4@1.0 # <- linkable, and runtime dep of build dep + # [ r ] ^dtrun1@1.0 # <- run-only dep is pruned (should it be in PATH?) + # [bl ] ^dtlink5@1.0 # <- children too + # [ r ] ^dtrun3@1.0 # <- children too + # [b ] ^dtbuild3@1.0 # <- children too + + expected_flags = { + "dttop": UseMode.ROOT, + "dtbuild1": UseMode.BUILDTIME_DIRECT, + "dtlink1": UseMode.BUILDTIME_DIRECT | UseMode.BUILDTIME, + "dtlink3": UseMode.BUILDTIME | UseMode.RUNTIME, + "dtlink4": UseMode.BUILDTIME | UseMode.RUNTIME, + "dtrun2": UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE, + "dtlink2": UseMode.RUNTIME, + } + + for spec, effective_type in spack.build_environment.effective_deptypes( + s, context=Context.BUILD + ): + assert effective_type & expected_flags.pop(spec.name) == effective_type + assert not expected_flags, f"Missing {expected_flags.keys()} from effective_deptypes" + + +def test_effective_deptype_run_environment(default_mock_concretization): + s = default_mock_concretization("dttop") + + # [ ] dttop@1.0 # + # [b ] ^dtbuild1@1.0 # <- direct build-only dep is pruned + # [b ] ^dtbuild2@1.0 # <- children too + # [bl ] ^dtlink2@1.0 # <- children too + # [ r ] ^dtrun2@1.0 # <- children too + # [bl ] ^dtlink1@1.0 # <- runtime, not executable + # [bl ] ^dtlink3@1.0 # <- runtime, not executable + # [b ] ^dtbuild2@1.0 # <- indirect build only dep is pruned + # [bl ] ^dtlink4@1.0 # <- runtime, not executable + # [ r ] ^dtrun1@1.0 # <- runtime and executable + # [bl ] ^dtlink5@1.0 # <- runtime, not executable + # [ r ] ^dtrun3@1.0 # <- runtime and executable + # [b ] ^dtbuild3@1.0 # <- indirect build-only dep is pruned + + expected_flags = { + "dttop": UseMode.ROOT, + "dtlink1": UseMode.RUNTIME, + "dtlink3": UseMode.BUILDTIME | UseMode.RUNTIME, + "dtlink4": UseMode.BUILDTIME | UseMode.RUNTIME, + "dtrun1": UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE, + "dtlink5": UseMode.RUNTIME, + "dtrun3": UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE, + } + + for spec, effective_type in spack.build_environment.effective_deptypes(s, context=Context.RUN): + assert effective_type & expected_flags.pop(spec.name) == effective_type + assert not expected_flags, f"Missing {expected_flags.keys()} from effective_deptypes" + + +def test_monkey_patching_works_across_virtual(default_mock_concretization): + """Assert that a monkeypatched attribute is found regardless we access through the + real name or the virtual name. + """ + s = default_mock_concretization("mpileaks ^mpich") + s["mpich"].foo = "foo" + assert s["mpich"].foo == "foo" + assert s["mpi"].foo == "foo" + + +def test_clear_compiler_related_runtime_variables_of_build_deps(default_mock_concretization): + """Verify that Spack drops CC, CXX, FC and F77 from the dependencies related build environment + variable changes if they are set in setup_run_environment. Spack manages those variables + elsewhere.""" + s = default_mock_concretization("build-env-compiler-var-a") + ctx = spack.build_environment.SetupContext(s, context=Context.BUILD) + result = {} + ctx.get_env_modifications().apply_modifications(result) + assert "CC" not in result + assert "CXX" not in result + assert "FC" not in result + assert "F77" not in result + assert result["ANOTHER_VAR"] == "this-should-be-present" diff --git a/lib/spack/spack/test/build_systems.py b/lib/spack/spack/test/build_systems.py index afb9f51b773fdf..ccb7a08f6cc29c 100644 --- a/lib/spack/spack/test/build_systems.py +++ b/lib/spack/spack/test/build_systems.py @@ -5,7 +5,6 @@ import glob import os -import sys import py.path import pytest @@ -23,8 +22,6 @@ DATA_PATH = os.path.join(spack.paths.test_path, "data") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") - @pytest.fixture() def concretize_and_setup(default_mock_concretization): @@ -45,6 +42,7 @@ def _func(dir_str): return _func +@pytest.mark.not_on_windows("make not available on Windows") @pytest.mark.usefixtures("config", "mock_packages", "working_env") class TestTargets: @pytest.mark.parametrize( @@ -93,6 +91,7 @@ def test_negative_ninja_check(self, input_dir, test_dir, concretize_and_setup): s.package._if_ninja_target_execute("check") +@pytest.mark.not_on_windows("autotools not available on windows") @pytest.mark.usefixtures("config", "mock_packages") class TestAutotoolsPackage: def test_with_or_without(self, default_mock_concretization): @@ -311,6 +310,16 @@ def test_define_from_variant(self): with pytest.raises(KeyError, match="not a variant"): s.package.define_from_variant("NONEXISTENT") + def test_cmake_std_args_cuda(self, default_mock_concretization): + s = default_mock_concretization("vtk-m +cuda cuda_arch=70 ^cmake@3.23") + option = spack.build_systems.cmake.CMakeBuilder.define_cuda_architectures(s.package) + assert "-DCMAKE_CUDA_ARCHITECTURES:STRING=70" == option + + def test_cmake_std_args_hip(self, default_mock_concretization): + s = default_mock_concretization("vtk-m +rocm amdgpu_target=gfx900 ^cmake@3.23") + option = spack.build_systems.cmake.CMakeBuilder.define_hip_architectures(s.package) + assert "-DCMAKE_HIP_ARCHITECTURES:STRING=gfx900" == option + @pytest.mark.usefixtures("config", "mock_packages") class TestDownloadMixins: diff --git a/lib/spack/spack/test/builder.py b/lib/spack/spack/test/builder.py index 96944dc3ac1b51..9be5b1ecbb338d 100644 --- a/lib/spack/spack/test/builder.py +++ b/lib/spack/spack/test/builder.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os.path -import sys import pytest @@ -106,10 +105,7 @@ def test_old_style_compatibility_with_super(spec_str, method_name, expected): assert value == expected -@pytest.mark.skipif( - sys.platform == "win32", - reason="log_ouput cannot currently be used outside of subprocess on Windows", -) +@pytest.mark.not_on_windows("log_ouput cannot currently be used outside of subprocess on Windows") @pytest.mark.regression("33928") @pytest.mark.usefixtures("builder_test_repository", "config", "working_env") @pytest.mark.disable_clean_stage_check @@ -153,7 +149,7 @@ def test_monkey_patching_test_log_file(): # Windows context manager's __exit__ fails with ValueError ("I/O operation # on closed file"). -@pytest.mark.skipif(sys.platform == "win32", reason="Does not run on windows") +@pytest.mark.not_on_windows("Does not run on windows") def test_install_time_test_callback(tmpdir, config, mock_packages, mock_stage): """Confirm able to run stand-alone test as a post-install callback.""" s = spack.spec.Spec("py-test-callback").concretized() diff --git a/lib/spack/spack/test/buildrequest.py b/lib/spack/spack/test/buildrequest.py index 465ec19606212e..6a022e4fb53e4e 100644 --- a/lib/spack/spack/test/buildrequest.py +++ b/lib/spack/spack/test/buildrequest.py @@ -3,27 +3,20 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys - import pytest +import spack.deptypes as dt import spack.installer as inst import spack.repo import spack.spec -# Spack functionality tested here should work on Windows, -# however, tests are currently failing because support -# for Spack on Windows has not been extended to this -# module yet. -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") - def test_build_request_errors(install_mockery): with pytest.raises(ValueError, match="must be a package"): inst.BuildRequest("abc", {}) spec = spack.spec.Spec("trivial-install-test-package") - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) with pytest.raises(ValueError, match="must have a concrete spec"): inst.BuildRequest(pkg_cls(spec), {}) @@ -67,10 +60,10 @@ def test_build_request_strings(install_mockery): @pytest.mark.parametrize( "package_cache_only,dependencies_cache_only,package_deptypes,dependencies_deptypes", [ - (False, False, ["build", "link", "run"], ["build", "link", "run"]), - (True, False, ["link", "run"], ["build", "link", "run"]), - (False, True, ["build", "link", "run"], ["link", "run"]), - (True, True, ["link", "run"], ["link", "run"]), + (False, False, dt.BUILD | dt.LINK | dt.RUN, dt.BUILD | dt.LINK | dt.RUN), + (True, False, dt.LINK | dt.RUN, dt.BUILD | dt.LINK | dt.RUN), + (False, True, dt.BUILD | dt.LINK | dt.RUN, dt.LINK | dt.RUN), + (True, True, dt.LINK | dt.RUN, dt.LINK | dt.RUN), ], ) def test_build_request_deptypes( @@ -90,8 +83,8 @@ def test_build_request_deptypes( }, ) - actual_package_deptypes = build_request.get_deptypes(s.package) - actual_dependency_deptypes = build_request.get_deptypes(s["dependency-install"].package) + actual_package_deptypes = build_request.get_depflags(s.package) + actual_dependency_deptypes = build_request.get_depflags(s["dependency-install"].package) - assert sorted(actual_package_deptypes) == package_deptypes - assert sorted(actual_dependency_deptypes) == dependencies_deptypes + assert actual_package_deptypes == package_deptypes + assert actual_dependency_deptypes == dependencies_deptypes diff --git a/lib/spack/spack/test/buildtask.py b/lib/spack/spack/test/buildtask.py index 45fba6a20abf07..6e25c3d6accefa 100644 --- a/lib/spack/spack/test/buildtask.py +++ b/lib/spack/spack/test/buildtask.py @@ -15,7 +15,7 @@ def test_build_task_errors(install_mockery): inst.BuildTask("abc", None, False, 0, 0, 0, []) spec = spack.spec.Spec("trivial-install-test-package") - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) with pytest.raises(ValueError, match="must have a concrete spec"): inst.BuildTask(pkg_cls(spec), None, False, 0, 0, 0, []) diff --git a/lib/spack/spack/test/cache_fetch.py b/lib/spack/spack/test/cache_fetch.py index 7fb5efc8951de8..f4bf0ba62ded23 100644 --- a/lib/spack/spack/test/cache_fetch.py +++ b/lib/spack/spack/test/cache_fetch.py @@ -31,13 +31,16 @@ def test_fetch_missing_cache(tmpdir, _fetch_method): @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) def test_fetch(tmpdir, _fetch_method): """Ensure a fetch after expanding is effectively a no-op.""" - testpath = str(tmpdir) - cache = os.path.join(testpath, "cache.tar.gz") + cache_dir = tmpdir.join("cache") + stage_dir = tmpdir.join("stage") + mkdirp(cache_dir) + mkdirp(stage_dir) + cache = os.path.join(cache_dir, "cache.tar.gz") touch(cache) url = url_util.path_to_file_url(cache) with spack.config.override("config:url_fetch_method", _fetch_method): fetcher = CacheURLFetchStrategy(url=url) - with Stage(fetcher, path=testpath) as stage: + with Stage(fetcher, path=str(stage_dir)) as stage: source_path = stage.source_path mkdirp(source_path) fetcher.fetch() diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 9c153c2d06ec44..6ba50004424e19 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -8,7 +8,6 @@ arguments correctly. """ import os -import sys import pytest @@ -145,7 +144,7 @@ + test_args_without_paths ) -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.fixture(scope="function") diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index 3ce64f77953749..53ed8382520745 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -5,7 +5,6 @@ import itertools import os import subprocess -import sys import pytest @@ -35,7 +34,7 @@ def test_urlencode_string(): assert ci._url_encode_string("Spack Test Project") == "Spack+Test+Project" -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_import_signing_key(mock_gnupghome): signing_key_dir = spack_paths.mock_gpg_keys_path signing_key_path = os.path.join(signing_key_dir, "package-signing-key") @@ -427,18 +426,14 @@ def test_affected_specs_on_first_concretization(mutable_mock_env_path, mock_pack assert len(mpileaks_specs) == 2, e.all_specs() -@pytest.mark.skipif( - sys.platform == "win32", reason="Reliance on bash script not supported on Windows" -) +@pytest.mark.not_on_windows("Reliance on bash script not supported on Windows") def test_ci_process_command(repro_dir): result = ci.process_command("help", commands=[], repro_dir=str(repro_dir)) help_sh = repro_dir / "help.sh" assert help_sh.exists() and not result -@pytest.mark.skipif( - sys.platform == "win32", reason="Reliance on bash script not supported on Windows" -) +@pytest.mark.not_on_windows("Reliance on bash script not supported on Windows") def test_ci_process_command_fail(repro_dir, monkeypatch): msg = "subprocess wait exception" @@ -456,10 +451,7 @@ def test_ci_create_buildcache(tmpdir, working_env, config, mock_packages, monkey monkeypatch.setattr(spack.ci, "push_mirror_contents", lambda a, b, c: True) results = ci.create_buildcache( - None, - pr_pipeline=True, - buildcache_mirror_url="file:///fake-url-one", - pipeline_mirror_url="file:///fake-url-two", + None, destination_mirror_urls=["file:///fake-url-one", "file:///fake-url-two"] ) assert len(results) == 2 @@ -469,9 +461,7 @@ def test_ci_create_buildcache(tmpdir, working_env, config, mock_packages, monkey assert result2.success assert result2.url == "file:///fake-url-two" - results = ci.create_buildcache( - None, pr_pipeline=True, buildcache_mirror_url="file:///fake-url-one" - ) + results = ci.create_buildcache(None, destination_mirror_urls=["file:///fake-url-one"]) assert len(results) == 1 assert results[0].success @@ -492,9 +482,7 @@ def test_ci_run_standalone_tests_missing_requirements( assert "Reproduction directory is required" in err -@pytest.mark.skipif( - sys.platform == "win32", reason="Reliance on bash script not supported on Windows" -) +@pytest.mark.not_on_windows("Reliance on bash script not supported on Windows") def test_ci_run_standalone_tests_not_installed_junit( tmp_path, repro_dir, working_env, default_mock_concretization, mock_test_stage, capfd ): @@ -512,9 +500,7 @@ def test_ci_run_standalone_tests_not_installed_junit( assert os.path.getsize(log_file) > 0 -@pytest.mark.skipif( - sys.platform == "win32", reason="Reliance on bash script not supported on Windows" -) +@pytest.mark.not_on_windows("Reliance on bash script not supported on Windows") def test_ci_run_standalone_tests_not_installed_cdash( tmp_path, repro_dir, working_env, default_mock_concretization, mock_test_stage, capfd ): diff --git a/lib/spack/spack/test/cmd/blame.py b/lib/spack/spack/test/cmd/blame.py index f34bc142e05d37..05e475eed300d2 100644 --- a/lib/spack/spack/test/cmd/blame.py +++ b/lib/spack/spack/test/cmd/blame.py @@ -3,8 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys - import pytest from llnl.util.filesystem import working_dir @@ -35,7 +33,7 @@ def test_blame_by_percent(mock_packages): assert "EMAIL" in out -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_blame_file(mock_packages): """Sanity check the blame command to make sure it works.""" with working_dir(spack.paths.prefix): @@ -68,7 +66,7 @@ def test_blame_json(mock_packages): assert key in loaded["authors"][0] -@pytest.mark.skipif(sys.platform == "win32", reason="git hangs") +@pytest.mark.not_on_windows("git hangs") def test_blame_by_git(mock_packages, capfd): """Sanity check the blame command to make sure it works.""" with capfd.disabled(): diff --git a/lib/spack/spack/test/cmd/bootstrap.py b/lib/spack/spack/test/cmd/bootstrap.py index d0b572e21c68d0..277a279222b8d7 100644 --- a/lib/spack/spack/test/cmd/bootstrap.py +++ b/lib/spack/spack/test/cmd/bootstrap.py @@ -7,13 +7,14 @@ import pytest +from llnl.path import convert_to_posix_path + import spack.bootstrap import spack.bootstrap.core import spack.config import spack.environment as ev import spack.main import spack.mirror -from spack.util.path import convert_to_posix_path _bootstrap = spack.main.SpackCommand("bootstrap") @@ -50,7 +51,7 @@ def test_reset_in_file_scopes(mutable_config, scopes): bootstrap_yaml_files = [] for s in scopes: _bootstrap("disable", "--scope={0}".format(s)) - scope_path = spack.config.config.scopes[s].path + scope_path = spack.config.CONFIG.scopes[s].path bootstrap_yaml = os.path.join(scope_path, "bootstrap.yaml") assert os.path.exists(bootstrap_yaml) bootstrap_yaml_files.append(bootstrap_yaml) @@ -80,7 +81,7 @@ def test_reset_in_environment(mutable_mock_env_path, mutable_config): def test_reset_in_file_scopes_overwrites_backup_files(mutable_config): # Create a bootstrap.yaml with some config _bootstrap("disable", "--scope=site") - scope_path = spack.config.config.scopes["site"].path + scope_path = spack.config.CONFIG.scopes["site"].path bootstrap_yaml = os.path.join(scope_path, "bootstrap.yaml") assert os.path.exists(bootstrap_yaml) @@ -99,7 +100,7 @@ def test_reset_in_file_scopes_overwrites_backup_files(mutable_config): assert os.path.exists(backup_file) -def test_list_sources(capsys): +def test_list_sources(config, capsys): # Get the merged list and ensure we get our defaults with capsys.disabled(): output = _bootstrap("list") @@ -168,13 +169,13 @@ def test_remove_and_add_a_source(mutable_config): assert not sources # Add it back and check we restored the initial state - _bootstrap("add", "github-actions", "$spack/share/spack/bootstrap/github-actions-v0.3") + _bootstrap("add", "github-actions", "$spack/share/spack/bootstrap/github-actions-v0.5") sources = spack.bootstrap.core.bootstrapping_sources() assert len(sources) == 1 @pytest.mark.maybeslow -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_bootstrap_mirror_metadata(mutable_config, linux_os, monkeypatch, tmpdir): """Test that `spack bootstrap mirror` creates a folder that can be ingested by `spack bootstrap add`. Here we don't download data, since that would be an diff --git a/lib/spack/spack/test/cmd/build_env.py b/lib/spack/spack/test/cmd/build_env.py index c27b292cb199dc..286fcd5e1101b3 100644 --- a/lib/spack/spack/test/cmd/build_env.py +++ b/lib/spack/spack/test/cmd/build_env.py @@ -54,7 +54,7 @@ def test_pickle(tmpdir): with tmpdir.as_cwd(): build_env("--pickle", _out_file, "zlib") environment = pickle.load(open(_out_file, "rb")) - assert type(environment) == dict + assert isinstance(environment, dict) assert "PATH" in environment diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index 85765c96acf259..55ec605913b3f4 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -5,9 +5,7 @@ import errno import os -import platform import shutil -import sys import pytest @@ -27,7 +25,7 @@ mirror = spack.main.SpackCommand("mirror") uninstall = spack.main.SpackCommand("uninstall") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.fixture() @@ -49,11 +47,8 @@ def mock_get_specs_multiarch(database, monkeypatch): monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", lambda: specs) -@pytest.mark.skipif( - platform.system().lower() != "linux", reason="implementation for MacOS still missing" -) -@pytest.mark.db -def test_buildcache_preview_just_runs(database): +def test_buildcache_preview_just_runs(): + # TODO: remove in Spack 0.21 buildcache("preview", "mpileaks") @@ -159,7 +154,7 @@ def test_update_key_index( # Put installed package in the buildcache, which, because we're signing # it, should result in the public key getting pushed to the buildcache # as well. - buildcache("push", "-a", mirror_dir.strpath, s.name) + buildcache("push", mirror_dir.strpath, s.name) # Now make sure that when we pass the "--keys" argument to update-index # it causes the index to get update. @@ -213,13 +208,13 @@ def verify_mirror_contents(): # Install a package and put it in the buildcache s = Spec(out_env_pkg).concretized() install(s.name) - buildcache("push", "-u", "-f", "-a", src_mirror_url, s.name) + buildcache("push", "-u", "-f", src_mirror_url, s.name) env("create", "test") with ev.read("test"): add(in_env_pkg) install() - buildcache("push", "-u", "-f", "-a", src_mirror_url, in_env_pkg) + buildcache("push", "-u", "-f", src_mirror_url, in_env_pkg) # Now run the spack buildcache sync command with all the various options # for specifying mirrors @@ -331,4 +326,8 @@ def fake_push(node, push_url, options): buildcache(*buildcache_create_args) - assert packages_to_push == expected + # Order is not guaranteed, so we can't just compare lists + assert set(packages_to_push) == set(expected) + + # Ensure no duplicates + assert len(set(packages_to_push)) == len(packages_to_push) diff --git a/lib/spack/spack/test/cmd/checksum.py b/lib/spack/spack/test/cmd/checksum.py index 848d31252ada40..0dbaa88053070a 100644 --- a/lib/spack/spack/test/cmd/checksum.py +++ b/lib/spack/spack/test/cmd/checksum.py @@ -4,15 +4,16 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse -import sys import pytest -import llnl.util.tty as tty - import spack.cmd.checksum +import spack.parser import spack.repo +import spack.spec from spack.main import SpackCommand +from spack.stage import interactive_version_filter +from spack.version import Version spack_checksum = SpackCommand("checksum") @@ -20,60 +21,220 @@ @pytest.mark.parametrize( "arguments,expected", [ - (["--batch", "patch"], (True, False, False, False)), - (["--latest", "patch"], (False, True, False, False)), - (["--preferred", "patch"], (False, False, True, False)), - (["--add-to-package", "patch"], (False, False, False, True)), + (["--batch", "patch"], (True, False, False, False, False)), + (["--latest", "patch"], (False, True, False, False, False)), + (["--preferred", "patch"], (False, False, True, False, False)), + (["--add-to-package", "patch"], (False, False, False, True, False)), + (["--verify", "patch"], (False, False, False, False, True)), ], ) def test_checksum_args(arguments, expected): parser = argparse.ArgumentParser() spack.cmd.checksum.setup_parser(parser) args = parser.parse_args(arguments) - check = args.batch, args.latest, args.preferred, args.add_to_package + check = args.batch, args.latest, args.preferred, args.add_to_package, args.verify assert check == expected -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.parametrize( "arguments,expected", [ (["--batch", "preferred-test"], "version of preferred-test"), (["--latest", "preferred-test"], "Found 1 version"), (["--preferred", "preferred-test"], "Found 1 version"), - (["--add-to-package", "preferred-test"], "Added 1 new versions to"), + (["--add-to-package", "preferred-test"], "Added 0 new versions to"), + (["--verify", "preferred-test"], "Verified 1 of 1"), + (["--verify", "zlib", "1.2.13"], "1.2.13 [-] No previous checksum"), ], ) def test_checksum(arguments, expected, mock_packages, mock_clone_repo, mock_stage): output = spack_checksum(*arguments) assert expected in output - assert "version(" in output + # --verify doesn't print versions strings like other flags + if "--verify" not in arguments: + assert "version(" in output -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") -def test_checksum_interactive(mock_packages, mock_fetch, mock_stage, monkeypatch): - # TODO: mock_fetch doesn't actually work with stage, working around with ignoring - # fail_on_error for now - def _get_number(*args, **kwargs): - return 1 - monkeypatch.setattr(tty, "get_number", _get_number) +def input_from_commands(*commands): + """Create a function that returns the next command from a list of inputs for interactive spack + checksum. If None is encountered, this is equivalent to EOF / ^D.""" + commands = iter(commands) + + def _input(prompt): + cmd = next(commands) + if cmd is None: + raise EOFError + assert isinstance(cmd, str) + return cmd + + return _input + + +def test_checksum_interactive_filter(): + # Filter effectively by 1:1.0, then checksum. + input = input_from_commands("f", "@1:", "f", "@:1.0", "c") + assert interactive_version_filter( + { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + }, + input=input, + ) == { + Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + } + + +def test_checksum_interactive_return_from_filter_prompt(): + # Enter and then exit filter subcommand. + input = input_from_commands("f", None, "c") + assert interactive_version_filter( + { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + }, + input=input, + ) == { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + } + + +def test_checksum_interactive_quit_returns_none(): + # Quit after filtering something out (y to confirm quit) + input = input_from_commands("f", "@1:", "q", "y") + assert ( + interactive_version_filter( + { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + }, + input=input, + ) + is None + ) + + +def test_checksum_interactive_reset_resets(): + # Filter 1:, then reset, then filter :0, should just given 0.9 (it was filtered out + # before reset) + input = input_from_commands("f", "@1:", "r", "f", ":0", "c") + assert interactive_version_filter( + { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + }, + input=input, + ) == {Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz"} + + +def test_checksum_interactive_ask_each(): + # Ask each should run on the filtered list. First select 1.x, then select only the second + # entry, which is 1.0.1. + input = input_from_commands("f", "@1:", "a", "n", "y", "n") + assert interactive_version_filter( + { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + }, + input=input, + ) == {Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz"} - output = spack_checksum("preferred-test", fail_on_error=False) - assert "version of preferred-test" in output - assert "version(" in output + +def test_checksum_interactive_quit_from_ask_each(): + # Enter ask each mode, select the second item, then quit from submenu, then checksum, which + # should still include the last item at which ask each stopped. + input = input_from_commands("a", "n", "y", None, "c") + assert interactive_version_filter( + { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + }, + input=input, + ) == { + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + } + + +def test_checksum_interactive_nothing_left(): + """If nothing is left after interactive filtering, return an empty dict.""" + input = input_from_commands("f", "@2", "c") + assert ( + interactive_version_filter( + { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + }, + input=input, + ) + == {} + ) + + +def test_checksum_interactive_new_only(): + # The 1.0 version is known already, and should be dropped on `n`. + input = input_from_commands("n", "c") + assert interactive_version_filter( + { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + }, + known_versions=[Version("1.0")], + input=input, + ) == { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + } + + +def test_checksum_interactive_top_n(): + """Test integers select top n versions""" + input = input_from_commands("2", "c") + assert interactive_version_filter( + { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", + }, + input=input, + ) == { + Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", + Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", + } + + +def test_checksum_interactive_unrecognized_command(): + """Unrecognized commands should be ignored""" + input = input_from_commands("-1", "0", "hello", "c") + v = {Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz"} + assert interactive_version_filter(v.copy(), input=input) == v def test_checksum_versions(mock_packages, mock_clone_repo, mock_fetch, mock_stage): - pkg_cls = spack.repo.path.get_pkg_class("preferred-test") - versions = [str(v) for v in pkg_cls.versions if not v.isdevelop()] - output = spack_checksum("preferred-test", versions[0]) - assert "Found 1 version" in output + pkg_cls = spack.repo.PATH.get_pkg_class("zlib") + versions = [str(v) for v in pkg_cls.versions] + output = spack_checksum("zlib", *versions) + assert "Found 3 versions" in output assert "version(" in output - output = spack_checksum("--add-to-package", "preferred-test", versions[0]) - assert "Found 1 version" in output - assert "version(" in output - assert "Added 1 new versions to" in output + output = spack_checksum("--add-to-package", "zlib", *versions) + assert "Found 3 versions" in output + assert "Added 0 new versions to" in output def test_checksum_missing_version(mock_packages, mock_clone_repo, mock_fetch, mock_stage): @@ -91,4 +252,23 @@ def test_checksum_deprecated_version(mock_packages, mock_clone_repo, mock_fetch, "--add-to-package", "deprecated-versions", "1.1.0", fail_on_error=False ) assert "Version 1.1.0 is deprecated" in output - assert "Added 1 new versions to" not in output + assert "Added 0 new versions to" not in output + + +def test_checksum_url(mock_packages): + pkg_cls = spack.repo.PATH.get_pkg_class("zlib") + with pytest.raises(spack.parser.SpecSyntaxError): + spack_checksum(f"{pkg_cls.url}") + + +def test_checksum_verification_fails(install_mockery, capsys): + spec = spack.spec.Spec("zlib").concretized() + pkg = spec.package + versions = list(pkg.versions.keys()) + version_hashes = {versions[0]: "abadhash", spack.version.Version("0.1"): "123456789"} + with pytest.raises(SystemExit): + spack.cmd.checksum.print_checksum_status(pkg, version_hashes) + out = str(capsys.readouterr()) + assert out.count("Correct") == 0 + assert "No previous checksum" in out + assert "Invalid checksum" in out diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 818bf1850e6bed..84e9e66bf05576 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -7,7 +7,6 @@ import json import os import shutil -import sys import jsonschema import pytest @@ -41,10 +40,7 @@ uninstall_cmd = spack.main.SpackCommand("uninstall") buildcache_cmd = spack.main.SpackCommand("buildcache") -pytestmark = [ - pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows"), - pytest.mark.maybeslow, -] +pytestmark = [pytest.mark.not_on_windows("does not run on windows"), pytest.mark.maybeslow] @pytest.fixture() @@ -215,6 +211,10 @@ def test_ci_generate_with_env( with open(outputfile) as f: contents = f.read() yaml_contents = syaml.load(contents) + assert "workflow" in yaml_contents + assert "rules" in yaml_contents["workflow"] + assert yaml_contents["workflow"]["rules"] == [{"when": "always"}] + assert "stages" in yaml_contents assert len(yaml_contents["stages"]) == 5 assert yaml_contents["stages"][0] == "stage-0" @@ -856,6 +856,43 @@ def mystrip(s): env_cmd("deactivate") +def test_ci_require_signing( + tmpdir, working_env, mutable_mock_env_path, mock_gnupghome, ci_base_environment +): + spack_yaml_contents = """ +spack: + specs: + - archive-files + mirrors: + test-mirror: file:///no-such-mirror + ci: + pipeline-gen: + - submapping: + - match: + - archive-files + build-job: + tags: + - donotcare + image: donotcare +""" + filename = str(tmpdir.join("spack.yaml")) + with open(filename, "w") as f: + f.write(spack_yaml_contents) + + with tmpdir.as_cwd(): + env_cmd("activate", "--without-view", "--sh", "-d", ".") + + # Run without the variable to make sure we don't accidentally require signing + output = ci_cmd("rebuild", output=str, fail_on_error=False) + assert "spack must have exactly one signing key" not in output + + # Now run with the variable to make sure it works + os.environ.update({"SPACK_REQUIRE_SIGNING": "True"}) + output = ci_cmd("rebuild", output=str, fail_on_error=False) + + assert "spack must have exactly one signing key" in output + + def test_ci_nothing_to_rebuild( tmpdir, working_env, @@ -895,7 +932,7 @@ def test_ci_nothing_to_rebuild( ) install_cmd("archive-files") - buildcache_cmd("push", "-a", "-f", "-u", mirror_url, "archive-files") + buildcache_cmd("push", "-f", "-u", mirror_url, "archive-files") filename = str(tmpdir.join("spack.yaml")) with open(filename, "w") as f: @@ -1043,14 +1080,17 @@ def test_push_mirror_contents( ci.import_signing_key(_signing_key()) - spack_yaml_contents = """ + with tmpdir.as_cwd(): + with open("spack.yaml", "w") as f: + f.write( + f"""\ spack: definitions: - packages: [patchelf] specs: - $packages mirrors: - test-mirror: {0} + test-mirror: {mirror_url} ci: enable-artifacts-buildcache: True pipeline-gen: @@ -1070,15 +1110,8 @@ def test_push_mirror_contents( - nonbuildtag image: basicimage custom_attribute: custom! -""".format( - mirror_url - ) - - filename = str(tmpdir.join("spack.yaml")) - with open(filename, "w") as f: - f.write(spack_yaml_contents) - - with tmpdir.as_cwd(): +""" + ) env_cmd("create", "test", "./spack.yaml") with ev.read("test"): concrete_spec = Spec("patchelf").concretized() @@ -1089,7 +1122,8 @@ def test_push_mirror_contents( install_cmd("--add", "--keep-stage", json_path) - ci.push_mirror_contents(concrete_spec, mirror_url, True) + for s in concrete_spec.traverse(): + ci.push_mirror_contents(s, mirror_url, True) buildcache_path = os.path.join(mirror_dir.strpath, "build_cache") @@ -1102,9 +1136,9 @@ def test_push_mirror_contents( with open(outputfile_pruned) as f: contents = f.read() yaml_contents = syaml.load(contents) - assert "no-specs-to-rebuild" in yaml_contents # Make sure there are no other spec jobs or rebuild-index - assert len(yaml_contents.keys()) == 1 + assert set(yaml_contents.keys()) == {"no-specs-to-rebuild", "workflow"} + the_elt = yaml_contents["no-specs-to-rebuild"] assert "tags" in the_elt assert "nonbuildtag" in the_elt["tags"] @@ -1112,6 +1146,9 @@ def test_push_mirror_contents( assert the_elt["image"] == "basicimage" assert the_elt["custom_attribute"] == "custom!" + assert "rules" in yaml_contents["workflow"] + assert yaml_contents["workflow"]["rules"] == [{"when": "always"}] + outputfile_not_pruned = str(tmpdir.join("unpruned_pipeline.yml")) ci_cmd("generate", "--no-prune-dag", "--output-file", outputfile_not_pruned) @@ -1458,7 +1495,7 @@ def test_ci_rebuild_index( ypfd.write(spec_json) install_cmd("--add", "--keep-stage", "-f", json_path) - buildcache_cmd("push", "-u", "-a", "-f", mirror_url, "callpath") + buildcache_cmd("push", "-u", "-f", mirror_url, "callpath") ci_cmd("rebuild-index") buildcache_path = os.path.join(mirror_dir.strpath, "build_cache") @@ -1950,8 +1987,7 @@ def test_ci_reproduce( ci_cmd("generate", "--output-file", pipeline_path, "--artifacts-root", artifacts_root) - target_name = spack.platforms.test.Test.default - job_name = ci.get_job_name(job_spec, "test-debian6-%s" % target_name, None) + job_name = ci.get_job_name(job_spec) repro_file = os.path.join(working_dir.strpath, "repro.json") repro_details = { @@ -1985,10 +2021,10 @@ def fake_download_and_extract_artifacts(url, work_dir): working_dir.strpath, output=str, ) - expect_out = "docker run --rm --name spack_reproducer -v {0}:{0}:Z -ti {1}".format( - os.path.realpath(working_dir.strpath), image_name - ) - + # Make sure the script was generated + assert os.path.exists(os.path.join(os.path.realpath(working_dir.strpath), "start.sh")) + # Make sure we tell the suer where it is when not in interactive mode + expect_out = "$ {0}/start.sh".format(os.path.realpath(working_dir.strpath)) assert expect_out in rep_out @@ -2173,3 +2209,50 @@ def test_gitlab_config_scopes( assert all([t in rebuild_tags for t in ["spack", "service"]]) expected_vars = ["CI_JOB_SIZE", "KUBERNETES_CPU_REQUEST", "KUBERNETES_MEMORY_REQUEST"] assert all([v in rebuild_vars for v in expected_vars]) + + +def test_ci_generate_mirror_config( + tmpdir, + mutable_mock_env_path, + install_mockery, + mock_packages, + monkeypatch, + ci_base_environment, + mock_binary_index, +): + """Make sure the correct mirror gets used as the buildcache destination""" + filename = str(tmpdir.join("spack.yaml")) + with open(filename, "w") as f: + f.write( + """\ +spack: + specs: + - archive-files + mirrors: + some-mirror: file:///this/is/a/source/mirror + buildcache-destination: file:///push/binaries/here + ci: + pipeline-gen: + - submapping: + - match: + - archive-files + build-job: + tags: + - donotcare + image: donotcare +""" + ) + + with tmpdir.as_cwd(): + env_cmd("create", "test", "./spack.yaml") + outputfile = str(tmpdir.join(".gitlab-ci.yml")) + + with ev.read("test"): + ci_cmd("generate", "--output-file", outputfile) + with open(outputfile) as of: + pipeline_doc = syaml.load(of.read()) + assert "rebuild-index" in pipeline_doc + reindex_job = pipeline_doc["rebuild-index"] + assert "script" in reindex_job + reindex_step = reindex_job["script"][0] + assert "file:///push/binaries/here" in reindex_step diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index 36710cd546ed22..f0cd58ee8434f2 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -4,21 +4,20 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import sys import pytest import llnl.util.filesystem as fs import spack.caches +import spack.cmd.clean import spack.main import spack.package_base import spack.stage +import spack.store clean = spack.main.SpackCommand("clean") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") - @pytest.fixture() def mock_calls_for_clean(monkeypatch): @@ -34,9 +33,9 @@ def __call__(self, *args, **kwargs): monkeypatch.setattr(spack.package_base.PackageBase, "do_clean", Counter("package")) monkeypatch.setattr(spack.stage, "purge", Counter("stages")) - monkeypatch.setattr(spack.caches.fetch_cache, "destroy", Counter("downloads"), raising=False) - monkeypatch.setattr(spack.caches.misc_cache, "destroy", Counter("caches")) - monkeypatch.setattr(spack.installer, "clear_failures", Counter("failures")) + monkeypatch.setattr(spack.caches.FETCH_CACHE, "destroy", Counter("downloads"), raising=False) + monkeypatch.setattr(spack.caches.MISC_CACHE, "destroy", Counter("caches")) + monkeypatch.setattr(spack.store.STORE.failure_tracker, "clear_all", Counter("failures")) monkeypatch.setattr(spack.cmd.clean, "remove_python_cache", Counter("python_cache")) yield counts diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index 1477aa4859f587..3288b092d4d3c0 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -7,14 +7,13 @@ import os import shutil import subprocess -import sys import pytest import spack.cmd import spack.main import spack.paths -from spack.cmd.commands import _positional_to_subroutine +from spack.cmd.commands import _dest_to_fish_complete, _positional_to_subroutine commands = spack.main.SpackCommand("commands", subprocess=True) @@ -59,6 +58,24 @@ def test_subcommands(): assert "spack compiler add" in out2 +@pytest.mark.not_on_windows("subprocess not supported on Windows") +def test_override_alias(): + """Test that spack commands cannot be overriden by aliases.""" + + install = spack.main.SpackCommand("install", subprocess=True) + instal = spack.main.SpackCommand("instal", subprocess=True) + + out = install(fail_on_error=False, global_args=["-c", "config:aliases:install:find"]) + assert "install requires a package argument or active environment" in out + assert "Alias 'install' (mapping to 'find') attempts to override built-in command" in out + + out = install(fail_on_error=False, global_args=["-c", "config:aliases:foo bar:find"]) + assert "Alias 'foo bar' (mapping to 'find') contains a space, which is not supported" in out + + out = instal(fail_on_error=False, global_args=["-c", "config:aliases:instal:find"]) + assert "install requires a package argument or active environment" not in out + + def test_rst(): """Do some simple sanity checks of the rst writer.""" out1 = commands("--format=rst") @@ -185,26 +202,60 @@ def test_bash_completion(): assert "_spack_compiler_add() {" in out2 -def test_update_completion_arg(tmpdir, monkeypatch): - mock_infile = tmpdir.join("spack-completion.in") - mock_bashfile = tmpdir.join("spack-completion.bash") +def test_fish_completion(): + """Test the fish completion writer.""" + out1 = commands("--format=fish") + + # Make sure header not included + assert "function __fish_spack_argparse" not in out1 + assert "complete -c spack --erase" not in out1 + + # Make sure subcommands appear + assert "__fish_spack_using_command remove" in out1 + assert "__fish_spack_using_command compiler find" in out1 + + # Make sure aliases don't appear + assert "__fish_spack_using_command rm" not in out1 + assert "__fish_spack_using_command compiler add" not in out1 + + # Make sure options appear + assert "-s h -l help" in out1 + + # Make sure subcommands are called + for complete_cmd in _dest_to_fish_complete.values(): + assert complete_cmd in out1 + + out2 = commands("--aliases", "--format=fish") + + # Make sure aliases appear + assert "__fish_spack_using_command rm" in out2 + assert "__fish_spack_using_command compiler add" in out2 + + +@pytest.mark.parametrize("shell", ["bash", "fish"]) +def test_update_completion_arg(shell, tmpdir, monkeypatch): + """Test the update completion flag.""" + + tmpdir.join(shell).mkdir() + mock_infile = tmpdir.join(shell).join(f"spack-completion.{shell}") + mock_outfile = tmpdir.join(f"spack-completion.{shell}") mock_args = { - "bash": { + shell: { "aliases": True, - "format": "bash", + "format": shell, "header": str(mock_infile), - "update": str(mock_bashfile), + "update": str(mock_outfile), } } # make a mock completion file missing the --update-completion argument real_args = spack.cmd.commands.update_completion_args - shutil.copy(real_args["bash"]["header"], mock_args["bash"]["header"]) - with open(real_args["bash"]["update"]) as old: + shutil.copy(real_args[shell]["header"], mock_args[shell]["header"]) + with open(real_args[shell]["update"]) as old: old_file = old.read() - with open(mock_args["bash"]["update"], "w") as mock: - mock.write(old_file.replace("--update-completion", "")) + with open(mock_args[shell]["update"], "w") as mock: + mock.write(old_file.replace("update-completion", "")) monkeypatch.setattr(spack.cmd.commands, "update_completion_args", mock_args) @@ -214,16 +265,15 @@ def test_update_completion_arg(tmpdir, monkeypatch): local_commands("--update-completion", "-a") # ensure arg is restored - assert "--update-completion" not in mock_bashfile.read() + assert "update-completion" not in mock_outfile.read() local_commands("--update-completion") - assert "--update-completion" in mock_bashfile.read() + assert "update-completion" in mock_outfile.read() # Note: this test is never expected to be supported on Windows -@pytest.mark.skipif( - sys.platform == "win32", reason="bash completion script generator fails on windows" -) -def test_updated_completion_scripts(tmpdir): +@pytest.mark.not_on_windows("Shell completion script generator fails on windows") +@pytest.mark.parametrize("shell", ["bash", "fish"]) +def test_updated_completion_scripts(shell, tmpdir): """Make sure our shell tab completion scripts remain up-to-date.""" msg = ( @@ -233,12 +283,11 @@ def test_updated_completion_scripts(tmpdir): "and adding the changed files to your pull request." ) - for shell in ["bash"]: # 'zsh', 'fish']: - header = os.path.join(spack.paths.share_path, shell, "spack-completion.in") - script = "spack-completion.{0}".format(shell) - old_script = os.path.join(spack.paths.share_path, script) - new_script = str(tmpdir.join(script)) + header = os.path.join(spack.paths.share_path, shell, f"spack-completion.{shell}") + script = "spack-completion.{0}".format(shell) + old_script = os.path.join(spack.paths.share_path, script) + new_script = str(tmpdir.join(script)) - commands("--aliases", "--format", shell, "--header", header, "--update", new_script) + commands("--aliases", "--format", shell, "--header", header, "--update", new_script) - assert filecmp.cmp(old_script, new_script), msg + assert filecmp.cmp(old_script, new_script), msg diff --git a/lib/spack/spack/test/cmd/compiler.py b/lib/spack/spack/test/cmd/compiler.py index 87eb7e4daf1dcc..1cea72d3b25ad6 100644 --- a/lib/spack/spack/test/cmd/compiler.py +++ b/lib/spack/spack/test/cmd/compiler.py @@ -4,12 +4,14 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import shutil -import sys import pytest +import spack.cmd.compiler import spack.compilers import spack.main +import spack.spec +import spack.util.pattern import spack.version compiler = spack.main.SpackCommand("compiler") @@ -64,7 +66,7 @@ def compilers_dir(mock_executable): return clang_path.parent -@pytest.mark.skipif(sys.platform == "win32", reason="Cannot execute bash script on Windows") +@pytest.mark.not_on_windows("Cannot execute bash script on Windows") @pytest.mark.regression("11678,13138") def test_compiler_find_without_paths(no_compilers_yaml, working_env, mock_executable): """Tests that 'spack compiler find' looks into PATH by default, if no specific path @@ -127,7 +129,7 @@ def test_removing_compilers_from_multiple_scopes(mutable_config, mock_packages): assert spack.spec.CompilerSpec("gcc@=4.5.0") not in spack.compilers.all_compiler_specs() -@pytest.mark.skipif(sys.platform == "win32", reason="Cannot execute bash script on Windows") +@pytest.mark.not_on_windows("Cannot execute bash script on Windows") def test_compiler_add(mutable_config, mock_packages, mock_executable): """Tests that we can add a compiler to configuration.""" expected_version = "4.5.3" @@ -146,7 +148,7 @@ def test_compiler_add(mutable_config, mock_packages, mock_executable): compilers_before_find = set(spack.compilers.all_compiler_specs()) args = spack.util.pattern.Bunch( - all=None, compiler_spec=None, add_paths=[str(root_dir)], scope=None + all=None, compiler_spec=None, add_paths=[str(root_dir)], scope=None, mixed_toolchain=False ) spack.cmd.compiler.compiler_find(args) compilers_after_find = set(spack.compilers.all_compiler_specs()) @@ -157,12 +159,17 @@ def test_compiler_add(mutable_config, mock_packages, mock_executable): assert new_compiler.version == spack.version.Version(expected_version) -@pytest.mark.skipif(sys.platform == "win32", reason="Cannot execute bash script on Windows") +@pytest.mark.not_on_windows("Cannot execute bash script on Windows") @pytest.mark.regression("17590") -def test_compiler_find_mixed_suffixes(no_compilers_yaml, working_env, compilers_dir): +@pytest.mark.parametrize("mixed_toolchain", [True, False]) +def test_compiler_find_mixed_suffixes( + mixed_toolchain, no_compilers_yaml, working_env, compilers_dir +): """Ensure that we'll mix compilers with different suffixes when necessary.""" os.environ["PATH"] = str(compilers_dir) - output = compiler("find", "--scope=site") + output = compiler( + "find", "--scope=site", "--mixed-toolchain" if mixed_toolchain else "--no-mixed-toolchain" + ) assert "clang@11.0.0" in output assert "gcc@8.4.0" in output @@ -176,9 +183,8 @@ def test_compiler_find_mixed_suffixes(no_compilers_yaml, working_env, compilers_ assert clang["paths"] == { "cc": str(compilers_dir / "clang"), "cxx": str(compilers_dir / "clang++"), - # we only auto-detect mixed clang on macos - "f77": gfortran_path if sys.platform == "darwin" else None, - "fc": gfortran_path if sys.platform == "darwin" else None, + "f77": gfortran_path if mixed_toolchain else None, + "fc": gfortran_path if mixed_toolchain else None, } assert gcc["paths"] == { @@ -189,7 +195,7 @@ def test_compiler_find_mixed_suffixes(no_compilers_yaml, working_env, compilers_ } -@pytest.mark.skipif(sys.platform == "win32", reason="Cannot execute bash script on Windows") +@pytest.mark.not_on_windows("Cannot execute bash script on Windows") @pytest.mark.regression("17590") def test_compiler_find_prefer_no_suffix(no_compilers_yaml, working_env, compilers_dir): """Ensure that we'll pick 'clang' over 'clang-gpu' when there is a choice.""" @@ -210,7 +216,7 @@ def test_compiler_find_prefer_no_suffix(no_compilers_yaml, working_env, compiler assert clang["paths"]["cxx"] == str(compilers_dir / "clang++") -@pytest.mark.skipif(sys.platform == "win32", reason="Cannot execute bash script on Windows") +@pytest.mark.not_on_windows("Cannot execute bash script on Windows") def test_compiler_find_path_order(no_compilers_yaml, working_env, compilers_dir): """Ensure that we look for compilers in the same order as PATH, when there are duplicates""" new_dir = compilers_dir / "first_in_path" diff --git a/lib/spack/spack/test/cmd/config.py b/lib/spack/spack/test/cmd/config.py index c410fcfc768715..7247ce97531dcd 100644 --- a/lib/spack/spack/test/cmd/config.py +++ b/lib/spack/spack/test/cmd/config.py @@ -24,7 +24,7 @@ def _create_config(scope=None, data={}, section="packages"): scope = scope or spack.config.default_modify_scope() - cfg_file = spack.config.config.get_config_filename(scope, section) + cfg_file = spack.config.CONFIG.get_config_filename(scope, section) with open(cfg_file, "w") as f: syaml.dump(data, stream=f) return cfg_file @@ -80,8 +80,8 @@ def test_config_edit(mutable_config, working_env): """Ensure `spack config edit` edits the right paths.""" dms = spack.config.default_modify_scope("compilers") - dms_path = spack.config.config.scopes[dms].path - user_path = spack.config.config.scopes["user"].path + dms_path = spack.config.CONFIG.scopes[dms].path + user_path = spack.config.CONFIG.scopes["user"].path comp_path = os.path.join(dms_path, "compilers.yaml") repos_path = os.path.join(user_path, "repos.yaml") @@ -215,10 +215,10 @@ def test_config_add_override_leaf(mutable_empty_config): def test_config_add_update_dict(mutable_empty_config): - config("add", "packages:all:version:[1.0.0]") + config("add", "packages:hdf5:version:[1.0.0]") output = config("get", "packages") - expected = "packages:\n all:\n version: [1.0.0]\n" + expected = "packages:\n hdf5:\n version: [1.0.0]\n" assert output == expected @@ -352,8 +352,7 @@ def test_config_add_update_dict_from_file(mutable_empty_config, tmpdir): contents = """spack: packages: all: - version: - - 1.0.0 + target: [x86_64] """ # create temp file and add it to config @@ -368,8 +367,7 @@ def test_config_add_update_dict_from_file(mutable_empty_config, tmpdir): # added config comes before prior config expected = """packages: all: - version: - - 1.0.0 + target: [x86_64] compiler: [gcc] """ @@ -381,7 +379,7 @@ def test_config_add_invalid_file_fails(tmpdir): # invalid because version requires a list contents = """spack: packages: - all: + hdf5: version: 1.0.0 """ @@ -544,7 +542,7 @@ def test_config_update_not_needed(mutable_config): def test_config_update_can_handle_comments(mutable_config): # Create an outdated config file with comments scope = spack.config.default_modify_scope() - cfg_file = spack.config.config.get_config_filename(scope, "config") + cfg_file = spack.config.CONFIG.get_config_filename(scope, "config") with open(cfg_file, mode="w") as f: f.write( """ @@ -574,7 +572,7 @@ def test_config_update_can_handle_comments(mutable_config): @pytest.mark.regression("18050") def test_config_update_works_for_empty_paths(mutable_config): scope = spack.config.default_modify_scope() - cfg_file = spack.config.config.get_config_filename(scope, "config") + cfg_file = spack.config.CONFIG.get_config_filename(scope, "config") with open(cfg_file, mode="w") as f: f.write( """ @@ -623,22 +621,19 @@ def test_config_prefer_upstream( downstream_db_root = str(tmpdir_factory.mktemp("mock_downstream_db_root")) db_for_test = spack.database.Database(downstream_db_root, upstream_dbs=[prepared_db]) - monkeypatch.setattr(spack.store, "db", db_for_test) + monkeypatch.setattr(spack.store.STORE, "db", db_for_test) output = config("prefer-upstream") scope = spack.config.default_modify_scope("packages") - cfg_file = spack.config.config.get_config_filename(scope, "packages") + cfg_file = spack.config.CONFIG.get_config_filename(scope, "packages") packages = syaml.load(open(cfg_file))["packages"] # Make sure only the non-default variants are set. - assert packages["boost"] == { - "compiler": ["gcc@=10.2.1"], - "variants": "+debug +graph", - "version": ["1.63.0"], - } - assert packages["dependency-install"] == {"compiler": ["gcc@=10.2.1"], "version": ["2.0"]} + assert packages["all"] == {"compiler": ["gcc@=10.2.1"]} + assert packages["boost"] == {"variants": "+debug +graph", "version": ["1.63.0"]} + assert packages["dependency-install"] == {"version": ["2.0"]} # Ensure that neither variant gets listed for hdf5, since they conflict - assert packages["hdf5"] == {"compiler": ["gcc@=10.2.1"], "version": ["2.3"]} + assert packages["hdf5"] == {"version": ["2.3"]} # Make sure a message about the conflicting hdf5's was given. assert "- hdf5" in output diff --git a/lib/spack/spack/test/cmd/create.py b/lib/spack/spack/test/cmd/create.py index ed7a6404bbe114..089dc8b0c52440 100644 --- a/lib/spack/spack/test/cmd/create.py +++ b/lib/spack/spack/test/cmd/create.py @@ -27,6 +27,7 @@ [r"TestNamedPackage(Package)", r"def install(self"], ), (["file://example.tar.gz"], "example", [r"Example(Package)", r"def install(self"]), + (["-n", "test-license"], "test-license", [r'license("UNKNOWN")']), # Template-specific cases ( ["-t", "autoreconf", "/test-autoreconf"], @@ -91,12 +92,7 @@ ( ["-t", "python", "/test-python"], "py-test-python", - [ - r"PyTestPython(PythonPackage)", - r'depends_on("py-', - r"def global_options(self", - r"def install_options(self", - ], + [r"PyTestPython(PythonPackage)", r'depends_on("py-', r"def config_settings(self"], ), ( ["-t", "qmake", "/test-qmake"], diff --git a/lib/spack/spack/test/cmd/debug.py b/lib/spack/spack/test/cmd/debug.py index 993e89954c46f2..92fe4a2d57888f 100644 --- a/lib/spack/spack/test/cmd/debug.py +++ b/lib/spack/spack/test/cmd/debug.py @@ -6,7 +6,6 @@ import os import os.path import platform -import sys import pytest @@ -17,7 +16,7 @@ debug = SpackCommand("debug") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.mark.db diff --git a/lib/spack/spack/test/cmd/deconcretize.py b/lib/spack/spack/test/cmd/deconcretize.py new file mode 100644 index 00000000000000..30e39604bf4d4d --- /dev/null +++ b/lib/spack/spack/test/cmd/deconcretize.py @@ -0,0 +1,78 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +import spack.environment as ev +from spack.main import SpackCommand, SpackCommandError + +deconcretize = SpackCommand("deconcretize") + + +@pytest.fixture(scope="function") +def test_env(mutable_mock_env_path, config, mock_packages): + ev.create("test") + with ev.read("test") as e: + e.add("a@2.0 foobar=bar ^b@1.0") + e.add("a@1.0 foobar=bar ^b@0.9") + e.concretize() + e.write() + + +def test_deconcretize_dep(test_env): + with ev.read("test") as e: + deconcretize("-y", "b@1.0") + specs = [s for s, _ in e.concretized_specs()] + + assert len(specs) == 1 + assert specs[0].satisfies("a@1.0") + + +def test_deconcretize_all_dep(test_env): + with ev.read("test") as e: + with pytest.raises(SpackCommandError): + deconcretize("-y", "b") + deconcretize("-y", "--all", "b") + specs = [s for s, _ in e.concretized_specs()] + + assert len(specs) == 0 + + +def test_deconcretize_root(test_env): + with ev.read("test") as e: + output = deconcretize("-y", "--root", "b@1.0") + assert "No matching specs to deconcretize" in output + assert len(e.concretized_order) == 2 + + deconcretize("-y", "--root", "a@2.0") + specs = [s for s, _ in e.concretized_specs()] + + assert len(specs) == 1 + assert specs[0].satisfies("a@1.0") + + +def test_deconcretize_all_root(test_env): + with ev.read("test") as e: + with pytest.raises(SpackCommandError): + deconcretize("-y", "--root", "a") + + output = deconcretize("-y", "--root", "--all", "b") + assert "No matching specs to deconcretize" in output + assert len(e.concretized_order) == 2 + + deconcretize("-y", "--root", "--all", "a") + specs = [s for s, _ in e.concretized_specs()] + + assert len(specs) == 0 + + +def test_deconcretize_all(test_env): + with ev.read("test") as e: + with pytest.raises(SpackCommandError): + deconcretize() + deconcretize("-y", "--all") + specs = [s for s, _ in e.concretized_specs()] + + assert len(specs) == 0 diff --git a/lib/spack/spack/test/cmd/dependencies.py b/lib/spack/spack/test/cmd/dependencies.py index 3d4671f63893b0..bc615c7a3a10d7 100644 --- a/lib/spack/spack/test/cmd/dependencies.py +++ b/lib/spack/spack/test/cmd/dependencies.py @@ -14,7 +14,14 @@ dependencies = SpackCommand("dependencies") -mpis = ["low-priority-provider", "mpich", "mpich2", "multi-provider-mpi", "zmpi"] +mpis = [ + "intel-parallel-studio", + "low-priority-provider", + "mpich", + "mpich2", + "multi-provider-mpi", + "zmpi", +] mpi_deps = ["fake"] @@ -54,7 +61,9 @@ def test_direct_installed_dependencies(mock_packages, database): lines = [line for line in out.strip().split("\n") if not line.startswith("--")] hashes = set([re.split(r"\s+", line)[0] for line in lines]) - expected = set([spack.store.db.query_one(s).dag_hash(7) for s in ["mpich", "callpath^mpich"]]) + expected = set( + [spack.store.STORE.db.query_one(s).dag_hash(7) for s in ["mpich", "callpath^mpich"]] + ) assert expected == hashes @@ -69,7 +78,7 @@ def test_transitive_installed_dependencies(mock_packages, database): expected = set( [ - spack.store.db.query_one(s).dag_hash(7) + spack.store.STORE.db.query_one(s).dag_hash(7) for s in ["zmpi", "callpath^zmpi", "fake", "dyninst", "libdwarf", "libelf"] ] ) diff --git a/lib/spack/spack/test/cmd/dependents.py b/lib/spack/spack/test/cmd/dependents.py index 16b81f1361ea6b..9d124c0a2c4ca2 100644 --- a/lib/spack/spack/test/cmd/dependents.py +++ b/lib/spack/spack/test/cmd/dependents.py @@ -57,9 +57,11 @@ def test_immediate_installed_dependents(mock_packages, database): lines = [li for li in out.strip().split("\n") if not li.startswith("--")] hashes = set([re.split(r"\s+", li)[0] for li in lines]) - expected = set([spack.store.db.query_one(s).dag_hash(7) for s in ["dyninst", "libdwarf"]]) + expected = set( + [spack.store.STORE.db.query_one(s).dag_hash(7) for s in ["dyninst", "libdwarf"]] + ) - libelf = spack.store.db.query_one("libelf") + libelf = spack.store.STORE.db.query_one("libelf") expected = set([d.dag_hash(7) for d in libelf.dependents()]) assert expected == hashes @@ -75,7 +77,7 @@ def test_transitive_installed_dependents(mock_packages, database): expected = set( [ - spack.store.db.query_one(s).dag_hash(7) + spack.store.STORE.db.query_one(s).dag_hash(7) for s in ["zmpi", "callpath^zmpi", "mpileaks^zmpi"] ] ) diff --git a/lib/spack/spack/test/cmd/deprecate.py b/lib/spack/spack/test/cmd/deprecate.py index 0978bc66ead7f1..7584827e769c24 100644 --- a/lib/spack/spack/test/cmd/deprecate.py +++ b/lib/spack/spack/test/cmd/deprecate.py @@ -3,8 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys - import pytest import spack.store @@ -16,22 +14,22 @@ deprecate = SpackCommand("deprecate") find = SpackCommand("find") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") def test_deprecate(mock_packages, mock_archive, mock_fetch, install_mockery): install("libelf@0.8.13") install("libelf@0.8.10") - all_installed = spack.store.db.query() + all_installed = spack.store.STORE.db.query() assert len(all_installed) == 2 deprecate("-y", "libelf@0.8.10", "libelf@0.8.13") - non_deprecated = spack.store.db.query() - all_available = spack.store.db.query(installed=any) + non_deprecated = spack.store.STORE.db.query() + all_available = spack.store.STORE.db.query(installed=any) assert all_available == all_installed - assert non_deprecated == spack.store.db.query("libelf@0.8.13") + assert non_deprecated == spack.store.STORE.db.query("libelf@0.8.13") def test_deprecate_fails_no_such_package(mock_packages, mock_archive, mock_fetch, install_mockery): @@ -53,13 +51,13 @@ def test_deprecate_install(mock_packages, mock_archive, mock_fetch, install_mock that is not yet installed.""" install("libelf@0.8.10") - to_deprecate = spack.store.db.query() + to_deprecate = spack.store.STORE.db.query() assert len(to_deprecate) == 1 deprecate("-y", "-i", "libelf@0.8.10", "libelf@0.8.13") - non_deprecated = spack.store.db.query() - deprecated = spack.store.db.query(installed=InstallStatuses.DEPRECATED) + non_deprecated = spack.store.STORE.db.query() + deprecated = spack.store.STORE.db.query(installed=InstallStatuses.DEPRECATED) assert deprecated == to_deprecate assert len(non_deprecated) == 1 assert non_deprecated[0].satisfies("libelf@0.8.13") @@ -73,13 +71,13 @@ def test_deprecate_deps(mock_packages, mock_archive, mock_fetch, install_mockery new_spec = spack.spec.Spec("libdwarf@20130729^libelf@0.8.13").concretized() old_spec = spack.spec.Spec("libdwarf@20130207^libelf@0.8.10").concretized() - all_installed = spack.store.db.query() + all_installed = spack.store.STORE.db.query() deprecate("-y", "-d", "libdwarf@20130207", "libdwarf@20130729") - non_deprecated = spack.store.db.query() - all_available = spack.store.db.query(installed=any) - deprecated = spack.store.db.query(installed=InstallStatuses.DEPRECATED) + non_deprecated = spack.store.STORE.db.query() + all_available = spack.store.STORE.db.query(installed=any) + deprecated = spack.store.STORE.db.query(installed=InstallStatuses.DEPRECATED) assert all_available == all_installed assert sorted(all_available) == sorted(deprecated + non_deprecated) @@ -95,12 +93,12 @@ def test_uninstall_deprecated(mock_packages, mock_archive, mock_fetch, install_m deprecate("-y", "libelf@0.8.10", "libelf@0.8.13") - non_deprecated = spack.store.db.query() + non_deprecated = spack.store.STORE.db.query() uninstall("-y", "libelf@0.8.10") - assert spack.store.db.query() == spack.store.db.query(installed=any) - assert spack.store.db.query() == non_deprecated + assert spack.store.STORE.db.query() == spack.store.STORE.db.query(installed=any) + assert spack.store.STORE.db.query() == non_deprecated def test_deprecate_already_deprecated(mock_packages, mock_archive, mock_fetch, install_mockery): @@ -113,17 +111,17 @@ def test_deprecate_already_deprecated(mock_packages, mock_archive, mock_fetch, i deprecate("-y", "libelf@0.8.10", "libelf@0.8.12") - deprecator = spack.store.db.deprecator(deprecated_spec) + deprecator = spack.store.STORE.db.deprecator(deprecated_spec) assert deprecator == spack.spec.Spec("libelf@0.8.12").concretized() deprecate("-y", "libelf@0.8.10", "libelf@0.8.13") - non_deprecated = spack.store.db.query() - all_available = spack.store.db.query(installed=any) + non_deprecated = spack.store.STORE.db.query() + all_available = spack.store.STORE.db.query(installed=any) assert len(non_deprecated) == 2 assert len(all_available) == 3 - deprecator = spack.store.db.deprecator(deprecated_spec) + deprecator = spack.store.STORE.db.deprecator(deprecated_spec) assert deprecator == spack.spec.Spec("libelf@0.8.13").concretized() @@ -140,19 +138,19 @@ def test_deprecate_deprecator(mock_packages, mock_archive, mock_fetch, install_m deprecate("-y", "libelf@0.8.10", "libelf@0.8.12") - deprecator = spack.store.db.deprecator(first_deprecated_spec) + deprecator = spack.store.STORE.db.deprecator(first_deprecated_spec) assert deprecator == second_deprecated_spec deprecate("-y", "libelf@0.8.12", "libelf@0.8.13") - non_deprecated = spack.store.db.query() - all_available = spack.store.db.query(installed=any) + non_deprecated = spack.store.STORE.db.query() + all_available = spack.store.STORE.db.query(installed=any) assert len(non_deprecated) == 1 assert len(all_available) == 3 - first_deprecator = spack.store.db.deprecator(first_deprecated_spec) + first_deprecator = spack.store.STORE.db.deprecator(first_deprecated_spec) assert first_deprecator == final_deprecator - second_deprecator = spack.store.db.deprecator(second_deprecated_spec) + second_deprecator = spack.store.STORE.db.deprecator(second_deprecated_spec) assert second_deprecator == final_deprecator diff --git a/lib/spack/spack/test/cmd/dev_build.py b/lib/spack/spack/test/cmd/dev_build.py index 00848e8524dc68..85199eddd66da2 100644 --- a/lib/spack/spack/test/cmd/dev_build.py +++ b/lib/spack/spack/test/cmd/dev_build.py @@ -4,26 +4,27 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import sys import pytest import llnl.util.filesystem as fs +import spack.build_environment import spack.environment as ev +import spack.error import spack.spec +import spack.store from spack.main import SpackCommand dev_build = SpackCommand("dev-build") install = SpackCommand("install") env = SpackCommand("env") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") -def test_dev_build_basics(tmpdir, mock_packages, install_mockery): - spec = spack.spec.Spec("dev-build-test-install@0.0.0 dev_path=%s" % tmpdir) - spec.concretize() +def test_dev_build_basics(tmpdir, install_mockery): + spec = spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmpdir}").concretized() assert "dev_path" in spec.variants @@ -40,9 +41,8 @@ def test_dev_build_basics(tmpdir, mock_packages, install_mockery): assert os.path.exists(str(tmpdir)) -def test_dev_build_before(tmpdir, mock_packages, install_mockery): - spec = spack.spec.Spec("dev-build-test-install@0.0.0 dev_path=%s" % tmpdir) - spec.concretize() +def test_dev_build_before(tmpdir, install_mockery): + spec = spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmpdir}").concretized() with tmpdir.as_cwd(): with open(spec.package.filename, "w") as f: @@ -57,9 +57,8 @@ def test_dev_build_before(tmpdir, mock_packages, install_mockery): assert not os.path.exists(spec.prefix) -def test_dev_build_until(tmpdir, mock_packages, install_mockery): - spec = spack.spec.Spec("dev-build-test-install@0.0.0 dev_path=%s" % tmpdir) - spec.concretize() +def test_dev_build_until(tmpdir, install_mockery): + spec = spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmpdir}").concretized() with tmpdir.as_cwd(): with open(spec.package.filename, "w") as f: @@ -72,13 +71,12 @@ def test_dev_build_until(tmpdir, mock_packages, install_mockery): assert f.read() == spec.package.replacement_string assert not os.path.exists(spec.prefix) - assert not spack.store.db.query(spec, installed=True) + assert not spack.store.STORE.db.query(spec, installed=True) -def test_dev_build_until_last_phase(tmpdir, mock_packages, install_mockery): +def test_dev_build_until_last_phase(tmpdir, install_mockery): # Test that we ignore the last_phase argument if it is already last - spec = spack.spec.Spec("dev-build-test-install@0.0.0 dev_path=%s" % tmpdir) - spec.concretize() + spec = spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmpdir}").concretized() with tmpdir.as_cwd(): with open(spec.package.filename, "w") as f: @@ -91,13 +89,12 @@ def test_dev_build_until_last_phase(tmpdir, mock_packages, install_mockery): assert f.read() == spec.package.replacement_string assert os.path.exists(spec.prefix) - assert spack.store.db.query(spec, installed=True) + assert spack.store.STORE.db.query(spec, installed=True) assert os.path.exists(str(tmpdir)) -def test_dev_build_before_until(tmpdir, mock_packages, install_mockery, capsys): - spec = spack.spec.Spec("dev-build-test-install@0.0.0 dev_path=%s" % tmpdir) - spec.concretize() +def test_dev_build_before_until(tmpdir, install_mockery, capsys): + spec = spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmpdir}").concretized() with tmpdir.as_cwd(): with open(spec.package.filename, "w") as f: @@ -135,7 +132,6 @@ def mock_module_noop(*args): def test_dev_build_drop_in(tmpdir, mock_packages, monkeypatch, install_mockery, working_env): monkeypatch.setattr(os, "execvp", print_spack_cc) - monkeypatch.setattr(spack.build_environment, "module", mock_module_noop) with tmpdir.as_cwd(): @@ -143,7 +139,7 @@ def test_dev_build_drop_in(tmpdir, mock_packages, monkeypatch, install_mockery, assert "lib/spack/env" in output -def test_dev_build_fails_already_installed(tmpdir, mock_packages, install_mockery): +def test_dev_build_fails_already_installed(tmpdir, install_mockery): spec = spack.spec.Spec("dev-build-test-install@0.0.0 dev_path=%s" % tmpdir) spec.concretize() @@ -167,8 +163,15 @@ def test_dev_build_fails_multiple_specs(mock_packages): def test_dev_build_fails_nonexistent_package_name(mock_packages): - output = dev_build("no_such_package", fail_on_error=False) - assert "No package for 'no_such_package' was found" in output + output = "" + + try: + dev_build("no_such_package") + assert False, "no exception was raised!" + except spack.repo.UnknownPackageError as e: + output = e.message + + assert "Package 'no_such_package' not found" in output def test_dev_build_fails_no_version(mock_packages): @@ -176,7 +179,7 @@ def test_dev_build_fails_no_version(mock_packages): assert "dev-build spec must have a single, concrete version" in output -def test_dev_build_env(tmpdir, mock_packages, install_mockery, mutable_mock_env_path): +def test_dev_build_env(tmpdir, install_mockery, mutable_mock_env_path): """Test Spack does dev builds for packages in develop section of env.""" # setup dev-build-test-install package for dev build build_dir = tmpdir.mkdir("build") @@ -192,7 +195,7 @@ def test_dev_build_env(tmpdir, mock_packages, install_mockery, mutable_mock_env_ with envdir.as_cwd(): with open("spack.yaml", "w") as f: f.write( - """\ + f"""\ spack: specs: - dev-build-test-install@0.0.0 @@ -200,11 +203,9 @@ def test_dev_build_env(tmpdir, mock_packages, install_mockery, mutable_mock_env_ develop: dev-build-test-install: spec: dev-build-test-install@0.0.0 - path: %s + path: {os.path.relpath(str(build_dir), start=str(envdir))} """ - % os.path.relpath(str(build_dir), start=str(envdir)) ) - env("create", "test", "./spack.yaml") with ev.read("test"): install() @@ -214,9 +215,7 @@ def test_dev_build_env(tmpdir, mock_packages, install_mockery, mutable_mock_env_ assert f.read() == spec.package.replacement_string -def test_dev_build_env_version_mismatch( - tmpdir, mock_packages, install_mockery, mutable_mock_env_path -): +def test_dev_build_env_version_mismatch(tmpdir, install_mockery, mutable_mock_env_path): """Test Spack constraints concretization by develop specs.""" # setup dev-build-test-install package for dev build build_dir = tmpdir.mkdir("build") @@ -232,7 +231,7 @@ def test_dev_build_env_version_mismatch( with envdir.as_cwd(): with open("spack.yaml", "w") as f: f.write( - """\ + f"""\ spack: specs: - dev-build-test-install@0.0.0 @@ -240,20 +239,17 @@ def test_dev_build_env_version_mismatch( develop: dev-build-test-install: spec: dev-build-test-install@1.1.1 - path: %s + path: {build_dir} """ - % build_dir ) env("create", "test", "./spack.yaml") with ev.read("test"): - with pytest.raises(RuntimeError): + with pytest.raises((RuntimeError, spack.error.UnsatisfiableSpecError)): install() -def test_dev_build_multiple( - tmpdir, mock_packages, install_mockery, mutable_mock_env_path, mock_fetch -): +def test_dev_build_multiple(tmpdir, install_mockery, mutable_mock_env_path, mock_fetch): """Test spack install with multiple developer builds Test that only the root needs to be specified in the environment @@ -266,7 +262,7 @@ def test_dev_build_multiple( # root and dependency if they wanted a dev build for both. leaf_dir = tmpdir.mkdir("leaf") leaf_spec = spack.spec.Spec("dev-build-test-install@=1.0.0") # non-existing version - leaf_pkg_cls = spack.repo.path.get_pkg_class(leaf_spec.name) + leaf_pkg_cls = spack.repo.PATH.get_pkg_class(leaf_spec.name) with leaf_dir.as_cwd(): with open(leaf_pkg_cls.filename, "w") as f: f.write(leaf_pkg_cls.original_string) @@ -275,7 +271,7 @@ def test_dev_build_multiple( # don't concretize outside environment -- dev info will be wrong root_dir = tmpdir.mkdir("root") root_spec = spack.spec.Spec("dev-build-test-dependent@0.0.0") - root_pkg_cls = spack.repo.path.get_pkg_class(root_spec.name) + root_pkg_cls = spack.repo.PATH.get_pkg_class(root_spec.name) with root_dir.as_cwd(): with open(root_pkg_cls.filename, "w") as f: f.write(root_pkg_cls.original_string) @@ -285,20 +281,19 @@ def test_dev_build_multiple( with envdir.as_cwd(): with open("spack.yaml", "w") as f: f.write( - """\ + f"""\ spack: specs: - dev-build-test-dependent@0.0.0 develop: dev-build-test-install: - path: %s + path: {leaf_dir} spec: dev-build-test-install@=1.0.0 dev-build-test-dependent: spec: dev-build-test-dependent@0.0.0 - path: %s + path: {root_dir} """ - % (leaf_dir, root_dir) ) env("create", "test", "./spack.yaml") @@ -317,9 +312,7 @@ def test_dev_build_multiple( assert f.read() == spec.package.replacement_string -def test_dev_build_env_dependency( - tmpdir, mock_packages, install_mockery, mock_fetch, mutable_mock_env_path -): +def test_dev_build_env_dependency(tmpdir, install_mockery, mock_fetch, mutable_mock_env_path): """ Test non-root specs in an environment are properly marked for dev builds. """ @@ -329,7 +322,7 @@ def test_dev_build_env_dependency( dep_spec = spack.spec.Spec("dev-build-test-install") with build_dir.as_cwd(): - dep_pkg_cls = spack.repo.path.get_pkg_class(dep_spec.name) + dep_pkg_cls = spack.repo.PATH.get_pkg_class(dep_spec.name) with open(dep_pkg_cls.filename, "w") as f: f.write(dep_pkg_cls.original_string) @@ -338,7 +331,7 @@ def test_dev_build_env_dependency( with envdir.as_cwd(): with open("spack.yaml", "w") as f: f.write( - """\ + f"""\ spack: specs: - dependent-of-dev-build@0.0.0 @@ -346,11 +339,9 @@ def test_dev_build_env_dependency( develop: dev-build-test-install: spec: dev-build-test-install@0.0.0 - path: %s + path: {os.path.relpath(str(build_dir), start=str(envdir))} """ - % os.path.relpath(str(build_dir), start=str(envdir)) ) - env("create", "test", "./spack.yaml") with ev.read("test"): # concretize in the environment to get the dev build info @@ -372,7 +363,7 @@ def test_dev_build_env_dependency( @pytest.mark.parametrize("test_spec", ["dev-build-test-install", "dependent-of-dev-build"]) def test_dev_build_rebuild_on_source_changes( - test_spec, tmpdir, mock_packages, install_mockery, mutable_mock_env_path, mock_fetch + test_spec, tmpdir, install_mockery, mutable_mock_env_path, mock_fetch ): """Test dev builds rebuild on changes to source code. @@ -417,4 +408,4 @@ def reset_string(): fs.touch(os.path.join(str(build_dir), "test")) output = install() - assert "Installing %s" % test_spec in output + assert f"Installing {test_spec}" in output diff --git a/lib/spack/spack/test/cmd/develop.py b/lib/spack/spack/test/cmd/develop.py index 2d339f56a85113..f80038c0edd97f 100644 --- a/lib/spack/spack/test/cmd/develop.py +++ b/lib/spack/spack/test/cmd/develop.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import shutil -import sys import pytest @@ -17,7 +16,7 @@ develop = SpackCommand("develop") env = SpackCommand("env") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.mark.usefixtures("mutable_mock_env_path", "mock_packages", "mock_fetch", "config") @@ -146,3 +145,48 @@ def check_path(stage, dest): # Check modifications actually worked assert spack.spec.Spec("mpich@1.0").concretized().satisfies("dev_path=%s" % abspath) + + +def _git_commit_list(git_repo_dir): + git = spack.util.git.git() + with fs.working_dir(git_repo_dir): + output = git("log", "--pretty=format:%h", "-n", "20", output=str) + return output.strip().split() + + +def test_develop_full_git_repo( + mutable_mock_env_path, + mock_git_version_info, + install_mockery, + mock_packages, + monkeypatch, + tmpdir, + mutable_config, + request, +): + repo_path, filename, commits = mock_git_version_info + monkeypatch.setattr( + spack.package_base.PackageBase, "git", "file://%s" % repo_path, raising=False + ) + + spec = spack.spec.Spec("git-test-commit@1.2").concretized() + try: + spec.package.do_stage() + commits = _git_commit_list(spec.package.stage[0].source_path) + # Outside of "spack develop" Spack will only pull exactly the commit it + # needs, with no additional history + assert len(commits) == 1 + finally: + spec.package.do_clean() + + # Now use "spack develop": look at the resulting stage directory and make + # sure the git repo pulled includes the full branch history (or rather, + # more than just one commit). + env("create", "test") + with ev.read("test"): + develop("git-test-commit@1.2") + + location = SpackCommand("location") + develop_stage_dir = location("git-test-commit").strip() + commits = _git_commit_list(develop_stage_dir) + assert len(commits) > 1 diff --git a/lib/spack/spack/test/cmd/diff.py b/lib/spack/spack/test/cmd/diff.py index d32a8b9e1db825..b90c61fbc8258a 100644 --- a/lib/spack/spack/test/cmd/diff.py +++ b/lib/spack/spack/test/cmd/diff.py @@ -3,8 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys - import pytest import spack.cmd.diff @@ -45,7 +43,7 @@ def test_diff_cmd(install_mockery, mock_fetch, mock_archive, mock_packages): assert ["hash", "mpileaks %s" % specB.dag_hash()] in c["b_not_a"] -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_load_first(install_mockery, mock_fetch, mock_archive, mock_packages): """Test with and without the --first option""" install_cmd("mpileaks") diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 5aeb93ad58db01..c3a7551e944ddf 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -8,21 +8,23 @@ import os import pathlib import shutil -import sys from argparse import Namespace import pytest import llnl.util.filesystem as fs import llnl.util.link_tree +import llnl.util.tty as tty import spack.cmd.env import spack.config import spack.environment as ev +import spack.environment.depfile as depfile import spack.environment.environment import spack.environment.shell import spack.error import spack.modules +import spack.package_base import spack.paths import spack.repo import spack.util.spack_json as sjson @@ -39,7 +41,7 @@ pytestmark = [ pytest.mark.usefixtures("mutable_mock_env_path", "config", "mutable_mock_repo"), pytest.mark.maybeslow, - pytest.mark.skipif(sys.platform == "win32", reason="Envs unsupported on Window"), + pytest.mark.not_on_windows("Envs unsupported on Window"), ] env = SpackCommand("env") @@ -148,7 +150,7 @@ def test_env_list(mutable_mock_env_path): assert "baz" in out # make sure `spack env list` skips invalid things in var/spack/env - mutable_mock_env_path.join(".DS_Store").ensure(file=True) + (mutable_mock_env_path / ".DS_Store").touch() out = env("list") assert "foo" in out @@ -167,7 +169,7 @@ def test_env_remove(capfd): foo = ev.read("foo") with foo: - with pytest.raises(spack.main.SpackCommandError): + with pytest.raises(SpackCommandError): with capfd.disabled(): env("remove", "-y", "foo") assert "foo" in env("list") @@ -248,8 +250,8 @@ def test_env_roots_marked_explicit(install_mockery, mock_fetch): install("dependent-install") # Check one explicit, one implicit install - dependent = spack.store.db.query(explicit=True) - dependency = spack.store.db.query(explicit=False) + dependent = spack.store.STORE.db.query(explicit=True) + dependency = spack.store.STORE.db.query(explicit=False) assert len(dependent) == 1 assert len(dependency) == 1 @@ -260,7 +262,7 @@ def test_env_roots_marked_explicit(install_mockery, mock_fetch): e.concretize() e.install_all() - explicit = spack.store.db.query(explicit=True) + explicit = spack.store.STORE.db.query(explicit=True) assert len(explicit) == 2 @@ -275,14 +277,14 @@ def test_env_modifications_error_on_activate(install_mockery, mock_fetch, monkey def setup_error(pkg, env): raise RuntimeError("cmake-client had issues!") - pkg = spack.repo.path.get_pkg_class("cmake-client") + pkg = spack.repo.PATH.get_pkg_class("cmake-client") monkeypatch.setattr(pkg, "setup_run_environment", setup_error) spack.environment.shell.activate(e) _, err = capfd.readouterr() assert "cmake-client had issues!" in err - assert "Warning: couldn't get environment settings" in err + assert "Warning: could not load runtime environment" in err def test_activate_adds_transitive_run_deps_to_path(install_mockery, mock_fetch, monkeypatch): @@ -360,10 +362,10 @@ def test_env_install_two_specs_same_dep(install_mockery, mock_fetch, tmpdir, cap assert "depb: Executing phase:" in out assert "a: Executing phase:" in out - depb = spack.store.db.query_one("depb", installed=True) + depb = spack.store.STORE.db.query_one("depb", installed=True) assert depb, "Expected depb to be installed" - a = spack.store.db.query_one("a", installed=True) + a = spack.store.STORE.db.query_one("a", installed=True) assert a, "Expected a to be installed" @@ -499,11 +501,14 @@ def test_env_activate_broken_view( # switch to a new repo that doesn't include the installed package # test that Spack detects the missing package and fails gracefully with spack.repo.use_repositories(mock_custom_repository): - with pytest.raises(SpackCommandError): - env("activate", "--sh", "test") + wrong_repo = env("activate", "--sh", "test") + assert "Warning: could not load runtime environment" in wrong_repo + assert "Unknown namespace: builtin.mock" in wrong_repo # test replacing repo fixes it - env("activate", "--sh", "test") + normal_repo = env("activate", "--sh", "test") + assert "Warning: could not load runtime environment" not in normal_repo + assert "Unknown namespace: builtin.mock" not in normal_repo def test_to_lockfile_dict(): @@ -627,7 +632,7 @@ def test_env_view_external_prefix(tmp_path, mutable_database, mock_packages): manifest_dir.mkdir(parents=True, exist_ok=False) manifest_file = manifest_dir / ev.manifest_name manifest_file.write_text( - """ + """\ spack: specs: - a @@ -662,7 +667,7 @@ def test_env_view_external_prefix(tmp_path, mutable_database, mock_packages): e.write() env_mod = spack.util.environment.EnvironmentModifications() - e.add_default_view_to_env(env_mod) + e.add_view_to_env(env_mod, "default") env_variables = {} env_mod.apply_modifications(env_variables) assert str(fake_bin) in env_variables["PATH"] @@ -715,38 +720,25 @@ def test_env_with_config(environment_from_manifest): def test_with_config_bad_include(environment_from_manifest): """Confirm missing include paths raise expected exception and error.""" - e = environment_from_manifest( - """ + with pytest.raises(spack.config.ConfigFileError, match="2 missing include path"): + e = environment_from_manifest( + """ spack: include: - /no/such/directory - no/such/file.yaml """ - ) - with pytest.raises(spack.config.ConfigFileError, match="2 missing include path"): + ) with e: e.concretize() assert ev.active_environment() is None -def test_env_with_include_config_files_same_basename(environment_from_manifest): - e = environment_from_manifest( - """ -spack: - include: - - ./path/to/included-config.yaml - - ./second/path/to/include-config.yaml - specs: - - libelf - - mpileaks -""" - ) - - e = ev.read("test") - - fs.mkdirp(os.path.join(e.path, "path", "to")) - with open(os.path.join(e.path, "./path/to/included-config.yaml"), "w") as f: +def test_env_with_include_config_files_same_basename(tmp_path, environment_from_manifest): + file1 = fs.join_path(tmp_path, "path", "to", "included-config.yaml") + fs.mkdirp(os.path.dirname(file1)) + with open(file1, "w") as f: f.write( """\ packages: @@ -755,8 +747,9 @@ def test_env_with_include_config_files_same_basename(environment_from_manifest): """ ) - fs.mkdirp(os.path.join(e.path, "second", "path", "to")) - with open(os.path.join(e.path, "./second/path/to/include-config.yaml"), "w") as f: + file2 = fs.join_path(tmp_path, "second", "path", "included-config.yaml") + fs.mkdirp(os.path.dirname(file2)) + with open(file2, "w") as f: f.write( """\ packages: @@ -765,6 +758,18 @@ def test_env_with_include_config_files_same_basename(environment_from_manifest): """ ) + e = environment_from_manifest( + f""" +spack: + include: + - {file1} + - {file2} + specs: + - libelf + - mpileaks +""" + ) + with e: e.concretize() @@ -801,12 +806,18 @@ def mpileaks_env_config(include_path): ) -def test_env_with_included_config_file(environment_from_manifest, packages_file): +def test_env_with_included_config_file(mutable_mock_env_path, packages_file): """Test inclusion of a relative packages configuration file added to an existing environment. """ + env_root = mutable_mock_env_path + fs.mkdirp(env_root) include_filename = "included-config.yaml" - e = environment_from_manifest( + included_path = env_root / include_filename + shutil.move(packages_file.strpath, included_path) + + spack_yaml = env_root / ev.manifest_name + spack_yaml.write_text( f"""\ spack: include: @@ -816,9 +827,7 @@ def test_env_with_included_config_file(environment_from_manifest, packages_file) """ ) - included_path = os.path.join(e.path, include_filename) - shutil.move(packages_file.strpath, included_path) - + e = ev.Environment(env_root) with e: e.concretize() @@ -851,68 +860,67 @@ def test_env_with_included_config_missing_file(tmpdir, mutable_empty_config): with spack_yaml.open("w") as f: f.write("spack:\n include:\n - {0}\n".format(missing_file.strpath)) - env = ev.Environment(tmpdir.strpath) with pytest.raises(spack.config.ConfigError, match="missing include path"): - ev.activate(env) + ev.Environment(tmpdir.strpath) -def test_env_with_included_config_scope(environment_from_manifest, packages_file): +def test_env_with_included_config_scope(mutable_mock_env_path, packages_file): """Test inclusion of a package file from the environment's configuration stage directory. This test is intended to represent a case where a remote file has already been staged.""" - config_scope_path = os.path.join(ev.root("test"), "config") - - # Configure the environment to include file(s) from the environment's - # remote configuration stage directory. - e = environment_from_manifest(mpileaks_env_config(config_scope_path)) + env_root = mutable_mock_env_path + config_scope_path = env_root / "config" # Copy the packages.yaml file to the environment configuration # directory, so it is picked up during concretization. (Using # copy instead of rename in case the fixture scope changes.) fs.mkdirp(config_scope_path) include_filename = os.path.basename(packages_file.strpath) - included_path = os.path.join(config_scope_path, include_filename) + included_path = config_scope_path / include_filename fs.copy(packages_file.strpath, included_path) + # Configure the environment to include file(s) from the environment's + # remote configuration stage directory. + spack_yaml = env_root / ev.manifest_name + spack_yaml.write_text(mpileaks_env_config(config_scope_path)) + # Ensure the concretized environment reflects contents of the # packages.yaml file. + e = ev.Environment(env_root) with e: e.concretize() assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs()) -def test_env_with_included_config_var_path(environment_from_manifest, packages_file): +def test_env_with_included_config_var_path(tmpdir, packages_file): """Test inclusion of a package configuration file with path variables "staged" in the environment's configuration stage directory.""" - config_var_path = os.path.join("$tempdir", "included-config.yaml") - e = environment_from_manifest(mpileaks_env_config(config_var_path)) + included_file = packages_file.strpath + env_path = pathlib.PosixPath(tmpdir) + config_var_path = os.path.join("$tempdir", "included-packages.yaml") + + spack_yaml = env_path / ev.manifest_name + spack_yaml.write_text(mpileaks_env_config(config_var_path)) config_real_path = substitute_path_variables(config_var_path) - fs.mkdirp(os.path.dirname(config_real_path)) - shutil.move(packages_file.strpath, config_real_path) + shutil.move(included_file, config_real_path) assert os.path.exists(config_real_path) + e = ev.Environment(env_path) with e: e.concretize() assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs()) -def test_env_config_precedence(environment_from_manifest): - e = environment_from_manifest( - """ -spack: - packages: - libelf: - version: ["0.8.12"] - include: - - ./included-config.yaml - specs: - - mpileaks -""" - ) - with open(os.path.join(e.path, "included-config.yaml"), "w") as f: +def test_env_with_included_config_precedence(tmp_path): + """Test included scope and manifest precedence when including a package + configuration file.""" + + included_file = "included-packages.yaml" + included_path = tmp_path / included_file + with open(included_path, "w") as f: f.write( """\ packages: @@ -923,29 +931,50 @@ def test_env_config_precedence(environment_from_manifest): """ ) + spack_yaml = tmp_path / ev.manifest_name + spack_yaml.write_text( + f"""\ +spack: + packages: + libelf: + version: ["0.8.12"] + include: + - {os.path.join(".", included_file)} + specs: + - mpileaks +""" + ) + + e = ev.Environment(tmp_path) with e: e.concretize() + specs = e._get_environment_specs() # ensure included scope took effect - assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs()) + assert any(x.satisfies("mpileaks@2.2") for x in specs) # ensure env file takes precedence - assert any(x.satisfies("libelf@0.8.12") for x in e._get_environment_specs()) + assert any(x.satisfies("libelf@0.8.12") for x in specs) -def test_included_config_precedence(environment_from_manifest): - e = environment_from_manifest( - """ +def test_env_with_included_configs_precedence(tmp_path): + """Test precendence of multiple included configuration files.""" + file1 = "high-config.yaml" + file2 = "low-config.yaml" + + spack_yaml = tmp_path / ev.manifest_name + spack_yaml.write_text( + f"""\ spack: include: - - ./high-config.yaml # this one should take precedence - - ./low-config.yaml + - {os.path.join(".", file1)} # this one should take precedence + - {os.path.join(".", file2)} specs: - mpileaks """ ) - with open(os.path.join(e.path, "high-config.yaml"), "w") as f: + with open(tmp_path / file1, "w") as f: f.write( """\ packages: @@ -954,7 +983,7 @@ def test_included_config_precedence(environment_from_manifest): """ ) - with open(os.path.join(e.path, "low-config.yaml"), "w") as f: + with open(tmp_path / file2, "w") as f: f.write( """\ packages: @@ -965,30 +994,115 @@ def test_included_config_precedence(environment_from_manifest): """ ) + e = ev.Environment(tmp_path) with e: e.concretize() + specs = e._get_environment_specs() - assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs()) + # ensure included package spec took precedence over manifest spec + assert any(x.satisfies("mpileaks@2.2") for x in specs) - assert any([x.satisfies("libelf@0.8.10") for x in e._get_environment_specs()]) + # ensure first included package spec took precedence over one from second + assert any(x.satisfies("libelf@0.8.10") for x in specs) -def test_bad_env_yaml_format(tmpdir): - filename = str(tmpdir.join("spack.yaml")) +@pytest.mark.regression("39248") +def test_bad_env_yaml_format_remove(mutable_mock_env_path): + badenv = "badenv" + env("create", badenv) + filename = mutable_mock_env_path / "spack.yaml" with open(filename, "w") as f: f.write( """\ -spack: - spacks: - mpileaks """ ) - with tmpdir.as_cwd(): - with pytest.raises(spack.config.ConfigFormatError) as e: - env("create", "test", "./spack.yaml") - assert "spack.yaml:2" in str(e) - assert "'spacks' was unexpected" in str(e) + assert badenv in env("list") + env("remove", "-y", badenv) + assert badenv not in env("list") + + +@pytest.mark.regression("39248") +@pytest.mark.parametrize( + "error,message,contents", + [ + ( + spack.config.ConfigFormatError, + "not of type", + """\ +spack: + specs: mpi@2.0 +""", + ), + ( + ev.SpackEnvironmentConfigError, + "duplicate key", + """\ +spack: + packages: + all: + providers: + mpi: [mvapich2] + mpi: [mpich] +""", + ), + ( + spack.config.ConfigFormatError, + "'specks' was unexpected", + """\ +spack: + specks: + - libdwarf +""", + ), + ], +) +def test_bad_env_yaml_create_fails(tmp_path, mutable_mock_env_path, error, message, contents): + """Ensure creation with invalid yaml does NOT create or leave the environment.""" + filename = tmp_path / ev.manifest_name + filename.write_text(contents) + env_name = "bad_env" + with pytest.raises(error, match=message): + env("create", env_name, str(filename)) + + assert env_name not in env("list") + manifest = mutable_mock_env_path / env_name / ev.manifest_name + assert not os.path.exists(str(manifest)) + + +@pytest.mark.regression("39248") +@pytest.mark.parametrize("answer", ["-y", ""]) +def test_multi_env_remove(mutable_mock_env_path, monkeypatch, answer): + """Test removal (or not) of a valid and invalid environment""" + remove_environment = answer == "-y" + monkeypatch.setattr(tty, "get_yes_or_no", lambda prompt, default: remove_environment) + + environments = ["goodenv", "badenv"] + for e in environments: + env("create", e) + + # Ensure the bad environment contains invalid yaml + filename = mutable_mock_env_path / environments[1] / ev.manifest_name + filename.write_text( + """\ + - libdwarf +""" + ) + + assert all(e in env("list") for e in environments) + + args = [answer] if answer else [] + args.extend(environments) + output = env("remove", *args, fail_on_error=False) + + if remove_environment is True: + # Successfully removed (and reported removal) of *both* environments + assert not all(e in env("list") for e in environments) + assert output.count("Successfully removed") == len(environments) + else: + # Not removing any of the environments + assert all(e in env("list") for e in environments) def test_env_loads(install_mockery, mock_fetch): @@ -1043,7 +1157,7 @@ def test_env_commands_die_with_no_env_arg(): env("remove") # these have an optional env arg and raise errors via tty.die - with pytest.raises(spack.main.SpackCommandError): + with pytest.raises(SpackCommandError): env("loads") # This should NOT raise an error with no environment @@ -1116,12 +1230,12 @@ def test_uninstall_removes_from_env(mock_stage, mock_fetch, install_mockery): @pytest.mark.usefixtures("config") -def test_indirect_build_dep(tmpdir): +def test_indirect_build_dep(tmp_path): """Simple case of X->Y->Z where Y is a build/link dep and Z is a build-only dep. Make sure this concrete DAG is preserved when writing the environment out and reading it back. """ - builder = spack.repo.MockRepositoryBuilder(tmpdir) + builder = spack.repo.MockRepositoryBuilder(tmp_path / "repo") builder.add_package("z") builder.add_package("y", dependencies=[("z", "build", None)]) builder.add_package("x", dependencies=[("y", None, None)]) @@ -1144,7 +1258,7 @@ def test_indirect_build_dep(tmpdir): @pytest.mark.usefixtures("config") -def test_store_different_build_deps(tmpdir): +def test_store_different_build_deps(tmp_path): r"""Ensure that an environment can store two instances of a build-only dependency:: @@ -1155,7 +1269,7 @@ def test_store_different_build_deps(tmpdir): z1 """ - builder = spack.repo.MockRepositoryBuilder(tmpdir) + builder = spack.repo.MockRepositoryBuilder(tmp_path / "mirror") builder.add_package("z") builder.add_package("y", dependencies=[("z", "build", None)]) builder.add_package("x", dependencies=[("y", None, None), ("z", "build", None)]) @@ -1527,11 +1641,10 @@ def test_stack_yaml_remove_from_list(tmpdir): assert Spec("callpath") in test.user_specs -def test_stack_yaml_remove_from_list_force(tmpdir): - filename = str(tmpdir.join("spack.yaml")) - with open(filename, "w") as f: - f.write( - """\ +def test_stack_yaml_remove_from_list_force(tmp_path): + spack_yaml = tmp_path / ev.manifest_name + spack_yaml.write_text( + """\ spack: definitions: - packages: [mpileaks, callpath] @@ -1540,20 +1653,20 @@ def test_stack_yaml_remove_from_list_force(tmpdir): - [$packages] - [^mpich, ^zmpi] """ - ) - with tmpdir.as_cwd(): - env("create", "test", "./spack.yaml") - with ev.read("test"): - concretize() - remove("-f", "-l", "packages", "mpileaks") - find_output = find("-c") + ) - assert "mpileaks" not in find_output + env("create", "test", str(spack_yaml)) + with ev.read("test"): + concretize() + remove("-f", "-l", "packages", "mpileaks") + find_output = find("-c") - test = ev.read("test") - assert len(test.user_specs) == 2 - assert Spec("callpath ^zmpi") in test.user_specs - assert Spec("callpath ^mpich") in test.user_specs + assert "mpileaks" not in find_output + + test = ev.read("test") + assert len(test.user_specs) == 2 + assert Spec("callpath ^zmpi") in test.user_specs + assert Spec("callpath ^mpich") in test.user_specs def test_stack_yaml_remove_from_matrix_no_effect(tmpdir): @@ -1599,7 +1712,7 @@ def test_stack_yaml_force_remove_from_matrix(tmpdir): with tmpdir.as_cwd(): env("create", "test", "./spack.yaml") with ev.read("test") as e: - concretize() + e.concretize() before_user = e.user_specs.specs before_conc = e.concretized_user_specs @@ -2355,7 +2468,7 @@ def test_env_activate_sh_prints_shell_output(tmpdir, mock_stage, mock_fetch, ins This is a cursory check; ``share/spack/qa/setup-env-test.sh`` checks for correctness. """ - env("create", "test", add_view=True) + env("create", "test") out = env("activate", "--sh", "test") assert "export SPACK_ENV=" in out @@ -2370,7 +2483,7 @@ def test_env_activate_sh_prints_shell_output(tmpdir, mock_stage, mock_fetch, ins def test_env_activate_csh_prints_shell_output(tmpdir, mock_stage, mock_fetch, install_mockery): """Check the shell commands output by ``spack env activate --csh``.""" - env("create", "test", add_view=True) + env("create", "test") out = env("activate", "--csh", "test") assert "setenv SPACK_ENV" in out @@ -2387,7 +2500,7 @@ def test_env_activate_csh_prints_shell_output(tmpdir, mock_stage, mock_fetch, in def test_env_activate_default_view_root_unconditional(mutable_mock_env_path): """Check that the root of the default view in the environment is added to the shell unconditionally.""" - env("create", "test", add_view=True) + env("create", "test") with ev.read("test") as e: viewdir = e.default_view.root @@ -2402,6 +2515,27 @@ def test_env_activate_default_view_root_unconditional(mutable_mock_env_path): ) +def test_env_activate_custom_view(tmp_path: pathlib.Path, mock_packages): + """Check that an environment can be activated with a non-default view.""" + env_template = tmp_path / "spack.yaml" + default_dir = tmp_path / "defaultdir" + nondefaultdir = tmp_path / "nondefaultdir" + with open(env_template, "w") as f: + f.write( + f"""\ +spack: + specs: [a] + view: + default: + root: {default_dir} + nondefault: + root: {nondefaultdir}""" + ) + env("create", "test", str(env_template)) + shell = env("activate", "--sh", "--with-view", "nondefault", "test") + assert os.path.join(nondefaultdir, "bin") in shell + + def test_concretize_user_specs_together(): e = ev.create("coconcretization") e.unify = True @@ -2418,8 +2552,12 @@ def test_concretize_user_specs_together(): e.remove("mpich") e.add("mpich2") + exc_cls = spack.error.SpackError + if spack.config.get("config:concretizer") == "clingo": + exc_cls = spack.error.UnsatisfiableSpecError + # Concretizing without invalidating the concrete spec for mpileaks fails - with pytest.raises(spack.error.UnsatisfiableSpecError): + with pytest.raises(exc_cls): e.concretize() e.concretize(force=True) @@ -2451,9 +2589,12 @@ def test_duplicate_packages_raise_when_concretizing_together(): e.add("mpileaks~opt") e.add("mpich") - with pytest.raises( - spack.error.UnsatisfiableSpecError, match=r"You could consider setting `concretizer:unify`" - ): + exc_cls, match = spack.error.SpackError, None + if spack.config.get("config:concretizer") == "clingo": + exc_cls = spack.error.UnsatisfiableSpecError + match = r"You could consider setting `concretizer:unify`" + + with pytest.raises(exc_cls, match=match): e.concretize() @@ -2480,7 +2621,7 @@ def test_env_write_only_non_default_nested(tmpdir): - matrix: - [mpileaks] packages: - mpileaks: + all: compiler: [gcc] view: true """ @@ -2800,11 +2941,11 @@ def test_custom_store_in_environment(mutable_config, tmpdir): install_root ) ) - current_store_root = str(spack.store.root) + current_store_root = str(spack.store.STORE.root) assert str(current_store_root) != install_root with spack.environment.Environment(str(tmpdir)): - assert str(spack.store.root) == install_root - assert str(spack.store.root) == current_store_root + assert str(spack.store.STORE.root) == install_root + assert str(spack.store.STORE.root) == current_store_root def test_activate_temp(monkeypatch, tmpdir): @@ -2818,6 +2959,25 @@ def test_activate_temp(monkeypatch, tmpdir): assert ev.is_env_dir(str(tmpdir)) +def test_activate_default(monkeypatch): + """Tests whether `spack env activate` creates / activates the default + environment""" + assert not ev.exists("default") + + # Activating it the first time should create it + env("activate", "--sh") + env("deactivate", "--sh") + assert ev.exists("default") + + # Activating it while it already exists should work + env("activate", "--sh") + env("deactivate", "--sh") + assert ev.exists("default") + + env("remove", "-y", "default") + assert not ev.exists("default") + + def test_env_view_fail_if_symlink_points_elsewhere(tmpdir, install_mockery, mock_fetch): view = str(tmpdir.join("view")) # Put a symlink to an actual directory in view @@ -3136,6 +3296,57 @@ def test_environment_depfile_makefile(depfile_flags, expected_installs, tmpdir, assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install)) +def test_depfile_safe_format(): + """Test that depfile.MakefileSpec.safe_format() escapes target names.""" + + class SpecLike: + def format(self, _): + return "abc@def=ghi" + + spec = depfile.MakefileSpec(SpecLike()) + assert spec.safe_format("{name}") == "abc_def_ghi" + assert spec.unsafe_format("{name}") == "abc@def=ghi" + + +def test_depfile_works_with_gitversions(tmpdir, mock_packages, monkeypatch): + """Git versions may contain = chars, which should be escaped in targets, + otherwise they're interpreted as makefile variable assignments.""" + monkeypatch.setattr(spack.package_base.PackageBase, "git", "repo.git", raising=False) + env("create", "test") + + make = Executable("make") + makefile = str(tmpdir.join("Makefile")) + + # Create an environment with dttop and dtlink1 both at a git version, + # and generate a depfile + with ev.read("test"): + add(f"dttop@{'a' * 40}=1.0 ^dtlink1@{'b' * 40}=1.0") + concretize() + env("depfile", "-o", makefile, "--make-disable-jobserver", "--make-prefix=prefix") + + # Do a dry run on the generated depfile + out = make("-n", "-f", makefile, output=str) + + # Check that all specs are there (without duplicates) + specs_that_make_would_install = _parse_dry_run_package_installs(out) + expected_installs = [ + "dtbuild1", + "dtbuild2", + "dtbuild3", + "dtlink1", + "dtlink2", + "dtlink3", + "dtlink4", + "dtlink5", + "dtrun1", + "dtrun2", + "dtrun3", + "dttop", + ] + assert set(specs_that_make_would_install) == set(expected_installs) + assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install)) + + @pytest.mark.parametrize( "picked_package,expected_installs", [ @@ -3252,6 +3463,20 @@ def test_spack_package_ids_variable(tmpdir, mock_packages): assert "post-install: {}".format(s.dag_hash()) in out +def test_depfile_empty_does_not_error(tmp_path): + # For empty environments Spack should create a depfile that does nothing + make = Executable("make") + makefile = str(tmp_path / "Makefile") + + env("create", "test") + with ev.read("test"): + env("depfile", "-o", makefile) + + make("-f", makefile) + + assert make.returncode == 0 + + def test_unify_when_possible_works_around_conflicts(): e = ev.create("coconcretization") e.unify = "when_possible" @@ -3297,12 +3522,11 @@ def test_relative_view_path_on_command_line_is_made_absolute(tmp_path, config): assert os.path.samefile("view", environment.default_view.root) -def test_environment_created_in_users_location(mutable_config, tmpdir): +def test_environment_created_in_users_location(mutable_mock_env_path, tmp_path): """Test that an environment is created in a location based on the config""" - spack.config.set("config:environments_root", str(tmpdir.join("envs"))) - env_dir = spack.config.get("config:environments_root") + env_dir = str(mutable_mock_env_path) - assert tmpdir.strpath in env_dir + assert str(tmp_path) in env_dir assert not os.path.isdir(env_dir) dir_name = "user_env" diff --git a/lib/spack/spack/test/cmd/extensions.py b/lib/spack/spack/test/cmd/extensions.py index c1bfc2b153fbb0..e3f82b9439c138 100644 --- a/lib/spack/spack/test/cmd/extensions.py +++ b/lib/spack/spack/test/cmd/extensions.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys import pytest @@ -23,7 +22,7 @@ def python_database(mock_packages, mutable_database): yield -@pytest.mark.skipif(sys.platform == "win32", reason="All Fetchers Failed") +@pytest.mark.not_on_windows("All Fetchers Failed") @pytest.mark.db def test_extensions(mock_packages, python_database, config, capsys): ext2 = Spec("py-extension2").concretized() diff --git a/lib/spack/spack/test/cmd/external.py b/lib/spack/spack/test/cmd/external.py index 135ed026cbe386..e9a387aac03e66 100644 --- a/lib/spack/spack/test/cmd/external.py +++ b/lib/spack/spack/test/cmd/external.py @@ -28,59 +28,41 @@ def _mock_search(path_hints=None): return _factory -@pytest.fixture -def _platform_executables(monkeypatch): - def _win_exe_ext(): - return ".bat" - - monkeypatch.setattr(spack.util.path, "win_exe_ext", _win_exe_ext) - - def define_plat_exe(exe): if sys.platform == "win32": exe += ".bat" return exe -def test_find_external_single_package(mock_executable, executables_found, _platform_executables): - pkgs_to_check = [spack.repo.path.get_pkg_class("cmake")] +def test_find_external_single_package(mock_executable): cmake_path = mock_executable("cmake", output="echo cmake version 1.foo") - executables_found({str(cmake_path): define_plat_exe("cmake")}) + search_dir = cmake_path.parent.parent - pkg_to_entries = spack.detection.by_executable(pkgs_to_check) + specs_by_package = spack.detection.by_path(["cmake"], path_hints=[str(search_dir)]) - pkg, entries = next(iter(pkg_to_entries.items())) - single_entry = next(iter(entries)) + assert len(specs_by_package) == 1 and "cmake" in specs_by_package + detected_spec = specs_by_package["cmake"] + assert len(detected_spec) == 1 and detected_spec[0].spec == Spec("cmake@1.foo") - assert single_entry.spec == Spec("cmake@1.foo") - - -def test_find_external_two_instances_same_package( - mock_executable, executables_found, _platform_executables -): - pkgs_to_check = [spack.repo.path.get_pkg_class("cmake")] +def test_find_external_two_instances_same_package(mock_executable): # Each of these cmake instances is created in a different prefix # In Windows, quoted strings are echo'd with quotes includes # we need to avoid that for proper regex. - cmake_path1 = mock_executable( - "cmake", output="echo cmake version 1.foo", subdir=("base1", "bin") - ) - cmake_path2 = mock_executable( - "cmake", output="echo cmake version 3.17.2", subdir=("base2", "bin") - ) - cmake_exe = define_plat_exe("cmake") - executables_found({str(cmake_path1): cmake_exe, str(cmake_path2): cmake_exe}) + cmake1 = mock_executable("cmake", output="echo cmake version 1.foo", subdir=("base1", "bin")) + cmake2 = mock_executable("cmake", output="echo cmake version 3.17.2", subdir=("base2", "bin")) + search_paths = [str(cmake1.parent.parent), str(cmake2.parent.parent)] - pkg_to_entries = spack.detection.by_executable(pkgs_to_check) + finder = spack.detection.path.ExecutablesFinder() + detected_specs = finder.find(pkg_name="cmake", initial_guess=search_paths) - pkg, entries = next(iter(pkg_to_entries.items())) - spec_to_path = dict((e.spec, e.prefix) for e in entries) + assert len(detected_specs) == 2 + spec_to_path = {e.spec: e.prefix for e in detected_specs} assert spec_to_path[Spec("cmake@1.foo")] == ( - spack.detection.executable_prefix(os.path.dirname(cmake_path1)) + spack.detection.executable_prefix(str(cmake1.parent)) ) assert spec_to_path[Spec("cmake@3.17.2")] == ( - spack.detection.executable_prefix(os.path.dirname(cmake_path2)) + spack.detection.executable_prefix(str(cmake2.parent)) ) @@ -112,23 +94,6 @@ def test_get_executables(working_env, mock_executable): external = SpackCommand("external") -def test_find_external_cmd(mutable_config, working_env, mock_executable, _platform_executables): - """Test invoking 'spack external find' with additional package arguments, - which restricts the set of packages that Spack looks for. - """ - cmake_path1 = mock_executable("cmake", output="echo cmake version 1.foo") - prefix = os.path.dirname(os.path.dirname(cmake_path1)) - - os.environ["PATH"] = os.pathsep.join([os.path.dirname(cmake_path1)]) - external("find", "cmake") - - pkgs_cfg = spack.config.get("packages") - cmake_cfg = pkgs_cfg["cmake"] - cmake_externals = cmake_cfg["externals"] - - assert {"spec": "cmake@1.foo", "prefix": prefix} in cmake_externals - - def test_find_external_cmd_not_buildable(mutable_config, working_env, mock_executable): """When the user invokes 'spack external find --not-buildable', the config for any package where Spack finds an external version should be marked as @@ -138,50 +103,30 @@ def test_find_external_cmd_not_buildable(mutable_config, working_env, mock_execu os.environ["PATH"] = os.pathsep.join([os.path.dirname(cmake_path1)]) external("find", "--not-buildable", "cmake") pkgs_cfg = spack.config.get("packages") + assert "cmake" in pkgs_cfg assert not pkgs_cfg["cmake"]["buildable"] -def test_find_external_cmd_full_repo( - mutable_config, working_env, mock_executable, mutable_mock_repo, _platform_executables -): - """Test invoking 'spack external find' with no additional arguments, which - iterates through each package in the repository. - """ - exe_path1 = mock_executable("find-externals1-exe", output="echo find-externals1 version 1.foo") - prefix = os.path.dirname(os.path.dirname(exe_path1)) - os.environ["PATH"] = os.pathsep.join([os.path.dirname(exe_path1)]) - external("find", "--all") - - pkgs_cfg = spack.config.get("packages") - pkg_cfg = pkgs_cfg["find-externals1"] - pkg_externals = pkg_cfg["externals"] - - assert {"spec": "find-externals1@1.foo", "prefix": prefix} in pkg_externals - - -def test_find_external_cmd_exclude( - mutable_config, working_env, mock_executable, mutable_mock_repo, _platform_executables -): - """Test invoking 'spack external find --all --exclude', to ensure arbitary - external packages can be ignored. - """ - exe_path1 = mock_executable("find-externals1-exe", output="echo find-externals1 version 1.foo") - os.environ["PATH"] = os.pathsep.join([os.path.dirname(exe_path1)]) - external("find", "--all", "--exclude=find-externals1") - - pkgs_cfg = spack.config.get("packages") - - assert "find-externals1" not in pkgs_cfg.keys() - - -def test_find_external_no_manifest( - mutable_config, - working_env, - mock_executable, - mutable_mock_repo, - _platform_executables, - monkeypatch, -): +@pytest.mark.parametrize( + "names,tags,exclude,expected", + [ + # find --all + (None, ["detectable"], [], ["builtin.mock.find-externals1"]), + # find --all --exclude find-externals1 + (None, ["detectable"], ["builtin.mock.find-externals1"], []), + (None, ["detectable"], ["find-externals1"], []), + # find cmake (and cmake is not detectable) + (["cmake"], ["detectable"], [], []), + ], +) +def test_package_selection(names, tags, exclude, expected, mutable_mock_repo): + """Tests various cases of selecting packages""" + # In the mock repo we only have 'find-externals1' that is detectable + result = spack.cmd.external.packages_to_search_for(names=names, tags=tags, exclude=exclude) + assert set(result) == set(expected) + + +def test_find_external_no_manifest(mutable_config, working_env, mutable_mock_repo, monkeypatch): """The user runs 'spack external find'; the default path for storing manifest files does not exist. Ensure that the command does not fail. @@ -194,13 +139,7 @@ def test_find_external_no_manifest( def test_find_external_empty_default_manifest_dir( - mutable_config, - working_env, - mock_executable, - mutable_mock_repo, - _platform_executables, - tmpdir, - monkeypatch, + mutable_config, working_env, mutable_mock_repo, tmpdir, monkeypatch ): """The user runs 'spack external find'; the default path for storing manifest files exists but is empty. Ensure that the command does not @@ -212,16 +151,10 @@ def test_find_external_empty_default_manifest_dir( external("find") -@pytest.mark.skipif(sys.platform == "win32", reason="Can't chmod on Windows") +@pytest.mark.not_on_windows("Can't chmod on Windows") @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_find_external_manifest_with_bad_permissions( - mutable_config, - working_env, - mock_executable, - mutable_mock_repo, - _platform_executables, - tmpdir, - monkeypatch, + mutable_config, working_env, mutable_mock_repo, tmpdir, monkeypatch ): """The user runs 'spack external find'; the default path for storing manifest files exists but with insufficient permissions. Check that @@ -261,24 +194,6 @@ def fail(): assert "Skipping manifest and continuing" in output -def test_find_external_nonempty_default_manifest_dir( - mutable_database, - mutable_mock_repo, - _platform_executables, - tmpdir, - monkeypatch, - directory_with_manifest, -): - """The user runs 'spack external find'; the default manifest directory - contains a manifest file. Ensure that the specs are read. - """ - monkeypatch.setenv("PATH", "") - monkeypatch.setattr(spack.cray_manifest, "default_path", str(directory_with_manifest)) - external("find") - specs = spack.store.db.query("hwloc") - assert any(x.dag_hash() == "hwlocfakehashaaa" for x in specs) - - def test_find_external_merge(mutable_config, mutable_mock_repo): """Check that 'spack find external' doesn't overwrite an existing spec entry in packages.yaml. @@ -312,59 +227,28 @@ def test_list_detectable_packages(mutable_config, mutable_mock_repo): assert external.returncode == 0 -def test_packages_yaml_format(mock_executable, mutable_config, monkeypatch, _platform_executables): - # Prepare an environment to detect a fake gcc +def test_overriding_prefix(mock_executable, mutable_config, monkeypatch): gcc_exe = mock_executable("gcc", output="echo 4.2.1") - prefix = os.path.dirname(gcc_exe) - monkeypatch.setenv("PATH", prefix) - - # Find the external spec - external("find", "gcc") - - # Check entries in 'packages.yaml' - packages_yaml = spack.config.get("packages") - assert "gcc" in packages_yaml - assert "externals" in packages_yaml["gcc"] - externals = packages_yaml["gcc"]["externals"] - assert len(externals) == 1 - external_gcc = externals[0] - assert external_gcc["spec"] == "gcc@4.2.1 languages=c" - assert external_gcc["prefix"] == os.path.dirname(prefix) - assert "extra_attributes" in external_gcc - extra_attributes = external_gcc["extra_attributes"] - assert "prefix" not in extra_attributes - assert extra_attributes["compilers"]["c"] == str(gcc_exe) - - -def test_overriding_prefix(mock_executable, mutable_config, monkeypatch, _platform_executables): - # Prepare an environment to detect a fake gcc that - # override its external prefix - gcc_exe = mock_executable("gcc", output="echo 4.2.1") - prefix = os.path.dirname(gcc_exe) - monkeypatch.setenv("PATH", prefix) + search_dir = gcc_exe.parent @classmethod def _determine_variants(cls, exes, version_str): return "languages=c", {"prefix": "/opt/gcc/bin", "compilers": {"c": exes[0]}} - gcc_cls = spack.repo.path.get_pkg_class("gcc") + gcc_cls = spack.repo.PATH.get_pkg_class("gcc") monkeypatch.setattr(gcc_cls, "determine_variants", _determine_variants) - # Find the external spec - external("find", "gcc") + finder = spack.detection.path.ExecutablesFinder() + detected_specs = finder.find(pkg_name="gcc", initial_guess=[str(search_dir)]) - # Check entries in 'packages.yaml' - packages_yaml = spack.config.get("packages") - assert "gcc" in packages_yaml - assert "externals" in packages_yaml["gcc"] - externals = packages_yaml["gcc"]["externals"] - assert len(externals) == 1 - assert externals[0]["prefix"] == os.path.sep + os.path.join("opt", "gcc", "bin") + assert len(detected_specs) == 1 + gcc = detected_specs[0].spec + assert gcc.name == "gcc" + assert gcc.external_path == os.path.sep + os.path.join("opt", "gcc", "bin") -def test_new_entries_are_reported_correctly( - mock_executable, mutable_config, monkeypatch, _platform_executables -): + +def test_new_entries_are_reported_correctly(mock_executable, mutable_config, monkeypatch): # Prepare an environment to detect a fake gcc gcc_exe = mock_executable("gcc", output="echo 4.2.1") prefix = os.path.dirname(gcc_exe) @@ -399,7 +283,7 @@ def test_use_tags_for_detection(command_args, mock_executable, mutable_config, m @pytest.mark.regression("38733") -@pytest.mark.skipif(sys.platform == "win32", reason="the test uses bash scripts") +@pytest.mark.not_on_windows("the test uses bash scripts") def test_failures_in_scanning_do_not_result_in_an_error( mock_executable, monkeypatch, mutable_config ): diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index 249f93257f9320..afeb53388d495b 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -117,13 +117,13 @@ def test_tag2_tag3(parser, specs): assert len(specs) == 0 +@pytest.mark.parametrize( + "args,with_namespace", [([], False), (["--namespace"], True), (["--namespaces"], True)] +) @pytest.mark.db -def test_namespaces_shown_correctly(database): - out = find() - assert "builtin.mock.zmpi" not in out - - out = find("--namespace") - assert "builtin.mock.zmpi" in out +def test_namespaces_shown_correctly(args, with_namespace, database): + """Test that --namespace(s) works. Old syntax is --namespace""" + assert ("builtin.mock.zmpi" in find(*args)) == with_namespace @pytest.mark.db @@ -332,7 +332,7 @@ def test_find_command_basic_usage(database): assert "mpileaks" in output -@pytest.mark.skipif(sys.platform == "win32", reason="envirnment is not yet supported on windows") +@pytest.mark.not_on_windows("envirnment is not yet supported on windows") @pytest.mark.regression("9875") def test_find_prefix_in_env( mutable_mock_env_path, install_mockery, mock_fetch, mock_packages, mock_archive, config @@ -352,7 +352,7 @@ def test_find_loaded(database, working_env): assert output == "" os.environ[uenv.spack_loaded_hashes_var] = ":".join( - [x.dag_hash() for x in spack.store.db.query()] + [x.dag_hash() for x in spack.store.STORE.db.query()] ) output = find("--loaded") expected = find() diff --git a/lib/spack/spack/test/cmd/gc.py b/lib/spack/spack/test/cmd/gc.py index 15f2a3242b8415..d692628e106fe0 100644 --- a/lib/spack/spack/test/cmd/gc.py +++ b/lib/spack/spack/test/cmd/gc.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys import pytest @@ -13,7 +12,7 @@ gc = spack.main.SpackCommand("gc") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.mark.db diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py index 0c5f087c9eed62..78a2a9ece9f952 100644 --- a/lib/spack/spack/test/cmd/gpg.py +++ b/lib/spack/spack/test/cmd/gpg.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import sys import pytest @@ -22,7 +21,7 @@ bootstrap = SpackCommand("bootstrap") mirror = SpackCommand("mirror") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") # test gpg command detection diff --git a/lib/spack/spack/test/cmd/info.py b/lib/spack/spack/test/cmd/info.py index 4b2f5d2b3980ac..5748323d8cba98 100644 --- a/lib/spack/spack/test/cmd/info.py +++ b/lib/spack/spack/test/cmd/info.py @@ -25,7 +25,7 @@ def parser(): def print_buffer(monkeypatch): buffer = [] - def _print(*args): + def _print(*args, **kwargs): buffer.extend(args) monkeypatch.setattr(spack.cmd.info.color, "cprint", _print, raising=False) @@ -88,6 +88,7 @@ def test_info_fields(pkg_query, parser, print_buffer): "Installation Phases:", "Virtual Packages:", "Tags:", + "Licenses:", ) args = parser.parse_args(["--all", pkg_query]) diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index e9503d80d82f71..ef9d19d77884a8 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -7,8 +7,8 @@ import filecmp import itertools import os +import pathlib import re -import sys import time import pytest @@ -23,6 +23,7 @@ import spack.environment as ev import spack.hash_types as ht import spack.package_base +import spack.store import spack.util.executable from spack.error import SpackError from spack.main import SpackCommand @@ -37,8 +38,6 @@ buildcache = SpackCommand("buildcache") find = SpackCommand("find") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") - @pytest.fixture() def noop_install(monkeypatch): @@ -203,7 +202,7 @@ def test_show_log_on_error( assert isinstance(install.error, spack.build_environment.ChildError) assert install.error.pkg.name == "build-error" - assert "==> Installing build-error" in out + assert "Installing build-error" in out assert "See build log for details:" in out @@ -216,7 +215,9 @@ def test_install_overwrite(mock_packages, mock_archive, mock_fetch, config, inst # Ignore manifest and install times manifest = os.path.join( - spec.prefix, spack.store.layout.metadata_dir, spack.store.layout.manifest_file_name + spec.prefix, + spack.store.STORE.layout.metadata_dir, + spack.store.STORE.layout.manifest_file_name, ) ignores = [manifest, spec.package.times_log_path] @@ -260,9 +261,9 @@ def test_install_commit(mock_git_version_info, install_mockery, mock_packages, m """ repo_path, filename, commits = mock_git_version_info - monkeypatch.setattr( - spack.package_base.PackageBase, "git", "file://%s" % repo_path, raising=False - ) + file_url = pathlib.Path(repo_path).as_uri() + + monkeypatch.setattr(spack.package_base.PackageBase, "git", file_url, raising=False) # Use the earliest commit in the respository spec = Spec(f"git-test-commit@{commits[-1]}").concretized() @@ -291,7 +292,9 @@ def test_install_overwrite_multiple( install("cmake") ld_manifest = os.path.join( - libdwarf.prefix, spack.store.layout.metadata_dir, spack.store.layout.manifest_file_name + libdwarf.prefix, + spack.store.STORE.layout.metadata_dir, + spack.store.STORE.layout.manifest_file_name, ) ld_ignores = [ld_manifest, libdwarf.package.times_log_path] @@ -300,7 +303,9 @@ def test_install_overwrite_multiple( expected_libdwarf_md5 = fs.hash_directory(libdwarf.prefix, ignore=ld_ignores) cm_manifest = os.path.join( - cmake.prefix, spack.store.layout.metadata_dir, spack.store.layout.manifest_file_name + cmake.prefix, + spack.store.STORE.layout.metadata_dir, + spack.store.STORE.layout.manifest_file_name, ) cm_ignores = [cm_manifest, cmake.package.times_log_path] @@ -512,7 +517,7 @@ def test_extra_files_are_archived( install("archive-files") - archive_dir = os.path.join(spack.store.layout.metadata_path(s), "archived-files") + archive_dir = os.path.join(spack.store.STORE.layout.metadata_path(s), "archived-files") config_log = os.path.join(archive_dir, mock_archive.expanded_archive_basedir, "config.log") assert os.path.exists(config_log) @@ -541,6 +546,7 @@ def test_cdash_report_concretization_error( assert any(x in content for x in expected_messages) +@pytest.mark.not_on_windows("Windows log_output logs phase header out of order") @pytest.mark.disable_clean_stage_check def test_cdash_upload_build_error(tmpdir, mock_fetch, install_mockery, capfd): # capfd interferes with Spack's capturing @@ -699,9 +705,11 @@ def test_cache_only_fails(tmpdir, mock_fetch, install_mockery, capfd): assert "was not installed" in out # Check that failure prefix locks are still cached - failure_lock_prefixes = ",".join(spack.store.db._prefix_failures.keys()) - assert "libelf" in failure_lock_prefixes - assert "libdwarf" in failure_lock_prefixes + failed_packages = [ + pkg_name for dag_hash, pkg_name in spack.store.STORE.failure_tracker.locker.locks.keys() + ] + assert "libelf" in failed_packages + assert "libdwarf" in failed_packages def test_install_only_dependencies(tmpdir, mock_fetch, install_mockery): @@ -738,6 +746,7 @@ def test_install_deps_then_package(tmpdir, mock_fetch, install_mockery): assert os.path.exists(root.prefix) +@pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") @pytest.mark.regression("12002") def test_install_only_dependencies_in_env( tmpdir, mock_fetch, install_mockery, mutable_mock_env_path @@ -887,7 +896,7 @@ def test_install_help_does_not_show_cdash_options(capsys): assert "CDash URL" not in captured.out -def test_install_help_cdash(capsys): +def test_install_help_cdash(): """Make sure `spack install --help-cdash` describes CDash arguments""" install_cmd = SpackCommand("install") out = install_cmd("--help-cdash") @@ -904,6 +913,7 @@ def test_cdash_auth_token(tmpdir, mock_fetch, install_mockery, capfd): assert "Using CDash auth token from environment" in out +@pytest.mark.not_on_windows("Windows log_output logs phase header out of order") @pytest.mark.disable_clean_stage_check def test_cdash_configure_warning(tmpdir, mock_fetch, install_mockery, capfd): # capfd interferes with Spack's capturing of e.g., Build.xml output @@ -929,6 +939,7 @@ def test_cdash_configure_warning(tmpdir, mock_fetch, install_mockery, capfd): assert "foo: No such file or directory" in content +@pytest.mark.not_on_windows("ArchSpec gives test platform debian rather than windows") def test_compiler_bootstrap( install_mockery_mutable_config, mock_packages, @@ -945,6 +956,7 @@ def test_compiler_bootstrap( install("a%gcc@=12.0") +@pytest.mark.not_on_windows("Binary mirrors not supported on windows") def test_compiler_bootstrap_from_binary_mirror( install_mockery_mutable_config, mock_packages, @@ -966,7 +978,7 @@ def test_compiler_bootstrap_from_binary_mirror( install("gcc@=10.2.0") # Put installed compiler in the buildcache - buildcache("push", "-u", "-a", "-f", mirror_dir.strpath, "gcc@10.2.0") + buildcache("push", "-u", "-f", mirror_dir.strpath, "gcc@10.2.0") # Now uninstall the compiler uninstall("-y", "gcc@10.2.0") @@ -985,6 +997,7 @@ def test_compiler_bootstrap_from_binary_mirror( install("--no-cache", "--only", "package", "b%gcc@10.2.0") +@pytest.mark.not_on_windows("ArchSpec gives test platform debian rather than windows") @pytest.mark.regression("16221") def test_compiler_bootstrap_already_installed( install_mockery_mutable_config, @@ -1028,6 +1041,7 @@ def test_install_fails_no_args_suggests_env_activation(tmpdir): assert "using the `spack.yaml` in this directory" in output +@pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") def test_install_env_with_tests_all( tmpdir, mock_packages, mock_fetch, install_mockery, mutable_mock_env_path ): @@ -1039,6 +1053,7 @@ def test_install_env_with_tests_all( assert os.path.exists(test_dep.prefix) +@pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") def test_install_env_with_tests_root( tmpdir, mock_packages, mock_fetch, install_mockery, mutable_mock_env_path ): @@ -1050,6 +1065,7 @@ def test_install_env_with_tests_root( assert not os.path.exists(test_dep.prefix) +@pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") def test_install_empty_env( tmpdir, mock_packages, mock_fetch, install_mockery, mutable_mock_env_path ): @@ -1063,6 +1079,7 @@ def test_install_empty_env( assert "no specs to install" in out +@pytest.mark.not_on_windows("Windows logger I/O operation on closed file when install fails") @pytest.mark.disable_clean_stage_check @pytest.mark.parametrize( "name,method", @@ -1086,6 +1103,7 @@ def test_installation_fail_tests(install_mockery, mock_fetch, name, method): assert "See test log for details" in output +@pytest.mark.not_on_windows("Buildcache not supported on windows") def test_install_use_buildcache( capsys, mock_packages, @@ -1138,7 +1156,7 @@ def install_use_buildcache(opt): # Populate the buildcache install(package_name) - buildcache("push", "-u", "-a", "-f", mirror_dir.strpath, package_name, dependency_name) + buildcache("push", "-u", "-f", mirror_dir.strpath, package_name, dependency_name) # Uninstall the all of the packages for clean slate uninstall("-y", "-a") @@ -1163,6 +1181,7 @@ def install_use_buildcache(opt): install_use_buildcache(opt) +@pytest.mark.not_on_windows("Windows logger I/O operation on closed file when install fails") @pytest.mark.regression("34006") @pytest.mark.disable_clean_stage_check def test_padded_install_runtests_root(install_mockery_mutable_config, mock_fetch): diff --git a/lib/spack/spack/test/cmd/license.py b/lib/spack/spack/test/cmd/license.py index a2005de8cc9022..32c6f0846ddcc5 100644 --- a/lib/spack/spack/test/cmd/license.py +++ b/lib/spack/spack/test/cmd/license.py @@ -5,7 +5,6 @@ import os.path import re -import sys import pytest @@ -17,7 +16,7 @@ license = SpackCommand("license") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") def test_list_files(): diff --git a/lib/spack/spack/test/cmd/load.py b/lib/spack/spack/test/cmd/load.py index 54a2f8cb085518..26fa374a05d34d 100644 --- a/lib/spack/spack/test/cmd/load.py +++ b/lib/spack/spack/test/cmd/load.py @@ -4,12 +4,12 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import re -import sys import pytest import spack.spec import spack.user_environment as uenv +import spack.util.environment from spack.main import SpackCommand load = SpackCommand("load") @@ -17,7 +17,7 @@ install = SpackCommand("install") location = SpackCommand("location") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") def test_manpath_trailing_colon( @@ -28,74 +28,63 @@ def test_manpath_trailing_colon( manpath search path via a trailing colon""" install("mpileaks") - sh_out = load("--sh", "--only", "package", "mpileaks") + sh_out = load("--sh", "mpileaks") lines = sh_out.split("\n") assert any(re.match(r"export MANPATH=.*:;", ln) for ln in lines) os.environ["MANPATH"] = "/tmp/man:" - sh_out = load("--sh", "--only", "package", "mpileaks") + sh_out = load("--sh", "mpileaks") lines = sh_out.split("\n") assert any(re.match(r"export MANPATH=.*:/tmp/man:;", ln) for ln in lines) -def test_load(install_mockery, mock_fetch, mock_archive, mock_packages): - """Test that the commands generated by load add the specified prefix - inspections. Also test that Spack records loaded specs by hash in the - user environment. - - CMAKE_PREFIX_PATH is the only prefix inspection guaranteed for fake - packages, since it keys on the prefix instead of a subdir.""" - install_out = install("mpileaks", output=str, fail_on_error=False) - print("spack install mpileaks") - print(install_out) +def test_load_recursive(install_mockery, mock_fetch, mock_archive, mock_packages, working_env): + """Test that `spack load` applies prefix inspections of its required runtime deps in + topo-order""" + install("mpileaks") mpileaks_spec = spack.spec.Spec("mpileaks").concretized() - sh_out = load("--sh", "--only", "package", "mpileaks") - csh_out = load("--csh", "--only", "package", "mpileaks") + # Ensure our reference variable is cleed. + os.environ["CMAKE_PREFIX_PATH"] = "/hello:/world" + + sh_out = load("--sh", "mpileaks") + csh_out = load("--csh", "mpileaks") + + def extract_cmake_prefix_path(output, prefix): + return next(cmd for cmd in output.split(";") if cmd.startswith(prefix))[ + len(prefix) : + ].split(":") - # Test prefix inspections - sh_out_test = "export CMAKE_PREFIX_PATH=%s" % mpileaks_spec.prefix - csh_out_test = "setenv CMAKE_PREFIX_PATH %s" % mpileaks_spec.prefix - assert sh_out_test in sh_out - assert csh_out_test in csh_out + # Map a prefix found in CMAKE_PREFIX_PATH back to a package name in mpileaks' DAG. + prefix_to_pkg = lambda prefix: next( + s.name for s in mpileaks_spec.traverse() if s.prefix == prefix + ) - # Test hashes recorded properly - hash_test_replacements = (uenv.spack_loaded_hashes_var, mpileaks_spec.dag_hash()) - sh_hash_test = "export %s=%s" % hash_test_replacements - csh_hash_test = "setenv %s %s" % hash_test_replacements - assert sh_hash_test in sh_out - assert csh_hash_test in csh_out + paths_sh = extract_cmake_prefix_path(sh_out, prefix="export CMAKE_PREFIX_PATH=") + paths_csh = extract_cmake_prefix_path(csh_out, prefix="setenv CMAKE_PREFIX_PATH ") + # Shouldn't be a difference between loading csh / sh, so check they're the same. + assert paths_sh == paths_csh -def test_load_recursive(install_mockery, mock_fetch, mock_archive, mock_packages): - """Test that the '-r' option to the load command prepends dependency prefix - inspections in post-order""" - install("mpileaks") - mpileaks_spec = spack.spec.Spec("mpileaks").concretized() + # We should've prepended new paths, and keep old ones. + assert paths_sh[-2:] == ["/hello", "/world"] - sh_out = load("--sh", "mpileaks") - csh_out = load("--csh", "mpileaks") + # All but the last two paths are added by spack load; lookup what packages they're from. + pkgs = [prefix_to_pkg(p) for p in paths_sh[:-2]] - # Test prefix inspections - prefix_test_replacement = ":".join( - reversed([s.prefix for s in mpileaks_spec.traverse(order="post")]) + # Do we have all the runtime packages? + assert set(pkgs) == set( + s.name for s in mpileaks_spec.traverse(deptype=("link", "run"), root=True) ) - sh_prefix_test = "export CMAKE_PREFIX_PATH=%s" % prefix_test_replacement - csh_prefix_test = "setenv CMAKE_PREFIX_PATH %s" % prefix_test_replacement - assert sh_prefix_test in sh_out - assert csh_prefix_test in csh_out + # Finally, do we list them in topo order? + for i, pkg in enumerate(pkgs): + set(s.name for s in mpileaks_spec[pkg].traverse(direction="parents")) in set(pkgs[:i]) - # Test spack records loaded hashes properly - hash_test_replacement = ( - uenv.spack_loaded_hashes_var, - ":".join(reversed([s.dag_hash() for s in mpileaks_spec.traverse(order="post")])), - ) - sh_hash_test = "export %s=%s" % hash_test_replacement - csh_hash_test = "setenv %s %s" % hash_test_replacement - assert sh_hash_test in sh_out - assert csh_hash_test in csh_out + # Lastly, do we keep track that mpileaks was loaded? + assert f"export {uenv.spack_loaded_hashes_var}={mpileaks_spec.dag_hash()}" in sh_out + assert f"setenv {uenv.spack_loaded_hashes_var} {mpileaks_spec.dag_hash()}" in csh_out def test_load_includes_run_env(install_mockery, mock_fetch, mock_archive, mock_packages): diff --git a/lib/spack/spack/test/cmd/location.py b/lib/spack/spack/test/cmd/location.py index 1e42141199f43d..11450e8828cb84 100644 --- a/lib/spack/spack/test/cmd/location.py +++ b/lib/spack/spack/test/cmd/location.py @@ -5,7 +5,6 @@ import os import shutil -import sys import pytest @@ -19,7 +18,7 @@ # Everything here uses (or can use) the mock config and database. pytestmark = [ pytest.mark.usefixtures("config", "database"), - pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows"), + pytest.mark.not_on_windows("does not run on windows"), ] # location prints out "locations of packages and spack directories" location = SpackCommand("location") @@ -97,7 +96,7 @@ def test_location_with_active_env(mutable_mock_env_path): assert location("--env").strip() == e.path -def test_location_env_flag_interference(mutable_mock_env_path, tmpdir): +def test_location_env_flag_interference(mutable_mock_env_path): """ Tests that specifying an active environment using `spack -e x location ...` does not interfere with the location command flags. diff --git a/lib/spack/spack/test/cmd/maintainers.py b/lib/spack/spack/test/cmd/maintainers.py index 3a80877c63cc12..5063877a109246 100644 --- a/lib/spack/spack/test/cmd/maintainers.py +++ b/lib/spack/spack/test/cmd/maintainers.py @@ -54,12 +54,10 @@ def test_all(mock_packages, capfd): "user3", "py-extension1:", "adamjstewart,", - "pradyunsg,", "user1,", "user2", "py-extension2:", - "adamjstewart,", - "pradyunsg", + "adamjstewart", ] with capfd.disabled(): @@ -74,9 +72,6 @@ def test_all_by_user(mock_packages, capfd): "adamjstewart:", "py-extension1,", "py-extension2", - "pradyunsg:", - "py-extension1,", - "py-extension2", "user0:", "maintainers-3", "user1:", diff --git a/lib/spack/spack/test/cmd/mark.py b/lib/spack/spack/test/cmd/mark.py index 1779ed80833bb6..d7d93c267cdba1 100644 --- a/lib/spack/spack/test/cmd/mark.py +++ b/lib/spack/spack/test/cmd/mark.py @@ -30,7 +30,7 @@ def test_mark_spec_required(mutable_database): def test_mark_all_explicit(mutable_database): mark("-e", "-a") gc("-y") - all_specs = spack.store.layout.all_specs() + all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 15 @@ -38,7 +38,7 @@ def test_mark_all_explicit(mutable_database): def test_mark_all_implicit(mutable_database): mark("-i", "-a") gc("-y") - all_specs = spack.store.layout.all_specs() + all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 0 @@ -47,7 +47,7 @@ def test_mark_one_explicit(mutable_database): mark("-e", "libelf") uninstall("-y", "-a", "mpileaks") gc("-y") - all_specs = spack.store.layout.all_specs() + all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 3 @@ -55,7 +55,7 @@ def test_mark_one_explicit(mutable_database): def test_mark_one_implicit(mutable_database): mark("-i", "externaltest") gc("-y") - all_specs = spack.store.layout.all_specs() + all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 14 @@ -64,5 +64,5 @@ def test_mark_all_implicit_then_explicit(mutable_database): mark("-i", "-a") mark("-e", "-a") gc("-y") - all_specs = spack.store.layout.all_specs() + all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 15 diff --git a/lib/spack/spack/test/cmd/mirror.py b/lib/spack/spack/test/cmd/mirror.py index 6bb3ae218f04fb..029765df93b5ac 100644 --- a/lib/spack/spack/test/cmd/mirror.py +++ b/lib/spack/spack/test/cmd/mirror.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import sys import pytest @@ -23,7 +22,7 @@ buildcache = SpackCommand("buildcache") uninstall = SpackCommand("uninstall") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.mark.disable_clean_stage_check @@ -36,8 +35,8 @@ def test_regression_8083(tmpdir, capfd, mock_packages, mock_fetch, config): @pytest.mark.regression("12345") -def test_mirror_from_env(tmpdir, mock_packages, mock_fetch, config, mutable_mock_env_path): - mirror_dir = str(tmpdir) +def test_mirror_from_env(tmp_path, mock_packages, mock_fetch, config, mutable_mock_env_path): + mirror_dir = str(tmp_path / "mirror") env_name = "test" env("create", env_name) @@ -149,7 +148,7 @@ def test_mirror_crud(mutable_config, capsys): assert "No changes made" in output output = mirror("set-url", "--push", "mirror", "s3://spack-public") - assert "Changed (push) url" in output + assert not output # no-op output = mirror("set-url", "--push", "mirror", "s3://spack-public") @@ -235,7 +234,7 @@ def test_mirror_destroy( # Put a binary package in a buildcache install("--no-cache", spec_name) - buildcache("push", "-u", "-a", "-f", mirror_dir.strpath, spec_name) + buildcache("push", "-u", "-f", mirror_dir.strpath, spec_name) contents = os.listdir(mirror_dir.strpath) assert "build_cache" in contents @@ -245,7 +244,7 @@ def test_mirror_destroy( assert not os.path.exists(mirror_dir.strpath) - buildcache("push", "-u", "-a", "-f", mirror_dir.strpath, spec_name) + buildcache("push", "-u", "-f", mirror_dir.strpath, spec_name) contents = os.listdir(mirror_dir.strpath) assert "build_cache" in contents @@ -348,3 +347,54 @@ def test_versions_per_spec_produces_concrete_specs(self, input_specs, nversions, args = MockMirrorArgs(specs=input_specs, versions_per_spec=nversions) specs = spack.cmd.mirror.concrete_specs_from_user(args) assert all(s.concrete for s in specs) + + +def test_mirror_type(mutable_config): + """Test the mirror set command""" + mirror("add", "example", "--type", "binary", "http://example.com") + assert spack.config.get("mirrors:example") == { + "url": "http://example.com", + "source": False, + "binary": True, + } + + mirror("set", "example", "--type", "source") + assert spack.config.get("mirrors:example") == { + "url": "http://example.com", + "source": True, + "binary": False, + } + + mirror("set", "example", "--type", "binary") + assert spack.config.get("mirrors:example") == { + "url": "http://example.com", + "source": False, + "binary": True, + } + mirror("set", "example", "--type", "binary", "--type", "source") + assert spack.config.get("mirrors:example") == { + "url": "http://example.com", + "source": True, + "binary": True, + } + + +def test_mirror_set_2(mutable_config): + """Test the mirror set command""" + mirror("add", "example", "http://example.com") + mirror( + "set", + "example", + "--push", + "--url", + "http://example2.com", + "--s3-access-key-id", + "username", + "--s3-access-key-secret", + "password", + ) + + assert spack.config.get("mirrors:example") == { + "url": "http://example.com", + "push": {"url": "http://example2.com", "access_pair": ["username", "password"]}, + } diff --git a/lib/spack/spack/test/cmd/module.py b/lib/spack/spack/test/cmd/module.py index e893ca1d89f12c..a885de79eeab91 100644 --- a/lib/spack/spack/test/cmd/module.py +++ b/lib/spack/spack/test/cmd/module.py @@ -5,7 +5,6 @@ import os.path import re -import sys import pytest @@ -16,7 +15,7 @@ module = spack.main.SpackCommand("module") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") #: make sure module files are generated for all the tests here diff --git a/lib/spack/spack/test/cmd/pkg.py b/lib/spack/spack/test/cmd/pkg.py index 0820b1c08027c1..56a1f1f78676e8 100644 --- a/lib/spack/spack/test/cmd/pkg.py +++ b/lib/spack/spack/test/cmd/pkg.py @@ -5,7 +5,6 @@ import re import shutil -import sys import pytest @@ -82,7 +81,7 @@ def mock_pkg_git_repo(git, tmpdir_factory): @pytest.fixture(scope="module") def mock_pkg_names(): - repo = spack.repo.path.get_repo("builtin.mock") + repo = spack.repo.PATH.get_repo("builtin.mock") # Be sure to include virtual packages since packages with stand-alone # tests may inherit additional tests from the virtuals they provide, @@ -105,11 +104,11 @@ def split(output): def test_packages_path(): - assert spack.repo.packages_path() == spack.repo.path.get_repo("builtin").packages_path + assert spack.repo.packages_path() == spack.repo.PATH.get_repo("builtin").packages_path def test_mock_packages_path(mock_packages): - assert spack.repo.packages_path() == spack.repo.path.get_repo("builtin.mock").packages_path + assert spack.repo.packages_path() == spack.repo.PATH.get_repo("builtin.mock").packages_path def test_pkg_add(git, mock_pkg_git_repo): @@ -126,14 +125,14 @@ def test_pkg_add(git, mock_pkg_git_repo): finally: shutil.rmtree("pkg-e") # Removing a package mid-run disrupts Spack's caching - if spack.repo.path.repos[0]._fast_package_checker: - spack.repo.path.repos[0]._fast_package_checker.invalidate() + if spack.repo.PATH.repos[0]._fast_package_checker: + spack.repo.PATH.repos[0]._fast_package_checker.invalidate() with pytest.raises(spack.main.SpackCommandError): pkg("add", "does-not-exist") -@pytest.mark.skipif(sys.platform == "win32", reason="stdout format conflict") +@pytest.mark.not_on_windows("stdout format conflict") def test_pkg_list(mock_pkg_git_repo, mock_pkg_names): out = split(pkg("list", "HEAD^^")) assert sorted(mock_pkg_names) == sorted(out) @@ -149,7 +148,7 @@ def test_pkg_list(mock_pkg_git_repo, mock_pkg_names): assert sorted(mock_pkg_names) == sorted(out) -@pytest.mark.skipif(sys.platform == "win32", reason="stdout format conflict") +@pytest.mark.not_on_windows("stdout format conflict") def test_pkg_diff(mock_pkg_git_repo, mock_pkg_names): out = split(pkg("diff", "HEAD^^", "HEAD^")) assert out == ["HEAD^:", "pkg-a", "pkg-b", "pkg-c"] @@ -161,7 +160,7 @@ def test_pkg_diff(mock_pkg_git_repo, mock_pkg_names): assert out == ["HEAD^:", "pkg-c", "HEAD:", "pkg-d"] -@pytest.mark.skipif(sys.platform == "win32", reason="stdout format conflict") +@pytest.mark.not_on_windows("stdout format conflict") def test_pkg_added(mock_pkg_git_repo): out = split(pkg("added", "HEAD^^", "HEAD^")) assert ["pkg-a", "pkg-b", "pkg-c"] == out @@ -176,7 +175,7 @@ def test_pkg_added(mock_pkg_git_repo): assert out == [] -@pytest.mark.skipif(sys.platform == "win32", reason="stdout format conflict") +@pytest.mark.not_on_windows("stdout format conflict") def test_pkg_removed(mock_pkg_git_repo): out = split(pkg("removed", "HEAD^^", "HEAD^")) assert out == [] @@ -188,7 +187,7 @@ def test_pkg_removed(mock_pkg_git_repo): assert out == ["pkg-c"] -@pytest.mark.skipif(sys.platform == "win32", reason="stdout format conflict") +@pytest.mark.not_on_windows("stdout format conflict") def test_pkg_changed(mock_pkg_git_repo): out = split(pkg("changed", "HEAD^^", "HEAD^")) assert out == [] @@ -248,7 +247,7 @@ def test_pkg_source_requires_one_arg(mock_packages): def test_pkg_source(mock_packages): fake_source = pkg("source", "fake") - fake_file = spack.repo.path.filename_for_package_name("fake") + fake_file = spack.repo.PATH.filename_for_package_name("fake") with open(fake_file) as f: contents = f.read() assert fake_source == contents @@ -303,7 +302,7 @@ def test_pkg_grep(mock_packages, capfd): pkg("grep", "-l", "splice", output=str) output, _ = capfd.readouterr() assert output.strip() == "\n".join( - spack.repo.path.get_pkg_class(name).module.__file__ + spack.repo.PATH.get_pkg_class(name).module.__file__ for name in ["splice-a", "splice-h", "splice-t", "splice-vh", "splice-z"] ) diff --git a/lib/spack/spack/test/cmd/providers.py b/lib/spack/spack/test/cmd/providers.py index e853a1a54c9296..6c6467378b103e 100644 --- a/lib/spack/spack/test/cmd/providers.py +++ b/lib/spack/spack/test/cmd/providers.py @@ -4,17 +4,13 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys - import pytest from spack.main import SpackCommand providers = SpackCommand("providers") -pytestmark = pytest.mark.skipif( - sys.platform == "win32", reason="Providers not currently supported on Windows" -) +pytestmark = pytest.mark.not_on_windows("Providers not currently supported on Windows") @pytest.mark.parametrize( diff --git a/lib/spack/spack/test/cmd/reindex.py b/lib/spack/spack/test/cmd/reindex.py index 3b611c03b6d53d..e33d48aedd5e29 100644 --- a/lib/spack/spack/test/cmd/reindex.py +++ b/lib/spack/spack/test/cmd/reindex.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import sys import pytest @@ -14,30 +13,30 @@ deprecate = SpackCommand("deprecate") reindex = SpackCommand("reindex") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") def test_reindex_basic(mock_packages, mock_archive, mock_fetch, install_mockery): install("libelf@0.8.13") install("libelf@0.8.12") - all_installed = spack.store.db.query() + all_installed = spack.store.STORE.db.query() reindex() - assert spack.store.db.query() == all_installed + assert spack.store.STORE.db.query() == all_installed def test_reindex_db_deleted(mock_packages, mock_archive, mock_fetch, install_mockery): install("libelf@0.8.13") install("libelf@0.8.12") - all_installed = spack.store.db.query() + all_installed = spack.store.STORE.db.query() - os.remove(spack.store.db._index_path) + os.remove(spack.store.STORE.db._index_path) reindex() - assert spack.store.db.query() == all_installed + assert spack.store.STORE.db.query() == all_installed def test_reindex_with_deprecated_packages( @@ -48,11 +47,11 @@ def test_reindex_with_deprecated_packages( deprecate("-y", "libelf@0.8.12", "libelf@0.8.13") - all_installed = spack.store.db.query(installed=any) - non_deprecated = spack.store.db.query(installed=True) + all_installed = spack.store.STORE.db.query(installed=any) + non_deprecated = spack.store.STORE.db.query(installed=True) - os.remove(spack.store.db._index_path) + os.remove(spack.store.STORE.db._index_path) reindex() - assert spack.store.db.query(installed=any) == all_installed - assert spack.store.db.query(installed=True) == non_deprecated + assert spack.store.STORE.db.query(installed=any) == all_installed + assert spack.store.STORE.db.query(installed=True) == non_deprecated diff --git a/lib/spack/spack/test/cmd/spec.py b/lib/spack/spack/test/cmd/spec.py index 7e4401f3139234..66dfce93089718 100644 --- a/lib/spack/spack/test/cmd/spec.py +++ b/lib/spack/spack/test/cmd/spec.py @@ -14,7 +14,6 @@ import spack.spec import spack.store from spack.main import SpackCommand, SpackCommandError -from spack.util.web import FetchError pytestmark = pytest.mark.usefixtures("config", "mutable_mock_repo") @@ -32,6 +31,7 @@ def test_spec(): assert "mpich@3.0.4" in output +@pytest.mark.only_clingo("Known failure of the original concretizer") def test_spec_concretizer_args(mutable_config, mutable_database): """End-to-end test of CLI concretizer prefs. @@ -39,9 +39,6 @@ def test_spec_concretizer_args(mutable_config, mutable_database): options to `solver.py`, and that config options are not lost along the way. """ - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - # remove two non-preferred mpileaks installations # so that reuse will pick up the zmpi one uninstall = SpackCommand("uninstall") @@ -49,7 +46,7 @@ def test_spec_concretizer_args(mutable_config, mutable_database): uninstall("-y", "mpileaks^mpich2") # get the hash of mpileaks^zmpi - mpileaks_zmpi = spack.store.db.query_one("mpileaks^zmpi") + mpileaks_zmpi = spack.store.STORE.db.query_one("mpileaks^zmpi") h = mpileaks_zmpi.dag_hash()[:7] output = spec("--fresh", "-l", "mpileaks") @@ -210,7 +207,7 @@ def test_env_aware_spec(mutable_mock_env_path): [ ("develop-branch-version", "f3c7206350ac8ee364af687deaae5c574dcfca2c=develop", None), ("develop-branch-version", "git." + "a" * 40 + "=develop", None), - ("callpath", "f3c7206350ac8ee364af687deaae5c574dcfca2c=1.0", FetchError), + ("callpath", "f3c7206350ac8ee364af687deaae5c574dcfca2c=1.0", spack.error.FetchError), ("develop-branch-version", "git.foo=0.2.15", None), ], ) diff --git a/lib/spack/spack/test/cmd/stage.py b/lib/spack/spack/test/cmd/stage.py index 9fff89afb40bac..cdf5ca9fb4c512 100644 --- a/lib/spack/spack/test/cmd/stage.py +++ b/lib/spack/spack/test/cmd/stage.py @@ -4,14 +4,16 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import sys import pytest import spack.config import spack.environment as ev +import spack.package_base import spack.repo -from spack.main import SpackCommand +import spack.stage +import spack.traverse +from spack.main import SpackCommand, SpackCommandError from spack.version import Version stage = SpackCommand("stage") @@ -20,7 +22,8 @@ pytestmark = pytest.mark.usefixtures("install_mockery", "mock_packages") -@pytest.mark.skipif(sys.platform == "win32", reason="not implemented on windows") +@pytest.mark.not_on_windows("not implemented on windows") +@pytest.mark.disable_clean_stage_check def test_stage_spec(monkeypatch): """Verify that staging specs works.""" @@ -48,7 +51,7 @@ def fake_stage(pkg, mirror_only=False): return expected_path -@pytest.mark.skipif(sys.platform == "win32", reason="PermissionError") +@pytest.mark.not_on_windows("PermissionError") def test_stage_path(check_stage_path): """Verify that --path only works with single specs.""" stage("--path={0}".format(check_stage_path), "trivial-install-test-package") @@ -56,11 +59,12 @@ def test_stage_path(check_stage_path): def test_stage_path_errors_multiple_specs(check_stage_path): """Verify that --path only works with single specs.""" - with pytest.raises(spack.main.SpackCommandError): - stage("--path={0}".format(check_stage_path), "trivial-install-test-package", "mpileaks") + with pytest.raises(SpackCommandError): + stage(f"--path={check_stage_path}", "trivial-install-test-package", "mpileaks") -@pytest.mark.skipif(sys.platform == "win32", reason="not implemented on windows") +@pytest.mark.not_on_windows("not implemented on windows") +@pytest.mark.disable_clean_stage_check def test_stage_with_env_outside_env(mutable_mock_env_path, monkeypatch): """Verify that stage concretizes specs not in environment instead of erroring.""" @@ -78,7 +82,8 @@ def fake_stage(pkg, mirror_only=False): stage("trivial-install-test-package") -@pytest.mark.skipif(sys.platform == "win32", reason="not implemented on windows") +@pytest.mark.not_on_windows("not implemented on windows") +@pytest.mark.disable_clean_stage_check def test_stage_with_env_inside_env(mutable_mock_env_path, monkeypatch): """Verify that stage filters specs in environment instead of reconcretizing.""" @@ -96,7 +101,8 @@ def fake_stage(pkg, mirror_only=False): stage("mpileaks") -@pytest.mark.skipif(sys.platform == "win32", reason="not implemented on windows") +@pytest.mark.not_on_windows("not implemented on windows") +@pytest.mark.disable_clean_stage_check def test_stage_full_env(mutable_mock_env_path, monkeypatch): """Verify that stage filters specs in environment.""" @@ -105,10 +111,7 @@ def test_stage_full_env(mutable_mock_env_path, monkeypatch): e.concretize() # list all the package names that should be staged - expected = set() - for spec in e.specs_by_hash.values(): - for dep in spec.traverse(): - expected.add(dep.name) + expected = set(dep.name for dep in spack.traverse.traverse_nodes(e.concrete_roots())) # pop the package name from the list instead of actually staging def fake_stage(pkg, mirror_only=False): diff --git a/lib/spack/spack/test/cmd/style.py b/lib/spack/spack/test/cmd/style.py index 9c233c1e74fe7b..55911169686fb0 100644 --- a/lib/spack/spack/test/cmd/style.py +++ b/lib/spack/spack/test/cmd/style.py @@ -120,7 +120,7 @@ def test_changed_files_all_files(): assert len(files) > 6000 # a builtin package - zlib = spack.repo.path.get_pkg_class("zlib") + zlib = spack.repo.PATH.get_pkg_class("zlib") zlib_file = zlib.module.__file__ if zlib_file.endswith("pyc"): zlib_file = zlib_file[:-1] diff --git a/lib/spack/spack/test/cmd/tags.py b/lib/spack/spack/test/cmd/tags.py index 79e1c7ea4c28ec..17a63432a05847 100644 --- a/lib/spack/spack/test/cmd/tags.py +++ b/lib/spack/spack/test/cmd/tags.py @@ -41,7 +41,7 @@ def test_tags_no_tags(monkeypatch): class tag_path: tag_index = dict() - monkeypatch.setattr(spack.repo, "path", tag_path) + monkeypatch.setattr(spack.repo, "PATH", tag_path) out = tags() assert "No tagged" in out diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py index 20b54ea8c2bdd7..254baa253b5843 100644 --- a/lib/spack/spack/test/cmd/test.py +++ b/lib/spack/spack/test/cmd/test.py @@ -5,7 +5,6 @@ import argparse import os -import sys import pytest @@ -22,7 +21,7 @@ install = SpackCommand("install") spack_test = SpackCommand("test") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") def test_test_package_not_installed( diff --git a/lib/spack/spack/test/cmd/undevelop.py b/lib/spack/spack/test/cmd/undevelop.py index 892b3f8f4753ae..93b7c7ef90d75b 100644 --- a/lib/spack/spack/test/cmd/undevelop.py +++ b/lib/spack/spack/test/cmd/undevelop.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys import pytest @@ -15,7 +14,7 @@ env = SpackCommand("env") concretize = SpackCommand("concretize") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") def test_undevelop(tmpdir, config, mock_packages, mutable_mock_env_path): diff --git a/lib/spack/spack/test/cmd/uninstall.py b/lib/spack/spack/test/cmd/uninstall.py index 48a2028f24ba37..abad976c8f4e91 100644 --- a/lib/spack/spack/test/cmd/uninstall.py +++ b/lib/spack/spack/test/cmd/uninstall.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys import pytest @@ -46,7 +45,7 @@ def test_correct_installed_dependents(mutable_database): # Test whether we return the right dependents. # Take callpath from the database - callpath = spack.store.db.query_local("callpath")[0] + callpath = spack.store.STORE.db.query_local("callpath")[0] # Ensure it still has dependents and dependencies dependents = callpath.dependents(deptype=("run", "link")) @@ -78,7 +77,7 @@ def test_recursive_uninstall(mutable_database): """Test recursive uninstall.""" uninstall("-y", "-a", "--dependents", "callpath") - all_specs = spack.store.layout.all_specs() + all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 9 # query specs with multiple configurations mpileaks_specs = [s for s in all_specs if s.satisfies("mpileaks")] @@ -98,7 +97,7 @@ def test_uninstall_spec_with_multiple_roots( ): uninstall("-y", "-a", "--dependents", constraint) - all_specs = spack.store.layout.all_specs() + all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == expected_number_of_specs @@ -109,7 +108,7 @@ def test_force_uninstall_spec_with_ref_count_not_zero( ): uninstall("-f", "-y", constraint) - all_specs = spack.store.layout.all_specs() + all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == expected_number_of_specs @@ -117,41 +116,41 @@ def test_force_uninstall_spec_with_ref_count_not_zero( def test_force_uninstall_and_reinstall_by_hash(mutable_database): """Test forced uninstall and reinstall of old specs.""" # this is the spec to be removed - callpath_spec = spack.store.db.query_one("callpath ^mpich") + callpath_spec = spack.store.STORE.db.query_one("callpath ^mpich") dag_hash = callpath_spec.dag_hash() # ensure can look up by hash and that it's a dependent of mpileaks def validate_callpath_spec(installed): assert installed is True or installed is False - specs = spack.store.db.get_by_hash(dag_hash, installed=installed) + specs = spack.store.STORE.db.get_by_hash(dag_hash, installed=installed) assert len(specs) == 1 and specs[0] == callpath_spec - specs = spack.store.db.get_by_hash(dag_hash[:7], installed=installed) + specs = spack.store.STORE.db.get_by_hash(dag_hash[:7], installed=installed) assert len(specs) == 1 and specs[0] == callpath_spec - specs = spack.store.db.get_by_hash(dag_hash, installed=any) + specs = spack.store.STORE.db.get_by_hash(dag_hash, installed=any) assert len(specs) == 1 and specs[0] == callpath_spec - specs = spack.store.db.get_by_hash(dag_hash[:7], installed=any) + specs = spack.store.STORE.db.get_by_hash(dag_hash[:7], installed=any) assert len(specs) == 1 and specs[0] == callpath_spec - specs = spack.store.db.get_by_hash(dag_hash, installed=not installed) + specs = spack.store.STORE.db.get_by_hash(dag_hash, installed=not installed) assert specs is None - specs = spack.store.db.get_by_hash(dag_hash[:7], installed=not installed) + specs = spack.store.STORE.db.get_by_hash(dag_hash[:7], installed=not installed) assert specs is None - mpileaks_spec = spack.store.db.query_one("mpileaks ^mpich") + mpileaks_spec = spack.store.STORE.db.query_one("mpileaks ^mpich") assert callpath_spec in mpileaks_spec - spec = spack.store.db.query_one("callpath ^mpich", installed=installed) + spec = spack.store.STORE.db.query_one("callpath ^mpich", installed=installed) assert spec == callpath_spec - spec = spack.store.db.query_one("callpath ^mpich", installed=any) + spec = spack.store.STORE.db.query_one("callpath ^mpich", installed=any) assert spec == callpath_spec - spec = spack.store.db.query_one("callpath ^mpich", installed=not installed) + spec = spack.store.STORE.db.query_one("callpath ^mpich", installed=not installed) assert spec is None validate_callpath_spec(True) @@ -164,7 +163,7 @@ def validate_callpath_spec(installed): # BUT, make sure that the removed callpath spec is not in queries def db_specs(): - all_specs = spack.store.layout.all_specs() + all_specs = spack.store.STORE.layout.all_specs() return ( all_specs, [s for s in all_specs if s.satisfies("mpileaks")], @@ -206,7 +205,7 @@ def _warn(*args, **kwargs): # Note: I want to use https://docs.pytest.org/en/7.1.x/how-to/skipping.html#skip-all-test-functions-of-a-class-or-module # the style formatter insists on separating these two lines. -@pytest.mark.skipif(sys.platform == "win32", reason="Envs unsupported on Windows") +@pytest.mark.not_on_windows("Envs unsupported on Windows") class TestUninstallFromEnv: """Tests an installation with two environments e1 and e2, which each have shared package installations: diff --git a/lib/spack/spack/test/cmd/verify.py b/lib/spack/spack/test/cmd/verify.py index c66559e8dbeaa5..1293df78e21d7f 100644 --- a/lib/spack/spack/test/cmd/verify.py +++ b/lib/spack/spack/test/cmd/verify.py @@ -22,7 +22,7 @@ def test_single_file_verify_cmd(tmpdir): # Test the verify command interface to verifying a single file. filedir = os.path.join(str(tmpdir), "a", "b", "c", "d") filepath = os.path.join(filedir, "file") - metadir = os.path.join(str(tmpdir), spack.store.layout.metadata_dir) + metadir = os.path.join(str(tmpdir), spack.store.STORE.layout.metadata_dir) fs.mkdirp(filedir) fs.mkdirp(metadir) @@ -32,7 +32,7 @@ def test_single_file_verify_cmd(tmpdir): data = spack.verify.create_manifest_entry(filepath) - manifest_file = os.path.join(metadir, spack.store.layout.manifest_file_name) + manifest_file = os.path.join(metadir, spack.store.STORE.layout.manifest_file_name) with open(manifest_file, "w") as f: sjson.dump({filepath: data}, f) diff --git a/lib/spack/spack/test/cmd/view.py b/lib/spack/spack/test/cmd/view.py index 3651dccd9b286c..75840d4e6b3308 100644 --- a/lib/spack/spack/test/cmd/view.py +++ b/lib/spack/spack/test/cmd/view.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os.path -import sys import pytest @@ -16,7 +15,7 @@ install = SpackCommand("install") view = SpackCommand("view") -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") def create_projection_file(tmpdir, projection): diff --git a/lib/spack/spack/test/compilers/basics.py b/lib/spack/spack/test/compilers/basics.py index 76c4c820840291..512defb195370f 100644 --- a/lib/spack/spack/test/compilers/basics.py +++ b/lib/spack/spack/test/compilers/basics.py @@ -4,8 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test basic behavior of compilers in Spack""" import os -import shutil -import sys from copy import copy import pytest @@ -193,7 +191,7 @@ def call_compiler(exe, *args, **kwargs): return no_flag_output -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.parametrize( "exe,flagname", [ @@ -249,7 +247,7 @@ def test_get_compiler_link_paths_no_verbose_flag(): assert dirs == [] -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.enable_compiler_link_paths def test_get_compiler_link_paths_load_env(working_env, monkeypatch, tmpdir): gcc = str(tmpdir.join("gcc")) @@ -694,7 +692,7 @@ def test_raising_if_compiler_target_is_over_specific(config): spack.compilers.get_compilers(cfg, spack.spec.CompilerSpec("gcc@9.0.1"), arch_spec) -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_compiler_get_real_version(working_env, monkeypatch, tmpdir): # Test variables test_version = "2.2.2" @@ -797,9 +795,7 @@ def _call(*args, **kwargs): assert "SPACK_TEST_CMP_ON" not in os.environ -@pytest.mark.skipif( - sys.platform == "win32", reason="Bash scripting unsupported on Windows (for now)" -) +@pytest.mark.not_on_windows("Bash scripting unsupported on Windows (for now)") def test_compiler_flags_use_real_version(working_env, monkeypatch, tmpdir): # Create compiler gcc = str(tmpdir.join("gcc")) @@ -832,119 +828,6 @@ def test_compiler_flags_use_real_version(working_env, monkeypatch, tmpdir): assert flag == "-std=c++0x" -@pytest.mark.skipif(sys.platform == "win32", reason="Apple Clang and XCode unsupported on Windows") -def test_apple_clang_setup_environment(mock_executable, monkeypatch): - """Test a code path that is taken only if the package uses - Xcode on MacOS. - """ - - class MockPackage: - use_xcode = False - - apple_clang_cls = spack.compilers.class_for_compiler_name("apple-clang") - compiler = apple_clang_cls( - spack.spec.CompilerSpec("apple-clang@=11.0.0"), - "catalina", - "x86_64", - ["/usr/bin/clang", "/usr/bin/clang++", None, None], - ) - env = spack.util.environment.EnvironmentModifications() - # Check a package that doesn't use xcode and ensure we don't add changes - # to the environment - pkg = MockPackage() - compiler.setup_custom_environment(pkg, env) - assert not env - - # Prepare mock executables to fake the Xcode environment - xcrun = mock_executable( - "xcrun", - """ -if [[ "$2" == "clang" ]] ; then - echo "/Library/Developer/CommandLineTools/usr/bin/clang" -fi -if [[ "$2" == "clang++" ]] ; then - echo "/Library/Developer/CommandLineTools/usr/bin/clang++" -fi -""", - ) - mock_executable( - "xcode-select", - """ -echo "/Library/Developer" -""", - ) - bin_dir = os.path.dirname(xcrun) - monkeypatch.setenv("PATH", bin_dir, prepend=os.pathsep) - - def noop(*args, **kwargs): - pass - - real_listdir = os.listdir - - def _listdir(path): - if not os.path.exists(path): - return [] - return real_listdir(path) - - # Set a few operations to noop - monkeypatch.setattr(shutil, "copytree", noop) - monkeypatch.setattr(os, "unlink", noop) - monkeypatch.setattr(os, "symlink", noop) - monkeypatch.setattr(os, "listdir", _listdir) - - # Qt is so far the only package that uses this code path, change - # introduced in https://github.com/spack/spack/pull/1832 - pkg.use_xcode = True - compiler.setup_custom_environment(pkg, env) - assert len(env) == 3 - assert env.env_modifications[0].name == "SPACK_CC" - assert env.env_modifications[1].name == "SPACK_CXX" - assert env.env_modifications[2].name == "DEVELOPER_DIR" - - -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") -@pytest.mark.parametrize("xcode_select_output", ["", "/Library/Developer/CommandLineTools"]) -def test_xcode_not_available(xcode_select_output, mock_executable, monkeypatch): - # Prepare mock executables to fake the Xcode environment - xcrun = mock_executable( - "xcrun", - """ - if [[ "$2" == "clang" ]] ; then - echo "/Library/Developer/CommandLineTools/usr/bin/clang" - fi - if [[ "$2" == "clang++" ]] ; then - echo "/Library/Developer/CommandLineTools/usr/bin/clang++" - fi - """, - ) - mock_executable( - "xcode-select", - """ - echo "{0}" - """.format( - xcode_select_output - ), - ) - bin_dir = os.path.dirname(xcrun) - monkeypatch.setenv("PATH", bin_dir, prepend=os.pathsep) - # Prepare compiler - apple_clang_cls = spack.compilers.class_for_compiler_name("apple-clang") - compiler = apple_clang_cls( - spack.spec.CompilerSpec("apple-clang@11.0.0"), - "catalina", - "x86_64", - ["/usr/bin/clang", "/usr/bin/clang++", None, None], - ) - env = spack.util.environment.EnvironmentModifications() - - class MockPackage: - use_xcode = True - - pkg = MockPackage() - with pytest.raises(OSError): - compiler.setup_custom_environment(pkg, env) - - @pytest.mark.enable_compiler_verification def test_compiler_executable_verification_raises(tmpdir): compiler = MockCompiler() diff --git a/lib/spack/spack/test/compilers/detection.py b/lib/spack/spack/test/compilers/detection.py index 93741bf4c283b4..cf197914b9dc8e 100644 --- a/lib/spack/spack/test/compilers/detection.py +++ b/lib/spack/spack/test/compilers/detection.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test detection of compiler version""" import os -import sys import pytest @@ -281,7 +280,7 @@ def test_oneapi_version_detection(version_str, expected_version): ( "NAG Fortran Compiler Release 6.0(Hibiya) Build 1037\n" "Product NPL6A60NA for x86-64 Linux\n", - "6.0", + "6.0.1037", ) ], ) @@ -414,7 +413,7 @@ def test_xl_version_detection(version_str, expected_version): assert version == expected_version -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.parametrize( "compiler,version", [ diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index b6c0e019cad9a4..eba86d14fcf7f0 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -16,15 +16,17 @@ import spack.compilers import spack.concretize import spack.config +import spack.deptypes as dt import spack.detection import spack.error import spack.hash_types as ht import spack.platforms import spack.repo +import spack.solver.asp import spack.variant as vt from spack.concretize import find_spec from spack.spec import CompilerSpec, Spec -from spack.version import ver +from spack.version import Version, ver def check_spec(abstract, concrete): @@ -43,7 +45,7 @@ def check_spec(abstract, concrete): cflag = concrete.compiler_flags[flag] assert set(aflag) <= set(cflag) - for name in spack.repo.path.get_pkg_class(abstract.name).variants: + for name in spack.repo.PATH.get_pkg_class(abstract.name).variants: assert name in concrete.variants for flag in concrete.compiler_flags.valid_compiler_flags(): @@ -152,7 +154,7 @@ class Root(Package): version("1.0", sha256="abcde") depends_on("changing") - conflicts("changing~foo") + conflicts("^changing~foo") """ packages_dir.join("root", "package.py").write(root_pkg_str, ensure=True) @@ -234,13 +236,13 @@ def test_concretize_mention_build_dep(self): # Check parent's perspective of child to_dependencies = spec.edges_to_dependencies(name="cmake") assert len(to_dependencies) == 1 - assert set(to_dependencies[0].deptypes) == set(["build"]) + assert to_dependencies[0].depflag == dt.BUILD # Check child's perspective of parent cmake = spec["cmake"] from_dependents = cmake.edges_from_dependents(name="cmake-client") assert len(from_dependents) == 1 - assert set(from_dependents[0].deptypes) == set(["build"]) + assert from_dependents[0].depflag == dt.BUILD def test_concretize_preferred_version(self): spec = check_concretize("python") @@ -292,7 +294,7 @@ def test_concretize_with_provides_when(self): """Make sure insufficient versions of MPI are not in providers list when we ask for some advanced version. """ - repo = spack.repo.path + repo = spack.repo.PATH assert not any(s.intersects("mpich2@:1.0") for s in repo.providers_for("mpi@2.1")) assert not any(s.intersects("mpich2@:1.1") for s in repo.providers_for("mpi@2.2")) assert not any(s.intersects("mpich@:1") for s in repo.providers_for("mpi@2")) @@ -301,7 +303,7 @@ def test_concretize_with_provides_when(self): def test_provides_handles_multiple_providers_of_same_version(self): """ """ - providers = spack.repo.path.providers_for("mpi@3.0") + providers = spack.repo.PATH.providers_for("mpi@3.0") # Note that providers are repo-specific, so we don't misinterpret # providers, but vdeps are not namespace-specific, so we can @@ -347,15 +349,17 @@ def test_compiler_flags_differ_identical_compilers(self): spec.concretize() assert spec.satisfies("cflags=-O2") + @pytest.mark.only_clingo( + "Optional compiler propagation isn't deprecated for original concretizer" + ) def test_concretize_compiler_flag_propagate(self): spec = Spec("hypre cflags=='-g' ^openblas") spec.concretize() assert spec.satisfies("^openblas cflags='-g'") - @pytest.mark.skipif( - os.environ.get("SPACK_TEST_SOLVER") == "original", - reason="Optional compiler propagation isn't deprecated for original concretizer", + @pytest.mark.only_clingo( + "Optional compiler propagation isn't deprecated for original concretizer" ) def test_concretize_compiler_flag_does_not_propagate(self): spec = Spec("hypre cflags='-g' ^openblas") @@ -363,9 +367,8 @@ def test_concretize_compiler_flag_does_not_propagate(self): assert not spec.satisfies("^openblas cflags='-g'") - @pytest.mark.skipif( - os.environ.get("SPACK_TEST_SOLVER") == "original", - reason="Optional compiler propagation isn't deprecated for original concretizer", + @pytest.mark.only_clingo( + "Optional compiler propagation isn't deprecated for original concretizer" ) def test_concretize_propagate_compiler_flag_not_passed_to_dependent(self): spec = Spec("hypre cflags=='-g' ^openblas cflags='-O3'") @@ -394,14 +397,12 @@ def test_architecture_inheritance(self): spec.concretize() assert spec["cmake"].architecture == spec.architecture + @pytest.mark.only_clingo("Fixing the parser broke this test for the original concretizer") def test_architecture_deep_inheritance(self, mock_targets): """Make sure that indirect dependencies receive architecture information from the root even when partial architecture information is provided by an intermediate dependency. """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Fixing the parser broke this test for the original concretizer.") - spec_str = "mpileaks %gcc@4.5.0 os=CNL target=nocona" " ^dyninst os=CNL ^callpath os=CNL" spec = Spec(spec_str).concretized() for s in spec.traverse(root=False): @@ -457,23 +458,69 @@ def test_concretize_two_virtuals_with_dual_provider_and_a_conflict(self): with pytest.raises(spack.error.SpackError): s.concretize() - @pytest.mark.skipif( - os.environ.get("SPACK_TEST_SOLVER") == "original", - reason="Optional compiler propagation isn't deprecated for original concretizer", + @pytest.mark.only_clingo( + "Optional compiler propagation isn't deprecated for original concretizer" + ) + @pytest.mark.parametrize( + "spec_str,expected_propagation", + [ + ("hypre~~shared ^openblas+shared", [("hypre", "~shared"), ("openblas", "+shared")]), + # Propagates past a node that doesn't have the variant + ("hypre~~shared ^openblas", [("hypre", "~shared"), ("openblas", "~shared")]), + ( + "ascent~~shared +adios2", + [("ascent", "~shared"), ("adios2", "~shared"), ("bzip2", "~shared")], + ), + # Propagates below a node that uses the other value explicitly + ( + "ascent~~shared +adios2 ^adios2+shared", + [("ascent", "~shared"), ("adios2", "+shared"), ("bzip2", "~shared")], + ), + ( + "ascent++shared +adios2 ^adios2~shared", + [("ascent", "+shared"), ("adios2", "~shared"), ("bzip2", "+shared")], + ), + ], + ) + def test_concretize_propagate_disabled_variant(self, spec_str, expected_propagation): + """Tests various patterns of boolean variant propagation""" + spec = Spec(spec_str).concretized() + for key, expected_satisfies in expected_propagation: + spec[key].satisfies(expected_satisfies) + + @pytest.mark.only_clingo( + "Optional compiler propagation isn't deprecated for original concretizer" ) - def test_concretize_propagate_disabled_variant(self): + def test_concretize_propagated_variant_is_not_passed_to_dependent(self): """Test a package variant value was passed from its parent.""" - spec = Spec("hypre~~shared ^openblas") + spec = Spec("ascent~~shared +adios2 ^adios2+shared") spec.concretize() - assert spec.satisfies("^openblas~shared") + assert spec.satisfies("^adios2+shared") + assert spec.satisfies("^bzip2~shared") - def test_concretize_propagated_variant_is_not_passed_to_dependent(self): - """Test a package variant value was passed from its parent.""" - spec = Spec("hypre~~shared ^openblas+shared") + @pytest.mark.only_clingo( + "Optional compiler propagation isn't deprecated for original concretizer" + ) + def test_concretize_propagate_specified_variant(self): + """Test that only the specified variant is propagated to the dependencies""" + spec = Spec("parent-foo-bar ~~foo") spec.concretize() - assert spec.satisfies("^openblas+shared") + assert spec.satisfies("~foo") and spec.satisfies("^dependency-foo-bar~foo") + assert spec.satisfies("+bar") and not spec.satisfies("^dependency-foo-bar+bar") + + @pytest.mark.only_clingo("Original concretizer is allowed to forego variant propagation") + def test_concretize_propagate_multivalue_variant(self): + """Test that multivalue variants are propagating the specified value(s) + to their dependecies. The dependencies should not have the default value""" + spec = Spec("multivalue-variant foo==baz,fee") + spec.concretize() + + assert spec.satisfies("^a foo=baz,fee") + assert spec.satisfies("^b foo=baz,fee") + assert not spec.satisfies("^a foo=bar") + assert not spec.satisfies("^b foo=bar") def test_no_matching_compiler_specs(self, mock_low_high_config): # only relevant when not building compilers as needed @@ -619,10 +666,8 @@ def test_conflicts_in_spec(self, conflict_spec): with pytest.raises(spack.error.SpackError): s.concretize() + @pytest.mark.only_clingo("Testing debug statements specific to new concretizer") def test_conflicts_show_cores(self, conflict_spec, monkeypatch): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Testing debug statements specific to new concretizer") - s = Spec(conflict_spec) with pytest.raises(spack.error.SpackError) as e: s.concretize() @@ -742,7 +787,7 @@ def test_noversion_pkg(self, spec): with pytest.raises(spack.error.SpackError): Spec(spec).concretized() - @pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") + @pytest.mark.not_on_windows("Not supported on Windows (yet)") # Include targets to prevent regression on 20537 @pytest.mark.parametrize( "spec, best_achievable", @@ -795,10 +840,8 @@ def test_concretize_anonymous_dep(self, spec_str): ("bowtie@1.2.2 os=redhat6", "%gcc@11.1.0"), ], ) + @pytest.mark.only_clingo("Original concretizer cannot work around conflicts") def test_compiler_conflicts_in_package_py(self, spec_str, expected_str): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Original concretizer cannot work around conflicts") - s = Spec(spec_str).concretized() assert s.satisfies(expected_str) @@ -902,10 +945,8 @@ def test_patching_dependencies(self, spec_str, patched_deps): ("quantum-espresso~veritas", ["^libelf@0.8.13"]), ], ) + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_working_around_conflicting_defaults(self, spec_str, expected): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - s = Spec(spec_str).concretized() assert s.concrete @@ -917,10 +958,8 @@ def test_working_around_conflicting_defaults(self, spec_str, expected): "spec_str,expected", [("cmake", ["%clang"]), ("cmake %gcc", ["%gcc"]), ("cmake %clang", ["%clang"])], ) + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_external_package_and_compiler_preferences(self, spec_str, expected): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - packages_yaml = { "all": {"compiler": ["clang", "gcc"]}, "cmake": { @@ -936,14 +975,12 @@ def test_external_package_and_compiler_preferences(self, spec_str, expected): assert s.satisfies(condition) @pytest.mark.regression("5651") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_package_with_constraint_not_met_by_external(self): """Check that if we have an external package A at version X.Y in packages.yaml, but our spec doesn't allow X.Y as a version, then a new version of A is built that meets the requirements. """ - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - packages_yaml = {"libelf": {"externals": [{"spec": "libelf@0.8.13", "prefix": "/usr"}]}} spack.config.set("packages", packages_yaml) @@ -953,10 +990,8 @@ def test_package_with_constraint_not_met_by_external(self): assert not s["libelf"].external @pytest.mark.regression("9744") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_cumulative_version_ranges_with_different_length(self): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - s = Spec("cumulative-vrange-root").concretized() assert s.concrete assert s.satisfies("^cumulative-vrange-bottom@2.2") @@ -983,10 +1018,8 @@ def test_dependency_conditional_on_another_dependency_state(self): @pytest.mark.parametrize( "spec_str,expected", [("cmake %gcc", "%gcc"), ("cmake %clang", "%clang")] ) + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_compiler_constraint_with_external_package(self, spec_str, expected): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - packages_yaml = { "cmake": {"externals": [{"spec": "cmake@3.4.3", "prefix": "/usr"}], "buildable": False} } @@ -1035,10 +1068,8 @@ def test_compiler_in_nonbuildable_external_package( assert s.satisfies(expected) assert "external-common-perl" not in [d.name for d in s.dependencies()] + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_external_packages_have_consistent_hash(self): - if spack.config.get("config:concretizer") == "original": - pytest.skip("This tests needs the ASP-based concretizer") - s, t = Spec("externaltool"), Spec("externaltool") s._old_concretize(), t._new_concretize() @@ -1062,10 +1093,8 @@ def test_transitive_conditional_virtual_dependency(self): assert "externalvirtual" in s @pytest.mark.regression("20040") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_conditional_provides_or_depends_on(self): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - # Check that we can concretize correctly a spec that can either # provide a virtual or depend on it based on the value of a variant s = Spec("conditional-provider +disable-v1").concretized() @@ -1103,10 +1132,8 @@ def test_activating_test_dependencies(self, spec_str, tests_arg, with_dep, witho assert not node.dependencies(deptype="test"), msg.format(pkg_name) @pytest.mark.regression("20019") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_compiler_match_is_preferred_to_newer_version(self): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - # This spec depends on openblas. Openblas has a conflict # that doesn't allow newer versions with gcc@4.4.0. Check # that an old version of openblas is selected, rather than @@ -1122,9 +1149,8 @@ def test_target_ranges_in_conflicts(self): with pytest.raises(spack.error.SpackError): Spec("impossible-concretization").concretized() + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_target_compatibility(self): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") with pytest.raises(spack.error.SpackError): Spec("libdwarf target=x86_64 ^libelf target=x86_64_v2").concretized() @@ -1140,10 +1166,8 @@ def test_variant_not_default(self): assert "+foo+bar+baz" in d @pytest.mark.regression("20055") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_custom_compiler_version(self): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - s = Spec("a %gcc@10foo os=redhat6").concretized() assert "%gcc@10foo" in s @@ -1196,7 +1220,7 @@ def test_external_package_versions(self, spec_str, is_external, expected): ) @pytest.mark.parametrize("mock_db", [True, False]) def test_reuse_does_not_overwrite_dev_specs( - self, dev_first, spec, mock_db, tmpdir, monkeypatch + self, dev_first, spec, mock_db, tmpdir, temporary_store, monkeypatch ): """Test that reuse does not mix dev specs with non-dev specs. @@ -1208,8 +1232,7 @@ def test_reuse_does_not_overwrite_dev_specs( # dev and non-dev specs that are otherwise identical spec = Spec(spec) dev_spec = spec.copy() - dev_constraint = "dev_path=%s" % tmpdir.strpath - dev_spec["dev-build-test-install"].constrain(dev_constraint) + dev_spec["dev-build-test-install"].constrain(f"dev_path={tmpdir.strpath}") # run the test in both orders first_spec = dev_spec if dev_first else spec @@ -1222,7 +1245,7 @@ def mock_fn(*args, **kwargs): return [first_spec] if mock_db: - monkeypatch.setattr(spack.store.db, "query", mock_fn) + temporary_store.db.add(first_spec, None) else: monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", mock_fn) @@ -1240,12 +1263,10 @@ def mock_fn(*args, **kwargs): {"add_variant": True, "delete_variant": True}, ], ) + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_reuse_installed_packages_when_package_def_changes( self, context, mutable_database, repo_with_changing_recipe ): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - # Install a spec root = Spec("root").concretized() dependency = root["changing"].copy() @@ -1269,13 +1290,11 @@ def test_reuse_installed_packages_when_package_def_changes( # Structure and package hash will be different without reuse assert root.dag_hash() != new_root_without_reuse.dag_hash() + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_reuse_with_flags(self, mutable_database, mutable_config): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Original concretizer does not reuse") - spack.config.set("concretizer:reuse", True) spec = Spec("a cflags=-g cxxflags=-g").concretized() - spack.store.db.add(spec, None) + spack.store.STORE.db.add(spec, None) testspec = Spec("a cflags=-g") testspec.concretize() @@ -1292,10 +1311,8 @@ def test_concretization_of_test_dependencies(self): @pytest.mark.parametrize( "spec_str", ["wrong-variant-in-conflicts", "wrong-variant-in-depends-on"] ) + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_error_message_for_inconsistent_variants(self, spec_str): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - s = Spec(spec_str) with pytest.raises(RuntimeError, match="not found in package"): s.concretize() @@ -1388,18 +1405,15 @@ def test_multivalued_variants_from_cli(self, spec_str, expected_dict): [ # Version 1.1.0 is deprecated and should not be selected, unless we # explicitly asked for that - ("deprecated-versions", ["deprecated-versions@1.0.0"]), - ("deprecated-versions@1.1.0", ["deprecated-versions@1.1.0"]), + ("deprecated-versions", "deprecated-versions@1.0.0"), + ("deprecated-versions@=1.1.0", "deprecated-versions@1.1.0"), ], ) + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_deprecated_versions_not_selected(self, spec_str, expected): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - - s = Spec(spec_str).concretized() - - for abstract_spec in expected: - assert abstract_spec in s + with spack.config.override("config:deprecated", True): + s = Spec(spec_str).concretized() + s.satisfies(expected) @pytest.mark.regression("24196") def test_version_badness_more_important_than_default_mv_variants(self): @@ -1446,7 +1460,7 @@ def test_non_default_provider_of_multiple_virtuals(self): assert s["lapack"].name == "low-priority-provider" for virtual_pkg in ("mpi", "lapack"): - for pkg in spack.repo.path.providers_for(virtual_pkg): + for pkg in spack.repo.PATH.providers_for(virtual_pkg): if pkg.name == "low-priority-provider": continue assert pkg not in s @@ -1456,12 +1470,10 @@ def test_non_default_provider_of_multiple_virtuals(self): "spec_str,expect_installed", [("mpich", True), ("mpich+debug", False), ("mpich~debug", True)], ) + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_concrete_specs_are_not_modified_on_reuse( self, mutable_database, spec_str, expect_installed, config ): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Original concretizer cannot reuse specs") - # Test the internal consistency of solve + DAG reconstruction # when reused specs are added to the mix. This prevents things # like additional constraints being added to concrete specs in @@ -1472,10 +1484,8 @@ def test_concrete_specs_are_not_modified_on_reuse( assert s.satisfies(spec_str) @pytest.mark.regression("26721,19736") + @pytest.mark.only_clingo("Original concretizer cannot use sticky variants") def test_sticky_variant_in_package(self): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Original concretizer cannot use sticky variants") - # Here we test that a sticky variant cannot be changed from its default value # by the ASP solver if not set explicitly. The package used in the test needs # to have +allow-gcc set to be concretized with %gcc and clingo is not allowed @@ -1489,10 +1499,8 @@ def test_sticky_variant_in_package(self): s = Spec("sticky-variant %clang").concretized() assert s.satisfies("%clang") and s.satisfies("~allow-gcc") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_do_not_invent_new_concrete_versions_unless_necessary(self): - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Original concretizer doesn't resolve concrete versions to known ones") - # ensure we select a known satisfying version rather than creating # a new '2.7' version. assert ver("=2.7.11") == Spec("python@2.7").concretized().version @@ -1513,30 +1521,24 @@ def test_do_not_invent_new_concrete_versions_unless_necessary(self): ("conditional-values-in-variant foo=foo", True), ], ) + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_conditional_values_in_variants(self, spec_str, valid): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer doesn't resolve conditional values in variants") - s = Spec(spec_str) raises = pytest.raises((RuntimeError, spack.error.UnsatisfiableSpecError)) with llnl.util.lang.nullcontext() if valid else raises: s.concretize() + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_conditional_values_in_conditional_variant(self): """Test that conditional variants play well with conditional possible values""" - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer doesn't resolve conditional values in variants") - s = Spec("conditional-values-in-variant@1.50.0").concretized() assert "cxxstd" not in s.variants s = Spec("conditional-values-in-variant@1.60.0").concretized() assert "cxxstd" in s.variants + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_target_granularity(self): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer cannot account for target granularity") - # The test architecture uses core2 as the default target. Check that when # we configure Spack for "generic" granularity we concretize for x86_64 default_target = spack.platforms.test.Test.default @@ -1546,10 +1548,8 @@ def test_target_granularity(self): with spack.config.override("concretizer:targets", {"granularity": "generic"}): assert s.concretized().satisfies("target=%s" % generic_target) + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_host_compatible_concretization(self): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer cannot account for host compatibility") - # Check that after setting "host_compatible" to false we cannot concretize. # Here we use "k10" to set a target non-compatible with the current host # to avoid a lot of boilerplate when mocking the test platform. The issue @@ -1561,10 +1561,8 @@ def test_host_compatible_concretization(self): with pytest.raises(spack.error.SpackError): s.concretized() + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_add_microarchitectures_on_explicit_request(self): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer cannot account for host compatibility") - # Check that if we consider only "generic" targets, we can still solve for # specific microarchitectures on explicit requests with spack.config.override("concretizer:targets", {"granularity": "generic"}): @@ -1572,13 +1570,11 @@ def test_add_microarchitectures_on_explicit_request(self): assert s.satisfies("target=k10") @pytest.mark.regression("29201") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_delete_version_and_reuse(self, mutable_database, repo_with_changing_recipe): """Test that we can reuse installed specs with versions not declared in package.py """ - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - root = Spec("root").concretized() root.package.do_install(fake=True, explicit=True) repo_with_changing_recipe.change({"delete_version": True}) @@ -1589,17 +1585,15 @@ def test_delete_version_and_reuse(self, mutable_database, repo_with_changing_rec assert root.dag_hash() == new_root.dag_hash() @pytest.mark.regression("29201") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_installed_version_is_selected_only_for_reuse( self, mutable_database, repo_with_changing_recipe ): """Test that a version coming from an installed spec is a possible version only for reuse """ - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known failure of the original concretizer") - # Install a dependency that cannot be reused with "root" - # because of a conflict a variant, then delete its version + # because of a conflict in a variant, then delete its version dependency = Spec("changing@1.0~foo").concretized() dependency.package.do_install(fake=True, explicit=True) repo_with_changing_recipe.change({"delete_version": True}) @@ -1653,12 +1647,8 @@ def test_reuse_with_unknown_package_dont_raise(self, tmpdir, monkeypatch): (["mpi", "mpich"], 1), ], ) + @pytest.mark.only_clingo("Original concretizer cannot concretize in rounds") def test_best_effort_coconcretize(self, specs, expected): - import spack.solver.asp - - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer cannot concretize in rounds") - specs = [Spec(s) for s in specs] solver = spack.solver.asp.Solver() solver.reuse = False @@ -1699,13 +1689,9 @@ def test_best_effort_coconcretize(self, specs, expected): (["hdf5+mpi", "zmpi", "mpich"], "mpich", 2), ], ) + @pytest.mark.only_clingo("Original concretizer cannot concretize in rounds") def test_best_effort_coconcretize_preferences(self, specs, expected_spec, occurances): """Test package preferences during coconcretization.""" - import spack.solver.asp - - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer cannot concretize in rounds") - specs = [Spec(s) for s in specs] solver = spack.solver.asp.Solver() solver.reuse = False @@ -1719,20 +1705,14 @@ def test_best_effort_coconcretize_preferences(self, specs, expected_spec, occura counter += 1 assert counter == occurances, concrete_specs + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_coconcretize_reuse_and_virtuals(self): - import spack.solver.asp - - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer cannot reuse") - reusable_specs = [] for s in ["mpileaks ^mpich", "zmpi"]: reusable_specs.extend(Spec(s).concretized().traverse(root=True)) root_specs = [Spec("mpileaks"), Spec("zmpi")] - import spack.solver.asp - with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() @@ -1742,15 +1722,11 @@ def test_coconcretize_reuse_and_virtuals(self): assert "zmpi" in spec @pytest.mark.regression("30864") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_misleading_error_message_on_version(self, mutable_database): # For this bug to be triggered we need a reusable dependency # that is not optimal in terms of optimization scores. # We pick an old version of "b" - import spack.solver.asp - - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer cannot reuse") - reusable_specs = [Spec("non-existing-conditional-dep@1.0").concretized()] root_spec = Spec("non-existing-conditional-dep@2.0") @@ -1763,13 +1739,9 @@ def test_misleading_error_message_on_version(self, mutable_database): solver.driver.solve(setup, [root_spec], reuse=reusable_specs) @pytest.mark.regression("31148") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_version_weight_and_provenance(self): """Test package preferences during coconcretization.""" - import spack.solver.asp - - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer cannot reuse") - reusable_specs = [Spec(spec_str).concretized() for spec_str in ("b@0.9", "b@1.0")] root_spec = Spec("a foobar=bar") @@ -1799,12 +1771,8 @@ def test_version_weight_and_provenance(self): assert result_spec.satisfies("^b@1.0") @pytest.mark.regression("31169") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_not_reusing_incompatible_os_or_compiler(self): - import spack.solver.asp - - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer cannot reuse") - root_spec = Spec("b") s = root_spec.concretized() wrong_compiler, wrong_os = s.copy(), s.copy() @@ -1826,9 +1794,8 @@ def test_git_hash_assigned_version_is_preferred(self): assert hash in str(c) @pytest.mark.parametrize("git_ref", ("a" * 40, "0.2.15", "main")) + @pytest.mark.only_clingo("Original concretizer cannot account for git hashes") def test_git_ref_version_is_equivalent_to_specified_version(self, git_ref): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer cannot account for git hashes") s = Spec("develop-branch-version@git.%s=develop" % git_ref) c = s.concretized() assert git_ref in str(c) @@ -1837,21 +1804,17 @@ def test_git_ref_version_is_equivalent_to_specified_version(self, git_ref): assert s.satisfies("@0.1:") @pytest.mark.parametrize("git_ref", ("a" * 40, "0.2.15", "fbranch")) + @pytest.mark.only_clingo("Original concretizer cannot account for git hashes") def test_git_ref_version_succeeds_with_unknown_version(self, git_ref): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer cannot account for git hashes") # main is not defined in the package.py for this file s = Spec("develop-branch-version@git.%s=main" % git_ref) s.concretize() assert s.satisfies("develop-branch-version@main") @pytest.mark.regression("31484") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_installed_externals_are_reused(self, mutable_database, repo_with_changing_recipe): """Test that external specs that are in the DB can be reused.""" - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Use case not supported by the original concretizer") - - # Configuration to be added to packages.yaml external_conf = { "changing": { "buildable": False, @@ -1879,12 +1842,9 @@ def test_installed_externals_are_reused(self, mutable_database, repo_with_changi assert external3.dag_hash() == external1.dag_hash() @pytest.mark.regression("31484") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_user_can_select_externals_with_require(self, mutable_database): """Test that users have means to select an external even in presence of reusable specs.""" - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Use case not supported by the original concretizer") - - # Configuration to be added to packages.yaml external_conf = { "mpi": {"buildable": False}, "multi-provider-mpi": { @@ -1911,15 +1871,13 @@ def test_user_can_select_externals_with_require(self, mutable_database): assert mpi_spec.name == "multi-provider-mpi" @pytest.mark.regression("31484") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_installed_specs_disregard_conflicts(self, mutable_database, monkeypatch): """Test that installed specs do not trigger conflicts. This covers for the rare case where a conflict is added on a package after a spec matching the conflict was installed. """ - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Use case not supported by the original concretizer") - # Add a conflict to "mpich" that match an already installed "mpich~debug" - pkg_cls = spack.repo.path.get_pkg_class("mpich") + pkg_cls = spack.repo.PATH.get_pkg_class("mpich") monkeypatch.setitem(pkg_cls.conflicts, "~debug", [(Spec(), None)]) # If we concretize with --fresh the conflict is taken into account @@ -1930,14 +1888,13 @@ def test_installed_specs_disregard_conflicts(self, mutable_database, monkeypatch # If we concretize with --reuse it is not, since "mpich~debug" was already installed with spack.config.override("concretizer:reuse", True): s = Spec("mpich").concretized() - assert s.satisfies("~debug") + assert s.installed + assert s.satisfies("~debug"), s @pytest.mark.regression("32471") + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_require_targets_are_allowed(self, mutable_database): """Test that users can set target constraints under the require attribute.""" - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Use case not supported by the original concretizer") - # Configuration to be added to packages.yaml external_conf = {"all": {"require": "target=%s" % spack.platforms.test.Test.front_end}} spack.config.set("packages", external_conf) @@ -2013,7 +1970,7 @@ def test_external_python_extension_find_dependency_from_installed(self, monkeypa # install python external python = Spec("python").concretized() - monkeypatch.setattr(spack.store.db, "query", lambda x: [python]) + monkeypatch.setattr(spack.store.STORE.db, "query", lambda x: [python]) # ensure that we can't be faking this by getting it from config external_conf.pop("python") @@ -2037,7 +1994,7 @@ def test_external_python_extension_find_dependency_from_detection(self, monkeypa def find_fake_python(classes, path_hints): return {"python": [spack.detection.DetectedPackage(python_spec, prefix=path_hints[0])]} - monkeypatch.setattr(spack.detection, "by_executable", find_fake_python) + monkeypatch.setattr(spack.detection, "by_path", find_fake_python) external_conf = { "py-extension1": { "buildable": False, @@ -2125,7 +2082,7 @@ def test_compiler_match_constraints_when_selected(self): assert s.compiler.version == ver("=11.1.0"), s @pytest.mark.regression("36339") - @pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows") + @pytest.mark.not_on_windows("Not supported on Windows") def test_compiler_with_custom_non_numeric_version(self, mock_executable): """Test that, when a compiler has a completely made up version, we can use its 'real version' to detect targets and don't raise during concretization. @@ -2181,3 +2138,288 @@ def test_virtuals_are_annotated_on_edges(self, spec_str, default_mock_concretiza assert len(edges) == 1 and edges[0].virtuals == ("mpi",) edges = spec.edges_to_dependencies(name="callpath") assert len(edges) == 1 and edges[0].virtuals == () + + @pytest.mark.only_clingo("Use case not supported by the original concretizer") + @pytest.mark.db + @pytest.mark.parametrize( + "spec_str,mpi_name", + [("mpileaks", "mpich"), ("mpileaks ^mpich2", "mpich2"), ("mpileaks ^zmpi", "zmpi")], + ) + def test_virtuals_are_reconstructed_on_reuse(self, spec_str, mpi_name, database): + """Tests that when we reuse a spec, virtual on edges are reconstructed correctly""" + with spack.config.override("concretizer:reuse", True): + spec = Spec(spec_str).concretized() + assert spec.installed + mpi_edges = spec.edges_to_dependencies(mpi_name) + assert len(mpi_edges) == 1 + assert "mpi" in mpi_edges[0].virtuals + + @pytest.mark.only_clingo("Use case not supported by the original concretizer") + def test_dont_define_new_version_from_input_if_checksum_required(self, working_env): + os.environ["SPACK_CONCRETIZER_REQUIRE_CHECKSUM"] = "yes" + with pytest.raises(spack.error.UnsatisfiableSpecError): + # normally spack concretizes to @=3.0 if it's not defined in package.py, except + # when checksums are required + Spec("a@=3.0").concretized() + + @pytest.mark.regression("39570") + @pytest.mark.db + def test_reuse_python_from_cli_and_extension_from_db(self, mutable_database): + """Tests that reusing python with and explicit request on the command line, when the spec + also reuses a python extension from the DB, doesn't fail. + """ + s = Spec("py-extension1").concretized() + python_hash = s["python"].dag_hash() + s.package.do_install(fake=True, explicit=True) + + with spack.config.override("concretizer:reuse", True): + with_reuse = Spec(f"py-extension2 ^/{python_hash}").concretized() + + with spack.config.override("concretizer:reuse", False): + without_reuse = Spec("py-extension2").concretized() + + assert with_reuse.dag_hash() == without_reuse.dag_hash() + + +@pytest.fixture() +def duplicates_test_repository(): + repository_path = os.path.join(spack.paths.repos_path, "duplicates.test") + with spack.repo.use_repositories(repository_path) as mock_repo: + yield mock_repo + + +@pytest.mark.usefixtures("mutable_config", "duplicates_test_repository") +@pytest.mark.only_clingo("Not supported by the original concretizer") +class TestConcretizeSeparately: + """Collects test on separate concretization""" + + @pytest.mark.parametrize("strategy", ["minimal", "full"]) + def test_two_gmake(self, strategy): + """Tests that we can concretize a spec with nodes using the same build + dependency pinned at different versions. + + o hdf5@1.0 + |\ + o | pinned-gmake@1.0 + o | gmake@3.0 + / + o gmake@4.1 + + """ + spack.config.CONFIG.set("concretizer:duplicates:strategy", strategy) + s = Spec("hdf5").concretized() + + # Check that hdf5 depends on gmake@=4.1 + hdf5_gmake = s["hdf5"].dependencies(name="gmake", deptype="build") + assert len(hdf5_gmake) == 1 and hdf5_gmake[0].satisfies("@=4.1") + + # Check that pinned-gmake depends on gmake@=3.0 + pinned_gmake = s["pinned-gmake"].dependencies(name="gmake", deptype="build") + assert len(pinned_gmake) == 1 and pinned_gmake[0].satisfies("@=3.0") + + @pytest.mark.parametrize("strategy", ["minimal", "full"]) + def test_two_setuptools(self, strategy): + """Tests that we can concretize separate build dependencies, when we are dealing + with extensions. + + o py-shapely@1.25.0 + |\ + | |\ + | o | py-setuptools@60 + |/ / + | o py-numpy@1.25.0 + |/| + | |\ + | o | py-setuptools@59 + |/ / + o | python@3.11.2 + o | gmake@3.0 + / + o gmake@4.1 + + """ + spack.config.CONFIG.set("concretizer:duplicates:strategy", strategy) + s = Spec("py-shapely").concretized() + # Requirements on py-shapely + setuptools = s["py-shapely"].dependencies(name="py-setuptools", deptype="build") + assert len(setuptools) == 1 and setuptools[0].satisfies("@=60") + + # Requirements on py-numpy + setuptools = s["py-numpy"].dependencies(name="py-setuptools", deptype="build") + assert len(setuptools) == 1 and setuptools[0].satisfies("@=59") + gmake = s["py-numpy"].dependencies(name="gmake", deptype="build") + assert len(gmake) == 1 and gmake[0].satisfies("@=4.1") + + # Requirements on python + gmake = s["python"].dependencies(name="gmake", deptype="build") + assert len(gmake) == 1 and gmake[0].satisfies("@=3.0") + + def test_solution_without_cycles(self): + """Tests that when we concretize a spec with cycles, a fallback kicks in to recompute + a solution without cycles. + """ + s = Spec("cycle-a").concretized() + assert s["cycle-a"].satisfies("+cycle") + assert s["cycle-b"].satisfies("~cycle") + + s = Spec("cycle-b").concretized() + assert s["cycle-a"].satisfies("~cycle") + assert s["cycle-b"].satisfies("+cycle") + + @pytest.mark.parametrize("strategy", ["minimal", "full"]) + def test_pure_build_virtual_dependency(self, strategy): + """Tests that we can concretize a pure build virtual dependency, and ensures that + pure build virtual dependencies are accounted in the list of possible virtual + dependencies. + + virtual-build@1.0 + | [type=build, virtual=pkgconfig] + pkg-config@1.0 + """ + spack.config.CONFIG.set("concretizer:duplicates:strategy", strategy) + + s = Spec("virtual-build").concretized() + assert s["pkgconfig"].name == "pkg-config" + + @pytest.mark.regression("40595") + def test_no_multiple_solutions_with_different_edges_same_nodes(self): + r"""Tests that the root node, which has a dependency on py-setuptools without constraint, + doesn't randomly pick one of the two setuptools (@=59, @=60) needed by its dependency. + + o py-floating@1.25.0/3baitsp + |\ + | |\ + | | |\ + | o | | py-shapely@1.25.0/4hep6my + |/| | | + | |\| | + | | |/ + | |/| + | | o py-setuptools@60/cwhbthc + | |/ + |/| + | o py-numpy@1.25.0/5q5fx4d + |/| + | |\ + | o | py-setuptools@59/jvsa7sd + |/ / + o | python@3.11.2/pdmjekv + o | gmake@3.0/jv7k2bl + / + o gmake@4.1/uo6ot3d + """ + spec_str = "py-floating" + + root = spack.spec.Spec(spec_str).concretized() + assert root["py-shapely"].satisfies("^py-setuptools@=60") + assert root["py-numpy"].satisfies("^py-setuptools@=59") + + edges = root.edges_to_dependencies("py-setuptools") + assert len(edges) == 1 + assert edges[0].spec.satisfies("@=60") + + +@pytest.mark.parametrize( + "v_str,v_opts,checksummed", + [ + ("1.2.3", {"sha256": f"{1:064x}"}, True), + # it's not about the version being "infinite", + # but whether it has a digest + ("develop", {"sha256": f"{1:064x}"}, True), + # other hash types + ("1.2.3", {"checksum": f"{1:064x}"}, True), + ("1.2.3", {"md5": f"{1:032x}"}, True), + ("1.2.3", {"sha1": f"{1:040x}"}, True), + ("1.2.3", {"sha224": f"{1:056x}"}, True), + ("1.2.3", {"sha384": f"{1:096x}"}, True), + ("1.2.3", {"sha512": f"{1:0128x}"}, True), + # no digest key + ("1.2.3", {"bogus": f"{1:064x}"}, False), + # git version with full commit sha + ("1.2.3", {"commit": f"{1:040x}"}, True), + (f"{1:040x}=1.2.3", {}, True), + # git version with short commit sha + ("1.2.3", {"commit": f"{1:07x}"}, False), + (f"{1:07x}=1.2.3", {}, False), + # git tag is a moving target + ("1.2.3", {"tag": "v1.2.3"}, False), + ("1.2.3", {"tag": "v1.2.3", "commit": f"{1:07x}"}, False), + # git branch is a moving target + ("1.2.3", {"branch": "releases/1.2"}, False), + # git ref is a moving target + ("git.branch=1.2.3", {}, False), + ], +) +def test_drop_moving_targets(v_str, v_opts, checksummed): + v = Version(v_str) + assert spack.solver.asp._is_checksummed_version((v, v_opts)) == checksummed + + +class TestConcreteSpecsByHash: + """Tests the container of concrete specs""" + + @pytest.mark.parametrize("input_specs", [["a"], ["a foobar=bar", "b"], ["a foobar=baz", "b"]]) + def test_adding_specs(self, input_specs, default_mock_concretization): + """Tests that concrete specs in the container are equivalent, but stored as different + objects in memory. + """ + container = spack.solver.asp.ConcreteSpecsByHash() + input_specs = [Spec(s).concretized() for s in input_specs] + for s in input_specs: + container.add(s) + + for root in input_specs: + for node in root.traverse(root=True): + assert node == container[node.dag_hash()] + assert node.dag_hash() in container + assert node is not container[node.dag_hash()] + + +@pytest.fixture() +def edges_test_repository(): + repository_path = os.path.join(spack.paths.repos_path, "edges.test") + with spack.repo.use_repositories(repository_path) as mock_repo: + yield mock_repo + + +@pytest.mark.usefixtures("mutable_config", "edges_test_repository") +@pytest.mark.only_clingo("Edge properties not supported by the original concretizer") +class TestConcretizeEdges: + """Collects tests on edge properties""" + + @pytest.mark.parametrize( + "spec_str,expected_satisfies,expected_not_satisfies", + [ + ("conditional-edge", ["^zlib@2.0"], ["^zlib-api"]), + ("conditional-edge~foo", ["^zlib@2.0"], ["^zlib-api"]), + ( + "conditional-edge+foo", + ["^zlib@1.0", "^zlib-api", "^[virtuals=zlib-api] zlib"], + ["^[virtuals=mpi] zlib"], + ), + ], + ) + def test_condition_triggered_by_edge_property( + self, spec_str, expected_satisfies, expected_not_satisfies + ): + """Tests that we can enforce constraints based on edge attributes""" + s = Spec(spec_str).concretized() + + for expected in expected_satisfies: + assert s.satisfies(expected), str(expected) + + for not_expected in expected_not_satisfies: + assert not s.satisfies(not_expected), str(not_expected) + + def test_virtuals_provided_together_but_only_one_required_in_dag(self): + """Tests that we can use a provider that provides more than one virtual together, + and is providing only one, iff the others are not needed in the DAG. + + o blas-only-client + | [virtual=blas] + o openblas (provides blas and lapack together) + + """ + s = Spec("blas-only-client ^openblas").concretized() + assert s.satisfies("^[virtuals=blas] openblas") + assert not s.satisfies("^[virtuals=blas,lapack] openblas") diff --git a/lib/spack/spack/test/concretize_errors.py b/lib/spack/spack/test/concretize_errors.py new file mode 100644 index 00000000000000..2a8be3e0457a46 --- /dev/null +++ b/lib/spack/spack/test/concretize_errors.py @@ -0,0 +1,68 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +import spack.solver.asp +import spack.spec + +pytestmark = [ + pytest.mark.not_on_windows("Windows uses old concretizer"), + pytest.mark.only_clingo("Original concretizer does not support configuration requirements"), +] + +version_error_messages = [ + "Cannot satisfy 'fftw@:1.0' and 'fftw@1.1:", + " required because quantum-espresso depends on fftw@:1.0", + " required because quantum-espresso ^fftw@1.1: requested from CLI", + " required because quantum-espresso ^fftw@1.1: requested from CLI", +] + +external_error_messages = [ + ( + "Attempted to build package quantum-espresso which is not buildable and does not have" + " a satisfying external" + ), + ( + " 'quantum-espresso~veritas' is an external constraint for quantum-espresso" + " which was not satisfied" + ), + " 'quantum-espresso+veritas' required", + " required because quantum-espresso+veritas requested from CLI", +] + +variant_error_messages = [ + "'fftw' required multiple values for single-valued variant 'mpi'", + " Requested '~mpi' and '+mpi'", + " required because quantum-espresso depends on fftw+mpi when +invino", + " required because quantum-espresso+invino ^fftw~mpi requested from CLI", + " required because quantum-espresso+invino ^fftw~mpi requested from CLI", +] + +external_config = { + "packages:quantum-espresso": { + "buildable": False, + "externals": [{"spec": "quantum-espresso@1.0~veritas", "prefix": "/path/to/qe"}], + } +} + + +@pytest.mark.parametrize( + "error_messages,config_set,spec", + [ + (version_error_messages, {}, "quantum-espresso^fftw@1.1:"), + (external_error_messages, external_config, "quantum-espresso+veritas"), + (variant_error_messages, {}, "quantum-espresso+invino^fftw~mpi"), + ], +) +def test_error_messages(error_messages, config_set, spec, mock_packages, mutable_config): + for path, conf in config_set.items(): + spack.config.set(path, conf) + + with pytest.raises(spack.solver.asp.UnsatisfiableSpecError) as e: + _ = spack.spec.Spec(spec).concretized() + + for em in error_messages: + assert em in str(e.value) diff --git a/lib/spack/spack/test/concretize_preferences.py b/lib/spack/spack/test/concretize_preferences.py index 457aa4af0089a3..d061f9a8f555c5 100644 --- a/lib/spack/spack/test/concretize_preferences.py +++ b/lib/spack/spack/test/concretize_preferences.py @@ -105,30 +105,22 @@ def test_preferred_variants_from_wildcard(self): @pytest.mark.parametrize( "compiler_str,spec_str", - [("gcc@4.5.0", "mpileaks"), ("clang@12.0.0", "mpileaks"), ("gcc@4.5.0", "openmpi")], + [("gcc@=4.5.0", "mpileaks"), ("clang@=12.0.0", "mpileaks"), ("gcc@=4.5.0", "openmpi")], ) def test_preferred_compilers(self, compiler_str, spec_str): """Test preferred compilers are applied correctly""" - spec = Spec(spec_str) - update_packages(spec.name, "compiler", [compiler_str]) - spec.concretize() - # note: lhs has concrete compiler version, rhs still abstract. - # Could be made more strict by checking for equality with `gcc@=4.5.0` - # etc. - assert spec.compiler.satisfies(CompilerSpec(compiler_str)) + update_packages("all", "compiler", [compiler_str]) + spec = spack.spec.Spec(spec_str).concretized() + assert spec.compiler == CompilerSpec(compiler_str) + @pytest.mark.only_clingo("Use case not supported by the original concretizer") def test_preferred_target(self, mutable_mock_repo): """Test preferred targets are applied correctly""" - # FIXME: This test was a false negative, since the default and - # FIXME: the preferred target were the same - if spack.config.get("config:concretizer") == "original": - pytest.xfail("Known bug in the original concretizer") - spec = concretize("mpich") default = str(spec.target) preferred = str(spec.target.family) - update_packages("mpich", "target", [preferred]) + update_packages("all", "target", [preferred]) spec = concretize("mpich") assert str(spec.target) == preferred @@ -136,7 +128,7 @@ def test_preferred_target(self, mutable_mock_repo): assert str(spec["mpileaks"].target) == preferred assert str(spec["mpich"].target) == preferred - update_packages("mpileaks", "target", [default]) + update_packages("all", "target", [default]) spec = concretize("mpileaks") assert str(spec["mpileaks"].target) == default assert str(spec["mpich"].target) == default @@ -151,9 +143,8 @@ def test_preferred_versions(self): spec = concretize("mpileaks") assert spec.version == Version("2.2") + @pytest.mark.only_clingo("This behavior is not enforced for the old concretizer") def test_preferred_versions_mixed_version_types(self): - if spack.config.get("config:concretizer") == "original": - pytest.skip("This behavior is not enforced for the old concretizer") update_packages("mixedversions", "version", ["=2.0"]) spec = concretize("mixedversions") assert spec.version == Version("2.0") @@ -178,11 +169,11 @@ def test_config_set_pkg_property_url(self, mutable_mock_repo): {"url": "http://www.somewhereelse.com/mpileaks-1.0.tar.gz"}, ) spec = concretize("mpileaks") - assert spec.package.fetcher[0].url == "http://www.somewhereelse.com/mpileaks-2.3.tar.gz" + assert spec.package.fetcher.url == "http://www.somewhereelse.com/mpileaks-2.3.tar.gz" update_packages("mpileaks", "package_attributes", {}) spec = concretize("mpileaks") - assert spec.package.fetcher[0].url == "http://www.llnl.gov/mpileaks-2.3.tar.gz" + assert spec.package.fetcher.url == "http://www.llnl.gov/mpileaks-2.3.tar.gz" def test_config_set_pkg_property_new(self, mutable_mock_repo): """Test that you can set arbitrary attributes on the Package class""" @@ -230,24 +221,20 @@ def test_preferred(self): spec.concretize() assert spec.version == Version("3.5.0") + @pytest.mark.only_clingo("This behavior is not enforced for the old concretizer") def test_preferred_undefined_raises(self): """Preference should not specify an undefined version""" - if spack.config.get("config:concretizer") == "original": - pytest.xfail("This behavior is not enforced for the old concretizer") - update_packages("python", "version", ["3.5.0.1"]) spec = Spec("python") with pytest.raises(spack.config.ConfigError): spec.concretize() + @pytest.mark.only_clingo("This behavior is not enforced for the old concretizer") def test_preferred_truncated(self): """Versions without "=" are treated as version ranges: if there is a satisfying version defined in the package.py, we should use that (don't define a new version). """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("This behavior is not enforced for the old concretizer") - update_packages("python", "version", ["3.5"]) spec = Spec("python") spec.concretize() diff --git a/lib/spack/spack/test/concretize_requirements.py b/lib/spack/spack/test/concretize_requirements.py index ff0e354965f8af..d5295691ce0a8b 100644 --- a/lib/spack/spack/test/concretize_requirements.py +++ b/lib/spack/spack/test/concretize_requirements.py @@ -2,8 +2,8 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os import pathlib -import sys import pytest @@ -14,11 +14,14 @@ import spack.repo import spack.util.spack_yaml as syaml import spack.version -from spack.solver.asp import UnsatisfiableSpecError +from spack.solver.asp import InternalConcretizerError, UnsatisfiableSpecError from spack.spec import Spec from spack.util.url import path_to_file_url -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="Windows uses old concretizer") +pytestmark = [ + pytest.mark.not_on_windows("Windows uses old concretizer"), + pytest.mark.only_clingo("Original concretizer does not support configuration requirements"), +] def update_packages_config(conf_str): @@ -134,9 +137,6 @@ def fake_installs(monkeypatch, tmpdir): def test_one_package_multiple_reqs(concretize_scope, test_repo): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - conf_str = """\ packages: y: @@ -153,9 +153,6 @@ def test_requirement_isnt_optional(concretize_scope, test_repo): """If a user spec requests something that directly conflicts with a requirement, make sure we get an error. """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - conf_str = """\ packages: x: @@ -173,9 +170,6 @@ def test_require_undefined_version(concretize_scope, test_repo): (it is assumed this is a typo, and raising the error here avoids a likely error when Spack attempts to fetch the version). """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - conf_str = """\ packages: x: @@ -192,9 +186,6 @@ def test_require_truncated(concretize_scope, test_repo): of the defined versions (vs. allowing the requirement to define a new version). """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - conf_str = """\ packages: x: @@ -256,9 +247,6 @@ def test_git_user_supplied_reference_satisfaction( def test_requirement_adds_new_version( concretize_scope, test_repo, mock_git_version_info, monkeypatch ): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", path_to_file_url(repo_path), raising=False @@ -289,9 +277,6 @@ def test_requirement_adds_version_satisfies( depends_on condition and make sure it is triggered (i.e. the dependency is added). """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration" " requirements") - repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", path_to_file_url(repo_path), raising=False @@ -315,11 +300,13 @@ def test_requirement_adds_version_satisfies( assert s1.satisfies("@2.2") +@pytest.mark.parametrize("require_checksum", (True, False)) def test_requirement_adds_git_hash_version( - concretize_scope, test_repo, mock_git_version_info, monkeypatch + require_checksum, concretize_scope, test_repo, mock_git_version_info, monkeypatch, working_env ): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") + # A full commit sha is a checksummed version, so this test should pass in both cases + if require_checksum: + os.environ["SPACK_CONCRETIZER_REQUIRE_CHECKSUM"] = "yes" repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( @@ -342,9 +329,6 @@ def test_requirement_adds_git_hash_version( def test_requirement_adds_multiple_new_versions( concretize_scope, test_repo, mock_git_version_info, monkeypatch ): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", path_to_file_url(repo_path), raising=False @@ -370,9 +354,6 @@ def test_preference_adds_new_version( """Normally a preference cannot define a new version, but that constraint is ignored if the version is a Git hash-based version. """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not enforce this constraint for preferences") - repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", path_to_file_url(repo_path), raising=False @@ -398,9 +379,6 @@ def test_external_adds_new_version_that_is_preferred(concretize_scope, test_repo """Test that we can use a version, not declared in package recipe, as the preferred version if that version appears in an external spec. """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not enforce this constraint for preferences") - conf_str = """\ packages: y: @@ -421,9 +399,6 @@ def test_requirement_is_successfully_applied(concretize_scope, test_repo): """If a simple requirement can be satisfied, make sure the concretization succeeds and the requirement spec is applied. """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - s1 = Spec("x").concretized() # Without any requirements/preferences, the later version is preferred assert s1.satisfies("@1.1") @@ -443,9 +418,6 @@ def test_multiple_packages_requirements_are_respected(concretize_scope, test_rep """Apply requirements to two packages; make sure the concretization succeeds and both requirements are respected. """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - conf_str = """\ packages: x: @@ -463,9 +435,6 @@ def test_oneof(concretize_scope, test_repo): """'one_of' allows forcing the concretizer to satisfy one of the specs in the group (but not all have to be satisfied). """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - conf_str = """\ packages: y: @@ -483,9 +452,6 @@ def test_one_package_multiple_oneof_groups(concretize_scope, test_repo): """One package has two 'one_of' groups; check that both are applied. """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - conf_str = """\ packages: y: @@ -503,19 +469,22 @@ def test_one_package_multiple_oneof_groups(concretize_scope, test_repo): @pytest.mark.regression("34241") -def test_require_cflags(concretize_scope, test_repo): +def test_require_cflags(concretize_scope, mock_packages): """Ensures that flags can be required from configuration.""" - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration" " requirements") - conf_str = """\ packages: - y: + mpich2: require: cflags="-g" + mpi: + require: mpich cflags="-O1" """ update_packages_config(conf_str) - spec = Spec("y").concretized() - assert spec.satisfies("cflags=-g") + + spec_mpich2 = Spec("mpich2").concretized() + assert spec_mpich2.satisfies("cflags=-g") + + spec_mpi = Spec("mpi").concretized() + assert spec_mpi.satisfies("mpich cflags=-O1") def test_requirements_for_package_that_is_not_needed(concretize_scope, test_repo): @@ -523,9 +492,6 @@ def test_requirements_for_package_that_is_not_needed(concretize_scope, test_repo a dependency of a concretized spec (in other words, none of the requirements are used for the requested spec). """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - # Note that the exact contents aren't important since this isn't # intended to be used, but the important thing is that a number of # packages have requirements applied @@ -549,9 +515,6 @@ def test_oneof_ordering(concretize_scope, test_repo): This priority should override default priority (e.g. choosing later versions). """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - conf_str = """\ packages: y: @@ -568,9 +531,6 @@ def test_oneof_ordering(concretize_scope, test_repo): def test_reuse_oneof(concretize_scope, create_test_repo, mutable_database, fake_installs): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - conf_str = """\ packages: y: @@ -589,32 +549,42 @@ def test_reuse_oneof(concretize_scope, create_test_repo, mutable_database, fake_ assert not s2.satisfies("@2.5 %gcc") -def test_requirements_are_higher_priority_than_deprecation(concretize_scope, test_repo): - """Test that users can override a deprecated version with a requirement.""" - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") +@pytest.mark.parametrize( + "allow_deprecated,expected,not_expected", + [(True, ["@=2.3", "%gcc"], []), (False, ["%gcc"], ["@=2.3"])], +) +def test_requirements_and_deprecated_versions( + allow_deprecated, expected, not_expected, concretize_scope, test_repo +): + """Tests the expected behavior of requirements and deprecated versions. + + If deprecated versions are not allowed, concretization should just pick + the other requirement. - # @2.3 is a deprecated versions. Ensure that any_of picks both constraints, + If deprecated versions are allowed, both requirements are honored. + """ + # 2.3 is a deprecated versions. Ensure that any_of picks both constraints, # since they are possible conf_str = """\ packages: y: require: - - any_of: ["@2.3", "%gcc"] + - any_of: ["@=2.3", "%gcc"] """ update_packages_config(conf_str) - s1 = Spec("y").concretized() - assert s1.satisfies("@2.3") - assert s1.satisfies("%gcc") + with spack.config.override("config:deprecated", allow_deprecated): + s1 = Spec("y").concretized() + for constrain in expected: + assert s1.satisfies(constrain) + + for constrain in not_expected: + assert not s1.satisfies(constrain) @pytest.mark.parametrize("spec_str,requirement_str", [("x", "%gcc"), ("x", "%clang")]) def test_default_requirements_with_all(spec_str, requirement_str, concretize_scope, test_repo): """Test that default requirements are applied to all packages.""" - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - conf_str = """\ packages: all: @@ -640,8 +610,6 @@ def test_default_and_package_specific_requirements( concretize_scope, requirements, expectations, test_repo ): """Test that specific package requirements override default package requirements.""" - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") generic_req, specific_req = requirements generic_exp, specific_exp = expectations conf_str = """\ @@ -663,8 +631,6 @@ def test_default_and_package_specific_requirements( @pytest.mark.parametrize("mpi_requirement", ["mpich", "mpich2", "zmpi"]) def test_requirements_on_virtual(mpi_requirement, concretize_scope, mock_packages): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") conf_str = """\ packages: mpi: @@ -686,8 +652,6 @@ def test_requirements_on_virtual(mpi_requirement, concretize_scope, mock_package def test_requirements_on_virtual_and_on_package( mpi_requirement, specific_requirement, concretize_scope, mock_packages ): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") conf_str = """\ packages: mpi: @@ -706,8 +670,6 @@ def test_requirements_on_virtual_and_on_package( def test_incompatible_virtual_requirements_raise(concretize_scope, mock_packages): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") conf_str = """\ packages: mpi: @@ -716,13 +678,12 @@ def test_incompatible_virtual_requirements_raise(concretize_scope, mock_packages update_packages_config(conf_str) spec = Spec("callpath ^zmpi") - with pytest.raises(UnsatisfiableSpecError): + # TODO (multiple nodes): recover a better error message later + with pytest.raises((UnsatisfiableSpecError, InternalConcretizerError)): spec.concretize() def test_non_existing_variants_under_all(concretize_scope, mock_packages): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") conf_str = """\ packages: all: @@ -805,9 +766,6 @@ def test_conditional_requirements_from_packages_yaml( """Test that conditional requirements are required when the condition is met, and optional when the condition is not met. """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - update_packages_config(packages_yaml) spec = Spec(spec_str).concretized() for match_str, expected in expected_satisfies: @@ -883,9 +841,6 @@ def test_requirements_fail_with_custom_message( """Test that specs failing due to requirements not being satisfiable fail with a custom error message. """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - update_packages_config(packages_yaml) with pytest.raises(spack.error.SpackError, match=expected_message): Spec(spec_str).concretized() @@ -899,9 +854,6 @@ def test_skip_requirement_when_default_requirement_condition_cannot_be_met( package. For those packages the requirement rule is not emitted, since it can be determined to be always false. """ - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") - packages_yaml = """ packages: all: @@ -919,8 +871,6 @@ def test_skip_requirement_when_default_requirement_condition_cannot_be_met( def test_requires_directive(concretize_scope, mock_packages): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer does not support configuration requirements") compilers_yaml = pathlib.Path(concretize_scope) / "compilers.yaml" compilers_yaml.write_text( """ @@ -937,7 +887,7 @@ def test_requires_directive(concretize_scope, mock_packages): modules: [] """ ) - spack.config.config.clear_caches() + spack.config.CONFIG.clear_caches() # This package requires either clang or gcc s = Spec("requires_clang_or_gcc").concretized() diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 3137d727a54765..5f544a31296a68 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -7,7 +7,6 @@ import getpass import io import os -import sys import tempfile from datetime import date @@ -79,7 +78,7 @@ def env_yaml(tmpdir): verify_ssl: False dirty: False packages: - libelf: + all: compiler: [ 'gcc@4.5.3' ] repos: - /x/y/z @@ -236,7 +235,7 @@ def test_write_key_to_disk(mock_low_high_config, compiler_specs): spack.config.set("compilers", b_comps["compilers"], scope="high") # Clear caches so we're forced to read from disk. - spack.config.config.clear_caches() + spack.config.CONFIG.clear_caches() # Same check again, to ensure consistency. check_compiler_config(a_comps["compilers"], *compiler_specs.a) @@ -249,7 +248,7 @@ def test_write_to_same_priority_file(mock_low_high_config, compiler_specs): spack.config.set("compilers", b_comps["compilers"], scope="low") # Clear caches so we're forced to read from disk. - spack.config.config.clear_caches() + spack.config.CONFIG.clear_caches() # Same check again, to ensure consistency. check_compiler_config(a_comps["compilers"], *compiler_specs.a) @@ -278,6 +277,25 @@ def test_add_config_path(mutable_config): compilers = spack.config.get("packages")["all"]["compiler"] assert "gcc" in compilers + # Try quotes to escape brackets + path = "config:install_tree:projections:cmake:\ +'{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + spack.config.add(path) + set_value = spack.config.get("config")["install_tree"]["projections"]["cmake"] + assert set_value == "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}" + + # NOTE: + # The config path: "config:install_tree:root:" is unique in that it can accept multiple + # schemas (such as a dropped "root" component) which is atypical and may lead to passing tests + # when the behavior is in reality incorrect. + # the config path below is such that no subkey accepts a string as a valid entry in our schema + + # try quotes to escape colons + path = "config:build_stage:'C:\\path\\to\\config.yaml'" + spack.config.add(path) + set_value = spack.config.get("config")["build_stage"] + assert "C:\\path\\to\\config.yaml" in set_value + @pytest.mark.regression("17543,23259") def test_add_config_path_with_enumerated_type(mutable_config): @@ -369,7 +387,7 @@ def test_substitute_config_variables(mock_low_high_config, monkeypatch): spack.config.set( "modules:default", {"roots": {"lmod": os.path.join("foo", "bar", "baz")}}, scope="low" ) - spack.config.config.clear_caches() + spack.config.CONFIG.clear_caches() path = spack.config.get("modules:default:roots:lmod") assert spack_path.canonicalize_path(path) == os.path.normpath( os.path.join(mock_low_high_config.scopes["low"].path, os.path.join("foo", "bar", "baz")) @@ -467,7 +485,7 @@ def test_substitute_date(mock_low_high_config): ], ) def test_parse_install_tree(config_settings, expected, mutable_config): - expected_root = expected[0] or spack.store.default_install_tree_root + expected_root = expected[0] or spack.store.DEFAULT_INSTALL_TREE_ROOT expected_unpadded_root = expected[1] or expected_root expected_proj = expected[2] or spack.directory_layout.default_projections @@ -484,7 +502,7 @@ def test_parse_install_tree(config_settings, expected, mutable_config): assert projections == expected_proj -@pytest.mark.skipif(sys.platform == "win32", reason="Padding unsupported on Windows") +@pytest.mark.not_on_windows("Padding unsupported on Windows") @pytest.mark.parametrize( "config_settings,expected", [ @@ -522,7 +540,7 @@ def test_parse_install_tree(config_settings, expected, mutable_config): ], ) def test_parse_install_tree_padded(config_settings, expected, mutable_config): - expected_root = expected[0] or spack.store.default_install_tree_root + expected_root = expected[0] or spack.store.DEFAULT_INSTALL_TREE_ROOT expected_unpadded_root = expected[1] or expected_root expected_proj = expected[2] or spack.directory_layout.default_projections @@ -816,7 +834,7 @@ def test_bad_config_section(mock_low_high_config): spack.config.get("foobar") -@pytest.mark.skipif(sys.platform == "win32", reason="chmod not supported on Windows") +@pytest.mark.not_on_windows("chmod not supported on Windows") @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_bad_command_line_scopes(tmpdir, config): cfg = spack.config.Configuration() @@ -854,18 +872,18 @@ def test_add_command_line_scopes(tmpdir, mutable_config): def test_nested_override(): """Ensure proper scope naming of nested overrides.""" - base_name = spack.config.overrides_base_name + base_name = spack.config._OVERRIDES_BASE_NAME def _check_scopes(num_expected, debug_values): scope_names = [ - s.name for s in spack.config.config.scopes.values() if s.name.startswith(base_name) + s.name for s in spack.config.CONFIG.scopes.values() if s.name.startswith(base_name) ] for i in range(num_expected): name = "{0}{1}".format(base_name, i) assert name in scope_names - data = spack.config.config.get_config("config", name) + data = spack.config.CONFIG.get_config("config", name) assert data["debug"] == debug_values[i] # Check results from single and nested override @@ -878,23 +896,23 @@ def _check_scopes(num_expected, debug_values): def test_alternate_override(monkeypatch): """Ensure proper scope naming of override when conflict present.""" - base_name = spack.config.overrides_base_name + base_name = spack.config._OVERRIDES_BASE_NAME def _matching_scopes(regexpr): return [spack.config.InternalConfigScope("{0}1".format(base_name))] # Check that the alternate naming works - monkeypatch.setattr(spack.config.config, "matching_scopes", _matching_scopes) + monkeypatch.setattr(spack.config.CONFIG, "matching_scopes", _matching_scopes) with spack.config.override("config:debug", False): name = "{0}2".format(base_name) scope_names = [ - s.name for s in spack.config.config.scopes.values() if s.name.startswith(base_name) + s.name for s in spack.config.CONFIG.scopes.values() if s.name.startswith(base_name) ] assert name in scope_names - data = spack.config.config.get_config("config", name) + data = spack.config.CONFIG.get_config("config", name) assert data["debug"] is False @@ -924,7 +942,7 @@ def test_single_file_scope(config, env_yaml): # from the single-file config assert spack.config.get("config:verify_ssl") is False assert spack.config.get("config:dirty") is False - assert spack.config.get("packages:libelf:compiler") == ["gcc@4.5.3"] + assert spack.config.get("packages:all:compiler") == ["gcc@4.5.3"] # from the lower config scopes assert spack.config.get("config:checksum") is True @@ -947,7 +965,7 @@ def test_single_file_scope_section_override(tmpdir, config): config: verify_ssl: False packages:: - libelf: + all: compiler: [ 'gcc@4.5.3' ] repos: - /x/y/z @@ -959,7 +977,7 @@ def test_single_file_scope_section_override(tmpdir, config): with spack.config.override(scope): # from the single-file config assert spack.config.get("config:verify_ssl") is False - assert spack.config.get("packages:libelf:compiler") == ["gcc@4.5.3"] + assert spack.config.get("packages:all:compiler") == ["gcc@4.5.3"] # from the lower config scopes assert spack.config.get("config:checksum") is True @@ -1158,13 +1176,13 @@ def test_license_dir_config(mutable_config, mock_packages): expected_dir = spack.paths.default_license_dir assert spack.config.get("config:license_dir") == expected_dir assert spack.package_base.PackageBase.global_license_dir == expected_dir - assert spack.repo.path.get_pkg_class("a").global_license_dir == expected_dir + assert spack.repo.PATH.get_pkg_class("a").global_license_dir == expected_dir rel_path = os.path.join(os.path.sep, "foo", "bar", "baz") spack.config.set("config:license_dir", rel_path) assert spack.config.get("config:license_dir") == rel_path assert spack.package_base.PackageBase.global_license_dir == rel_path - assert spack.repo.path.get_pkg_class("a").global_license_dir == rel_path + assert spack.repo.PATH.get_pkg_class("a").global_license_dir == rel_path @pytest.mark.regression("22547") @@ -1230,21 +1248,21 @@ def test_default_install_tree(monkeypatch): def test_local_config_can_be_disabled(working_env): os.environ["SPACK_DISABLE_LOCAL_CONFIG"] = "true" - cfg = spack.config._config() + cfg = spack.config.create() assert "defaults" in cfg.scopes assert "system" not in cfg.scopes assert "site" in cfg.scopes assert "user" not in cfg.scopes os.environ["SPACK_DISABLE_LOCAL_CONFIG"] = "" - cfg = spack.config._config() + cfg = spack.config.create() assert "defaults" in cfg.scopes assert "system" not in cfg.scopes assert "site" in cfg.scopes assert "user" not in cfg.scopes del os.environ["SPACK_DISABLE_LOCAL_CONFIG"] - cfg = spack.config._config() + cfg = spack.config.create() assert "defaults" in cfg.scopes assert "system" in cfg.scopes assert "site" in cfg.scopes @@ -1395,7 +1413,7 @@ def test_config_file_dir_failure(tmpdir, mutable_empty_config): spack.config.read_config_file(tmpdir.strpath) -@pytest.mark.skipif(sys.platform == "win32", reason="chmod not supported on Windows") +@pytest.mark.not_on_windows("chmod not supported on Windows") def test_config_file_read_perms_failure(tmpdir, mutable_empty_config): """Test reading a configuration file without permissions to ensure ConfigFileError is raised.""" diff --git a/lib/spack/spack/test/config_values.py b/lib/spack/spack/test/config_values.py index 163634ea4edd73..618d2c40a81796 100644 --- a/lib/spack/spack/test/config_values.py +++ b/lib/spack/spack/test/config_values.py @@ -13,12 +13,7 @@ @pytest.mark.usefixtures("mock_packages") def test_set_install_hash_length(hash_length, mutable_config, tmpdir): mutable_config.set("config:install_hash_length", hash_length) - mutable_config.set("config:install_tree", {"root": str(tmpdir)}) - # The call below is to reinitialize the directory layout associated - # with the store according to the configuration changes above (i.e. - # with the shortened hash) - store = spack.store._store() - with spack.store.use_store(store): + with spack.store.use_store(str(tmpdir)): spec = spack.spec.Spec("libelf").concretized() prefix = spec.prefix hash_str = prefix.rsplit("-")[-1] @@ -28,14 +23,7 @@ def test_set_install_hash_length(hash_length, mutable_config, tmpdir): @pytest.mark.usefixtures("mock_packages") def test_set_install_hash_length_upper_case(mutable_config, tmpdir): mutable_config.set("config:install_hash_length", 5) - mutable_config.set( - "config:install_tree", {"root": str(tmpdir), "projections": {"all": "{name}-{HASH}"}} - ) - # The call below is to reinitialize the directory layout associated - # with the store according to the configuration changes above (i.e. - # with the shortened hash and projection) - store = spack.store._store() - with spack.store.use_store(store): + with spack.store.use_store(str(tmpdir), extra_data={"projections": {"all": "{name}-{HASH}"}}): spec = spack.spec.Spec("libelf").concretized() prefix = spec.prefix hash_str = prefix.rsplit("-")[-1] diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 2582d1255d65e1..514b1e91542403 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -27,15 +27,17 @@ import llnl.util.lang import llnl.util.lock import llnl.util.tty as tty -from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, working_dir +from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, touchp, working_dir import spack.binary_distribution import spack.caches +import spack.cmd.buildcache import spack.compilers import spack.config import spack.database import spack.directory_layout import spack.environment as ev +import spack.error import spack.package_base import spack.package_prefs import spack.paths @@ -50,9 +52,8 @@ import spack.util.gpg import spack.util.spack_yaml as syaml import spack.util.url as url_util -from spack.fetch_strategy import FetchStrategyComposite, URLFetchStrategy +from spack.fetch_strategy import URLFetchStrategy from spack.util.pattern import Bunch -from spack.util.web import FetchError def ensure_configuration_fixture_run_before(request): @@ -472,7 +473,7 @@ def fetcher(self, target_path, digest, **kwargs): class MockCacheFetcher: def fetch(self): - raise FetchError("Mock cache always fails for tests") + raise spack.error.FetchError("Mock cache always fails for tests") def __str__(self): return "[mock fetch cache]" @@ -480,10 +481,10 @@ def __str__(self): @pytest.fixture(autouse=True) def mock_fetch_cache(monkeypatch): - """Substitutes spack.paths.fetch_cache with a mock object that does nothing + """Substitutes spack.paths.FETCH_CACHE with a mock object that does nothing and raises on fetch. """ - monkeypatch.setattr(spack.caches, "fetch_cache", MockCache()) + monkeypatch.setattr(spack.caches, "FETCH_CACHE", MockCache()) @pytest.fixture() @@ -494,7 +495,7 @@ def mock_binary_index(monkeypatch, tmpdir_factory): tmpdir = tmpdir_factory.mktemp("mock_binary_index") index_path = tmpdir.join("binary_index").strpath mock_index = spack.binary_distribution.BinaryCacheIndex(index_path) - monkeypatch.setattr(spack.binary_distribution, "binary_index", mock_index) + monkeypatch.setattr(spack.binary_distribution, "BINARY_INDEX", mock_index) yield @@ -565,6 +566,8 @@ def mock_repo_path(): def _pkg_install_fn(pkg, spec, prefix): # sanity_check_prefix requires something in the install directory mkdirp(prefix.bin) + if not os.path.exists(spec.package.build_log_path): + touchp(spec.package.build_log_path) @pytest.fixture @@ -716,7 +719,7 @@ def configuration_dir(tmpdir_factory, linux_os): def _create_mock_configuration_scopes(configuration_dir): """Create the configuration scopes used in `config` and `mutable_config`.""" - scopes = [spack.config.InternalConfigScope("_builtin", spack.config.config_defaults)] + scopes = [spack.config.InternalConfigScope("_builtin", spack.config.CONFIG_DEFAULTS)] scopes += [ spack.config.ConfigScope(name, str(configuration_dir.join(name))) for name in ["site", "system", "user"] @@ -773,7 +776,7 @@ def concretize_scope(mutable_config, tmpdir): yield str(tmpdir.join("concretize")) mutable_config.pop_scope() - spack.repo.path._provider_index = None + spack.repo.PATH._provider_index = None @pytest.fixture @@ -950,26 +953,20 @@ def disable_compiler_execution(monkeypatch, request): @pytest.fixture(scope="function") -def install_mockery(temporary_store, config, mock_packages): +def install_mockery(temporary_store: spack.store.Store, mutable_config, mock_packages): """Hooks a fake install directory, DB, and stage directory into Spack.""" # We use a fake package, so temporarily disable checksumming with spack.config.override("config:checksum", False): yield - # Also wipe out any cached prefix failure locks (associated with - # the session-scoped mock archive). - for pkg_id in list(temporary_store.db._prefix_failures.keys()): - lock = spack.store.db._prefix_failures.pop(pkg_id, None) - if lock: - try: - lock.release_write() - except Exception: - pass + # Wipe out any cached prefix failure locks (associated with the session-scoped mock archive) + temporary_store.failure_tracker.clear_all() @pytest.fixture(scope="function") -def temporary_store(tmpdir): +def temporary_store(tmpdir, request): """Hooks a temporary empty store for the test function.""" + ensure_configuration_fixture_run_before(request) temporary_store_path = tmpdir.join("opt") with spack.store.use_store(str(temporary_store_path)) as s: yield s @@ -992,10 +989,9 @@ def install_mockery_mutable_config(temporary_store, mutable_config, mock_package @pytest.fixture() def mock_fetch(mock_archive, monkeypatch): """Fake the URL for a package so it downloads from a file.""" - mock_fetcher = FetchStrategyComposite() - mock_fetcher.append(URLFetchStrategy(mock_archive.url)) - - monkeypatch.setattr(spack.package_base.PackageBase, "fetcher", mock_fetcher) + monkeypatch.setattr( + spack.package_base.PackageBase, "fetcher", URLFetchStrategy(mock_archive.url) + ) class MockLayout: @@ -1536,13 +1532,12 @@ def get_rev(): @pytest.fixture(scope="function") -def mutable_mock_env_path(tmpdir_factory, mutable_config): +def mutable_mock_env_path(tmp_path, mutable_config, monkeypatch): """Fixture for mocking the internal spack environments directory.""" - saved_path = ev.environment.default_env_path - mock_path = tmpdir_factory.mktemp("mock-env-path") - ev.environment.default_env_path = str(mock_path) - yield mock_path - ev.environment.default_env_path = saved_path + mock_path = tmp_path / "mock-env-path" + mutable_config.set("config:environments_root", str(mock_path)) + monkeypatch.setattr(ev.environment, "default_env_path", str(mock_path)) + return mock_path @pytest.fixture() @@ -1701,36 +1696,25 @@ def mock_test_stage(mutable_config, tmpdir): @pytest.fixture(autouse=True) def inode_cache(): - llnl.util.lock.file_tracker.purge() + llnl.util.lock.FILE_TRACKER.purge() yield # TODO: it is a bug when the file tracker is non-empty after a test, # since it means a lock was not released, or the inode was not purged # when acquiring the lock failed. So, we could assert that here, but # currently there are too many issues to fix, so look for the more # serious issue of having a closed file descriptor in the cache. - assert not any(f.fh.closed for f in llnl.util.lock.file_tracker._descriptors.values()) - llnl.util.lock.file_tracker.purge() + assert not any(f.fh.closed for f in llnl.util.lock.FILE_TRACKER._descriptors.values()) + llnl.util.lock.FILE_TRACKER.purge() @pytest.fixture(autouse=True) def brand_new_binary_cache(): yield - spack.binary_distribution.binary_index = llnl.util.lang.Singleton( - spack.binary_distribution._binary_index + spack.binary_distribution.BINARY_INDEX = llnl.util.lang.Singleton( + spack.binary_distribution.BinaryCacheIndex ) -@pytest.fixture -def directory_with_manifest(tmpdir): - """Create a manifest file in a directory. Used by 'spack external'.""" - with tmpdir.as_cwd(): - test_db_fname = "external-db.json" - with open(test_db_fname, "w") as db_file: - json.dump(spack.test.cray_manifest.create_manifest_content(), db_file) - - yield str(tmpdir) - - @pytest.fixture() def noncyclical_dir_structure(tmpdir): """ @@ -1938,3 +1922,48 @@ def shell_as(shell): # restore old shell if one was set if _shell: os.environ["SPACK_SHELL"] = _shell + + +@pytest.fixture() +def nullify_globals(request, monkeypatch): + ensure_configuration_fixture_run_before(request) + monkeypatch.setattr(spack.config, "CONFIG", None) + monkeypatch.setattr(spack.caches, "MISC_CACHE", None) + monkeypatch.setattr(spack.caches, "FETCH_CACHE", None) + monkeypatch.setattr(spack.repo, "PATH", None) + monkeypatch.setattr(spack.store, "STORE", None) + + +def pytest_runtest_setup(item): + # Skip tests if they are marked only clingo and are run with the original concretizer + only_clingo_marker = item.get_closest_marker(name="only_clingo") + if only_clingo_marker and os.environ.get("SPACK_TEST_SOLVER") == "original": + pytest.skip(*only_clingo_marker.args) + + # Skip tests if they are marked only original and are run with clingo + only_original_marker = item.get_closest_marker(name="only_original") + if only_original_marker and os.environ.get("SPACK_TEST_SOLVER", "clingo") == "clingo": + pytest.skip(*only_original_marker.args) + + # Skip test marked "not_on_windows" if they're run on Windows + not_on_windows_marker = item.get_closest_marker(name="not_on_windows") + if not_on_windows_marker and sys.platform == "win32": + pytest.skip(*not_on_windows_marker.args) + + +@pytest.fixture(scope="function") +def disable_parallel_buildcache_push(monkeypatch): + class MockPool: + def map(self, func, args): + return [func(a) for a in args] + + def starmap(self, func, args): + return [func(*a) for a in args] + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + monkeypatch.setattr(spack.cmd.buildcache, "_make_pool", MockPool) diff --git a/lib/spack/spack/test/container/cli.py b/lib/spack/spack/test/container/cli.py index 8353514b70a0c7..1ccd47d002a990 100644 --- a/lib/spack/spack/test/container/cli.py +++ b/lib/spack/spack/test/container/cli.py @@ -41,5 +41,7 @@ def test_bootstrap_phase(minimal_configuration, config_dumper, capsys): with fs.working_dir(spack_yaml_dir): output = containerize() - # Check for the presence of the clone command - assert "git clone" in output + # Check for the presence of the Git commands + assert "git init" in output + assert "git fetch" in output + assert "git checkout" in output diff --git a/lib/spack/spack/test/cray_manifest.py b/lib/spack/spack/test/cray_manifest.py index aa9b9c623ce068..123e2ac3f12fd6 100644 --- a/lib/spack/spack/test/cray_manifest.py +++ b/lib/spack/spack/test/cray_manifest.py @@ -23,53 +23,6 @@ import spack.store from spack.cray_manifest import compiler_from_entry, entries_to_specs -example_x_json_str = """\ -{ - "name": "packagex", - "hash": "hash-of-x", - "prefix": "/path/to/packagex-install/", - "version": "1.0", - "arch": { - "platform": "linux", - "platform_os": "centos8", - "target": { - "name": "haswell" - } - }, - "compiler": { - "name": "gcc", - "version": "10.2.0.cray" - }, - "dependencies": { - "packagey": { - "hash": "hash-of-y", - "type": ["link"] - } - }, - "parameters": { - "precision": ["double", "float"] - } -} -""" - - -example_compiler_entry = """\ -{ - "name": "gcc", - "prefix": "/path/to/compiler/", - "version": "7.5.0", - "arch": { - "os": "centos8", - "target": "x86_64" - }, - "executables": { - "cc": "/path/to/compiler/cc", - "cxx": "/path/to/compiler/cxx", - "fc": "/path/to/compiler/fc" - } -} -""" - class JsonSpecEntry: def __init__(self, name, hash, prefix, version, arch, compiler, dependencies, parameters): @@ -104,16 +57,19 @@ def __init__(self, platform, os, target): self.os = os self.target = target - def to_dict(self): + def spec_json(self): return {"platform": self.platform, "platform_os": self.os, "target": {"name": self.target}} + def compiler_json(self): + return {"os": self.os, "target": self.target} + class JsonCompilerEntry: def __init__(self, name, version, arch=None, executables=None): self.name = name self.version = version if not arch: - arch = {"os": "centos8", "target": "x86_64"} + arch = JsonArchEntry("anyplatform", "anyos", "anytarget") if not executables: executables = { "cc": "/path/to/compiler/cc", @@ -127,7 +83,7 @@ def compiler_json(self): return { "name": self.name, "version": self.version, - "arch": self.arch, + "arch": self.arch.compiler_json(), "executables": self.executables, } @@ -138,22 +94,58 @@ def spec_json(self): return {"name": self.name, "version": self.version} -_common_arch = JsonArchEntry(platform="linux", os="centos8", target="haswell").to_dict() +@pytest.fixture +def _common_arch(test_platform): + return JsonArchEntry( + platform=test_platform.name, + os=test_platform.front_os, + target=test_platform.target("fe").name, + ) + -# Intended to match example_compiler_entry above -_common_compiler = JsonCompilerEntry( - name="gcc", - version="10.2.0.cray", - arch={"os": "centos8", "target": "x86_64"}, - executables={ - "cc": "/path/to/compiler/cc", - "cxx": "/path/to/compiler/cxx", - "fc": "/path/to/compiler/fc", - }, -) +@pytest.fixture +def _common_compiler(_common_arch): + return JsonCompilerEntry( + name="gcc", + version="10.2.0.2112", + arch=_common_arch, + executables={ + "cc": "/path/to/compiler/cc", + "cxx": "/path/to/compiler/cxx", + "fc": "/path/to/compiler/fc", + }, + ) -def test_compatibility(): +@pytest.fixture +def _other_compiler(_common_arch): + return JsonCompilerEntry( + name="clang", + version="3.0.0", + arch=_common_arch, + executables={ + "cc": "/path/to/compiler/clang", + "cxx": "/path/to/compiler/clang++", + "fc": "/path/to/compiler/flang", + }, + ) + + +@pytest.fixture +def _raw_json_x(_common_arch): + return { + "name": "packagex", + "hash": "hash-of-x", + "prefix": "/path/to/packagex-install/", + "version": "1.0", + "arch": _common_arch.spec_json(), + "compiler": {"name": "gcc", "version": "10.2.0.2112"}, + "dependencies": {"packagey": {"hash": "hash-of-y", "type": ["link"]}}, + "parameters": {"precision": ["double", "float"]}, + } + + +def test_manifest_compatibility(_common_arch, _common_compiler, _raw_json_x): """Make sure that JsonSpecEntry outputs the expected JSON structure by comparing it with JSON parsed from an example string. This ensures that the testing objects like JsonSpecEntry produce the @@ -164,7 +156,7 @@ def test_compatibility(): hash="hash-of-y", prefix="/path/to/packagey-install/", version="1.0", - arch=_common_arch, + arch=_common_arch.spec_json(), compiler=_common_compiler.spec_json(), dependencies={}, parameters={}, @@ -175,23 +167,44 @@ def test_compatibility(): hash="hash-of-x", prefix="/path/to/packagex-install/", version="1.0", - arch=_common_arch, + arch=_common_arch.spec_json(), compiler=_common_compiler.spec_json(), dependencies=dict([y.as_dependency(deptypes=["link"])]), parameters={"precision": ["double", "float"]}, ) x_from_entry = x.to_dict() - x_from_str = json.loads(example_x_json_str) - assert x_from_entry == x_from_str + assert x_from_entry == _raw_json_x def test_compiler_from_entry(): - compiler_data = json.loads(example_compiler_entry) - compiler_from_entry(compiler_data) + compiler_data = json.loads( + """\ +{ + "name": "gcc", + "prefix": "/path/to/compiler/", + "version": "7.5.0", + "arch": { + "os": "centos8", + "target": "x86_64" + }, + "executables": { + "cc": "/path/to/compiler/cc", + "cxx": "/path/to/compiler/cxx", + "fc": "/path/to/compiler/fc" + } +} +""" + ) + compiler = compiler_from_entry(compiler_data, "/example/file") + assert compiler.cc == "/path/to/compiler/cc" + assert compiler.cxx == "/path/to/compiler/cxx" + assert compiler.fc == "/path/to/compiler/fc" + assert compiler.operating_system == "centos8" -def generate_openmpi_entries(): +@pytest.fixture +def generate_openmpi_entries(_common_arch, _common_compiler): """Generate two example JSON entries that refer to an OpenMPI installation and a hwloc dependency. """ @@ -202,7 +215,7 @@ def generate_openmpi_entries(): hash="hwlocfakehashaaa", prefix="/path/to/hwloc-install/", version="2.0.3", - arch=_common_arch, + arch=_common_arch.spec_json(), compiler=_common_compiler.spec_json(), dependencies={}, parameters={}, @@ -216,26 +229,25 @@ def generate_openmpi_entries(): hash="openmpifakehasha", prefix="/path/to/openmpi-install/", version="4.1.0", - arch=_common_arch, + arch=_common_arch.spec_json(), compiler=_common_compiler.spec_json(), dependencies=dict([hwloc.as_dependency(deptypes=["link"])]), parameters={"internal-hwloc": False, "fabrics": ["psm"], "missing_variant": True}, ) - return [openmpi, hwloc] + return list(x.to_dict() for x in [openmpi, hwloc]) -def test_generate_specs_from_manifest(): +def test_generate_specs_from_manifest(generate_openmpi_entries): """Given JSON entries, check that we can form a set of Specs including dependency references. """ - entries = list(x.to_dict() for x in generate_openmpi_entries()) - specs = entries_to_specs(entries) + specs = entries_to_specs(generate_openmpi_entries) (openmpi_spec,) = list(x for x in specs.values() if x.name == "openmpi") assert openmpi_spec["hwloc"] -def test_translate_cray_platform_to_linux(monkeypatch): +def test_translate_cray_platform_to_linux(monkeypatch, _common_compiler): """Manifests might list specs on newer Cray platforms as being "cray", but Spack identifies such platforms as "linux". Make sure we automaticaly transform these entries. @@ -247,13 +259,13 @@ def the_host_is_linux(): monkeypatch.setattr(spack.platforms, "host", the_host_is_linux) - cray_arch = JsonArchEntry(platform="cray", os="rhel8", target="x86_64").to_dict() + cray_arch = JsonArchEntry(platform="cray", os="rhel8", target="x86_64") spec_json = JsonSpecEntry( name="cray-mpich", hash="craympichfakehashaaa", prefix="/path/to/cray-mpich/", version="1.0.0", - arch=cray_arch, + arch=cray_arch.spec_json(), compiler=_common_compiler.spec_json(), dependencies={}, parameters={}, @@ -263,14 +275,15 @@ def the_host_is_linux(): assert spec.architecture.platform == "linux" -def test_translate_compiler_name(): +def test_translate_compiler_name(_common_arch): nvidia_compiler = JsonCompilerEntry( name="nvidia", version="19.1", + arch=_common_arch, executables={"cc": "/path/to/compiler/nvc", "cxx": "/path/to/compiler/nvc++"}, ) - compiler = compiler_from_entry(nvidia_compiler.compiler_json()) + compiler = compiler_from_entry(nvidia_compiler.compiler_json(), "/example/file") assert compiler.name == "nvhpc" spec_json = JsonSpecEntry( @@ -278,7 +291,7 @@ def test_translate_compiler_name(): hash="hwlocfakehashaaa", prefix="/path/to/hwloc-install/", version="2.0.3", - arch=_common_arch, + arch=_common_arch.spec_json(), compiler=nvidia_compiler.spec_json(), dependencies={}, parameters={}, @@ -288,18 +301,18 @@ def test_translate_compiler_name(): assert spec.compiler.name == "nvhpc" -def test_failed_translate_compiler_name(): +def test_failed_translate_compiler_name(_common_arch): unknown_compiler = JsonCompilerEntry(name="unknown", version="1.0") with pytest.raises(spack.compilers.UnknownCompilerError): - compiler_from_entry(unknown_compiler.compiler_json()) + compiler_from_entry(unknown_compiler.compiler_json(), "/example/file") spec_json = JsonSpecEntry( name="packagey", hash="hash-of-y", prefix="/path/to/packagey-install/", version="1.0", - arch=_common_arch, + arch=_common_arch.spec_json(), compiler=unknown_compiler.spec_json(), dependencies={}, parameters={}, @@ -309,7 +322,8 @@ def test_failed_translate_compiler_name(): entries_to_specs([spec_json]) -def create_manifest_content(): +@pytest.fixture +def manifest_content(generate_openmpi_entries, _common_compiler, _other_compiler): return { # Note: the cray_manifest module doesn't use the _meta section right # now, but it is anticipated to be useful @@ -319,47 +333,70 @@ def create_manifest_content(): "schema-version": "1.3", "cpe-version": "22.06", }, - "specs": list(x.to_dict() for x in generate_openmpi_entries()), - "compilers": [_common_compiler.compiler_json()], + "specs": generate_openmpi_entries, + "compilers": [_common_compiler.compiler_json(), _other_compiler.compiler_json()], } -def test_read_cray_manifest(tmpdir, mutable_config, mock_packages, mutable_database): +def test_read_cray_manifest( + tmpdir, mutable_config, mock_packages, mutable_database, manifest_content +): """Check that (a) we can read the cray manifest and add it to the Spack Database and (b) we can concretize specs based on that. """ - if spack.config.get("config:concretizer") == "clingo": - pytest.skip( - "The ASP-based concretizer is currently picky about " " OS matching and will fail." - ) - with tmpdir.as_cwd(): test_db_fname = "external-db.json" with open(test_db_fname, "w") as db_file: - json.dump(create_manifest_content(), db_file) + json.dump(manifest_content, db_file) cray_manifest.read(test_db_fname, True) - query_specs = spack.store.db.query("openmpi") + query_specs = spack.store.STORE.db.query("openmpi") assert any(x.dag_hash() == "openmpifakehasha" for x in query_specs) concretized_specs = spack.cmd.parse_specs( - "depends-on-openmpi %gcc@4.5.0 arch=test-redhat6-x86_64" " ^/openmpifakehasha".split(), - concretize=True, + "depends-on-openmpi ^/openmpifakehasha".split(), concretize=True ) assert concretized_specs[0]["hwloc"].dag_hash() == "hwlocfakehashaaa" -def test_read_cray_manifest_twice_no_compiler_duplicates( - tmpdir, mutable_config, mock_packages, mutable_database +def test_read_cray_manifest_add_compiler_failure( + tmpdir, mutable_config, mock_packages, mutable_database, manifest_content, monkeypatch ): - if spack.config.get("config:concretizer") == "clingo": - pytest.skip( - "The ASP-based concretizer is currently picky about OS matching and will fail." - ) + """Check that cray manifest can be read even if some compilers cannot + be added. + """ + orig_add_compilers_to_config = spack.compilers.add_compilers_to_config + + class fail_for_clang: + def __init__(self): + self.called_with_clang = False + + def __call__(self, compilers, **kwargs): + if any(x.name == "clang" for x in compilers): + self.called_with_clang = True + raise Exception() + return orig_add_compilers_to_config(compilers, **kwargs) + + checker = fail_for_clang() + monkeypatch.setattr(spack.compilers, "add_compilers_to_config", checker) + + with tmpdir.as_cwd(): + test_db_fname = "external-db.json" + with open(test_db_fname, "w") as db_file: + json.dump(manifest_content, db_file) + cray_manifest.read(test_db_fname, True) + query_specs = spack.store.STORE.db.query("openmpi") + assert any(x.dag_hash() == "openmpifakehasha" for x in query_specs) + + assert checker.called_with_clang + +def test_read_cray_manifest_twice_no_compiler_duplicates( + tmpdir, mutable_config, mock_packages, mutable_database, manifest_content +): with tmpdir.as_cwd(): test_db_fname = "external-db.json" with open(test_db_fname, "w") as db_file: - json.dump(create_manifest_content(), db_file) + json.dump(manifest_content, db_file) # Read the manifest twice cray_manifest.read(test_db_fname, True) @@ -367,7 +404,7 @@ def test_read_cray_manifest_twice_no_compiler_duplicates( compilers = spack.compilers.all_compilers() filtered = list( - c for c in compilers if c.spec == spack.spec.CompilerSpec("gcc@=10.2.0.cray") + c for c in compilers if c.spec == spack.spec.CompilerSpec("gcc@=10.2.0.2112") ) assert len(filtered) == 1 @@ -427,3 +464,27 @@ def test_convert_validation_error(tmpdir, mutable_config, mock_packages, mutable with pytest.raises(cray_manifest.ManifestValidationError) as e: cray_manifest.read(invalid_schema_path, True) str(e) + + +@pytest.fixture +def directory_with_manifest(tmpdir, manifest_content): + """Create a manifest file in a directory. Used by 'spack external'.""" + with tmpdir.as_cwd(): + test_db_fname = "external-db.json" + with open(test_db_fname, "w") as db_file: + json.dump(manifest_content, db_file) + + yield str(tmpdir) + + +def test_find_external_nonempty_default_manifest_dir( + mutable_database, mutable_mock_repo, tmpdir, monkeypatch, directory_with_manifest +): + """The user runs 'spack external find'; the default manifest directory + contains a manifest file. Ensure that the specs are read. + """ + monkeypatch.setenv("PATH", "") + monkeypatch.setattr(spack.cray_manifest, "default_path", str(directory_with_manifest)) + spack.cmd.external._collect_and_consume_cray_manifest_files(ignore_default_dir=False) + specs = spack.store.STORE.db.query("hwloc") + assert any(x.dag_hash() == "hwlocfakehashaaa" for x in specs) diff --git a/lib/spack/spack/test/data/config/bootstrap.yaml b/lib/spack/spack/test/data/config/bootstrap.yaml index 6adb7ab9967e78..4757b8729d23a8 100644 --- a/lib/spack/spack/test/data/config/bootstrap.yaml +++ b/lib/spack/spack/test/data/config/bootstrap.yaml @@ -1,5 +1,5 @@ bootstrap: sources: - name: 'github-actions' - metadata: $spack/share/spack/bootstrap/github-actions-v0.3 + metadata: $spack/share/spack/bootstrap/github-actions-v0.5 trusted: {} diff --git a/lib/spack/spack/test/data/config/concretizer.yaml b/lib/spack/spack/test/data/config/concretizer.yaml index b5b1c712dbe008..0dd810163dd77a 100644 --- a/lib/spack/spack/test/data/config/concretizer.yaml +++ b/lib/spack/spack/test/data/config/concretizer.yaml @@ -3,3 +3,5 @@ concretizer: targets: granularity: microarchitectures host_compatible: false + duplicates: + strategy: minimal diff --git a/lib/spack/spack/test/data/modules/lmod/blacklist_environment.yaml b/lib/spack/spack/test/data/modules/lmod/blacklist_environment.yaml deleted file mode 100644 index 997501e08ba454..00000000000000 --- a/lib/spack/spack/test/data/modules/lmod/blacklist_environment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# DEPRECATED: remove this in v0.20 -# See `alter_environment.yaml` for the new syntax -enable: - - lmod -lmod: - core_compilers: - - 'clang@3.3' - - hierarchy: - - mpi - - all: - autoload: none - filter: - environment_blacklist: - - CMAKE_PREFIX_PATH - environment: - set: - '{name}_ROOT': '{prefix}' - - 'platform=test target=x86_64': - environment: - set: - FOO: 'foo' - unset: - - BAR - - 'platform=test target=core2': - load: - - 'foo/bar' diff --git a/lib/spack/spack/test/data/modules/lmod/conflicts.yaml b/lib/spack/spack/test/data/modules/lmod/conflicts.yaml new file mode 100644 index 00000000000000..7336dc10a76b9c --- /dev/null +++ b/lib/spack/spack/test/data/modules/lmod/conflicts.yaml @@ -0,0 +1,10 @@ +enable: + - lmod +lmod: + core_compilers: + - 'clang@3.3' + all: + autoload: none + conflict: + - '{name}' + - 'intel/14.0.1' diff --git a/lib/spack/spack/test/data/modules/lmod/blacklist.yaml b/lib/spack/spack/test/data/modules/lmod/hide_implicits.yaml similarity index 51% rename from lib/spack/spack/test/data/modules/lmod/blacklist.yaml rename to lib/spack/spack/test/data/modules/lmod/hide_implicits.yaml index 8c88214380f28b..e9326ab42c4661 100644 --- a/lib/spack/spack/test/data/modules/lmod/blacklist.yaml +++ b/lib/spack/spack/test/data/modules/lmod/hide_implicits.yaml @@ -1,14 +1,12 @@ -# DEPRECATED: remove this in v0.20 -# See `exclude.yaml` for the new syntax enable: - lmod lmod: + hide_implicits: true + hash_length: 0 core_compilers: - 'clang@3.3' hierarchy: - mpi - blacklist: - - callpath all: autoload: direct diff --git a/lib/spack/spack/test/data/modules/lmod/wrong_conflicts.yaml b/lib/spack/spack/test/data/modules/lmod/wrong_conflicts.yaml new file mode 100644 index 00000000000000..f674ee631a88be --- /dev/null +++ b/lib/spack/spack/test/data/modules/lmod/wrong_conflicts.yaml @@ -0,0 +1,11 @@ +enable: + - lmod +lmod: + core_compilers: + - 'clang@3.3' + projections: + all: '{name}/{version}-{compiler.name}' + all: + autoload: none + conflict: + - '{name}/{compiler.name}' diff --git a/lib/spack/spack/test/data/modules/tcl/blacklist.yaml b/lib/spack/spack/test/data/modules/tcl/blacklist.yaml deleted file mode 100644 index 4ffeb135e95ebe..00000000000000 --- a/lib/spack/spack/test/data/modules/tcl/blacklist.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# DEPRECATED: remove this in v0.20 -# See `exclude.yaml` for the new syntax -enable: - - tcl -tcl: - whitelist: - - zmpi - blacklist: - - callpath - - mpi - all: - autoload: direct diff --git a/lib/spack/spack/test/data/modules/tcl/blacklist_environment.yaml b/lib/spack/spack/test/data/modules/tcl/blacklist_environment.yaml deleted file mode 100644 index 128200d6ec6f87..00000000000000 --- a/lib/spack/spack/test/data/modules/tcl/blacklist_environment.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# DEPRECATED: remove this in v0.20 -# See `alter_environment.yaml` for the new syntax -enable: - - tcl -tcl: - all: - autoload: none - filter: - environment_blacklist: - - CMAKE_PREFIX_PATH - environment: - set: - '{name}_ROOT': '{prefix}' - - 'platform=test target=x86_64': - environment: - set: - FOO: 'foo' - OMPI_MCA_mpi_leave_pinned: '1' - unset: - - BAR - - 'platform=test target=core2': - load: - - 'foo/bar' diff --git a/lib/spack/spack/test/data/modules/tcl/blacklist_implicits.yaml b/lib/spack/spack/test/data/modules/tcl/blacklist_implicits.yaml deleted file mode 100644 index b49bc80b5e82a7..00000000000000 --- a/lib/spack/spack/test/data/modules/tcl/blacklist_implicits.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# DEPRECATED: remove this in v0.20 -# See `exclude_implicits.yaml` for the new syntax -enable: - - tcl -tcl: - blacklist_implicits: true - all: - autoload: direct diff --git a/lib/spack/spack/test/data/modules/tcl/exclude_implicits.yaml b/lib/spack/spack/test/data/modules/tcl/exclude_implicits.yaml index 2d892c43513a51..4835b4ecd93f33 100644 --- a/lib/spack/spack/test/data/modules/tcl/exclude_implicits.yaml +++ b/lib/spack/spack/test/data/modules/tcl/exclude_implicits.yaml @@ -1,6 +1,9 @@ +# DEPRECATED: remove this in ? +# See `hide_implicits.yaml` for the new syntax enable: - tcl tcl: exclude_implicits: true + hash_length: 0 all: autoload: direct diff --git a/lib/spack/spack/test/data/modules/tcl/hide_implicits.yaml b/lib/spack/spack/test/data/modules/tcl/hide_implicits.yaml new file mode 100644 index 00000000000000..136c42f3c7cb50 --- /dev/null +++ b/lib/spack/spack/test/data/modules/tcl/hide_implicits.yaml @@ -0,0 +1,7 @@ +enable: + - tcl +tcl: + hide_implicits: true + hash_length: 0 + all: + autoload: direct diff --git a/lib/spack/spack/test/data/modules/tcl/invalid_token_in_env_var_name.yaml b/lib/spack/spack/test/data/modules/tcl/invalid_token_in_env_var_name.yaml index b03f966c7c1509..75b4cd09d2ec42 100644 --- a/lib/spack/spack/test/data/modules/tcl/invalid_token_in_env_var_name.yaml +++ b/lib/spack/spack/test/data/modules/tcl/invalid_token_in_env_var_name.yaml @@ -4,7 +4,7 @@ tcl: all: autoload: none filter: - environment_blacklist: + exclude_env_vars: - CMAKE_PREFIX_PATH environment: set: diff --git a/lib/spack/spack/test/data/templates/non_relocatable.c b/lib/spack/spack/test/data/templates/non_relocatable.c deleted file mode 100644 index c2e3724c2c07c0..00000000000000 --- a/lib/spack/spack/test/data/templates/non_relocatable.c +++ /dev/null @@ -1,5 +0,0 @@ -#include - -int main(){ - printf("Hello World from {{ prefix }} !"); -} diff --git a/lib/spack/spack/test/data/templates/relocatable.c b/lib/spack/spack/test/data/templates/relocatable.c deleted file mode 100644 index f3e49596922e3f..00000000000000 --- a/lib/spack/spack/test/data/templates/relocatable.c +++ /dev/null @@ -1,5 +0,0 @@ -#include - -int main(){ - printf("Hello World!"); -} diff --git a/lib/spack/spack/test/data/unparse/llvm.txt b/lib/spack/spack/test/data/unparse/llvm.txt index f16fd9cc4784df..834ab2dc92db00 100644 --- a/lib/spack/spack/test/data/unparse/llvm.txt +++ b/lib/spack/spack/test/data/unparse/llvm.txt @@ -235,7 +235,7 @@ class Llvm(CMakePackage, CudaPackage): depends_on("libffi", when="+cuda") # libomptarget # llvm-config --system-libs libraries. - depends_on("zlib") + depends_on("zlib-api") # lldb dependencies depends_on("swig", when="+lldb") diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 368b2c1b723b74..ee3e5da81ef679 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -18,7 +18,6 @@ _use_uuid = True except ImportError: _use_uuid = False - pass import jsonschema @@ -77,7 +76,7 @@ def test_spec_installed_upstream( upstream_write_db.add(spec, upstream_layout) upstream_db._read() - monkeypatch.setattr(spack.store, "db", downstream_db) + monkeypatch.setattr(spack.store.STORE, "db", downstream_db) assert spec.installed assert spec.installed_upstream assert spec.copy().installed @@ -195,11 +194,11 @@ def test_add_to_upstream_after_downstream(upstream_and_downstream_db, tmpdir): assert len(qresults) == 1 (queried_spec,) = qresults try: - orig_db = spack.store.db - spack.store.db = downstream_db + orig_db = spack.store.STORE.db + spack.store.STORE.db = downstream_db assert queried_spec.prefix == downstream_layout.path_for_spec(spec) finally: - spack.store.db = orig_db + spack.store.STORE.db = orig_db @pytest.mark.usefixtures("config", "temporary_store") @@ -294,16 +293,16 @@ def _print_ref_counts(): recs = [] def add_rec(spec): - cspecs = spack.store.db.query(spec, installed=any) + cspecs = spack.store.STORE.db.query(spec, installed=any) if not cspecs: recs.append("[ %-7s ] %-20s-" % ("", spec)) else: key = cspecs[0].dag_hash() - rec = spack.store.db.get_record(cspecs[0]) + rec = spack.store.STORE.db.get_record(cspecs[0]) recs.append("[ %-7s ] %-20s%d" % (key[:7], spec, rec.ref_count)) - with spack.store.db.read_transaction(): + with spack.store.STORE.db.read_transaction(): add_rec("mpileaks ^mpich") add_rec("callpath ^mpich") add_rec("mpich") @@ -326,7 +325,7 @@ def add_rec(spec): def _check_merkleiness(): """Ensure the spack database is a valid merkle graph.""" - all_specs = spack.store.db.query(installed=any) + all_specs = spack.store.STORE.db.query(installed=any) seen = {} for spec in all_specs: @@ -340,7 +339,7 @@ def _check_merkleiness(): def _check_db_sanity(database): """Utility function to check db against install layout.""" - pkg_in_layout = sorted(spack.store.layout.all_specs()) + pkg_in_layout = sorted(spack.store.STORE.layout.all_specs()) actual = sorted(database.query()) externals = sorted([x for x in actual if x.external]) @@ -376,7 +375,7 @@ def _check_remove_and_add_package(database, spec): assert concrete_spec not in remaining # add it back and make sure everything is ok. - database.add(concrete_spec, spack.store.layout) + database.add(concrete_spec, spack.store.STORE.layout) installed = database.query() assert concrete_spec in installed assert installed == original @@ -392,7 +391,7 @@ def _mock_install(spec): def _mock_remove(spec): - specs = spack.store.db.query(spec) + specs = spack.store.STORE.db.query(spec) assert len(specs) == 1 spec = specs[0] spec.package.do_uninstall(spec) @@ -454,7 +453,7 @@ def test_005_db_exists(database): def test_010_all_install_sanity(database): """Ensure that the install layout reflects what we think it does.""" - all_specs = spack.store.layout.all_specs() + all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 15 # Query specs with multiple configurations @@ -483,12 +482,12 @@ def test_010_all_install_sanity(database): def test_015_write_and_read(mutable_database): # write and read DB - with spack.store.db.write_transaction(): - specs = spack.store.db.query() - recs = [spack.store.db.get_record(s) for s in specs] + with spack.store.STORE.db.write_transaction(): + specs = spack.store.STORE.db.query() + recs = [spack.store.STORE.db.get_record(s) for s in specs] for spec, rec in zip(specs, recs): - new_rec = spack.store.db.get_record(spec) + new_rec = spack.store.STORE.db.get_record(spec) assert new_rec.ref_count == rec.ref_count assert new_rec.spec == rec.spec assert new_rec.path == rec.path @@ -498,12 +497,12 @@ def test_015_write_and_read(mutable_database): def test_017_write_and_read_without_uuid(mutable_database, monkeypatch): monkeypatch.setattr(spack.database, "_use_uuid", False) # write and read DB - with spack.store.db.write_transaction(): - specs = spack.store.db.query() - recs = [spack.store.db.get_record(s) for s in specs] + with spack.store.STORE.db.write_transaction(): + specs = spack.store.STORE.db.query() + recs = [spack.store.STORE.db.get_record(s) for s in specs] for spec, rec in zip(specs, recs): - new_rec = spack.store.db.get_record(spec) + new_rec = spack.store.STORE.db.get_record(spec) assert new_rec.ref_count == rec.ref_count assert new_rec.spec == rec.spec assert new_rec.path == rec.path @@ -517,7 +516,7 @@ def test_020_db_sanity(database): def test_025_reindex(mutable_database): """Make sure reindex works and ref counts are valid.""" - spack.store.store.reindex() + spack.store.STORE.reindex() _check_db_sanity(mutable_database) @@ -527,7 +526,7 @@ def test_026_reindex_after_deprecate(mutable_database): zmpi = mutable_database.query_one("zmpi") mutable_database.deprecate(mpich, zmpi) - spack.store.store.reindex() + spack.store.STORE.reindex() _check_db_sanity(mutable_database) @@ -538,8 +537,8 @@ class ReadModify: def __call__(self): # check that other process can read DB - _check_db_sanity(spack.store.db) - with spack.store.db.write_transaction(): + _check_db_sanity(spack.store.STORE.db) + with spack.store.STORE.db.write_transaction(): _mock_remove("mpileaks ^zmpi") @@ -571,7 +570,7 @@ def test_041_ref_counts_deprecate(mutable_database): def test_050_basic_query(database): """Ensure querying database is consistent with what is installed.""" # query everything - total_specs = len(spack.store.db.query()) + total_specs = len(spack.store.STORE.db.query()) assert total_specs == 17 # query specs with multiple configurations @@ -626,7 +625,7 @@ def test_080_root_ref_counts(mutable_database): assert mutable_database.get_record("mpich").ref_count == 1 # Put the spec back - mutable_database.add(rec.spec, spack.store.layout) + mutable_database.add(rec.spec, spack.store.STORE.layout) # record is present again assert len(mutable_database.query("mpileaks ^mpich", installed=any)) == 1 @@ -702,7 +701,7 @@ def test_115_reindex_with_packages_not_in_repo(mutable_database, tmpdir): # packages should not have to be defined in the repository once they # are installed with spack.repo.use_repositories(spack.repo.MockRepositoryBuilder(tmpdir).root): - spack.store.store.reindex() + spack.store.STORE.reindex() _check_db_sanity(mutable_database) @@ -740,7 +739,7 @@ def test_regression_issue_8036(mutable_database, usr_folder_exists): @pytest.mark.regression("11118") def test_old_external_entries_prefix(mutable_database): - with open(spack.store.db._index_path, "r") as f: + with open(spack.store.STORE.db._index_path, "r") as f: db_obj = json.loads(f.read()) jsonschema.validate(db_obj, schema) @@ -750,13 +749,13 @@ def test_old_external_entries_prefix(mutable_database): db_obj["database"]["installs"][s.dag_hash()]["path"] = "None" - with open(spack.store.db._index_path, "w") as f: + with open(spack.store.STORE.db._index_path, "w") as f: f.write(json.dumps(db_obj)) if _use_uuid: - with open(spack.store.db._verifier_path, "w") as f: + with open(spack.store.STORE.db._verifier_path, "w") as f: f.write(str(uuid.uuid4())) - record = spack.store.db.get_record(s) + record = spack.store.STORE.db.get_record(s) assert record.path is None assert record.spec._prefix is None @@ -779,7 +778,7 @@ def test_query_unused_specs(mutable_database): s.concretize() s.package.do_install(fake=True, explicit=True) - unused = spack.store.db.unused_specs + unused = spack.store.STORE.db.unused_specs assert len(unused) == 1 assert unused[0].name == "cmake" @@ -792,7 +791,7 @@ def test_query_spec_with_conditional_dependency(mutable_database): s.concretize() s.package.do_install(fake=True, explicit=True) - results = spack.store.db.query_local("hdf5 ^mpich") + results = spack.store.STORE.db.query_local("hdf5 ^mpich") assert not results @@ -800,29 +799,37 @@ def test_query_spec_with_conditional_dependency(mutable_database): def test_query_spec_with_non_conditional_virtual_dependency(database): # Ensure the same issue doesn't come up for virtual # dependency that are not conditional on variants - results = spack.store.db.query_local("mpileaks ^mpich") + results = spack.store.STORE.db.query_local("mpileaks ^mpich") assert len(results) == 1 +def test_query_virtual_spec(database): + """Make sure we can query for virtuals in the DB""" + results = spack.store.STORE.db.query_local("mpi") + assert len(results) == 3 + names = [s.name for s in results] + assert all(name in names for name in ["mpich", "mpich2", "zmpi"]) + + def test_failed_spec_path_error(database): """Ensure spec not concrete check is covered.""" s = spack.spec.Spec("a") - with pytest.raises(ValueError, match="Concrete spec required"): - spack.store.db._failed_spec_path(s) + with pytest.raises(AssertionError, match="concrete spec required"): + spack.store.STORE.failure_tracker.mark(s) @pytest.mark.db def test_clear_failure_keep(mutable_database, monkeypatch, capfd): """Add test coverage for clear_failure operation when to be retained.""" - def _is(db, spec): + def _is(self, spec): return True # Pretend the spec has been failure locked - monkeypatch.setattr(spack.database.Database, "prefix_failure_locked", _is) + monkeypatch.setattr(spack.database.FailureTracker, "lock_taken", _is) - s = spack.spec.Spec("a") - spack.store.db.clear_failure(s) + s = spack.spec.Spec("a").concretized() + spack.store.STORE.failure_tracker.clear(s) out = capfd.readouterr()[0] assert "Retaining failure marking" in out @@ -831,16 +838,16 @@ def _is(db, spec): def test_clear_failure_forced(default_mock_concretization, mutable_database, monkeypatch, capfd): """Add test coverage for clear_failure operation when force.""" - def _is(db, spec): + def _is(self, spec): return True # Pretend the spec has been failure locked - monkeypatch.setattr(spack.database.Database, "prefix_failure_locked", _is) + monkeypatch.setattr(spack.database.FailureTracker, "lock_taken", _is) # Ensure raise OSError when try to remove the non-existent marking - monkeypatch.setattr(spack.database.Database, "prefix_failure_marked", _is) + monkeypatch.setattr(spack.database.FailureTracker, "persistent_mark", _is) s = default_mock_concretization("a") - spack.store.db.clear_failure(s, force=True) + spack.store.STORE.failure_tracker.clear(s, force=True) out = capfd.readouterr()[1] assert "Removing failure marking despite lock" in out assert "Unable to remove failure marking" in out @@ -858,55 +865,34 @@ def _raise_exc(lock): with tmpdir.as_cwd(): s = default_mock_concretization("a") - spack.store.db.mark_failed(s) + spack.store.STORE.failure_tracker.mark(s) out = str(capsys.readouterr()[1]) assert "Unable to mark a as failed" in out - # Clean up the failure mark to ensure it does not interfere with other - # tests using the same spec. - del spack.store.db._prefix_failures[s.prefix] + spack.store.STORE.failure_tracker.clear_all() @pytest.mark.db def test_prefix_failed(default_mock_concretization, mutable_database, monkeypatch): - """Add coverage to prefix_failed operation.""" - - def _is(db, spec): - return True + """Add coverage to failed operation.""" s = default_mock_concretization("a") # Confirm the spec is not already marked as failed - assert not spack.store.db.prefix_failed(s) + assert not spack.store.STORE.failure_tracker.has_failed(s) # Check that a failure entry is sufficient - spack.store.db._prefix_failures[s.prefix] = None - assert spack.store.db.prefix_failed(s) + spack.store.STORE.failure_tracker.mark(s) + assert spack.store.STORE.failure_tracker.has_failed(s) # Remove the entry and check again - del spack.store.db._prefix_failures[s.prefix] - assert not spack.store.db.prefix_failed(s) + spack.store.STORE.failure_tracker.clear(s) + assert not spack.store.STORE.failure_tracker.has_failed(s) # Now pretend that the prefix failure is locked - monkeypatch.setattr(spack.database.Database, "prefix_failure_locked", _is) - assert spack.store.db.prefix_failed(s) - - -def test_prefix_read_lock_error(default_mock_concretization, mutable_database, monkeypatch): - """Cover the prefix read lock exception.""" - - def _raise(db, spec): - raise lk.LockError("Mock lock error") - - s = default_mock_concretization("a") - - # Ensure subsequent lock operations fail - monkeypatch.setattr(lk.Lock, "acquire_read", _raise) - - with pytest.raises(Exception): - with spack.store.db.prefix_read_lock(s): - assert False + monkeypatch.setattr(spack.database.FailureTracker, "lock_taken", lambda self, spec: True) + assert spack.store.STORE.failure_tracker.has_failed(s) def test_prefix_write_lock_error(default_mock_concretization, mutable_database, monkeypatch): @@ -921,7 +907,7 @@ def _raise(db, spec): monkeypatch.setattr(lk.Lock, "acquire_write", _raise) with pytest.raises(Exception): - with spack.store.db.prefix_write_lock(s): + with spack.store.STORE.prefix_locker.write_lock(s): assert False @@ -969,7 +955,7 @@ def test_reindex_removed_prefix_is_not_installed(mutable_database, mock_store, c shutil.rmtree(prefix) # Reindex should pick up libelf as a dependency of libdwarf - spack.store.store.reindex() + spack.store.STORE.reindex() # Reindexing should warn about libelf not being found on the filesystem err = capfd.readouterr()[1] @@ -982,7 +968,7 @@ def test_reindex_removed_prefix_is_not_installed(mutable_database, mock_store, c def test_reindex_when_all_prefixes_are_removed(mutable_database, mock_store): # Remove all non-external installations from the filesystem - for spec in spack.store.db.query_local(): + for spec in spack.store.STORE.db.query_local(): if not spec.external: assert spec.prefix.startswith(str(mock_store)) shutil.rmtree(spec.prefix) @@ -992,7 +978,7 @@ def test_reindex_when_all_prefixes_are_removed(mutable_database, mock_store): assert num > 0 # Reindex uses the current index to repopulate itself - spack.store.store.reindex() + spack.store.STORE.reindex() # Make sure all explicit specs are still there, but are now uninstalled. specs = mutable_database.query_local(installed=False, explicit=True) @@ -1060,8 +1046,18 @@ def test_error_message_when_using_too_new_db(database, monkeypatch): back to an older version of Spack. This test ensures that the error message for a too new database version stays comprehensible across refactoring of the database code. """ - monkeypatch.setattr(spack.database, "_db_version", vn.Version("0")) + monkeypatch.setattr(spack.database, "_DB_VERSION", vn.Version("0")) with pytest.raises( spack.database.InvalidDatabaseVersionError, match="you need a newer Spack version" ): spack.database.Database(database.root)._read() + + +@pytest.mark.parametrize( + "lock_cfg", + [spack.database.NO_LOCK, spack.database.NO_TIMEOUT, spack.database.DEFAULT_LOCK_CFG, None], +) +def test_database_construction_doesnt_use_globals(tmpdir, config, nullify_globals, lock_cfg): + lock_cfg = lock_cfg or spack.database.lock_configuration(config) + db = spack.database.Database(str(tmpdir), lock_cfg=lock_cfg) + assert os.path.exists(db.database_directory) diff --git a/lib/spack/spack/test/detection.py b/lib/spack/spack/test/detection.py new file mode 100644 index 00000000000000..6218bc87578f7c --- /dev/null +++ b/lib/spack/spack/test/detection.py @@ -0,0 +1,30 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import collections + +import spack.detection +import spack.spec + + +def test_detection_update_config(mutable_config): + # mock detected package + detected_packages = collections.defaultdict(list) + detected_packages["cmake"] = [ + spack.detection.common.DetectedPackage( + spec=spack.spec.Spec("cmake@3.27.5"), prefix="/usr/bin" + ) + ] + + # update config for new package + spack.detection.common.update_configuration(detected_packages) + # Check entries in 'packages.yaml' + packages_yaml = spack.config.get("packages") + assert "cmake" in packages_yaml + assert "externals" in packages_yaml["cmake"] + externals = packages_yaml["cmake"]["externals"] + assert len(externals) == 1 + external_gcc = externals[0] + assert external_gcc["spec"] == "cmake@3.27.5" + assert external_gcc["prefix"] == "/usr/bin" diff --git a/lib/spack/spack/test/directives.py b/lib/spack/spack/test/directives.py index bbb5789e2185d0..677eb043a9e6ed 100644 --- a/lib/spack/spack/test/directives.py +++ b/lib/spack/spack/test/directives.py @@ -16,7 +16,7 @@ def test_false_directives_do_not_exist(mock_packages): """Ensure directives that evaluate to False at import time are added to dicts on packages. """ - cls = spack.repo.path.get_pkg_class("when-directives-false") + cls = spack.repo.PATH.get_pkg_class("when-directives-false") assert not cls.dependencies assert not cls.resources assert not cls.patches @@ -26,7 +26,7 @@ def test_true_directives_exist(mock_packages): """Ensure directives that evaluate to True at import time are added to dicts on packages. """ - cls = spack.repo.path.get_pkg_class("when-directives-true") + cls = spack.repo.PATH.get_pkg_class("when-directives-true") assert cls.dependencies assert spack.spec.Spec() in cls.dependencies["extendee"] @@ -40,7 +40,7 @@ def test_true_directives_exist(mock_packages): def test_constraints_from_context(mock_packages): - pkg_cls = spack.repo.path.get_pkg_class("with-constraint-met") + pkg_cls = spack.repo.PATH.get_pkg_class("with-constraint-met") assert pkg_cls.dependencies assert spack.spec.Spec("@1.0") in pkg_cls.dependencies["b"] @@ -51,7 +51,7 @@ def test_constraints_from_context(mock_packages): @pytest.mark.regression("26656") def test_constraints_from_context_are_merged(mock_packages): - pkg_cls = spack.repo.path.get_pkg_class("with-constraint-met") + pkg_cls = spack.repo.PATH.get_pkg_class("with-constraint-met") assert pkg_cls.dependencies assert spack.spec.Spec("@0.14:15 ^b@3.8:4.0") in pkg_cls.dependencies["c"] @@ -68,7 +68,7 @@ def test_extends_spec(config, mock_packages): @pytest.mark.regression("34368") def test_error_on_anonymous_dependency(config, mock_packages): - pkg = spack.repo.path.get_pkg_class("a") + pkg = spack.repo.PATH.get_pkg_class("a") with pytest.raises(spack.directives.DependencyError): spack.directives._depends_on(pkg, "@4.5") @@ -78,17 +78,55 @@ def test_error_on_anonymous_dependency(config, mock_packages): "package_name,expected_maintainers", [ ("maintainers-1", ["user1", "user2"]), - # Reset from PythonPackage - ("py-extension1", ["adamjstewart", "pradyunsg", "user1", "user2"]), + # Extends PythonPackage + ("py-extension1", ["adamjstewart", "user1", "user2"]), # Extends maintainers-1 ("maintainers-3", ["user0", "user1", "user2", "user3"]), ], ) def test_maintainer_directive(config, mock_packages, package_name, expected_maintainers): - pkg_cls = spack.repo.path.get_pkg_class(package_name) + pkg_cls = spack.repo.PATH.get_pkg_class(package_name) assert pkg_cls.maintainers == expected_maintainers +@pytest.mark.parametrize( + "package_name,expected_licenses", [("licenses-1", [("MIT", "+foo"), ("Apache-2.0", "~foo")])] +) +def test_license_directive(config, mock_packages, package_name, expected_licenses): + pkg_cls = spack.repo.PATH.get_pkg_class(package_name) + for license in expected_licenses: + assert spack.spec.Spec(license[1]) in pkg_cls.licenses + assert license[0] == pkg_cls.licenses[spack.spec.Spec(license[1])] + + +def test_duplicate_exact_range_license(): + package = namedtuple("package", ["licenses", "name"]) + package.licenses = {spack.directives.make_when_spec("+foo"): "Apache-2.0"} + package.name = "test_package" + + msg = ( + r"test_package is specified as being licensed as MIT when \+foo, but it is also " + r"specified as being licensed under Apache-2.0 when \+foo, which conflict." + ) + + with pytest.raises(spack.directives.OverlappingLicenseError, match=msg): + spack.directives._execute_license(package, "MIT", "+foo") + + +def test_overlapping_duplicate_licenses(): + package = namedtuple("package", ["licenses", "name"]) + package.licenses = {spack.directives.make_when_spec("+foo"): "Apache-2.0"} + package.name = "test_package" + + msg = ( + r"test_package is specified as being licensed as MIT when \+bar, but it is also " + r"specified as being licensed under Apache-2.0 when \+foo, which conflict." + ) + + with pytest.raises(spack.directives.OverlappingLicenseError, match=msg): + spack.directives._execute_license(package, "MIT", "+bar") + + def test_version_type_validation(): # A version should be a string or an int, not a float, because it leads to subtle issues # such as 3.10 being interpreted as 3.1. diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 323c081e64fb8b..64676bfebcb8d9 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -8,14 +8,16 @@ """ import os import os.path +from pathlib import Path import pytest +from llnl.path import path_to_os_path + import spack.paths import spack.repo from spack.directory_layout import DirectoryLayout, InvalidDirectoryLayoutParametersError from spack.spec import Spec -from spack.util.path import path_to_os_path # number of packages to test (to reduce test time) max_packages = 10 @@ -29,8 +31,12 @@ def test_yaml_directory_layout_parameters(tmpdir, default_mock_concretization): # Ensure default layout matches expected spec format layout_default = DirectoryLayout(str(tmpdir)) path_default = layout_default.relative_path_for_spec(spec) - assert path_default == spec.format( - "{architecture}/" "{compiler.name}-{compiler.version}/" "{name}-{version}-{hash}" + assert path_default == str( + Path( + spec.format( + "{architecture}/" "{compiler.name}-{compiler.version}/" "{name}-{version}-{hash}" + ) + ) ) # Test hash_length parameter works correctly @@ -43,7 +49,7 @@ def test_yaml_directory_layout_parameters(tmpdir, default_mock_concretization): assert len(path_default) - len(path_7) == 25 # Test path_scheme - arch, compiler, package7 = path_7.split("/") + arch, compiler, package7 = path_7.split(os.sep) projections_package7 = {"all": "{name}-{version}-{hash:7}"} layout_package7 = DirectoryLayout(str(tmpdir), projections=projections_package7) path_package7 = layout_package7.relative_path_for_spec(spec) @@ -61,10 +67,10 @@ def test_yaml_directory_layout_parameters(tmpdir, default_mock_concretization): layout_arch_ns = DirectoryLayout(str(tmpdir), projections=arch_ns_scheme_projections) arch_path_spec2 = layout_arch_ns.relative_path_for_spec(spec2) - assert arch_path_spec2 == spec2.format(arch_scheme) + assert arch_path_spec2 == str(Path(spec2.format(arch_scheme))) ns_path_spec = layout_arch_ns.relative_path_for_spec(spec) - assert ns_path_spec == spec.format(ns_scheme) + assert ns_path_spec == str(Path(spec.format(ns_scheme))) # Ensure conflicting parameters caught with pytest.raises(InvalidDirectoryLayoutParametersError): @@ -79,7 +85,7 @@ def test_read_and_write_spec(temporary_store, config, mock_packages): layout. """ layout = temporary_store.layout - pkg_names = list(spack.repo.path.all_package_names())[:max_packages] + pkg_names = list(spack.repo.PATH.all_package_names())[:max_packages] for name in pkg_names: if name.startswith("external"): @@ -191,7 +197,7 @@ def test_handle_unknown_package(temporary_store, config, mock_packages): def test_find(temporary_store, config, mock_packages): """Test that finding specs within an install layout works.""" layout = temporary_store.layout - package_names = list(spack.repo.path.all_package_names())[:max_packages] + package_names = list(spack.repo.PATH.all_package_names())[:max_packages] # Create install prefixes for all packages in the list installed_specs = {} diff --git a/lib/spack/spack/test/env.py b/lib/spack/spack/test/env.py index 965e841d0c5790..7490a6e0b26204 100644 --- a/lib/spack/spack/test/env.py +++ b/lib/spack/spack/test/env.py @@ -6,7 +6,6 @@ import filecmp import os import pickle -import sys import pytest @@ -14,11 +13,14 @@ import spack.environment as ev import spack.spec -from spack.environment.environment import SpackEnvironmentViewError, _error_on_nonempty_view_dir - -pytestmark = pytest.mark.skipif( - sys.platform == "win32", reason="Envs are not supported on windows" +from spack.environment.environment import ( + EnvironmentManifestFile, + SpackEnvironmentViewError, + _error_on_nonempty_view_dir, ) +from spack.spec_list import UndefinedReferenceError + +pytestmark = pytest.mark.not_on_windows("Envs are not supported on windows") class TestDirectoryInitialization: @@ -500,3 +502,279 @@ def test_error_message_when_using_too_new_lockfile(tmp_path): ev.initialize_environment_dir(env_dir, init_file) with pytest.raises(ev.SpackEnvironmentError, match="You need to use a newer Spack version."): ev.Environment(env_dir) + + +@pytest.mark.regression("38240") +@pytest.mark.parametrize( + "unify_in_lower_scope,unify_in_spack_yaml", + [ + (True, False), + (True, "when_possible"), + (False, True), + (False, "when_possible"), + ("when_possible", False), + ("when_possible", True), + ], +) +def test_environment_concretizer_scheme_used(tmp_path, unify_in_lower_scope, unify_in_spack_yaml): + """Tests that "unify" settings in spack.yaml always take precedence over settings in lower + configuration scopes. + """ + manifest = tmp_path / "spack.yaml" + manifest.write_text( + f"""\ +spack: + specs: + - mpileaks + concretizer: + unify: {str(unify_in_spack_yaml).lower()} +""" + ) + + with spack.config.override("concretizer:unify", unify_in_lower_scope): + with ev.Environment(manifest.parent) as e: + assert e.unify == unify_in_spack_yaml + + +@pytest.mark.parametrize("unify_in_config", [True, False, "when_possible"]) +def test_environment_config_scheme_used(tmp_path, unify_in_config): + """Tests that "unify" settings in lower configuration scopes is taken into account, + if absent in spack.yaml. + """ + manifest = tmp_path / "spack.yaml" + manifest.write_text( + """\ +spack: + specs: + - mpileaks +""" + ) + + with spack.config.override("concretizer:unify", unify_in_config): + with ev.Environment(manifest.parent) as e: + assert e.unify == unify_in_config + + +@pytest.mark.parametrize( + "spec_str,expected_raise,expected_spec", + [ + # vendorsb vendors "b" only when @=1.1 + ("vendorsb", False, "vendorsb@=1.0"), + ("vendorsb@=1.1", True, None), + ], +) +def test_conflicts_with_packages_that_are_not_dependencies( + spec_str, expected_raise, expected_spec, tmp_path, mock_packages, config +): + """Tests that we cannot concretize two specs together, if one conflicts with the other, + even though they don't have a dependency relation. + """ + if spack.config.get("config:concretizer") == "original": + pytest.xfail("Known failure of the original concretizer") + + manifest = tmp_path / "spack.yaml" + manifest.write_text( + f"""\ +spack: + specs: + - {spec_str} + - b + concretizer: + unify: true +""" + ) + with ev.Environment(manifest.parent) as e: + if expected_raise: + with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): + e.concretize() + else: + e.concretize() + assert any(s.satisfies(expected_spec) for s in e.concrete_roots()) + + +@pytest.mark.regression("39455") +@pytest.mark.only_clingo("Known failure of the original concretizer") +@pytest.mark.parametrize( + "possible_mpi_spec,unify", [("mpich", False), ("mpich", True), ("zmpi", False), ("zmpi", True)] +) +def test_requires_on_virtual_and_potential_providers( + possible_mpi_spec, unify, tmp_path, mock_packages, config +): + """Tests that in an environment we can add packages explicitly, even though they provide + a virtual package, and we require the provider of the same virtual to be another package, + if they are added explicitly by their name. + """ + manifest = tmp_path / "spack.yaml" + manifest.write_text( + f"""\ + spack: + specs: + - {possible_mpi_spec} + - mpich2 + - mpileaks + packages: + mpi: + require: mpich2 + concretizer: + unify: {unify} + """ + ) + with ev.Environment(manifest.parent) as e: + e.concretize() + assert e.matching_spec(possible_mpi_spec) + assert e.matching_spec("mpich2") + + mpileaks = e.matching_spec("mpileaks") + assert mpileaks.satisfies("^mpich2") + assert mpileaks["mpi"].satisfies("mpich2") + assert not mpileaks.satisfies(f"^{possible_mpi_spec}") + + +@pytest.mark.regression("39387") +@pytest.mark.parametrize( + "spec_str", ["mpileaks +opt", "mpileaks +opt ~shared", "mpileaks ~shared +opt"] +) +def test_manifest_file_removal_works_if_spec_is_not_normalized(tmp_path, spec_str): + """Tests that we can remove a spec from a manifest file even if its string + representation is not normalized. + """ + manifest = tmp_path / "spack.yaml" + manifest.write_text( + f"""\ +spack: + specs: + - {spec_str} +""" + ) + s = spack.spec.Spec(spec_str) + spack_yaml = EnvironmentManifestFile(tmp_path) + # Doing a round trip str -> Spec -> str normalizes the representation + spack_yaml.remove_user_spec(str(s)) + spack_yaml.flush() + + assert spec_str not in manifest.read_text() + + +@pytest.mark.regression("39387") +@pytest.mark.parametrize( + "duplicate_specs,expected_number", + [ + # Swap variants, versions, etc. add spaces + (["foo +bar ~baz", "foo ~baz +bar"], 3), + (["foo @1.0 ~baz %gcc", "foo ~baz @1.0%gcc"], 3), + # Item 1 and 3 are exactly the same + (["zlib +shared", "zlib +shared", "zlib +shared"], 4), + ], +) +def test_removing_spec_from_manifest_with_exact_duplicates( + duplicate_specs, expected_number, tmp_path +): + """Tests that we can remove exact duplicates from a manifest file. + + Note that we can't get in a state with duplicates using only CLI, but this might happen + on user edited spack.yaml files. + """ + manifest = tmp_path / "spack.yaml" + manifest.write_text( + f"""\ + spack: + specs: [{", ".join(duplicate_specs)} , "zlib"] + """ + ) + + with ev.Environment(tmp_path) as env: + assert len(env.user_specs) == expected_number + env.remove(duplicate_specs[0]) + env.write() + + assert "+shared" not in manifest.read_text() + assert "zlib" in manifest.read_text() + with ev.Environment(tmp_path) as env: + assert len(env.user_specs) == 1 + + +@pytest.mark.regression("35298") +@pytest.mark.only_clingo("Propagation not supported in the original concretizer") +def test_variant_propagation_with_unify_false(tmp_path, mock_packages): + """Spack distributes concretizations to different processes, when unify:false is selected and + the number of roots is 2 or more. When that happens, the specs to be concretized need to be + properly reconstructed on the worker process, if variant propagation was requested. + """ + manifest = tmp_path / "spack.yaml" + manifest.write_text( + """ + spack: + specs: + - parent-foo ++foo + - c + concretizer: + unify: false + """ + ) + with ev.Environment(tmp_path) as env: + env.concretize() + + root = env.matching_spec("parent-foo") + for node in root.traverse(): + assert node.satisfies("+foo") + + +def test_env_with_include_defs(mutable_mock_env_path, mock_packages): + """Test environment with included definitions file.""" + env_path = mutable_mock_env_path + env_path.mkdir() + defs_file = env_path / "definitions.yaml" + defs_file.write_text( + """definitions: +- core_specs: [libdwarf, libelf] +- compilers: ['%gcc'] +""" + ) + + spack_yaml = env_path / ev.manifest_name + spack_yaml.write_text( + f"""spack: + include: + - file://{defs_file} + + definitions: + - my_packages: [zlib] + + specs: + - matrix: + - [$core_specs] + - [$compilers] + - $my_packages +""" + ) + + e = ev.Environment(env_path) + with e: + e.concretize() + + +def test_env_with_include_def_missing(mutable_mock_env_path, mock_packages): + """Test environment with included definitions file that is missing a definition.""" + env_path = mutable_mock_env_path + env_path.mkdir() + filename = "missing-def.yaml" + defs_file = env_path / filename + defs_file.write_text("definitions:\n- my_compilers: ['%gcc']\n") + + spack_yaml = env_path / ev.manifest_name + spack_yaml.write_text( + f"""spack: + include: + - file://{defs_file} + + specs: + - matrix: + - [$core_specs] + - [$my_compilers] +""" + ) + + e = ev.Environment(env_path) + with e: + with pytest.raises(UndefinedReferenceError, match=r"which does not appear"): + e.concretize() diff --git a/lib/spack/spack/test/environment_modifications.py b/lib/spack/spack/test/environment_modifications.py index 359a57a79df9e0..c2daac3cdb5b05 100644 --- a/lib/spack/spack/test/environment_modifications.py +++ b/lib/spack/spack/test/environment_modifications.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import sys import pytest @@ -159,7 +158,7 @@ def test_unset(env): os.environ["UNSET_ME"] -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_filter_system_paths(miscellaneous_paths): """Tests that the filtering of system paths works as expected.""" filtered = filter_system_paths(miscellaneous_paths) @@ -174,7 +173,7 @@ def test_filter_system_paths(miscellaneous_paths): # TODO 27021 -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_set_path(env): """Tests setting paths in an environment variable.""" @@ -190,7 +189,7 @@ def test_set_path(env): assert "foo;bar;baz" == os.environ["B"] -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_path_manipulation(env): """Tests manipulating list of paths in the environment.""" @@ -244,7 +243,7 @@ def test_extend(env): assert x is y -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.usefixtures("prepare_environment_for_tests") def test_source_files(files_to_be_sourced): """Tests the construction of a list of environment modifications that are @@ -311,7 +310,7 @@ def test_preserve_environment(prepare_environment_for_tests): assert os.environ["PATH_LIST"] == "/path/second:/path/third" -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.parametrize( "files,expected,deleted", [ @@ -486,7 +485,7 @@ def test_from_environment_diff(before, after, search_list): assert item in mod -@pytest.mark.skipif(sys.platform == "win32", reason="Lmod not supported on Windows") +@pytest.mark.not_on_windows("Lmod not supported on Windows") @pytest.mark.regression("15775") def test_exclude_lmod_variables(): # Construct the list of environment modifications @@ -498,7 +497,7 @@ def test_exclude_lmod_variables(): assert not any(x.startswith("LMOD_") for x in modifications) -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.regression("13504") def test_exclude_modules_variables(): # Construct the list of environment modifications diff --git a/lib/spack/spack/test/flag_handlers.py b/lib/spack/spack/test/flag_handlers.py index ae83b05885fa39..3e680c1ab7d037 100644 --- a/lib/spack/spack/test/flag_handlers.py +++ b/lib/spack/spack/test/flag_handlers.py @@ -121,7 +121,6 @@ def test_ld_flags_cmake(self, temp_env): "-DCMAKE_EXE_LINKER_FLAGS=-mthreads", "-DCMAKE_MODULE_LINKER_FLAGS=-mthreads", "-DCMAKE_SHARED_LINKER_FLAGS=-mthreads", - "-DCMAKE_STATIC_LINKER_FLAGS=-mthreads", } def test_ld_libs_cmake(self, temp_env): diff --git a/lib/spack/spack/test/gcs_fetch.py b/lib/spack/spack/test/gcs_fetch.py index 2122eeb36d70fe..76b99714714059 100644 --- a/lib/spack/spack/test/gcs_fetch.py +++ b/lib/spack/spack/test/gcs_fetch.py @@ -8,9 +8,9 @@ import pytest import spack.config +import spack.error import spack.fetch_strategy import spack.stage -from spack.util.web import FetchError @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) @@ -33,7 +33,7 @@ def test_gcsfetchstrategy_bad_url(tmpdir, _fetch_method): with spack.stage.Stage(fetcher, path=testpath) as stage: assert stage is not None assert fetcher.archive_file is None - with pytest.raises(FetchError): + with pytest.raises(spack.error.FetchError): fetcher.fetch() diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index a31648e2240179..e900628b7b983b 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -100,7 +100,7 @@ def test_fetch( t = mock_git_repository.checks[type_of_test] h = mock_git_repository.hash - pkg_class = spack.repo.path.get_pkg_class("git-test") + pkg_class = spack.repo.PATH.get_pkg_class("git-test") # This would fail using the default-no-per-version-git check but that # isn't included in this test monkeypatch.delattr(pkg_class, "git") @@ -147,7 +147,7 @@ def test_fetch_pkg_attr_submodule_init( """ t = mock_git_repository.checks["default-no-per-version-git"] - pkg_class = spack.repo.path.get_pkg_class("git-test") + pkg_class = spack.repo.PATH.get_pkg_class("git-test") # For this test, the version args don't specify 'git' (which is # the majority of version specifications) monkeypatch.setattr(pkg_class, "git", mock_git_repository.url) @@ -179,7 +179,7 @@ def test_adhoc_version_submodules( ): t = mock_git_repository.checks["tag"] # Construct the package under test - pkg_class = spack.repo.path.get_pkg_class("git-test") + pkg_class = spack.repo.PATH.get_pkg_class("git-test") monkeypatch.setitem(pkg_class.versions, Version("git"), t.args) monkeypatch.setattr(pkg_class, "git", "file://%s" % mock_git_repository.path, raising=False) diff --git a/lib/spack/spack/test/graph.py b/lib/spack/spack/test/graph.py index 3ab45df67f53b1..888d43d914bd5e 100644 --- a/lib/spack/spack/test/graph.py +++ b/lib/spack/spack/test/graph.py @@ -24,7 +24,7 @@ def test_static_graph_mpileaks(config, mock_packages): assert ' "libelf" [label="libelf"]\n' in dot assert ' "libdwarf" [label="libdwarf"]\n' in dot - mpi_providers = spack.repo.path.providers_for("mpi") + mpi_providers = spack.repo.PATH.providers_for("mpi") for spec in mpi_providers: assert ('"mpileaks" -> "%s"' % spec.name) in dot assert ('"callpath" -> "%s"' % spec.name) in dot diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 3939d460e9404e..48c57dbe6b56ba 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import sys import pytest @@ -22,7 +21,7 @@ # and expected to be fixed pytestmark = [ pytest.mark.skipif(not which("hg"), reason="requires mercurial to be installed"), - pytest.mark.skipif(sys.platform == "win32", reason="Failing on Win"), + pytest.mark.not_on_windows("Failing on Win"), ] diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 63dcd47f4ca316..467577cead8111 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -52,7 +52,7 @@ def test_uninstall_non_existing_package(install_mockery, mock_fetch, monkeypatch # Mock deletion of the package spec._package = None - monkeypatch.setattr(spack.repo.path, "get", find_nothing) + monkeypatch.setattr(spack.repo.PATH, "get", find_nothing) with pytest.raises(spack.repo.UnknownPackageError): spec.package @@ -159,7 +159,7 @@ def test_partial_install_delete_prefix_and_stage(install_mockery, mock_fetch, wo s.package.remove_prefix = rm_prefix_checker.remove_prefix # must clear failure markings for the package before re-installing it - spack.store.db.clear_failure(s, True) + spack.store.STORE.failure_tracker.clear(s, True) s.package.set_install_succeed() s.package.stage = MockStage(s.package.stage) @@ -288,17 +288,19 @@ def install_upstream(tmpdir_factory, gen_mock_layout, install_mockery): mock_db_root = str(tmpdir_factory.mktemp("mock_db_root")) prepared_db = spack.database.Database(mock_db_root) upstream_layout = gen_mock_layout("/a/") + spack.config.CONFIG.push_scope( + spack.config.InternalConfigScope( + name="install-upstream-fixture", + data={"upstreams": {"mock1": {"install_tree": prepared_db.root}}}, + ) + ) def _install_upstream(*specs): for spec_str in specs: s = spack.spec.Spec(spec_str).concretized() prepared_db.add(s, upstream_layout) - downstream_root = str(tmpdir_factory.mktemp("mock_downstream_db_root")) - db_for_test = spack.database.Database(downstream_root, upstream_dbs=[prepared_db]) - store = spack.store.Store(downstream_root) - store.db = db_for_test - return store, upstream_layout + return downstream_root, upstream_layout return _install_upstream @@ -307,8 +309,8 @@ def test_installed_upstream_external(install_upstream, mock_fetch): """Check that when a dependency package is recorded as installed in an upstream database that it is not reinstalled. """ - s, _ = install_upstream("externaltool") - with spack.store.use_store(s): + store_root, _ = install_upstream("externaltool") + with spack.store.use_store(store_root): dependent = spack.spec.Spec("externaltest") dependent.concretize() @@ -326,8 +328,8 @@ def test_installed_upstream(install_upstream, mock_fetch): """Check that when a dependency package is recorded as installed in an upstream database that it is not reinstalled. """ - s, upstream_layout = install_upstream("dependency-install") - with spack.store.use_store(s): + store_root, upstream_layout = install_upstream("dependency-install") + with spack.store.use_store(store_root): dependency = spack.spec.Spec("dependency-install").concretized() dependent = spack.spec.Spec("dependent-install").concretized() @@ -352,7 +354,7 @@ def test_partial_install_keep_prefix(install_mockery, mock_fetch, monkeypatch, w assert os.path.exists(s.package.prefix) # must clear failure markings for the package before re-installing it - spack.store.db.clear_failure(s, True) + spack.store.STORE.failure_tracker.clear(s, True) s.package.set_install_succeed() s.package.stage = MockStage(s.package.stage) @@ -379,9 +381,8 @@ def test_install_prefix_collision_fails(config, mock_fetch, mock_packages, tmpdi Test that different specs with coinciding install prefixes will fail to install. """ - projections = {"all": "all-specs-project-to-this-prefix"} - store = spack.store.Store(str(tmpdir), projections=projections) - with spack.store.use_store(store): + projections = {"projections": {"all": "all-specs-project-to-this-prefix"}} + with spack.store.use_store(str(tmpdir), extra_data=projections): with spack.config.override("config:checksum", False): pkg_a = Spec("libelf@0.8.13").concretized().package pkg_b = Spec("libelf@0.8.12").concretized().package @@ -615,7 +616,7 @@ def _install(src, dest): def test_unconcretized_install(install_mockery, mock_fetch, mock_packages): """Test attempts to perform install phases with unconcretized spec.""" spec = Spec("trivial-install-test-package") - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) with pytest.raises(ValueError, match="must have a concrete spec"): pkg_cls(spec).do_install() diff --git a/lib/spack/spack/test/installer.py b/lib/spack/spack/test/installer.py index 7b9c2b5184d365..d28d3f44c387bd 100644 --- a/lib/spack/spack/test/installer.py +++ b/lib/spack/spack/test/installer.py @@ -19,7 +19,10 @@ import spack.compilers import spack.concretize import spack.config +import spack.database +import spack.deptypes as dt import spack.installer as inst +import spack.package_base import spack.package_prefs as prefs import spack.repo import spack.spec @@ -50,7 +53,6 @@ def _mock_repo(root, namespace): def _noop(*args, **kwargs): """Generic monkeypatch no-op routine.""" - pass def _none(*args, **kwargs): @@ -148,15 +150,19 @@ def test_install_msg(monkeypatch): install_msg = "Installing {0}".format(name) monkeypatch.setattr(tty, "_debug", 0) - assert inst.install_msg(name, pid) == install_msg + assert inst.install_msg(name, pid, None) == install_msg + + install_status = inst.InstallStatus(1) + expected = "{0} [0/1]".format(install_msg) + assert inst.install_msg(name, pid, install_status) == expected monkeypatch.setattr(tty, "_debug", 1) - assert inst.install_msg(name, pid) == install_msg + assert inst.install_msg(name, pid, None) == install_msg # Expect the PID to be added at debug level 2 monkeypatch.setattr(tty, "_debug", 2) expected = "{0}: {1}".format(pid, install_msg) - assert inst.install_msg(name, pid) == expected + assert inst.install_msg(name, pid, None) == expected def test_install_from_cache_errors(install_mockery, capsys): @@ -359,7 +365,7 @@ def test_ensure_locked_err(install_mockery, monkeypatch, tmpdir, capsys): """Test _ensure_locked when a non-lock exception is raised.""" mock_err_msg = "Mock exception error" - def _raise(lock, timeout): + def _raise(lock, timeout=None): raise RuntimeError(mock_err_msg) const_arg = installer_args(["trivial-install-test-package"], {}) @@ -427,7 +433,7 @@ def test_ensure_locked_new_lock(install_mockery, tmpdir, lock_type, reads, write def test_ensure_locked_new_warn(install_mockery, monkeypatch, tmpdir, capsys): - orig_pl = spack.database.Database.prefix_lock + orig_pl = spack.database.SpecLocker.lock def _pl(db, spec, timeout): lock = orig_pl(db, spec, timeout) @@ -439,7 +445,7 @@ def _pl(db, spec, timeout): installer = create_installer(const_arg) spec = installer.build_requests[0].pkg.spec - monkeypatch.setattr(spack.database.Database, "prefix_lock", _pl) + monkeypatch.setattr(spack.database.SpecLocker, "lock", _pl) lock_type = "read" ltype, lock = installer._ensure_locked(lock_type, spec.package) @@ -452,7 +458,7 @@ def _pl(db, spec, timeout): def test_package_id_err(install_mockery): s = spack.spec.Spec("trivial-install-test-package") - pkg_cls = spack.repo.path.get_pkg_class(s.name) + pkg_cls = spack.repo.PATH.get_pkg_class(s.name) with pytest.raises(ValueError, match="spec is not concretized"): inst.package_id(pkg_cls(s)) @@ -553,7 +559,7 @@ def test_dump_packages_deps_ok(install_mockery, tmpdir, mock_packages): def test_dump_packages_deps_errs(install_mockery, tmpdir, monkeypatch, capsys): """Test error paths for dump_packages with dependencies.""" - orig_bpp = spack.store.layout.build_packages_path + orig_bpp = spack.store.STORE.layout.build_packages_path orig_dirname = spack.repo.Repo.dirname_for_package_name repo_err_msg = "Mock dirname_for_package_name" @@ -572,7 +578,7 @@ def _repoerr(repo, name): # Now mock the creation of the required directory structure to cover # the try-except block - monkeypatch.setattr(spack.store.layout, "build_packages_path", bpp_path) + monkeypatch.setattr(spack.store.STORE.layout, "build_packages_path", bpp_path) spec = spack.spec.Spec("simple-inheritance").concretized() path = str(tmpdir) @@ -592,58 +598,50 @@ def _repoerr(repo, name): assert "Couldn't copy in provenance for cmake" in out -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") -def test_clear_failures_success(install_mockery): +def test_clear_failures_success(tmpdir): """Test the clear_failures happy path.""" + failures = spack.database.FailureTracker(str(tmpdir), default_timeout=0.1) - # Set up a test prefix failure lock - lock = lk.Lock( - spack.store.db.prefix_fail_path, start=1, length=1, default_timeout=1e-9, desc="test" - ) - try: - lock.acquire_write() - except lk.LockTimeoutError: - tty.warn("Failed to write lock the test install failure") - spack.store.db._prefix_failures["test"] = lock + spec = spack.spec.Spec("a") + spec._mark_concrete() - # Set up a fake failure mark (or file) - fs.touch(os.path.join(spack.store.db._failure_dir, "test")) + # Set up a test prefix failure lock + failures.mark(spec) + assert failures.has_failed(spec) # Now clear failure tracking - inst.clear_failures() + failures.clear_all() # Ensure there are no cached failure locks or failure marks - assert len(spack.store.db._prefix_failures) == 0 - assert len(os.listdir(spack.store.db._failure_dir)) == 0 + assert len(failures.locker.locks) == 0 + assert len(os.listdir(failures.dir)) == 0 # Ensure the core directory and failure lock file still exist - assert os.path.isdir(spack.store.db._failure_dir) - assert os.path.isfile(spack.store.db.prefix_fail_path) + assert os.path.isdir(failures.dir) + # Locks on windows are a no-op + if sys.platform != "win32": + assert os.path.isfile(failures.locker.lock_path) -def test_clear_failures_errs(install_mockery, monkeypatch, capsys): - """Test the clear_failures exception paths.""" - orig_fn = os.remove - err_msg = "Mock os remove" - - def _raise_except(path): - raise OSError(err_msg) - # Set up a fake failure mark (or file) - fs.touch(os.path.join(spack.store.db._failure_dir, "test")) +@pytest.mark.xfail(sys.platform == "win32", reason="chmod does not prevent removal on Win") +def test_clear_failures_errs(tmpdir, capsys): + """Test the clear_failures exception paths.""" + failures = spack.database.FailureTracker(str(tmpdir), default_timeout=0.1) + spec = spack.spec.Spec("a") + spec._mark_concrete() + failures.mark(spec) - monkeypatch.setattr(os, "remove", _raise_except) + # Make the file marker not writeable, so that clearing_failures fails + failures.dir.chmod(0o000) # Clear failure tracking - inst.clear_failures() + failures.clear_all() # Ensure expected warning generated out = str(capsys.readouterr()[1]) assert "Unable to remove failure" in out - assert err_msg in out - - # Restore remove for teardown - monkeypatch.setattr(os, "remove", orig_fn) + failures.dir.chmod(0o750) def test_combine_phase_logs(tmpdir): @@ -688,14 +686,18 @@ def test_combine_phase_logs_does_not_care_about_encoding(tmpdir): assert f.read() == data * 2 -def test_check_deps_status_install_failure(install_mockery, monkeypatch): +def test_check_deps_status_install_failure(install_mockery): + """Tests that checking the dependency status on a request to install + 'a' fails, if we mark the dependency as failed. + """ + s = spack.spec.Spec("a").concretized() + for dep in s.traverse(root=False): + spack.store.STORE.failure_tracker.mark(dep) + const_arg = installer_args(["a"], {}) installer = create_installer(const_arg) request = installer.build_requests[0] - # Make sure the package is identified as failed - monkeypatch.setattr(spack.database.Database, "prefix_failed", _true) - with pytest.raises(inst.InstallError, match="install failure"): installer._check_deps_status(request) @@ -717,13 +719,12 @@ def test_check_deps_status_external(install_mockery, monkeypatch): installer = create_installer(const_arg) request = installer.build_requests[0] - # Mock the known dependent, b, as external so assumed to be installed + # Mock the dependencies as external so assumed to be installed monkeypatch.setattr(spack.spec.Spec, "external", True) installer._check_deps_status(request) - # exotic architectures will add dependencies on gnuconfig, which we want to ignore - installed = [x for x in installer.installed if not x.startswith("gnuconfig")] - assert installed[0].startswith("b") + for dep in request.spec.traverse(root=False): + assert inst.package_id(dep.package) in installer.installed def test_check_deps_status_upstream(install_mockery, monkeypatch): @@ -731,13 +732,12 @@ def test_check_deps_status_upstream(install_mockery, monkeypatch): installer = create_installer(const_arg) request = installer.build_requests[0] - # Mock the known dependent, b, as installed upstream + # Mock the known dependencies as installed upstream monkeypatch.setattr(spack.spec.Spec, "installed_upstream", True) installer._check_deps_status(request) - # exotic architectures will add dependencies on gnuconfig, which we want to ignore - installed = [x for x in installer.installed if not x.startswith("gnuconfig")] - assert installed[0].startswith("b") + for dep in request.spec.traverse(root=False): + assert inst.package_id(dep.package) in installer.installed def test_add_bootstrap_compilers(install_mockery, monkeypatch): @@ -795,7 +795,7 @@ def test_install_task_use_cache(install_mockery, monkeypatch): task = create_build_task(request.pkg) monkeypatch.setattr(inst, "_install_from_cache", _true) - installer._install_task(task) + installer._install_task(task, None) assert request.pkg_id in installer.installed @@ -817,7 +817,7 @@ def _add(_compilers): monkeypatch.setattr(spack.database.Database, "add", _noop) monkeypatch.setattr(spack.compilers, "add_compilers_to_config", _add) - installer._install_task(task) + installer._install_task(task, None) out = capfd.readouterr()[0] assert config_msg in out @@ -868,7 +868,7 @@ def test_requeue_task(install_mockery, capfd): # temporarily set tty debug messages on so we can test output current_debug_level = tty.debug_level() tty.set_debug(1) - installer._requeue_task(task) + installer._requeue_task(task, None) tty.set_debug(current_debug_level) ids = list(installer.build_tasks) @@ -927,7 +927,7 @@ def _chgrp(path, group, follow_symlinks=True): spec = installer.build_requests[0].pkg.spec fs.touchp(spec.prefix) - metadatadir = spack.store.layout.metadata_path(spec) + metadatadir = spack.store.STORE.layout.metadata_path(spec) # Regex matching with Windows style paths typically fails # so we skip the match check here if sys.platform == "win32": @@ -1000,7 +1000,7 @@ def test_install_failed(install_mockery, monkeypatch, capsys): installer = create_installer(const_arg) # Make sure the package is identified as failed - monkeypatch.setattr(spack.database.Database, "prefix_failed", _true) + monkeypatch.setattr(spack.database.FailureTracker, "has_failed", _true) with pytest.raises(inst.InstallError, match="request failed"): installer.install() @@ -1016,7 +1016,7 @@ def test_install_failed_not_fast(install_mockery, monkeypatch, capsys): installer = create_installer(const_arg) # Make sure the package is identified as failed - monkeypatch.setattr(spack.database.Database, "prefix_failed", _true) + monkeypatch.setattr(spack.database.FailureTracker, "has_failed", _true) with pytest.raises(inst.InstallError, match="request failed"): installer.install() @@ -1031,7 +1031,7 @@ def test_install_fail_on_interrupt(install_mockery, monkeypatch): spec_name = "a" err_msg = "mock keyboard interrupt for {0}".format(spec_name) - def _interrupt(installer, task, **kwargs): + def _interrupt(installer, task, install_status, **kwargs): if task.pkg.name == spec_name: raise KeyboardInterrupt(err_msg) else: @@ -1058,7 +1058,7 @@ def test_install_fail_single(install_mockery, monkeypatch): class MyBuildException(Exception): pass - def _install(installer, task, **kwargs): + def _install(installer, task, install_status, **kwargs): if task.pkg.name == spec_name: raise MyBuildException(err_msg) else: @@ -1085,7 +1085,7 @@ def test_install_fail_multi(install_mockery, monkeypatch): class MyBuildException(Exception): pass - def _install(installer, task, **kwargs): + def _install(installer, task, install_status, **kwargs): if task.pkg.name == spec_name: raise MyBuildException(err_msg) else: @@ -1115,7 +1115,7 @@ def test_install_fail_fast_on_detect(install_mockery, monkeypatch, capsys): # # This will prevent b from installing, which will cause the build of a # to be skipped. - monkeypatch.setattr(spack.database.Database, "prefix_failed", _true) + monkeypatch.setattr(spack.database.FailureTracker, "has_failed", _true) with pytest.raises(inst.InstallError, match="after first install failure"): installer.install() @@ -1134,6 +1134,7 @@ def _test_install_fail_fast_on_except_patch(installer, **kwargs): raise RuntimeError("mock patch failure") +@pytest.mark.disable_clean_stage_check def test_install_fail_fast_on_except(install_mockery, monkeypatch, capsys): """Test fail_fast install when an install failure results from an error.""" const_arg = installer_args(["a"], {"fail_fast": True}) @@ -1157,7 +1158,7 @@ def test_install_fail_fast_on_except(install_mockery, monkeypatch, capsys): def test_install_lock_failures(install_mockery, monkeypatch, capfd): """Cover basic install lock failure handling in a single pass.""" - def _requeued(installer, task): + def _requeued(installer, task, install_status): tty.msg("requeued {0}".format(task.pkg.spec.name)) const_arg = installer_args(["b"], {}) @@ -1192,7 +1193,7 @@ def _prep(installer, task): # also do not allow the package to be locked again monkeypatch.setattr(inst.PackageInstaller, "_ensure_locked", _not_locked) - def _requeued(installer, task): + def _requeued(installer, task, install_status): tty.msg("requeued {0}".format(inst.package_id(task.pkg))) # Flag the package as installed @@ -1224,7 +1225,7 @@ def _prep(installer, task): tty.msg("preparing {0}".format(task.pkg.spec.name)) assert task.pkg.spec.name not in installer.installed - def _requeued(installer, task): + def _requeued(installer, task, install_status): tty.msg("requeued {0}".format(task.pkg.spec.name)) # Force a read lock @@ -1289,7 +1290,7 @@ def test_overwrite_install_backup_success(temporary_store, config, mock_packages fs.touchp(installed_file) class InstallerThatWipesThePrefixDir: - def _install_task(self, task): + def _install_task(self, task, install_status): shutil.rmtree(task.pkg.prefix, ignore_errors=True) fs.mkdirp(task.pkg.prefix) raise Exception("Some fatal install error") @@ -1302,7 +1303,7 @@ def remove(self, spec): fake_installer = InstallerThatWipesThePrefixDir() fake_db = FakeDatabase() - overwrite_install = inst.OverwriteInstall(fake_installer, fake_db, task) + overwrite_install = inst.OverwriteInstall(fake_installer, fake_db, task, None) # Installation should throw the installation exception, not the backup # failure. @@ -1323,7 +1324,7 @@ def test_overwrite_install_backup_failure(temporary_store, config, mock_packages """ class InstallerThatAccidentallyDeletesTheBackupDir: - def _install_task(self, task): + def _install_task(self, task, install_status): # Remove the backup directory, which is at the same level as the prefix, # starting with .backup backup_glob = os.path.join( @@ -1351,7 +1352,7 @@ def remove(self, spec): fake_installer = InstallerThatAccidentallyDeletesTheBackupDir() fake_db = FakeDatabase() - overwrite_install = inst.OverwriteInstall(fake_installer, fake_db, task) + overwrite_install = inst.OverwriteInstall(fake_installer, fake_db, task, None) # Installation should throw the installation exception, not the backup # failure. @@ -1383,7 +1384,27 @@ def test_single_external_implicit_install(install_mockery, explicit_args, is_exp s = spack.spec.Spec(pkg).concretized() s.external_path = "/usr" create_installer([(s, explicit_args)]).install() - assert spack.store.db.get_record(pkg).explicit == is_explicit + assert spack.store.STORE.db.get_record(pkg).explicit == is_explicit + + +def test_overwrite_install_does_install_build_deps(install_mockery, mock_fetch): + """When overwrite installing something from sources, build deps should be installed.""" + s = spack.spec.Spec("dtrun3").concretized() + create_installer([(s, {})]).install() + + # Verify there is a pure build dep + edge = s.edges_to_dependencies(name="dtbuild3").pop() + assert edge.depflag == dt.BUILD + build_dep = edge.spec + + # Uninstall the build dep + build_dep.package.do_uninstall() + + # Overwrite install the root dtrun3 + create_installer([(s, {"overwrite": [s.dag_hash()]})]).install() + + # Verify that the build dep was also installed. + assert build_dep.installed @pytest.mark.parametrize("run_tests", [True, False]) diff --git a/lib/spack/spack/test/llnl/llnl_string.py b/lib/spack/spack/test/llnl/llnl_string.py new file mode 100644 index 00000000000000..93d7e662c57bea --- /dev/null +++ b/lib/spack/spack/test/llnl/llnl_string.py @@ -0,0 +1,43 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest + +import llnl.string + + +@pytest.mark.parametrize( + "arguments,expected", + [ + ((0, "thing"), "0 things"), + ((1, "thing"), "1 thing"), + ((2, "thing"), "2 things"), + ((1, "thing", "wombats"), "1 thing"), + ((2, "thing", "wombats"), "2 wombats"), + ((2, "thing", "wombats", False), "wombats"), + ], +) +def test_plural(arguments, expected): + assert llnl.string.plural(*arguments) == expected + + +@pytest.mark.parametrize( + "arguments,expected", + [((["one", "two"],), ["'one'", "'two'"]), ((["one", "two"], "^"), ["^one^", "^two^"])], +) +def test_quote(arguments, expected): + assert llnl.string.quote(*arguments) == expected + + +@pytest.mark.parametrize( + "input,expected_and,expected_or", + [ + (["foo"], "foo", "foo"), + (["foo", "bar"], "foo and bar", "foo or bar"), + (["foo", "bar", "baz"], "foo, bar, and baz", "foo, bar, or baz"), + ], +) +def test_comma_and_or(input, expected_and, expected_or): + assert llnl.string.comma_and(input) == expected_and + assert llnl.string.comma_or(input) == expected_or diff --git a/lib/spack/spack/test/llnl/url.py b/lib/spack/spack/test/llnl/url.py new file mode 100644 index 00000000000000..8da8e727ec5587 --- /dev/null +++ b/lib/spack/spack/test/llnl/url.py @@ -0,0 +1,167 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Tests for llnl.url functions""" +import itertools + +import pytest + +import llnl.url + + +@pytest.fixture(params=llnl.url.ALLOWED_ARCHIVE_TYPES) +def archive_and_expected(request): + archive_name = ".".join(["Foo", request.param]) + return archive_name, request.param + + +def test_get_extension(archive_and_expected): + """Tests that we can predict correctly known extensions for simple cases.""" + archive, expected = archive_and_expected + result = llnl.url.extension_from_path(archive) + assert result == expected + + +def test_get_bad_extension(): + """Tests that a bad extension returns None""" + result = llnl.url.extension_from_path("Foo.cxx") + assert result is None + + +@pytest.mark.parametrize( + "url,expected", + [ + # No suffix + ("rgb-1.0.6", "rgb-1.0.6"), + # Misleading prefix + ("jpegsrc.v9b", "jpegsrc.v9b"), + ("turbolinux702", "turbolinux702"), + ("converge_install_2.3.16", "converge_install_2.3.16"), + # Download type - code, source + ("cistem-1.0.0-beta-source-code", "cistem-1.0.0-beta"), + # Download type - src + ("apache-ant-1.9.7-src", "apache-ant-1.9.7"), + ("go1.7.4.src", "go1.7.4"), + # Download type - source + ("bowtie2-2.2.5-source", "bowtie2-2.2.5"), + ("grib_api-1.17.0-Source", "grib_api-1.17.0"), + # Download type - full + ("julia-0.4.3-full", "julia-0.4.3"), + # Download type - bin + ("apache-maven-3.3.9-bin", "apache-maven-3.3.9"), + # Download type - binary + ("Jmol-14.8.0-binary", "Jmol-14.8.0"), + # Download type - gem + ("rubysl-date-2.0.9.gem", "rubysl-date-2.0.9"), + # Download type - tar + ("gromacs-4.6.1-tar", "gromacs-4.6.1"), + # Download type - sh + ("Miniconda2-4.3.11-Linux-x86_64.sh", "Miniconda2-4.3.11"), + # Download version - release + ("v1.0.4-release", "v1.0.4"), + # Download version - stable + ("libevent-2.0.21-stable", "libevent-2.0.21"), + # Download version - final + ("2.6.7-final", "2.6.7"), + # Download version - rel + ("v1.9.5.1rel", "v1.9.5.1"), + # Download version - orig + ("dash_0.5.5.1.orig", "dash_0.5.5.1"), + # Download version - plus + ("ncbi-blast-2.6.0+-src", "ncbi-blast-2.6.0"), + # License + ("cppad-20170114.gpl", "cppad-20170114"), + # Arch + ("pcraster-4.1.0_x86-64", "pcraster-4.1.0"), + ("dislin-11.0.linux.i586_64", "dislin-11.0"), + ("PAGIT.V1.01.64bit", "PAGIT.V1.01"), + # OS - linux + ("astyle_2.04_linux", "astyle_2.04"), + # OS - unix + ("install-tl-unx", "install-tl"), + # OS - macos + ("astyle_1.23_macosx", "astyle_1.23"), + ("haxe-2.08-osx", "haxe-2.08"), + # PyPI - wheel + ("entrypoints-0.2.2-py2.py3-none-any.whl", "entrypoints-0.2.2"), + ( + "numpy-1.12.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel." + "macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", + "numpy-1.12.0", + ), + # PyPI - exe + ("PyYAML-3.12.win-amd64-py3.5.exe", "PyYAML-3.12"), + # Combinations of multiple patterns - bin, release + ("rocketmq-all-4.5.2-bin-release", "rocketmq-all-4.5.2"), + # Combinations of multiple patterns - all + ("p7zip_9.04_src_all", "p7zip_9.04"), + # Combinations of multiple patterns - run + ("cuda_8.0.44_linux.run", "cuda_8.0.44"), + # Combinations of multiple patterns - file + ("ack-2.14-single-file", "ack-2.14"), + # Combinations of multiple patterns - jar + ("antlr-3.4-complete.jar", "antlr-3.4"), + # Combinations of multiple patterns - oss + ("tbb44_20160128oss_src_0", "tbb44_20160128"), + # Combinations of multiple patterns - darwin + ("ghc-7.0.4-x86_64-apple-darwin", "ghc-7.0.4"), + ("ghc-7.0.4-i386-apple-darwin", "ghc-7.0.4"), + # Combinations of multiple patterns - centos + ("sratoolkit.2.8.2-1-centos_linux64", "sratoolkit.2.8.2-1"), + # Combinations of multiple patterns - arch + ( + "VizGlow_v2.2alpha17-R21November2016-Linux-x86_64-Install", + "VizGlow_v2.2alpha17-R21November2016", + ), + ("jdk-8u92-linux-x64", "jdk-8u92"), + ("cuda_6.5.14_linux_64.run", "cuda_6.5.14"), + ("Mathematica_12.0.0_LINUX.sh", "Mathematica_12.0.0"), + ("trf407b.linux64", "trf407b"), + # Combinations of multiple patterns - with + ("mafft-7.221-with-extensions-src", "mafft-7.221"), + ("spark-2.0.0-bin-without-hadoop", "spark-2.0.0"), + ("conduit-v0.3.0-src-with-blt", "conduit-v0.3.0"), + # Combinations of multiple patterns - rock + ("bitlib-23-2.src.rock", "bitlib-23-2"), + # Combinations of multiple patterns - public + ("dakota-6.3-public.src", "dakota-6.3"), + # Combinations of multiple patterns - universal + ("synergy-1.3.6p2-MacOSX-Universal", "synergy-1.3.6p2"), + # Combinations of multiple patterns - dynamic + ("snptest_v2.5.2_linux_x86_64_dynamic", "snptest_v2.5.2"), + # Combinations of multiple patterns - other + ("alglib-3.11.0.cpp.gpl", "alglib-3.11.0"), + ("hpcviewer-2019.08-linux.gtk.x86_64", "hpcviewer-2019.08"), + ("apache-mxnet-src-1.3.0-incubating", "apache-mxnet-src-1.3.0"), + ], +) +def test_url_strip_version_suffixes(url, expected): + stripped = llnl.url.strip_version_suffixes(url) + assert stripped == expected + + +def test_strip_compression_extension(archive_and_expected): + archive, extension = archive_and_expected + stripped = llnl.url.strip_compression_extension(archive) + if extension == "zip": + assert stripped == "Foo.zip" + stripped = llnl.url.strip_compression_extension(archive, "zip") + assert stripped == "Foo" + elif ( + extension.lower() == "tar" + or extension in llnl.url.CONTRACTION_MAP + or extension + in [ + ".".join(ext) + for ext in itertools.product(llnl.url.PREFIX_EXTENSIONS, llnl.url.EXTENSIONS) + ] + ): + assert stripped == "Foo.tar" or stripped == "Foo.TAR" + else: + assert stripped == "Foo" + + +def test_allowed_archive(archive_and_expected): + archive, _ = archive_and_expected + assert llnl.url.allowed_archive(archive) diff --git a/lib/spack/spack/test/llnl/util/argparsewriter.py b/lib/spack/spack/test/llnl/util/argparsewriter.py index a2455e03033870..433833c6a21147 100644 --- a/lib/spack/spack/test/llnl/util/argparsewriter.py +++ b/lib/spack/spack/test/llnl/util/argparsewriter.py @@ -22,13 +22,3 @@ def test_format_not_overridden(): with pytest.raises(TypeError): aw.ArgparseWriter("spack") - - -def test_completion_format_not_overridden(): - writer = aw.ArgparseCompletionWriter("spack") - - assert writer.positionals([]) == "" - assert writer.optionals([]) == "" - assert writer.subcommands([]) == "" - - writer.write(parser) diff --git a/lib/spack/spack/test/llnl/util/file_list.py b/lib/spack/spack/test/llnl/util/file_list.py index b1574db713c1ca..425ce641bd135d 100644 --- a/lib/spack/spack/test/llnl/util/file_list.py +++ b/lib/spack/spack/test/llnl/util/file_list.py @@ -134,7 +134,7 @@ def test_get_item(self, library_list): assert a == "/dir1/liblapack.%s" % plat_static_ext b = library_list[:] - assert type(b) == type(library_list) + assert type(b) is type(library_list) assert library_list == b assert library_list is not b @@ -152,8 +152,8 @@ def test_add(self, library_list): assert both == both + both # Always produce an instance of LibraryList - assert type(library_list + pylist) == type(library_list) - assert type(pylist + library_list) == type(library_list) + assert type(library_list + pylist) is type(library_list) + assert type(pylist + library_list) is type(library_list) class TestHeaderList: @@ -219,7 +219,7 @@ def test_get_item(self, header_list): assert a == "/dir1/Python.h" b = header_list[:] - assert type(b) == type(header_list) + assert type(b) is type(header_list) assert header_list == b assert header_list is not b @@ -237,8 +237,8 @@ def test_add(self, header_list): assert h == h + h # Always produce an instance of HeaderList - assert type(header_list + pylist) == type(header_list) - assert type(pylist + header_list) == type(header_list) + assert type(header_list + pylist) is type(header_list) + assert type(pylist + header_list) is type(header_list) #: Directory where the data for the test below is stored diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py index 9e22f96c657d89..eae5c732c6b302 100644 --- a/lib/spack/spack/test/llnl/util/filesystem.py +++ b/lib/spack/spack/test/llnl/util/filesystem.py @@ -13,7 +13,8 @@ import pytest import llnl.util.filesystem as fs -from llnl.util.symlink import islink, symlink +import llnl.util.symlink +from llnl.util.symlink import SymlinkError, _windows_can_symlink, islink, symlink import spack.paths @@ -150,7 +151,6 @@ def test_multiple_src_file_dest(self, stage): fs.install("source/a/*/*", "dest/1") -@pytest.mark.skipif(sys.platform == "win32", reason="Skip test on Windows") class TestCopyTree: """Tests for ``filesystem.copy_tree``""" @@ -189,7 +189,7 @@ def test_symlinks_true(self, stage): def test_symlinks_true_ignore(self, stage): """Test copying when specifying relative paths that should be ignored""" with fs.working_dir(str(stage)): - ignore = lambda p: p in ["c/d/e", "a"] + ignore = lambda p: p in [os.path.join("c", "d", "e"), "a"] fs.copy_tree("source", "dest", symlinks=True, ignore=ignore) assert not os.path.exists("dest/a") assert os.path.exists("dest/c/d") @@ -231,7 +231,6 @@ def test_parent_dir(self, stage): fs.copy_tree("source", "source/sub/directory") -@pytest.mark.skipif(sys.platform == "win32", reason="Skip test on Windows") class TestInstallTree: """Tests for ``filesystem.install_tree``""" @@ -275,6 +274,15 @@ def test_symlinks_false(self, stage): assert not os.path.islink("dest/2") check_added_exe_permissions("source/2", "dest/2") + @pytest.mark.skipif(sys.platform == "win32", reason="Broken symlinks not allowed on Windows") + def test_allow_broken_symlinks(self, stage): + """Test installing with a broken symlink.""" + with fs.working_dir(str(stage)): + symlink("nonexistant.txt", "source/broken", allow_broken_symlinks=True) + fs.install_tree("source", "dest", symlinks=True, allow_broken_symlinks=True) + assert os.path.islink("dest/broken") + assert not os.path.exists(os.readlink("dest/broken")) + def test_glob_src(self, stage): """Test using a glob as the source.""" @@ -502,7 +510,7 @@ def test_filter_files_with_different_encodings(regex, replacement, filename, tmp assert replacement in f.read() -@pytest.mark.skipif(sys.platform == "win32", reason="chgrp isn't used on Windows") +@pytest.mark.not_on_windows("chgrp isn't used on Windows") def test_chgrp_dont_set_group_if_already_set(tmpdir, monkeypatch): with fs.working_dir(tmpdir): os.mkdir("test-dir_chgrp_dont_set_group_if_already_set") @@ -722,7 +730,7 @@ def test_temporary_dir_context_manager(): assert os.path.realpath(str(tmp_dir)) == os.path.realpath(os.getcwd()) -@pytest.mark.skipif(sys.platform == "win32", reason="No shebang on Windows") +@pytest.mark.not_on_windows("No shebang on Windows") def test_is_nonsymlink_exe_with_shebang(tmpdir): with tmpdir.as_cwd(): # Create an executable with shebang. @@ -746,6 +754,7 @@ def test_is_nonsymlink_exe_with_shebang(tmpdir): assert not fs.is_nonsymlink_exe_with_shebang("symlink_to_executable_script") +@pytest.mark.skipif(sys.platform == "win32", reason="Unix-only test.") def test_lexists_islink_isdir(tmpdir): root = str(tmpdir) @@ -764,12 +773,12 @@ def test_lexists_islink_isdir(tmpdir): with open(file, "wb") as f: f.write(b"file") - os.symlink("dir", symlink_to_dir) - os.symlink("file", symlink_to_file) - os.symlink("does_not_exist", dangling_symlink) - os.symlink("dangling_symlink", symlink_to_dangling_symlink) - os.symlink("symlink_to_dir", symlink_to_symlink_to_dir) - os.symlink("symlink_to_file", symlink_to_symlink_to_file) + symlink("dir", symlink_to_dir) + symlink("file", symlink_to_file) + symlink("does_not_exist", dangling_symlink) + symlink("dangling_symlink", symlink_to_dangling_symlink) + symlink("symlink_to_dir", symlink_to_symlink_to_dir) + symlink("symlink_to_file", symlink_to_symlink_to_file) assert fs.lexists_islink_isdir(dir) == (True, False, True) assert fs.lexists_islink_isdir(file) == (True, False, False) @@ -781,6 +790,57 @@ def test_lexists_islink_isdir(tmpdir): assert fs.lexists_islink_isdir(symlink_to_symlink_to_file) == (True, True, False) +@pytest.mark.skipif(sys.platform != "win32", reason="For Windows Only") +@pytest.mark.parametrize("win_can_symlink", [True, False]) +def test_lexists_islink_isdir_windows(tmpdir, monkeypatch, win_can_symlink): + """Run on windows without elevated privileges to test junctions and hard links which have + different results from the lexists_islink_isdir method. + """ + if win_can_symlink and not _windows_can_symlink(): + pytest.skip("Cannot test dev mode behavior without dev mode enabled.") + with tmpdir.as_cwd(): + monkeypatch.setattr(llnl.util.symlink, "_windows_can_symlink", lambda: win_can_symlink) + dir = str(tmpdir.join("dir")) + file = str(tmpdir.join("file")) + nonexistent = str(tmpdir.join("does_not_exist")) + symlink_to_dir = str(tmpdir.join("symlink_to_dir")) + symlink_to_file = str(tmpdir.join("symlink_to_file")) + dangling_symlink = str(tmpdir.join("dangling_symlink")) + symlink_to_dangling_symlink = str(tmpdir.join("symlink_to_dangling_symlink")) + symlink_to_symlink_to_dir = str(tmpdir.join("symlink_to_symlink_to_dir")) + symlink_to_symlink_to_file = str(tmpdir.join("symlink_to_symlink_to_file")) + + os.mkdir(dir) + assert fs.lexists_islink_isdir(dir) == (True, False, True) + + symlink("dir", symlink_to_dir) + assert fs.lexists_islink_isdir(dir) == (True, False, True) + assert fs.lexists_islink_isdir(symlink_to_dir) == (True, True, True) + + with open(file, "wb") as f: + f.write(b"file") + assert fs.lexists_islink_isdir(file) == (True, False, False) + + symlink("file", symlink_to_file) + if win_can_symlink: + assert fs.lexists_islink_isdir(file) == (True, False, False) + else: + assert fs.lexists_islink_isdir(file) == (True, True, False) + assert fs.lexists_islink_isdir(symlink_to_file) == (True, True, False) + + with pytest.raises(SymlinkError): + symlink("does_not_exist", dangling_symlink) + symlink("dangling_symlink", symlink_to_dangling_symlink) + + symlink("symlink_to_dir", symlink_to_symlink_to_dir) + symlink("symlink_to_file", symlink_to_symlink_to_file) + + assert fs.lexists_islink_isdir(nonexistent) == (False, False, False) + assert fs.lexists_islink_isdir(symlink_to_dangling_symlink) == (False, False, False) + assert fs.lexists_islink_isdir(symlink_to_symlink_to_dir) == (True, True, True) + assert fs.lexists_islink_isdir(symlink_to_symlink_to_file) == (True, True, False) + + class RegisterVisitor(fs.BaseDirectoryVisitor): """A directory visitor that keeps track of all visited paths""" @@ -825,7 +885,7 @@ def after_visit_symlinked_dir(self, root, rel_path, depth): self.symlinked_dirs_after.append(rel_path) -@pytest.mark.skipif(sys.platform == "win32", reason="Requires symlinks") +@pytest.mark.not_on_windows("Requires symlinks") def test_visit_directory_tree_follow_all(noncyclical_dir_structure): root = str(noncyclical_dir_structure) visitor = RegisterVisitor(root, follow_dirs=True, follow_symlink_dirs=True) @@ -850,7 +910,7 @@ def test_visit_directory_tree_follow_all(noncyclical_dir_structure): assert visitor.symlinked_dirs_after == [j("a", "to_c"), j("b", "to_c"), j("b")] -@pytest.mark.skipif(sys.platform == "win32", reason="Requires symlinks") +@pytest.mark.not_on_windows("Requires symlinks") def test_visit_directory_tree_follow_dirs(noncyclical_dir_structure): root = str(noncyclical_dir_structure) visitor = RegisterVisitor(root, follow_dirs=True, follow_symlink_dirs=False) @@ -869,7 +929,7 @@ def test_visit_directory_tree_follow_dirs(noncyclical_dir_structure): assert not visitor.symlinked_dirs_after -@pytest.mark.skipif(sys.platform == "win32", reason="Requires symlinks") +@pytest.mark.not_on_windows("Requires symlinks") def test_visit_directory_tree_follow_none(noncyclical_dir_structure): root = str(noncyclical_dir_structure) visitor = RegisterVisitor(root, follow_dirs=False, follow_symlink_dirs=False) @@ -884,7 +944,7 @@ def test_visit_directory_tree_follow_none(noncyclical_dir_structure): @pytest.mark.regression("29687") @pytest.mark.parametrize("initial_mode", [stat.S_IRUSR | stat.S_IXUSR, stat.S_IWGRP]) -@pytest.mark.skipif(sys.platform == "win32", reason="Windows might change permissions") +@pytest.mark.not_on_windows("Windows might change permissions") def test_remove_linked_tree_doesnt_change_file_permission(tmpdir, initial_mode): # Here we test that a failed call to remove_linked_tree, due to passing a file # as an argument instead of a directory, doesn't leave the file with different diff --git a/lib/spack/spack/test/llnl/util/link_tree.py b/lib/spack/spack/test/llnl/util/link_tree.py index 3ae39bbb7924d4..9d154014b174c9 100644 --- a/lib/spack/spack/test/llnl/util/link_tree.py +++ b/lib/spack/spack/test/llnl/util/link_tree.py @@ -4,12 +4,14 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os +import sys import pytest +import llnl.util.symlink from llnl.util.filesystem import mkdirp, touchp, visit_directory_tree, working_dir from llnl.util.link_tree import DestinationMergeVisitor, LinkTree, SourceMergeVisitor -from llnl.util.symlink import islink +from llnl.util.symlink import _windows_can_symlink, islink, readlink, symlink from spack.stage import Stage @@ -44,77 +46,116 @@ def link_tree(stage): def check_file_link(filename, expected_target): assert os.path.isfile(filename) assert islink(filename) - assert os.path.abspath(os.path.realpath(filename)) == os.path.abspath(expected_target) + if sys.platform != "win32" or llnl.util.symlink._windows_can_symlink(): + assert os.path.abspath(os.path.realpath(filename)) == os.path.abspath(expected_target) def check_dir(filename): assert os.path.isdir(filename) -def test_merge_to_new_directory(stage, link_tree): +@pytest.mark.parametrize("run_as_root", [True, False]) +def test_merge_to_new_directory(stage, link_tree, monkeypatch, run_as_root): + if sys.platform != "win32": + if run_as_root: + pass + else: + pytest.skip("Skipping duplicate test.") + elif _windows_can_symlink() or not run_as_root: + monkeypatch.setattr(llnl.util.symlink, "_windows_can_symlink", lambda: run_as_root) + else: + # Skip if trying to run as dev-mode without having dev-mode. + pytest.skip("Skipping portion of test which required dev-mode privileges.") + with working_dir(stage.path): link_tree.merge("dest") - check_file_link("dest/1", "source/1") - check_file_link("dest/a/b/2", "source/a/b/2") - check_file_link("dest/a/b/3", "source/a/b/3") - check_file_link("dest/c/4", "source/c/4") - check_file_link("dest/c/d/5", "source/c/d/5") - check_file_link("dest/c/d/6", "source/c/d/6") - check_file_link("dest/c/d/e/7", "source/c/d/e/7") - - assert os.path.isabs(os.readlink("dest/1")) - assert os.path.isabs(os.readlink("dest/a/b/2")) - assert os.path.isabs(os.readlink("dest/a/b/3")) - assert os.path.isabs(os.readlink("dest/c/4")) - assert os.path.isabs(os.readlink("dest/c/d/5")) - assert os.path.isabs(os.readlink("dest/c/d/6")) - assert os.path.isabs(os.readlink("dest/c/d/e/7")) + files = [ + ("dest/1", "source/1"), + ("dest/a/b/2", "source/a/b/2"), + ("dest/a/b/3", "source/a/b/3"), + ("dest/c/4", "source/c/4"), + ("dest/c/d/5", "source/c/d/5"), + ("dest/c/d/6", "source/c/d/6"), + ("dest/c/d/e/7", "source/c/d/e/7"), + ] + + for dest, source in files: + check_file_link(dest, source) + assert os.path.isabs(readlink(dest)) link_tree.unmerge("dest") assert not os.path.exists("dest") -def test_merge_to_new_directory_relative(stage, link_tree): +@pytest.mark.parametrize("run_as_root", [True, False]) +def test_merge_to_new_directory_relative(stage, link_tree, monkeypatch, run_as_root): + if sys.platform != "win32": + if run_as_root: + pass + else: + pytest.skip("Skipping duplicate test.") + elif _windows_can_symlink() or not run_as_root: + monkeypatch.setattr(llnl.util.symlink, "_windows_can_symlink", lambda: run_as_root) + else: + # Skip if trying to run as dev-mode without having dev-mode. + pytest.skip("Skipping portion of test which required dev-mode privileges.") + with working_dir(stage.path): link_tree.merge("dest", relative=True) - check_file_link("dest/1", "source/1") - check_file_link("dest/a/b/2", "source/a/b/2") - check_file_link("dest/a/b/3", "source/a/b/3") - check_file_link("dest/c/4", "source/c/4") - check_file_link("dest/c/d/5", "source/c/d/5") - check_file_link("dest/c/d/6", "source/c/d/6") - check_file_link("dest/c/d/e/7", "source/c/d/e/7") - - assert not os.path.isabs(os.readlink("dest/1")) - assert not os.path.isabs(os.readlink("dest/a/b/2")) - assert not os.path.isabs(os.readlink("dest/a/b/3")) - assert not os.path.isabs(os.readlink("dest/c/4")) - assert not os.path.isabs(os.readlink("dest/c/d/5")) - assert not os.path.isabs(os.readlink("dest/c/d/6")) - assert not os.path.isabs(os.readlink("dest/c/d/e/7")) + files = [ + ("dest/1", "source/1"), + ("dest/a/b/2", "source/a/b/2"), + ("dest/a/b/3", "source/a/b/3"), + ("dest/c/4", "source/c/4"), + ("dest/c/d/5", "source/c/d/5"), + ("dest/c/d/6", "source/c/d/6"), + ("dest/c/d/e/7", "source/c/d/e/7"), + ] + + for dest, source in files: + check_file_link(dest, source) + # Hard links/junctions are inherently absolute. + if sys.platform != "win32" or run_as_root: + assert not os.path.isabs(readlink(dest)) link_tree.unmerge("dest") assert not os.path.exists("dest") -def test_merge_to_existing_directory(stage, link_tree): +@pytest.mark.parametrize("run_as_root", [True, False]) +def test_merge_to_existing_directory(stage, link_tree, monkeypatch, run_as_root): + if sys.platform != "win32": + if run_as_root: + pass + else: + pytest.skip("Skipping duplicate test.") + elif _windows_can_symlink() or not run_as_root: + monkeypatch.setattr(llnl.util.symlink, "_windows_can_symlink", lambda: run_as_root) + else: + # Skip if trying to run as dev-mode without having dev-mode. + pytest.skip("Skipping portion of test which required dev-mode privileges.") + with working_dir(stage.path): touchp("dest/x") touchp("dest/a/b/y") link_tree.merge("dest") - check_file_link("dest/1", "source/1") - check_file_link("dest/a/b/2", "source/a/b/2") - check_file_link("dest/a/b/3", "source/a/b/3") - check_file_link("dest/c/4", "source/c/4") - check_file_link("dest/c/d/5", "source/c/d/5") - check_file_link("dest/c/d/6", "source/c/d/6") - check_file_link("dest/c/d/e/7", "source/c/d/e/7") + files = [ + ("dest/1", "source/1"), + ("dest/a/b/2", "source/a/b/2"), + ("dest/a/b/3", "source/a/b/3"), + ("dest/c/4", "source/c/4"), + ("dest/c/d/5", "source/c/d/5"), + ("dest/c/d/6", "source/c/d/6"), + ("dest/c/d/e/7", "source/c/d/e/7"), + ] + for dest, source in files: + check_file_link(dest, source) assert os.path.isfile("dest/x") assert os.path.isfile("dest/a/b/y") @@ -124,13 +165,8 @@ def test_merge_to_existing_directory(stage, link_tree): assert os.path.isfile("dest/x") assert os.path.isfile("dest/a/b/y") - assert not os.path.isfile("dest/1") - assert not os.path.isfile("dest/a/b/2") - assert not os.path.isfile("dest/a/b/3") - assert not os.path.isfile("dest/c/4") - assert not os.path.isfile("dest/c/d/5") - assert not os.path.isfile("dest/c/d/6") - assert not os.path.isfile("dest/c/d/e/7") + for dest, _ in files: + assert not os.path.isfile(dest) def test_merge_with_empty_directories(stage, link_tree): @@ -192,9 +228,9 @@ def test_source_merge_visitor_does_not_follow_symlinked_dirs_at_depth(tmpdir): os.mkdir(j("a", "b")) os.mkdir(j("a", "b", "c")) os.mkdir(j("a", "b", "c", "d")) - os.symlink(j("b"), j("a", "symlink_b")) - os.symlink(j("c"), j("a", "b", "symlink_c")) - os.symlink(j("d"), j("a", "b", "c", "symlink_d")) + symlink(j("b"), j("a", "symlink_b")) + symlink(j("c"), j("a", "b", "symlink_c")) + symlink(j("d"), j("a", "b", "c", "symlink_d")) with open(j("a", "b", "c", "d", "file"), "wb"): pass @@ -236,10 +272,11 @@ def test_source_merge_visitor_cant_be_cyclical(tmpdir): j = os.path.join with tmpdir.as_cwd(): os.mkdir(j("a")) - os.symlink(j("..", "b"), j("a", "symlink_b")) - os.symlink(j("symlink_b"), j("a", "symlink_b_b")) os.mkdir(j("b")) - os.symlink(j("..", "a"), j("b", "symlink_a")) + + symlink(j("..", "b"), j("a", "symlink_b")) + symlink(j("symlink_b"), j("a", "symlink_b_b")) + symlink(j("..", "a"), j("b", "symlink_a")) visitor = SourceMergeVisitor() visit_directory_tree(str(tmpdir), visitor) diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py index 15129aa9b77dc9..9e7f3a3bde31b5 100644 --- a/lib/spack/spack/test/llnl/util/lock.py +++ b/lib/spack/spack/test/llnl/util/lock.py @@ -65,7 +65,7 @@ if sys.platform != "win32": import fcntl -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") # @@ -307,7 +307,7 @@ def __name__(self): return self.__class__.__name__ def __call__(self, barrier): - lock = lk.Lock(self.lock_path, self.start, self.length) + lock = lk.Lock(self.lock_path, start=self.start, length=self.length) lock.acquire_write() # grab exclusive lock barrier.wait() barrier.wait() # hold the lock until timeout in other procs. @@ -324,7 +324,7 @@ def __name__(self): return self.__class__.__name__ def __call__(self, barrier): - lock = lk.Lock(self.lock_path, self.start, self.length) + lock = lk.Lock(self.lock_path, start=self.start, length=self.length) lock.acquire_read() # grab shared lock barrier.wait() barrier.wait() # hold the lock until timeout in other procs. @@ -341,7 +341,7 @@ def __name__(self): return self.__class__.__name__ def __call__(self, barrier): - lock = lk.Lock(self.lock_path, self.start, self.length) + lock = lk.Lock(self.lock_path, start=self.start, length=self.length) barrier.wait() # wait for lock acquire in first process with pytest.raises(lk.LockTimeoutError): lock.acquire_write(lock_fail_timeout) @@ -359,7 +359,7 @@ def __name__(self): return self.__class__.__name__ def __call__(self, barrier): - lock = lk.Lock(self.lock_path, self.start, self.length) + lock = lk.Lock(self.lock_path, start=self.start, length=self.length) barrier.wait() # wait for lock acquire in first process with pytest.raises(lk.LockTimeoutError): lock.acquire_read(lock_fail_timeout) @@ -687,8 +687,8 @@ def test_upgrade_read_to_write_fails_with_readonly_file(private_lock_path): with pytest.raises(lk.LockROFileError): lock.acquire_write() - # TODO: lk.file_tracker does not release private_lock_path - lk.file_tracker.release_by_stat(os.stat(private_lock_path)) + # TODO: lk.FILE_TRACKER does not release private_lock_path + lk.FILE_TRACKER.release_by_stat(os.stat(private_lock_path)) class ComplexAcquireAndRelease: @@ -1345,8 +1345,7 @@ def _lockf(fd, cmd, len, start, whence): with tmpdir.as_cwd(): lockfile = "lockfile" lock = lk.Lock(lockfile) - - touch(lockfile) + lock.acquire_read() monkeypatch.setattr(fcntl, "lockf", _lockf) @@ -1356,6 +1355,9 @@ def _lockf(fd, cmd, len, start, whence): with pytest.raises(IOError, match=err_msg): lock._poll_lock(fcntl.LOCK_EX) + monkeypatch.undo() + lock.release_read() + def test_upgrade_read_okay(tmpdir): """Test the lock read-to-write upgrade operation.""" diff --git a/lib/spack/spack/test/llnl/util/symlink.py b/lib/spack/spack/test/llnl/util/symlink.py new file mode 100644 index 00000000000000..bbc69473f86648 --- /dev/null +++ b/lib/spack/spack/test/llnl/util/symlink.py @@ -0,0 +1,247 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Tests for ``llnl/util/symlink.py``""" +import os +import sys +import tempfile + +import pytest + +from llnl.util import symlink + + +def test_symlink_file(tmpdir): + """Test the symlink.symlink functionality on all operating systems for a file""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) + link_file = str(tmpdir.join("link.txt")) + assert os.path.exists(link_file) is False + symlink.symlink(source_path=real_file, link_path=link_file) + assert os.path.exists(link_file) + assert symlink.islink(link_file) + + +def test_symlink_dir(tmpdir): + """Test the symlink.symlink functionality on all operating systems for a directory""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + real_dir = os.path.join(test_dir, "real_dir") + link_dir = os.path.join(test_dir, "link_dir") + os.mkdir(real_dir) + symlink.symlink(source_path=real_dir, link_path=link_dir) + assert os.path.exists(link_dir) + assert symlink.islink(link_dir) + + +def test_symlink_source_not_exists(tmpdir): + """Test the symlink.symlink method for the case where a source path does not exist""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + real_dir = os.path.join(test_dir, "real_dir") + link_dir = os.path.join(test_dir, "link_dir") + with pytest.raises(symlink.SymlinkError): + symlink.symlink(source_path=real_dir, link_path=link_dir, allow_broken_symlinks=False) + + +def test_symlink_src_relative_to_link(tmpdir): + """Test the symlink.symlink functionality where the source value exists relative to the link + but not relative to the cwd""" + with tmpdir.as_cwd(): + subdir_1 = tmpdir.join("a") + subdir_2 = os.path.join(subdir_1, "b") + link_dir = os.path.join(subdir_1, "c") + + os.mkdir(subdir_1) + os.mkdir(subdir_2) + + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=subdir_2) + link_file = os.path.join(subdir_1, "link.txt") + + symlink.symlink( + source_path=f"b/{os.path.basename(real_file)}", + link_path=f"a/{os.path.basename(link_file)}", + ) + assert os.path.exists(link_file) + assert symlink.islink(link_file) + # Check dirs + assert not os.path.lexists(link_dir) + symlink.symlink(source_path="b", link_path="a/c") + assert os.path.lexists(link_dir) + + +def test_symlink_src_not_relative_to_link(tmpdir): + """Test the symlink.symlink functionality where the source value does not exist relative to + the link and not relative to the cwd. NOTE that this symlink api call is EXPECTED to raise + a symlink.SymlinkError exception that we catch.""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + subdir_1 = os.path.join(test_dir, "a") + subdir_2 = os.path.join(subdir_1, "b") + link_dir = os.path.join(subdir_1, "c") + os.mkdir(subdir_1) + os.mkdir(subdir_2) + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=subdir_2) + link_file = str(tmpdir.join("link.txt")) + # Expected SymlinkError because source path does not exist relative to link path + with pytest.raises(symlink.SymlinkError): + symlink.symlink( + source_path=f"d/{os.path.basename(real_file)}", + link_path=f"a/{os.path.basename(link_file)}", + allow_broken_symlinks=False, + ) + assert not os.path.exists(link_file) + # Check dirs + assert not os.path.lexists(link_dir) + with pytest.raises(symlink.SymlinkError): + symlink.symlink(source_path="d", link_path="a/c", allow_broken_symlinks=False) + assert not os.path.lexists(link_dir) + + +def test_symlink_link_already_exists(tmpdir): + """Test the symlink.symlink method for the case where a link already exists""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + real_dir = os.path.join(test_dir, "real_dir") + link_dir = os.path.join(test_dir, "link_dir") + os.mkdir(real_dir) + symlink.symlink(real_dir, link_dir, allow_broken_symlinks=False) + assert os.path.exists(link_dir) + with pytest.raises(symlink.SymlinkError): + symlink.symlink(source_path=real_dir, link_path=link_dir, allow_broken_symlinks=False) + + +@pytest.mark.skipif(not symlink._windows_can_symlink(), reason="Test requires elevated privileges") +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_symlink_win_file(tmpdir): + """Check that symlink.symlink makes a symlink file when run with elevated permissions""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) + link_file = str(tmpdir.join("link.txt")) + symlink.symlink(source_path=real_file, link_path=link_file) + # Verify that all expected conditions are met + assert os.path.exists(link_file) + assert symlink.islink(link_file) + assert os.path.islink(link_file) + assert not symlink._windows_is_hardlink(link_file) + assert not symlink._windows_is_junction(link_file) + + +@pytest.mark.skipif(not symlink._windows_can_symlink(), reason="Test requires elevated privileges") +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_symlink_win_dir(tmpdir): + """Check that symlink.symlink makes a symlink dir when run with elevated permissions""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + real_dir = os.path.join(test_dir, "real") + link_dir = os.path.join(test_dir, "link") + os.mkdir(real_dir) + symlink.symlink(source_path=real_dir, link_path=link_dir) + # Verify that all expected conditions are met + assert os.path.exists(link_dir) + assert symlink.islink(link_dir) + assert os.path.islink(link_dir) + assert not symlink._windows_is_hardlink(link_dir) + assert not symlink._windows_is_junction(link_dir) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_windows_create_junction(tmpdir): + """Test the symlink._windows_create_junction method""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + junction_real_dir = os.path.join(test_dir, "real_dir") + junction_link_dir = os.path.join(test_dir, "link_dir") + os.mkdir(junction_real_dir) + symlink._windows_create_junction(junction_real_dir, junction_link_dir) + # Verify that all expected conditions are met + assert os.path.exists(junction_link_dir) + assert symlink._windows_is_junction(junction_link_dir) + assert symlink.islink(junction_link_dir) + assert not os.path.islink(junction_link_dir) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_windows_create_hard_link(tmpdir): + """Test the symlink._windows_create_hard_link method""" + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) + link_file = str(tmpdir.join("link.txt")) + symlink._windows_create_hard_link(real_file, link_file) + # Verify that all expected conditions are met + assert os.path.exists(link_file) + assert symlink._windows_is_hardlink(real_file) + assert symlink._windows_is_hardlink(link_file) + assert symlink.islink(link_file) + assert not os.path.islink(link_file) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_windows_create_link_dir(tmpdir): + """Test the functionality of the windows_create_link method with a directory + which should result in making a junction. + """ + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + real_dir = os.path.join(test_dir, "real") + link_dir = os.path.join(test_dir, "link") + os.mkdir(real_dir) + symlink._windows_create_link(real_dir, link_dir) + # Verify that all expected conditions are met + assert os.path.exists(link_dir) + assert symlink.islink(link_dir) + assert not symlink._windows_is_hardlink(link_dir) + assert symlink._windows_is_junction(link_dir) + assert not os.path.islink(link_dir) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_windows_create_link_file(tmpdir): + """Test the functionality of the windows_create_link method with a file + which should result in the creation of a hard link. It also tests the + functionality of the symlink islink infrastructure. + """ + with tmpdir.as_cwd(): + test_dir = str(tmpdir) + fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) + link_file = str(tmpdir.join("link.txt")) + symlink._windows_create_link(real_file, link_file) + # Verify that all expected conditions are met + assert symlink._windows_is_hardlink(link_file) + assert symlink.islink(link_file) + assert not symlink._windows_is_junction(link_file) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows") +def test_windows_read_link(tmpdir): + """Makes sure symlink.readlink can read the link source for hard links and + junctions on windows.""" + with tmpdir.as_cwd(): + real_dir_1 = "real_dir_1" + real_dir_2 = "real_dir_2" + link_dir_1 = "link_dir_1" + link_dir_2 = "link_dir_2" + os.mkdir(real_dir_1) + os.mkdir(real_dir_2) + + # Create a file and a directory + _, real_file_1 = tempfile.mkstemp(prefix="real_1", suffix=".txt", dir=".") + _, real_file_2 = tempfile.mkstemp(prefix="real_2", suffix=".txt", dir=".") + link_file_1 = "link_1.txt" + link_file_2 = "link_2.txt" + + # Make hard link/junction + symlink._windows_create_hard_link(real_file_1, link_file_1) + symlink._windows_create_hard_link(real_file_2, link_file_2) + symlink._windows_create_junction(real_dir_1, link_dir_1) + symlink._windows_create_junction(real_dir_2, link_dir_2) + + assert symlink.readlink(link_file_1) == os.path.abspath(real_file_1) + assert symlink.readlink(link_file_2) == os.path.abspath(real_file_2) + assert symlink.readlink(link_dir_1) == os.path.abspath(real_dir_1) + assert symlink.readlink(link_dir_2) == os.path.abspath(real_dir_2) diff --git a/lib/spack/spack/test/llnl/util/tty/log.py b/lib/spack/spack/test/llnl/util/tty/log.py index 5a26cd86d57f3d..228c2f30448e06 100644 --- a/lib/spack/spack/test/llnl/util/tty/log.py +++ b/lib/spack/spack/test/llnl/util/tty/log.py @@ -29,7 +29,7 @@ pass -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") @contextlib.contextmanager diff --git a/lib/spack/spack/test/main.py b/lib/spack/spack/test/main.py index 5449cd52703425..b4d831cec8e733 100644 --- a/lib/spack/spack/test/main.py +++ b/lib/spack/spack/test/main.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys import pytest @@ -14,8 +13,8 @@ import spack.util.git from spack.main import get_version, main -pytestmark = pytest.mark.skipif( - sys.platform == "win32", reason="Test functionality supported but tests are failing on Win" +pytestmark = pytest.mark.not_on_windows( + "Test functionality supported but tests are failing on Win" ) diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 5d5970c2afe6b7..450a5b282ebc19 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -5,11 +5,10 @@ import filecmp import os -import sys import pytest -from llnl.util.filesystem import resolve_link_target_relative_to_the_link +from llnl.util.symlink import resolve_link_target_relative_to_the_link import spack.mirror import spack.repo @@ -22,7 +21,7 @@ from spack.util.spack_yaml import SpackYAMLError pytestmark = [ - pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows"), + pytest.mark.not_on_windows("does not run on windows"), pytest.mark.usefixtures("mutable_config", "mutable_mock_repo"), ] @@ -65,7 +64,7 @@ def check_mirror(): assert os.path.isdir(mirror_root) for spec in specs: - fetcher = spec.package.fetcher[0] + fetcher = spec.package.fetcher per_package_ref = os.path.join(spec.name, "-".join([spec.name, str(spec.version)])) mirror_paths = spack.mirror.mirror_archive_paths(fetcher, per_package_ref) expected_path = os.path.join(mirror_root, mirror_paths.storage_path) @@ -132,9 +131,14 @@ def test_all_mirror(mock_git_repository, mock_svn_repository, mock_hg_repository @pytest.mark.parametrize( - "mirror", [spack.mirror.Mirror("https://example.com/fetch", "https://example.com/push")] + "mirror", + [ + spack.mirror.Mirror( + {"fetch": "https://example.com/fetch", "push": "https://example.com/push"} + ) + ], ) -def test_roundtrip_mirror(mirror): +def test_roundtrip_mirror(mirror: spack.mirror.Mirror): mirror_yaml = mirror.to_yaml() assert spack.mirror.Mirror.from_yaml(mirror_yaml) == mirror mirror_json = mirror.to_json() @@ -224,6 +228,9 @@ def successful_expand(_class): def successful_apply(*args, **kwargs): pass + def successful_symlink(*args, **kwargs): + pass + with Stage("spack-mirror-test") as stage: mirror_root = os.path.join(stage.path, "test-mirror") @@ -231,6 +238,7 @@ def successful_apply(*args, **kwargs): monkeypatch.setattr(spack.fetch_strategy.URLFetchStrategy, "expand", successful_expand) monkeypatch.setattr(spack.patch, "apply_patch", successful_apply) monkeypatch.setattr(spack.caches.MirrorCache, "store", record_store) + monkeypatch.setattr(spack.caches.MirrorCache, "symlink", successful_symlink) with spack.config.override("config:checksum", False): spack.mirror.create(mirror_root, list(spec.traverse())) @@ -291,3 +299,70 @@ def test_get_all_versions(specs, expected_specs): output_list = [str(x) for x in output_list] # Compare sets since order is not important assert set(output_list) == set(expected_specs) + + +def test_update_1(): + # No change + m = spack.mirror.Mirror("https://example.com") + assert not m.update({"url": "https://example.com"}) + assert m.to_dict() == "https://example.com" + + +def test_update_2(): + # Change URL, shouldn't expand to {"url": ...} dict. + m = spack.mirror.Mirror("https://example.com") + assert m.update({"url": "https://example.org"}) + assert m.to_dict() == "https://example.org" + assert m.fetch_url == "https://example.org" + assert m.push_url == "https://example.org" + + +def test_update_3(): + # Change fetch url, ensure minimal config + m = spack.mirror.Mirror("https://example.com") + assert m.update({"url": "https://example.org"}, "fetch") + assert m.to_dict() == {"url": "https://example.com", "fetch": "https://example.org"} + assert m.fetch_url == "https://example.org" + assert m.push_url == "https://example.com" + + +def test_update_4(): + # Change push url, ensure minimal config + m = spack.mirror.Mirror("https://example.com") + assert m.update({"url": "https://example.org"}, "push") + assert m.to_dict() == {"url": "https://example.com", "push": "https://example.org"} + assert m.push_url == "https://example.org" + assert m.fetch_url == "https://example.com" + + +@pytest.mark.parametrize("direction", ["fetch", "push"]) +def test_update_connection_params(direction): + """Test whether new connection params expand the mirror config to a dict.""" + m = spack.mirror.Mirror("https://example.com") + + assert m.update( + { + "url": "http://example.org", + "access_pair": ["username", "password"], + "access_token": "token", + "profile": "profile", + "endpoint_url": "https://example.com", + }, + direction, + ) + + assert m.to_dict() == { + "url": "https://example.com", + direction: { + "url": "http://example.org", + "access_pair": ["username", "password"], + "access_token": "token", + "profile": "profile", + "endpoint_url": "https://example.com", + }, + } + + assert m.get_access_pair(direction) == ["username", "password"] + assert m.get_access_token(direction) == "token" + assert m.get_profile(direction) == "profile" + assert m.get_endpoint_url(direction) == "https://example.com" diff --git a/lib/spack/spack/test/module_parsing.py b/lib/spack/spack/test/module_parsing.py index b27a36d11d6763..196c18fe2deec5 100644 --- a/lib/spack/spack/test/module_parsing.py +++ b/lib/spack/spack/test/module_parsing.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import sys import pytest @@ -17,7 +16,7 @@ path_from_modules, ) -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="Tests fail on Windows") +pytestmark = pytest.mark.not_on_windows("Tests fail on Windows") test_module_lines = [ "prepend-path LD_LIBRARY_PATH /path/to/lib", diff --git a/lib/spack/spack/test/modules/common.py b/lib/spack/spack/test/modules/common.py index d26ac8ff1b3cf7..11b4305b4844df 100644 --- a/lib/spack/spack/test/modules/common.py +++ b/lib/spack/spack/test/modules/common.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import stat -import sys import pytest @@ -18,7 +17,7 @@ from spack.modules.common import UpstreamModuleIndex from spack.spec import Spec -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") def test_update_dictionary_extending_list(): @@ -181,7 +180,7 @@ def find_nothing(*args): # Mock deletion of the package spec._package = None - monkeypatch.setattr(spack.repo.path, "get", find_nothing) + monkeypatch.setattr(spack.repo.PATH, "get", find_nothing) with pytest.raises(spack.repo.UnknownPackageError): spec.package @@ -195,7 +194,6 @@ def find_nothing(*args): def test_check_module_set_name(mutable_config): """Tests that modules set name are validated correctly and an error is reported if the name we require does not exist or is reserved by the configuration.""" - # Minimal modules.yaml config. spack.config.set( "modules", diff --git a/lib/spack/spack/test/modules/conftest.py b/lib/spack/spack/test/modules/conftest.py index e6dfc024e2f366..210a88a65f8ad3 100644 --- a/lib/spack/spack/test/modules/conftest.py +++ b/lib/spack/spack/test/modules/conftest.py @@ -39,14 +39,6 @@ def _impl(spec_str, module_set_name="default", explicit=True): return _impl -@pytest.fixture() -def update_template_dirs(config, monkeypatch): - """Mocks the template directories for tests""" - dirs = spack.config.get_config("config")["template_dirs"] - dirs = [spack.util.path.canonicalize_path(x) for x in dirs] - monkeypatch.setattr(spack, "template_dirs", dirs) - - @pytest.fixture() def factory(request): """Function that, given a spec string, returns an instance of the writer diff --git a/lib/spack/spack/test/modules/lmod.py b/lib/spack/spack/test/modules/lmod.py index 1ccae6a90dda6d..ecb6ae6db80992 100644 --- a/lib/spack/spack/test/modules/lmod.py +++ b/lib/spack/spack/test/modules/lmod.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys +import os import pytest @@ -21,7 +21,7 @@ #: Class of the writer tested in this module writer_cls = spack.modules.lmod.LmodModulefileWriter -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.fixture(params=["clang@=12.0.0", "gcc@=10.2.1"]) @@ -242,6 +242,18 @@ def test_help_message(self, modulefile_content, module_configuration): ) assert help_msg in "".join(content) + content = modulefile_content("module-long-help target=core2") + + help_msg = ( + "help([[Name : module-long-help]])" + "help([[Version: 1.0]])" + "help([[Target : core2]])" + "help()" + "help([[Package to test long description message generated in modulefile." + "Message too long is wrapped over multiple lines.]])" + ) + assert help_msg in "".join(content) + def test_exclude(self, modulefile_content, module_configuration): """Tests excluding the generation of selected modules.""" module_configuration("exclude") @@ -298,6 +310,26 @@ def test_non_virtual_in_hierarchy(self, factory, module_configuration): with pytest.raises(spack.modules.lmod.NonVirtualInHierarchyError): module.write() + def test_conflicts(self, modulefile_content, module_configuration): + """Tests adding conflicts to the module.""" + + # This configuration has no error, so check the conflicts directives + # are there + module_configuration("conflicts") + content = modulefile_content("mpileaks") + + assert len([x for x in content if x.startswith("conflict")]) == 2 + assert len([x for x in content if x == 'conflict("mpileaks")']) == 1 + assert len([x for x in content if x == 'conflict("intel/14.0.1")']) == 1 + + def test_inconsistent_conflict_in_modules_yaml(self, modulefile_content, module_configuration): + """Tests inconsistent conflict definition in `modules.yaml`.""" + + # This configuration is inconsistent, check an error is raised + module_configuration("wrong_conflicts") + with pytest.raises(spack.modules.common.ModulesError): + modulefile_content("mpileaks") + def test_override_template_in_package(self, modulefile_content, module_configuration): """Tests overriding a template from and attribute in the package.""" @@ -410,3 +442,84 @@ def test_modules_no_arch(self, factory, module_configuration): path = module.layout.filename assert str(spec.os) not in path + + def test_hide_implicits(self, module_configuration, temporary_store): + """Tests the addition and removal of hide command in modulerc.""" + module_configuration("hide_implicits") + + spec = spack.spec.Spec("mpileaks@2.3").concretized() + + # mpileaks is defined as implicit, thus hide command should appear in modulerc + writer = writer_cls(spec, "default", False) + writer.write() + assert os.path.exists(writer.layout.modulerc) + with open(writer.layout.modulerc) as f: + content = [line.strip() for line in f.readlines()] + hide_implicit_mpileaks = f'hide_version("{writer.layout.use_name}")' + assert len([x for x in content if hide_implicit_mpileaks == x]) == 1 + + # The direct dependencies are all implicitly installed, and they should all be hidden, + # except for mpich, which is provider for mpi, which is in the hierarchy, and therefore + # can't be hidden. All other hidden modules should have a 7 character hash (the config + # hash_length = 0 only applies to exposed modules). + with open(writer.layout.filename) as f: + depends_statements = [line.strip() for line in f.readlines() if "depends_on" in line] + for dep in spec.dependencies(deptype=("link", "run")): + if dep.satisfies("mpi"): + assert not any(dep.dag_hash(7) in line for line in depends_statements) + else: + assert any(dep.dag_hash(7) in line for line in depends_statements) + + # when mpileaks becomes explicit, its file name changes (hash_length = 0), meaning an + # extra module file is created; the old one still exists and remains hidden. + writer = writer_cls(spec, "default", True) + writer.write() + assert os.path.exists(writer.layout.modulerc) + with open(writer.layout.modulerc) as f: + content = [line.strip() for line in f.readlines()] + assert hide_implicit_mpileaks in content # old, implicit mpileaks is still hidden + assert f'hide_version("{writer.layout.use_name}")' not in content + + # after removing both the implicit and explicit module, the modulerc file would be empty + # and should be removed. + writer_cls(spec, "default", False).remove() + writer_cls(spec, "default", True).remove() + assert not os.path.exists(writer.layout.modulerc) + assert not os.path.exists(writer.layout.filename) + + # implicit module is removed + writer = writer_cls(spec, "default", False) + writer.write() + assert os.path.exists(writer.layout.filename) + assert os.path.exists(writer.layout.modulerc) + writer.remove() + assert not os.path.exists(writer.layout.modulerc) + assert not os.path.exists(writer.layout.filename) + + # three versions of mpileaks are implicit + writer = writer_cls(spec, "default", False) + writer.write(overwrite=True) + spec_alt1 = spack.spec.Spec("mpileaks@2.2").concretized() + spec_alt2 = spack.spec.Spec("mpileaks@2.1").concretized() + writer_alt1 = writer_cls(spec_alt1, "default", False) + writer_alt1.write(overwrite=True) + writer_alt2 = writer_cls(spec_alt2, "default", False) + writer_alt2.write(overwrite=True) + assert os.path.exists(writer.layout.modulerc) + with open(writer.layout.modulerc) as f: + content = [line.strip() for line in f.readlines()] + hide_cmd = f'hide_version("{writer.layout.use_name}")' + hide_cmd_alt1 = f'hide_version("{writer_alt1.layout.use_name}")' + hide_cmd_alt2 = f'hide_version("{writer_alt2.layout.use_name}")' + assert len([x for x in content if hide_cmd == x]) == 1 + assert len([x for x in content if hide_cmd_alt1 == x]) == 1 + assert len([x for x in content if hide_cmd_alt2 == x]) == 1 + + # one version is removed + writer_alt1.remove() + assert os.path.exists(writer.layout.modulerc) + with open(writer.layout.modulerc) as f: + content = [line.strip() for line in f.readlines()] + assert len([x for x in content if hide_cmd == x]) == 1 + assert len([x for x in content if hide_cmd_alt1 == x]) == 0 + assert len([x for x in content if hide_cmd_alt2 == x]) == 1 diff --git a/lib/spack/spack/test/modules/tcl.py b/lib/spack/spack/test/modules/tcl.py index af511f72e9ad98..b69a4482c27160 100644 --- a/lib/spack/spack/test/modules/tcl.py +++ b/lib/spack/spack/test/modules/tcl.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import sys +import os import pytest @@ -18,7 +18,7 @@ #: Class of the writer tested in this module writer_cls = spack.modules.tcl.TclModulefileWriter -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.mark.usefixtures("config", "mock_packages", "mock_module_filename") @@ -29,7 +29,7 @@ def test_simple_case(self, modulefile_content, module_configuration): module_configuration("autoload_direct") content = modulefile_content(mpich_spec_string) - assert 'module-whatis "mpich @3.0.4"' in content + assert "module-whatis {mpich @3.0.4}" in content def test_autoload_direct(self, modulefile_content, module_configuration): """Tests the automatic loading of direct dependencies.""" @@ -112,9 +112,9 @@ def test_alter_environment(self, modulefile_content, module_configuration): content = modulefile_content("mpileaks platform=test target=x86_64") assert len([x for x in content if x.startswith("prepend-path CMAKE_PREFIX_PATH")]) == 0 - assert len([x for x in content if 'setenv FOO "foo"' in x]) == 1 - assert len([x for x in content if 'setenv OMPI_MCA_mpi_leave_pinned "1"' in x]) == 1 - assert len([x for x in content if 'setenv OMPI_MCA_MPI_LEAVE_PINNED "1"' in x]) == 0 + assert len([x for x in content if "setenv FOO {foo}" in x]) == 1 + assert len([x for x in content if "setenv OMPI_MCA_mpi_leave_pinned {1}" in x]) == 1 + assert len([x for x in content if "setenv OMPI_MCA_MPI_LEAVE_PINNED {1}" in x]) == 0 assert len([x for x in content if "unsetenv BAR" in x]) == 1 assert ( len([x for x in content if "setenv MPILEAKS_ROOT" in x]) == 1 @@ -124,7 +124,7 @@ def test_alter_environment(self, modulefile_content, module_configuration): content = modulefile_content("libdwarf platform=test target=core2") assert len([x for x in content if x.startswith("prepend-path CMAKE_PREFIX_PATH")]) == 0 - assert len([x for x in content if 'setenv FOO "foo"' in x]) == 0 + assert len([x for x in content if "setenv FOO {foo}" in x]) == 0 assert len([x for x in content if "unsetenv BAR" in x]) == 0 assert len([x for x in content if "depends-on foo/bar" in x]) == 1 assert len([x for x in content if "module load foo/bar" in x]) == 1 @@ -139,14 +139,14 @@ def test_prepend_path_separator(self, modulefile_content, module_configuration): module_configuration("module_path_separator") content = modulefile_content("module-path-separator") - assert len([x for x in content if 'append-path --delim ":" COLON "foo"' in x]) == 1 - assert len([x for x in content if 'prepend-path --delim ":" COLON "foo"' in x]) == 1 - assert len([x for x in content if 'remove-path --delim ":" COLON "foo"' in x]) == 1 - assert len([x for x in content if 'append-path --delim ";" SEMICOLON "bar"' in x]) == 1 - assert len([x for x in content if 'prepend-path --delim ";" SEMICOLON "bar"' in x]) == 1 - assert len([x for x in content if 'remove-path --delim ";" SEMICOLON "bar"' in x]) == 1 - assert len([x for x in content if 'append-path --delim " " SPACE "qux"' in x]) == 1 - assert len([x for x in content if 'remove-path --delim " " SPACE "qux"' in x]) == 1 + assert len([x for x in content if "append-path COLON {foo}" in x]) == 1 + assert len([x for x in content if "prepend-path COLON {foo}" in x]) == 1 + assert len([x for x in content if "remove-path COLON {foo}" in x]) == 1 + assert len([x for x in content if "append-path --delim {;} SEMICOLON {bar}" in x]) == 1 + assert len([x for x in content if "prepend-path --delim {;} SEMICOLON {bar}" in x]) == 1 + assert len([x for x in content if "remove-path --delim {;} SEMICOLON {bar}" in x]) == 1 + assert len([x for x in content if "append-path --delim { } SPACE {qux}" in x]) == 1 + assert len([x for x in content if "remove-path --delim { } SPACE {qux}" in x]) == 1 @pytest.mark.regression("11355") def test_manpath_setup(self, modulefile_content, module_configuration): @@ -156,37 +156,23 @@ def test_manpath_setup(self, modulefile_content, module_configuration): # no manpath set by module content = modulefile_content("mpileaks") - assert len([x for x in content if 'append-path --delim ":" MANPATH ""' in x]) == 0 + assert len([x for x in content if "append-path MANPATH {}" in x]) == 0 # manpath set by module with prepend-path content = modulefile_content("module-manpath-prepend") - assert ( - len([x for x in content if 'prepend-path --delim ":" MANPATH "/path/to/man"' in x]) - == 1 - ) - assert ( - len( - [ - x - for x in content - if 'prepend-path --delim ":" MANPATH "/path/to/share/man"' in x - ] - ) - == 1 - ) - assert len([x for x in content if 'append-path --delim ":" MANPATH ""' in x]) == 1 + assert len([x for x in content if "prepend-path MANPATH {/path/to/man}" in x]) == 1 + assert len([x for x in content if "prepend-path MANPATH {/path/to/share/man}" in x]) == 1 + assert len([x for x in content if "append-path MANPATH {}" in x]) == 1 # manpath set by module with append-path content = modulefile_content("module-manpath-append") - assert ( - len([x for x in content if 'append-path --delim ":" MANPATH "/path/to/man"' in x]) == 1 - ) - assert len([x for x in content if 'append-path --delim ":" MANPATH ""' in x]) == 1 + assert len([x for x in content if "append-path MANPATH {/path/to/man}" in x]) == 1 + assert len([x for x in content if "append-path MANPATH {}" in x]) == 1 # manpath set by module with setenv content = modulefile_content("module-manpath-setenv") - assert len([x for x in content if 'setenv MANPATH "/path/to/man"' in x]) == 1 - assert len([x for x in content if 'append-path --delim ":" MANPATH ""' in x]) == 0 + assert len([x for x in content if "setenv MANPATH {/path/to/man}" in x]) == 1 + assert len([x for x in content if "append-path MANPATH {}" in x]) == 0 @pytest.mark.regression("29578") def test_setenv_raw_value(self, modulefile_content, module_configuration): @@ -195,7 +181,7 @@ def test_setenv_raw_value(self, modulefile_content, module_configuration): module_configuration("autoload_direct") content = modulefile_content("module-setenv-raw") - assert len([x for x in content if 'setenv FOO "{{name}}, {name}, {{}}, {}"' in x]) == 1 + assert len([x for x in content if "setenv FOO {{{name}}, {name}, {{}}, {}}" in x]) == 1 def test_help_message(self, modulefile_content, module_configuration): """Tests the generation of module help message.""" @@ -205,11 +191,11 @@ def test_help_message(self, modulefile_content, module_configuration): help_msg = ( "proc ModulesHelp { } {" - ' puts stderr "Name : mpileaks"' - ' puts stderr "Version: 2.3"' - ' puts stderr "Target : core2"' - ' puts stderr ""' - ' puts stderr "Mpileaks is a mock package that passes audits"' + " puts stderr {Name : mpileaks}" + " puts stderr {Version: 2.3}" + " puts stderr {Target : core2}" + " puts stderr {}" + " puts stderr {Mpileaks is a mock package that passes audits}" "}" ) assert help_msg in "".join(content) @@ -218,9 +204,23 @@ def test_help_message(self, modulefile_content, module_configuration): help_msg = ( "proc ModulesHelp { } {" - ' puts stderr "Name : libdwarf"' - ' puts stderr "Version: 20130729"' - ' puts stderr "Target : core2"' + " puts stderr {Name : libdwarf}" + " puts stderr {Version: 20130729}" + " puts stderr {Target : core2}" + "}" + ) + assert help_msg in "".join(content) + + content = modulefile_content("module-long-help target=core2") + + help_msg = ( + "proc ModulesHelp { } {" + " puts stderr {Name : module-long-help}" + " puts stderr {Version: 1.0}" + " puts stderr {Target : core2}" + " puts stderr {}" + " puts stderr {Package to test long description message generated in modulefile.}" + " puts stderr {Message too long is wrapped over multiple lines.}" "}" ) assert help_msg in "".join(content) @@ -317,9 +317,12 @@ def test_conflicts(self, modulefile_content, module_configuration): assert len([x for x in content if x == "conflict mpileaks"]) == 1 assert len([x for x in content if x == "conflict intel/14.0.1"]) == 1 + def test_inconsistent_conflict_in_modules_yaml(self, modulefile_content, module_configuration): + """Tests inconsistent conflict definition in `modules.yaml`.""" + # This configuration is inconsistent, check an error is raised module_configuration("wrong_conflicts") - with pytest.raises(SystemExit): + with pytest.raises(spack.modules.common.ModulesError): modulefile_content("mpileaks") def test_module_index(self, module_configuration, factory, tmpdir_factory): @@ -375,14 +378,14 @@ def test_setup_environment(self, modulefile_content, module_configuration): content = modulefile_content("mpileaks") assert len([x for x in content if "setenv FOOBAR" in x]) == 1 - assert len([x for x in content if 'setenv FOOBAR "mpileaks"' in x]) == 1 + assert len([x for x in content if "setenv FOOBAR {mpileaks}" in x]) == 1 spec = spack.spec.Spec("mpileaks") spec.concretize() content = modulefile_content(str(spec["callpath"])) assert len([x for x in content if "setenv FOOBAR" in x]) == 1 - assert len([x for x in content if 'setenv FOOBAR "callpath"' in x]) == 1 + assert len([x for x in content if "setenv FOOBAR {callpath}" in x]) == 1 def test_override_config(self, module_configuration, factory): """Tests overriding some sections of the configuration file.""" @@ -423,12 +426,12 @@ def test_extend_context(self, modulefile_content, module_configuration): assert 'puts stderr "sentence from package"' in content - short_description = 'module-whatis "This package updates the context for Tcl modulefiles."' + short_description = "module-whatis {This package updates the context for Tcl modulefiles.}" assert short_description in content @pytest.mark.regression("4400") @pytest.mark.db - def test_exclude_implicits(self, module_configuration, database): + def test_hide_implicits_no_arg(self, module_configuration, database): module_configuration("exclude_implicits") # mpileaks has been installed explicitly when setting up @@ -446,7 +449,7 @@ def test_exclude_implicits(self, module_configuration, database): assert writer.conf.excluded @pytest.mark.regression("12105") - def test_exclude_implicits_with_arg(self, module_configuration): + def test_hide_implicits_with_arg(self, module_configuration): module_configuration("exclude_implicits") # mpileaks is defined as explicit with explicit argument set on writer @@ -488,3 +491,79 @@ def test_modules_no_arch(self, factory, module_configuration): path = module.layout.filename assert str(spec.os) not in path + + def test_hide_implicits(self, module_configuration, temporary_store): + """Tests the addition and removal of hide command in modulerc.""" + module_configuration("hide_implicits") + + spec = spack.spec.Spec("mpileaks@2.3").concretized() + + # mpileaks is defined as implicit, thus hide command should appear in modulerc + writer = writer_cls(spec, "default", False) + writer.write() + assert os.path.exists(writer.layout.modulerc) + with open(writer.layout.modulerc) as f: + content = [line.strip() for line in f.readlines()] + hide_implicit_mpileaks = f"module-hide --soft --hidden-loaded {writer.layout.use_name}" + assert len([x for x in content if hide_implicit_mpileaks == x]) == 1 + + # The direct dependencies are all implicit, and they should have depends-on with fixed + # 7 character hash, even though the config is set to hash_length = 0. + with open(writer.layout.filename) as f: + depends_statements = [line.strip() for line in f.readlines() if "depends-on" in line] + for dep in spec.dependencies(deptype=("link", "run")): + assert any(dep.dag_hash(7) in line for line in depends_statements) + + # when mpileaks becomes explicit, its file name changes (hash_length = 0), meaning an + # extra module file is created; the old one still exists and remains hidden. + writer = writer_cls(spec, "default", True) + writer.write() + assert os.path.exists(writer.layout.modulerc) + with open(writer.layout.modulerc) as f: + content = [line.strip() for line in f.readlines()] + assert hide_implicit_mpileaks in content # old, implicit mpileaks is still hidden + assert f"module-hide --soft --hidden-loaded {writer.layout.use_name}" not in content + + # after removing both the implicit and explicit module, the modulerc file would be empty + # and should be removed. + writer_cls(spec, "default", False).remove() + writer_cls(spec, "default", True).remove() + assert not os.path.exists(writer.layout.modulerc) + assert not os.path.exists(writer.layout.filename) + + # implicit module is removed + writer = writer_cls(spec, "default", False) + writer.write() + assert os.path.exists(writer.layout.filename) + assert os.path.exists(writer.layout.modulerc) + writer.remove() + assert not os.path.exists(writer.layout.modulerc) + assert not os.path.exists(writer.layout.filename) + + # three versions of mpileaks are implicit + writer = writer_cls(spec, "default", False) + writer.write(overwrite=True) + spec_alt1 = spack.spec.Spec("mpileaks@2.2").concretized() + spec_alt2 = spack.spec.Spec("mpileaks@2.1").concretized() + writer_alt1 = writer_cls(spec_alt1, "default", False) + writer_alt1.write(overwrite=True) + writer_alt2 = writer_cls(spec_alt2, "default", False) + writer_alt2.write(overwrite=True) + assert os.path.exists(writer.layout.modulerc) + with open(writer.layout.modulerc) as f: + content = [line.strip() for line in f.readlines()] + hide_cmd = f"module-hide --soft --hidden-loaded {writer.layout.use_name}" + hide_cmd_alt1 = f"module-hide --soft --hidden-loaded {writer_alt1.layout.use_name}" + hide_cmd_alt2 = f"module-hide --soft --hidden-loaded {writer_alt2.layout.use_name}" + assert len([x for x in content if hide_cmd == x]) == 1 + assert len([x for x in content if hide_cmd_alt1 == x]) == 1 + assert len([x for x in content if hide_cmd_alt2 == x]) == 1 + + # one version is removed + writer_alt1.remove() + assert os.path.exists(writer.layout.modulerc) + with open(writer.layout.modulerc) as f: + content = [line.strip() for line in f.readlines()] + assert len([x for x in content if hide_cmd == x]) == 1 + assert len([x for x in content if hide_cmd_alt1 == x]) == 0 + assert len([x for x in content if hide_cmd_alt2 == x]) == 1 diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 8d4cec1253c868..76e06ab3a7fa24 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -4,8 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test for multi_method dispatch.""" -import os -import sys import pytest @@ -16,10 +14,8 @@ pytestmark = [ pytest.mark.usefixtures("mock_packages", "config"), - pytest.mark.skipif( - os.environ.get("SPACK_TEST_SOLVER") == "original" or sys.platform == "win32", - reason="The original concretizer cannot concretize most of the specs", - ), + pytest.mark.only_clingo("The original concretizer cannot concretize most of the specs"), + pytest.mark.not_on_windows("Not running on windows"), ] diff --git a/lib/spack/spack/test/oci/image.py b/lib/spack/spack/test/oci/image.py new file mode 100644 index 00000000000000..b074cc679af0a6 --- /dev/null +++ b/lib/spack/spack/test/oci/image.py @@ -0,0 +1,105 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import re + +import pytest + +import spack.spec +from spack.oci.image import Digest, ImageReference, default_tag, tag + + +@pytest.mark.parametrize( + "image_ref, expected", + [ + ( + f"example.com:1234/a/b/c:tag@sha256:{'a'*64}", + ("example.com:1234", "a/b/c", "tag", Digest.from_sha256("a" * 64)), + ), + ("example.com:1234/a/b/c:tag", ("example.com:1234", "a/b/c", "tag", None)), + ("example.com:1234/a/b/c", ("example.com:1234", "a/b/c", "latest", None)), + ( + f"example.com:1234/a/b/c@sha256:{'a'*64}", + ("example.com:1234", "a/b/c", "latest", Digest.from_sha256("a" * 64)), + ), + # ipv4 + ("1.2.3.4:1234/a/b/c:tag", ("1.2.3.4:1234", "a/b/c", "tag", None)), + # ipv6 + ("[2001:db8::1]:1234/a/b/c:tag", ("[2001:db8::1]:1234", "a/b/c", "tag", None)), + # Follow docker rules for parsing + ("ubuntu:22.04", ("index.docker.io", "library/ubuntu", "22.04", None)), + ("myname/myimage:abc", ("index.docker.io", "myname/myimage", "abc", None)), + ("myname:1234/myimage:abc", ("myname:1234", "myimage", "abc", None)), + ("localhost/myimage:abc", ("localhost", "myimage", "abc", None)), + ("localhost:1234/myimage:abc", ("localhost:1234", "myimage", "abc", None)), + ( + "example.com/UPPERCASE/lowercase:AbC", + ("example.com", "uppercase/lowercase", "AbC", None), + ), + ], +) +def test_name_parsing(image_ref, expected): + x = ImageReference.from_string(image_ref) + assert (x.domain, x.name, x.tag, x.digest) == expected + + +@pytest.mark.parametrize( + "image_ref", + [ + # wrong order of tag and sha + f"example.com:1234/a/b/c@sha256:{'a'*64}:tag", + # double tag + "example.com:1234/a/b/c:tag:tag", + # empty tag + "example.com:1234/a/b/c:", + # empty digest + "example.com:1234/a/b/c@sha256:", + # unsupport digest algorithm + f"example.com:1234/a/b/c@sha512:{'a'*128}", + # invalid digest length + f"example.com:1234/a/b/c@sha256:{'a'*63}", + # whitespace + "example.com:1234/a/b/c :tag", + "example.com:1234/a/b/c: tag", + "example.com:1234/a/b/c:tag ", + " example.com:1234/a/b/c:tag", + # broken ipv4 + "1.2..3:1234/a/b/c:tag", + ], +) +def test_parsing_failure(image_ref): + with pytest.raises(ValueError): + ImageReference.from_string(image_ref) + + +def test_digest(): + valid_digest = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + + # Test string roundtrip + assert str(Digest.from_string(f"sha256:{valid_digest}")) == f"sha256:{valid_digest}" + + # Invalid digest length + with pytest.raises(ValueError): + Digest.from_string("sha256:abcdef") + + # Missing algorithm + with pytest.raises(ValueError): + Digest.from_string(valid_digest) + + +@pytest.mark.parametrize( + "spec", + [ + # Standard case + "short-name@=1.2.3", + # Unsupported characters in git version + f"git-version@{1:040x}=develop", + # Too long of a name + f"{'too-long':x<256}@=1.2.3", + ], +) +def test_default_tag(spec: str): + """Make sure that computed image tags are valid.""" + assert re.fullmatch(tag, default_tag(spack.spec.Spec(spec))) diff --git a/lib/spack/spack/test/oci/integration_test.py b/lib/spack/spack/test/oci/integration_test.py new file mode 100644 index 00000000000000..b2f9366c3a5fbf --- /dev/null +++ b/lib/spack/spack/test/oci/integration_test.py @@ -0,0 +1,148 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# These are slow integration tests that do concretization, install, tarballing +# and compression. They still use an in-memory OCI registry. + +import hashlib +import json +import os +from contextlib import contextmanager + +import spack.oci.opener +from spack.binary_distribution import gzip_compressed_tarfile +from spack.main import SpackCommand +from spack.oci.image import Digest, ImageReference, default_config, default_manifest +from spack.oci.oci import blob_exists, get_manifest_and_config, upload_blob, upload_manifest +from spack.test.oci.mock_registry import DummyServer, InMemoryOCIRegistry, create_opener + +buildcache = SpackCommand("buildcache") +mirror = SpackCommand("mirror") + + +@contextmanager +def oci_servers(*servers: DummyServer): + old_opener = spack.oci.opener.urlopen + spack.oci.opener.urlopen = create_opener(*servers).open + yield + spack.oci.opener.urlopen = old_opener + + +def test_buildcache_push_command(mutable_database, disable_parallel_buildcache_push): + with oci_servers(InMemoryOCIRegistry("example.com")): + mirror("add", "oci-test", "oci://example.com/image") + + # Push the package(s) to the OCI registry + buildcache("push", "--update-index", "oci-test", "mpileaks^mpich") + + # Remove mpileaks from the database + matches = mutable_database.query_local("mpileaks^mpich") + assert len(matches) == 1 + spec = matches[0] + spec.package.do_uninstall() + + # Reinstall mpileaks from the OCI registry + buildcache("install", "--unsigned", "mpileaks^mpich") + + # Now it should be installed again + assert spec.installed + + # And let's check that the bin/mpileaks executable is there + assert os.path.exists(os.path.join(spec.prefix, "bin", "mpileaks")) + + +def test_buildcache_push_with_base_image_command( + mutable_database, tmpdir, disable_parallel_buildcache_push +): + """Test that we can push a package with a base image to an OCI registry. + + This test is a bit involved, cause we have to create a small base image.""" + + registry_src = InMemoryOCIRegistry("src.example.com") + registry_dst = InMemoryOCIRegistry("dst.example.com") + + base_image = ImageReference.from_string("src.example.com/my-base-image:latest") + + with oci_servers(registry_src, registry_dst): + mirror("add", "oci-test", "oci://dst.example.com/image") + + # TODO: simplify creation of images... + # We create a rootfs.tar.gz, a config file and a manifest file, + # and upload those. + + config, manifest = default_config(architecture="amd64", os="linux"), default_manifest() + + # Create a small rootfs + rootfs = tmpdir.join("rootfs") + rootfs.ensure(dir=True) + rootfs.join("bin").ensure(dir=True) + rootfs.join("bin", "sh").ensure(file=True) + + # Create a tarball of it. + tarball = tmpdir.join("base.tar.gz") + with gzip_compressed_tarfile(tarball) as (tar, tar_gz_checksum, tar_checksum): + tar.add(rootfs, arcname=".") + + tar_gz_digest = Digest.from_sha256(tar_gz_checksum.hexdigest()) + tar_digest = Digest.from_sha256(tar_checksum.hexdigest()) + + # Save the config file + config["rootfs"]["diff_ids"] = [str(tar_digest)] + config_file = tmpdir.join("config.json") + with open(config_file, "w") as f: + f.write(json.dumps(config)) + + config_digest = Digest.from_sha256( + hashlib.sha256(open(config_file, "rb").read()).hexdigest() + ) + + # Register the layer in the manifest + manifest["layers"].append( + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": str(tar_gz_digest), + "size": tarball.size(), + } + ) + manifest["config"]["digest"] = str(config_digest) + manifest["config"]["size"] = config_file.size() + + # Upload the layer and config file + upload_blob(base_image, tarball, tar_gz_digest) + upload_blob(base_image, config_file, config_digest) + + # Upload the manifest + upload_manifest(base_image, manifest) + + # END TODO + + # Finally... use it as a base image + buildcache("push", "--base-image", str(base_image), "oci-test", "mpileaks^mpich") + + # Figure out what tag was produced + tag = next(tag for _, tag in registry_dst.manifests.keys() if tag.startswith("mpileaks-")) + assert tag is not None + + # Fetch the manifest and config + dst_image = ImageReference.from_string(f"dst.example.com/image:{tag}") + retrieved_manifest, retrieved_config = get_manifest_and_config(dst_image) + + # Check that the base image layer is first. + assert retrieved_manifest["layers"][0]["digest"] == str(tar_gz_digest) + assert retrieved_config["rootfs"]["diff_ids"][0] == str(tar_digest) + + # And also check that we have layers for each link-run dependency + matches = mutable_database.query_local("mpileaks^mpich") + assert len(matches) == 1 + spec = matches[0] + + num_runtime_deps = len(list(spec.traverse(root=True, deptype=("link", "run")))) + + # One base layer + num_runtime_deps + assert len(retrieved_manifest["layers"]) == 1 + num_runtime_deps + + # And verify that all layers including the base layer are present + for layer in retrieved_manifest["layers"]: + assert blob_exists(dst_image, digest=Digest.from_string(layer["digest"])) diff --git a/lib/spack/spack/test/oci/mock_registry.py b/lib/spack/spack/test/oci/mock_registry.py new file mode 100644 index 00000000000000..ec3e85c333ab7e --- /dev/null +++ b/lib/spack/spack/test/oci/mock_registry.py @@ -0,0 +1,410 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +import base64 +import email.message +import hashlib +import io +import json +import re +import urllib.error +import urllib.parse +import urllib.request +import uuid +from typing import Callable, Dict, List, Optional, Pattern, Tuple +from urllib.request import Request + +from spack.oci.image import Digest +from spack.oci.opener import OCIAuthHandler + + +class MockHTTPResponse(io.IOBase): + """This is a mock HTTP response, which implements part of http.client.HTTPResponse""" + + def __init__(self, status, reason, headers=None, body=None): + self.msg = None + self.version = 11 + self.url = None + self.headers = email.message.EmailMessage() + self.status = status + self.code = status + self.reason = reason + self.debuglevel = 0 + self._body = body + + if headers is not None: + for key, value in headers.items(): + self.headers[key] = value + + @classmethod + def with_json(cls, status, reason, headers=None, body=None): + """Create a mock HTTP response with JSON string as body""" + body = io.BytesIO(json.dumps(body).encode("utf-8")) + return cls(status, reason, headers, body) + + def read(self, *args, **kwargs): + return self._body.read(*args, **kwargs) + + def getheader(self, name, default=None): + self.headers.get(name, default) + + def getheaders(self): + return self.headers.items() + + def fileno(self): + return 0 + + def getcode(self): + return self.status + + def info(self): + return self.headers + + +class MiddlewareError(Exception): + """Thrown in a handler to return a response early.""" + + def __init__(self, response: MockHTTPResponse): + self.response = response + + +class Router: + """This class is a small router for requests to the OCI registry. + + It is used to dispatch requests to a handler, and middleware can be + used to transform requests, as well as return responses early + (e.g. for authentication).""" + + def __init__(self) -> None: + self.routes: List[Tuple[str, Pattern, Callable]] = [] + self.middleware: List[Callable[[Request], Request]] = [] + + def handle(self, req: Request) -> MockHTTPResponse: + """Dispatch a request to a handler.""" + result = urllib.parse.urlparse(req.full_url) + + # Apply middleware + try: + for handler in self.middleware: + req = handler(req) + except MiddlewareError as e: + return e.response + + for method, path_regex, handler in self.routes: + if method != req.get_method(): + continue + match = re.fullmatch(path_regex, result.path) + if not match: + continue + + return handler(req, **match.groupdict()) + + return MockHTTPResponse(404, "Not found") + + def register(self, method, path: str, handler: Callable): + self.routes.append((method, re.compile(path), handler)) + + def add_middleware(self, handler: Callable[[Request], Request]): + self.middleware.append(handler) + + +class DummyServer: + def __init__(self, domain: str) -> None: + # The domain of the server, e.g. "registry.example.com" + self.domain = domain + + # List of (method, url) tuples + self.requests: List[Tuple[str, str]] = [] + + # Dispatches requests to handlers + self.router = Router() + + # Always install a request logger + self.router.add_middleware(self.log_request) + + def handle(self, req: Request) -> MockHTTPResponse: + return self.router.handle(req) + + def log_request(self, req: Request): + path = urllib.parse.urlparse(req.full_url).path + self.requests.append((req.get_method(), path)) + return req + + def clear_log(self): + self.requests = [] + + +class InMemoryOCIRegistry(DummyServer): + """This implements the basic OCI registry API, but in memory. + + It supports two types of blob uploads: + 1. POST + PUT: the client first starts a session with POST, then does a large PUT request + 2. POST: the client does a single POST request with the whole blob + + Option 2 is not supported by all registries, so we allow to disable it, + with allow_single_post=False. + + A third option is to use the chunked upload, but this is not implemented here, because + it's typically a major performance hit in upload speed, so we're not using it in Spack.""" + + def __init__(self, domain: str, allow_single_post: bool = True) -> None: + super().__init__(domain) + self.router.register("GET", r"/v2/", self.index) + self.router.register("HEAD", r"/v2/(?P.+)/blobs/(?P.+)", self.head_blob) + self.router.register("POST", r"/v2/(?P.+)/blobs/uploads/", self.start_session) + self.router.register("PUT", r"/upload", self.put_session) + self.router.register("PUT", r"/v2/(?P.+)/manifests/(?P.+)", self.put_manifest) + self.router.register("GET", r"/v2/(?P.+)/manifests/(?P.+)", self.get_manifest) + self.router.register("GET", r"/v2/(?P.+)/blobs/(?P.+)", self.get_blob) + self.router.register("GET", r"/v2/(?P.+)/tags/list", self.list_tags) + + # If True, allow single POST upload, not all registries support this + self.allow_single_post = allow_single_post + + # Used for POST + PUT upload. This is a map from session ID to image name + self.sessions: Dict[str, str] = {} + + # Set of sha256:... digests that are known to the registry + self.blobs: Dict[str, bytes] = {} + + # Map from (name, tag) to manifest + self.manifests: Dict[Tuple[str, str], Dict] = {} + + def index(self, req: Request): + return MockHTTPResponse.with_json(200, "OK", body={}) + + def head_blob(self, req: Request, name: str, digest: str): + if digest in self.blobs: + return MockHTTPResponse(200, "OK", headers={"Content-Length": "1234"}) + return MockHTTPResponse(404, "Not found") + + def get_blob(self, req: Request, name: str, digest: str): + if digest in self.blobs: + return MockHTTPResponse(200, "OK", body=io.BytesIO(self.blobs[digest])) + return MockHTTPResponse(404, "Not found") + + def start_session(self, req: Request, name: str): + id = str(uuid.uuid4()) + self.sessions[id] = name + + # Check if digest is present (single monolithic upload) + result = urllib.parse.urlparse(req.full_url) + query = urllib.parse.parse_qs(result.query) + + if self.allow_single_post and "digest" in query: + return self.handle_upload( + req, name=name, digest=Digest.from_string(query["digest"][0]) + ) + + return MockHTTPResponse(202, "Accepted", headers={"Location": f"/upload?uuid={id}"}) + + def put_session(self, req: Request): + # Do the upload. + result = urllib.parse.urlparse(req.full_url) + query = urllib.parse.parse_qs(result.query) + + # uuid param should be preserved, and digest should be present + assert "uuid" in query and len(query["uuid"]) == 1 + assert "digest" in query and len(query["digest"]) == 1 + + id = query["uuid"][0] + assert id in self.sessions + + name, digest = self.sessions[id], Digest.from_string(query["digest"][0]) + + response = self.handle_upload(req, name=name, digest=digest) + + # End the session + del self.sessions[id] + + return response + + def put_manifest(self, req: Request, name: str, ref: str): + # In requests, Python runs header.capitalize(). + content_type = req.get_header("Content-type") + assert content_type in ( + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.oci.image.index.v1+json", + ) + + index_or_manifest = json.loads(self._require_data(req)) + + # Verify that we have all blobs (layers for manifest, manifests for index) + if content_type == "application/vnd.oci.image.manifest.v1+json": + for layer in index_or_manifest["layers"]: + assert layer["digest"] in self.blobs, "Missing blob while uploading manifest" + + else: + for manifest in index_or_manifest["manifests"]: + assert ( + name, + manifest["digest"], + ) in self.manifests, "Missing manifest while uploading index" + + self.manifests[(name, ref)] = index_or_manifest + + return MockHTTPResponse( + 201, "Created", headers={"Location": f"/v2/{name}/manifests/{ref}"} + ) + + def get_manifest(self, req: Request, name: str, ref: str): + if (name, ref) not in self.manifests: + return MockHTTPResponse(404, "Not found") + + manifest_or_index = self.manifests[(name, ref)] + + return MockHTTPResponse.with_json( + 200, + "OK", + headers={"Content-type": manifest_or_index["mediaType"]}, + body=manifest_or_index, + ) + + def _require_data(self, req: Request) -> bytes: + """Extract request.data, it's type remains a mystery""" + assert req.data is not None + + if hasattr(req.data, "read"): + return req.data.read() + elif isinstance(req.data, bytes): + return req.data + + raise ValueError("req.data should be bytes or have a read() method") + + def handle_upload(self, req: Request, name: str, digest: Digest): + """Verify the digest, save the blob, return created status""" + data = self._require_data(req) + assert hashlib.sha256(data).hexdigest() == digest.digest + self.blobs[str(digest)] = data + return MockHTTPResponse(201, "Created", headers={"Location": f"/v2/{name}/blobs/{digest}"}) + + def list_tags(self, req: Request, name: str): + # List all tags, exclude digests. + tags = [_tag for _name, _tag in self.manifests.keys() if _name == name and ":" not in _tag] + tags.sort() + return MockHTTPResponse.with_json(200, "OK", body={"tags": tags}) + + +class DummyServerUrllibHandler(urllib.request.BaseHandler): + """Glue between urllib and DummyServer, routing requests to + the correct mock server for a given domain.""" + + def __init__(self) -> None: + self.servers: Dict[str, DummyServer] = {} + + def add_server(self, domain: str, api: DummyServer): + self.servers[domain] = api + return self + + def https_open(self, req: Request): + domain = urllib.parse.urlparse(req.full_url).netloc + + if domain not in self.servers: + return MockHTTPResponse(404, "Not found") + + return self.servers[domain].handle(req) + + +class InMemoryOCIRegistryWithAuth(InMemoryOCIRegistry): + """This is another in-memory OCI registry, but it requires authentication.""" + + def __init__( + self, domain, token: Optional[str], realm: str, allow_single_post: bool = True + ) -> None: + super().__init__(domain, allow_single_post) + self.token = token # token to accept + self.realm = realm # url to the authorization server + self.router.add_middleware(self.authenticate) + + def authenticate(self, req: Request): + # Any request needs an Authorization header + authorization = req.get_header("Authorization") + + if authorization is None: + raise MiddlewareError(self.unauthorized()) + + # Ensure that the token is correct + assert authorization.startswith("Bearer ") + token = authorization[7:] + + if token != self.token: + raise MiddlewareError(self.unauthorized()) + + return req + + def unauthorized(self): + return MockHTTPResponse( + 401, + "Unauthorized", + { + "www-authenticate": f'Bearer realm="{self.realm}",' + f'service="{self.domain}",' + 'scope="repository:spack-registry:pull,push"' + }, + ) + + +class MockBearerTokenServer(DummyServer): + """Simulates a basic server that hands out bearer tokens + at the /login endpoint for the following services: + public.example.com, which doesn't require Basic Auth + private.example.com, which requires Basic Auth, with user:pass + """ + + def __init__(self, domain: str) -> None: + super().__init__(domain) + self.router.register("GET", "/login", self.login) + + def login(self, req: Request): + url = urllib.parse.urlparse(req.full_url) + query_params = urllib.parse.parse_qs(url.query) + + # Verify query params, from the www-authenticate header + assert query_params["client_id"] == ["spack"] + assert len(query_params["service"]) == 1 + assert query_params["scope"] == ["repository:spack-registry:pull,push"] + + service = query_params["service"][0] + + if service == "public.example.com": + return self.public_auth(req) + elif service == "private.example.com": + return self.private_auth(req) + + return MockHTTPResponse(404, "Not found") + + def public_auth(self, req: Request): + # No need to login with username and password for the public registry + assert req.get_header("Authorization") is None + return MockHTTPResponse.with_json(200, "OK", body={"token": "public_token"}) + + def private_auth(self, req: Request): + # For the private registry we need to login with username and password + auth_value = req.get_header("Authorization") + + if ( + auth_value is None + or not auth_value.startswith("Basic ") + or base64.b64decode(auth_value[6:]) != b"user:pass" + ): + return MockHTTPResponse(401, "Unauthorized") + + return MockHTTPResponse.with_json(200, "OK", body={"token": "private_token"}) + + +def create_opener(*servers: DummyServer, credentials_provider=None): + """Creates a mock opener, that can be used to fake requests to a list + of servers.""" + opener = urllib.request.OpenerDirector() + handler = DummyServerUrllibHandler() + for server in servers: + handler.add_server(server.domain, server) + opener.add_handler(handler) + opener.add_handler(urllib.request.HTTPDefaultErrorHandler()) + opener.add_handler(urllib.request.HTTPErrorProcessor()) + if credentials_provider is not None: + opener.add_handler(OCIAuthHandler(credentials_provider)) + return opener diff --git a/lib/spack/spack/test/oci/urlopen.py b/lib/spack/spack/test/oci/urlopen.py new file mode 100644 index 00000000000000..16efdfe12d9673 --- /dev/null +++ b/lib/spack/spack/test/oci/urlopen.py @@ -0,0 +1,672 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +import hashlib +import json +import urllib.error +import urllib.parse +import urllib.request +from urllib.request import Request + +import pytest + +import spack.mirror +from spack.oci.image import Digest, ImageReference, default_config, default_manifest +from spack.oci.oci import ( + copy_missing_layers, + get_manifest_and_config, + image_from_mirror, + upload_blob, + upload_manifest, +) +from spack.oci.opener import ( + Challenge, + RealmServiceScope, + UsernamePassword, + credentials_from_mirrors, + default_retry, + get_bearer_challenge, + parse_www_authenticate, +) +from spack.test.oci.mock_registry import ( + DummyServer, + DummyServerUrllibHandler, + InMemoryOCIRegistry, + InMemoryOCIRegistryWithAuth, + MiddlewareError, + MockBearerTokenServer, + MockHTTPResponse, + create_opener, +) + + +def test_parse_www_authenticate(): + """Test parsing of valid WWW-Authenticate header, check whether it's + decomposed into a list of challenges with correct scheme and parameters + according to RFC 7235 section 4.1""" + www_authenticate = 'Bearer realm="https://spack.io/authenticate",service="spack-registry",scope="repository:spack-registry:pull,push"' + assert parse_www_authenticate(www_authenticate) == [ + Challenge( + "Bearer", + [ + ("realm", "https://spack.io/authenticate"), + ("service", "spack-registry"), + ("scope", "repository:spack-registry:pull,push"), + ], + ) + ] + + assert parse_www_authenticate("Bearer") == [Challenge("Bearer")] + assert parse_www_authenticate("MethodA, MethodB,MethodC") == [ + Challenge("MethodA"), + Challenge("MethodB"), + Challenge("MethodC"), + ] + + assert parse_www_authenticate( + 'Digest realm="Digest Realm", nonce="1234567890", algorithm=MD5, qop="auth"' + ) == [ + Challenge( + "Digest", + [ + ("realm", "Digest Realm"), + ("nonce", "1234567890"), + ("algorithm", "MD5"), + ("qop", "auth"), + ], + ) + ] + + assert parse_www_authenticate( + r'Newauth realm="apps", type=1, title="Login to \"apps\"", Basic realm="simple"' + ) == [ + Challenge("Newauth", [("realm", "apps"), ("type", "1"), ("title", 'Login to "apps"')]), + Challenge("Basic", [("realm", "simple")]), + ] + + +@pytest.mark.parametrize( + "invalid_str", + [ + # Not comma separated + "SchemeA SchemeB SchemeC", + # Unexpected eof + "SchemeA, SchemeB, SchemeC, ", + # Invalid auth param or scheme + r"Scheme x=y, ", + # Unexpected eof + "Scheme key=", + # Invalid token + r'"Bearer"', + # Invalid token + r'Scheme"xyz"', + # No auth param + r"Scheme ", + ], +) +def test_invalid_www_authenticate(invalid_str): + with pytest.raises(ValueError): + parse_www_authenticate(invalid_str) + + +def test_get_bearer_challenge(): + """Test extracting Bearer challenge from a list of challenges""" + + # Only an incomplete bearer challenge, missing service and scope, not usable. + assert ( + get_bearer_challenge( + [ + Challenge("Bearer", [("realm", "https://spack.io/authenticate")]), + Challenge("Basic", [("realm", "simple")]), + Challenge( + "Digest", + [ + ("realm", "Digest Realm"), + ("nonce", "1234567890"), + ("algorithm", "MD5"), + ("qop", "auth"), + ], + ), + ] + ) + is None + ) + + # Multiple challenges, should pick the bearer one. + assert get_bearer_challenge( + [ + Challenge( + "Dummy", + [("realm", "https://example.com/"), ("service", "service"), ("scope", "scope")], + ), + Challenge( + "Bearer", + [ + ("realm", "https://spack.io/authenticate"), + ("service", "spack-registry"), + ("scope", "repository:spack-registry:pull,push"), + ], + ), + ] + ) == RealmServiceScope( + "https://spack.io/authenticate", "spack-registry", "repository:spack-registry:pull,push" + ) + + +@pytest.mark.parametrize( + "image_ref,token", + [ + ("public.example.com/spack-registry:latest", "public_token"), + ("private.example.com/spack-registry:latest", "private_token"), + ], +) +def test_automatic_oci_authentication(image_ref, token): + image = ImageReference.from_string(image_ref) + + def credentials_provider(domain: str): + return UsernamePassword("user", "pass") if domain == "private.example.com" else None + + opener = create_opener( + InMemoryOCIRegistryWithAuth( + image.domain, token=token, realm="https://auth.example.com/login" + ), + MockBearerTokenServer("auth.example.com"), + credentials_provider=credentials_provider, + ) + + # Run this twice, as it will triggers a code path that caches the bearer token + assert opener.open(image.endpoint()).status == 200 + assert opener.open(image.endpoint()).status == 200 + + +def test_wrong_credentials(): + """Test that when wrong credentials are rejected by the auth server, we + get a 401 error.""" + credentials_provider = lambda domain: UsernamePassword("wrong", "wrong") + image = ImageReference.from_string("private.example.com/image") + opener = create_opener( + InMemoryOCIRegistryWithAuth( + image.domain, token="something", realm="https://auth.example.com/login" + ), + MockBearerTokenServer("auth.example.com"), + credentials_provider=credentials_provider, + ) + + with pytest.raises(urllib.error.HTTPError) as e: + opener.open(image.endpoint()) + + assert e.value.getcode() == 401 + + +def test_wrong_bearer_token_returned_by_auth_server(): + """When the auth server returns a wrong bearer token, we should get a 401 error + when the request we attempt fails. We shouldn't go in circles getting a 401 from + the registry, then a non-working token from the auth server, then a 401 from the + registry, etc.""" + image = ImageReference.from_string("private.example.com/image") + opener = create_opener( + InMemoryOCIRegistryWithAuth( + image.domain, + token="other_token_than_token_server_provides", + realm="https://auth.example.com/login", + ), + MockBearerTokenServer("auth.example.com"), + credentials_provider=lambda domain: UsernamePassword("user", "pass"), + ) + + with pytest.raises(urllib.error.HTTPError) as e: + opener.open(image.endpoint()) + + assert e.value.getcode() == 401 + + +class TrivialAuthServer(DummyServer): + """A trivial auth server that hands out a bearer token at GET /login.""" + + def __init__(self, domain: str, token: str) -> None: + super().__init__(domain) + self.router.register("GET", "/login", self.login) + self.token = token + + def login(self, req: Request): + return MockHTTPResponse.with_json(200, "OK", body={"token": self.token}) + + +def test_registry_with_short_lived_bearer_tokens(): + """An issued bearer token is mostly opaque to the client, but typically + it embeds a short-lived expiration date. To speed up requests to a registry, + it's good not to authenticate on every request, but to cache the bearer token, + however: we have to deal with the case of an expired bearer token. + + Here we test that when the bearer token expires, we authenticate again, and + when the token is still valid, we don't re-authenticate.""" + + image = ImageReference.from_string("private.example.com/image") + credentials_provider = lambda domain: UsernamePassword("user", "pass") + + auth_server = TrivialAuthServer("auth.example.com", token="token") + registry_server = InMemoryOCIRegistryWithAuth( + image.domain, token="token", realm="https://auth.example.com/login" + ) + urlopen = create_opener( + registry_server, auth_server, credentials_provider=credentials_provider + ).open + + # First request, should work with token "token" + assert urlopen(image.endpoint()).status == 200 + + # Invalidate the token on the registry + registry_server.token = "new_token" + auth_server.token = "new_token" + + # Second request: reusing the cached token should fail + # but in the background we will get a new token from the auth server + assert urlopen(image.endpoint()).status == 200 + + # Subsequent requests should work with the same token, let's do two more + assert urlopen(image.endpoint()).status == 200 + assert urlopen(image.endpoint()).status == 200 + + # And finally, we should see that we've issues exactly two requests to the auth server + assert auth_server.requests == [("GET", "/login"), ("GET", "/login")] + + # Whereas we've done more requests to the registry + assert registry_server.requests == [ + ("GET", "/v2/"), # 1: without bearer token + ("GET", "/v2/"), # 2: retry with bearer token + ("GET", "/v2/"), # 3: with incorrect bearer token + ("GET", "/v2/"), # 4: retry with new bearer token + ("GET", "/v2/"), # 5: with recyled correct bearer token + ("GET", "/v2/"), # 6: with recyled correct bearer token + ] + + +class InMemoryRegistryWithUnsupportedAuth(InMemoryOCIRegistry): + """A registry that does set a WWW-Authenticate header, but + with a challenge we don't support.""" + + def __init__(self, domain: str, allow_single_post: bool = True, www_authenticate=None) -> None: + self.www_authenticate = www_authenticate + super().__init__(domain, allow_single_post) + self.router.add_middleware(self.unsupported_auth_method) + + def unsupported_auth_method(self, req: Request): + headers = {} + if self.www_authenticate: + headers["WWW-Authenticate"] = self.www_authenticate + raise MiddlewareError(MockHTTPResponse(401, "Unauthorized", headers=headers)) + + +@pytest.mark.parametrize( + "www_authenticate,error_message", + [ + # missing service and scope + ('Bearer realm="https://auth.example.com/login"', "unsupported authentication scheme"), + # we don't do basic auth + ('Basic realm="https://auth.example.com/login"', "unsupported authentication scheme"), + # multiple unsupported challenges + ( + "CustomChallenge method=unsupported, OtherChallenge method=x,param=y", + "unsupported authentication scheme", + ), + # no challenge + (None, "missing WWW-Authenticate header"), + # malformed challenge, missing quotes + ("Bearer realm=https://auth.example.com", "malformed WWW-Authenticate header"), + # http instead of https + ('Bearer realm="http://auth.example.com",scope=x,service=y', "insecure http connection"), + ], +) +def test_auth_method_we_cannot_handle_is_error(www_authenticate, error_message): + # We can only handle WWW-Authenticate with a Bearer challenge + image = ImageReference.from_string("private.example.com/image") + urlopen = create_opener( + InMemoryRegistryWithUnsupportedAuth(image.domain, www_authenticate=www_authenticate), + TrivialAuthServer("auth.example.com", token="token"), + credentials_provider=lambda domain: UsernamePassword("user", "pass"), + ).open + + with pytest.raises(urllib.error.HTTPError, match=error_message) as e: + urlopen(image.endpoint()) + assert e.value.getcode() == 401 + + +# Parametrize over single POST vs POST + PUT. +@pytest.mark.parametrize("client_single_request", [True, False]) +@pytest.mark.parametrize("server_single_request", [True, False]) +def test_oci_registry_upload(tmpdir, client_single_request, server_single_request): + opener = urllib.request.OpenerDirector() + opener.add_handler( + DummyServerUrllibHandler().add_server( + "example.com", InMemoryOCIRegistry(server_single_request) + ) + ) + opener.add_handler(urllib.request.HTTPDefaultErrorHandler()) + opener.add_handler(urllib.request.HTTPErrorProcessor()) + + # Create a small blob + blob = tmpdir.join("blob") + blob.write("Hello world!") + + image = ImageReference.from_string("example.com/image:latest") + digest = Digest.from_sha256(hashlib.sha256(blob.read_binary()).hexdigest()) + + # Set small file size larger than the blob iff we're doing single request + small_file_size = 1024 if client_single_request else 0 + + # Upload once, should actually upload + assert upload_blob( + ref=image, + file=blob.strpath, + digest=digest, + small_file_size=small_file_size, + _urlopen=opener.open, + ) + + # Second time should exit as it exists + assert not upload_blob( + ref=image, + file=blob.strpath, + digest=digest, + small_file_size=small_file_size, + _urlopen=opener.open, + ) + + # Force upload should upload again + assert upload_blob( + ref=image, + file=blob.strpath, + digest=digest, + force=True, + small_file_size=small_file_size, + _urlopen=opener.open, + ) + + +def test_copy_missing_layers(tmpdir, config): + """Test copying layers from one registry to another. + Creates 3 blobs, 1 config and 1 manifest in registry A + and copies layers to registry B. Then checks that all + layers are present in registry B. Finally it runs the copy + again and checks that no new layers are uploaded.""" + + # NOTE: config fixture is used to disable default source mirrors + # which are used in Stage(...). Otherwise this test doesn't really + # rely on globals. + + src = ImageReference.from_string("a.example.com/image:x") + dst = ImageReference.from_string("b.example.com/image:y") + + src_registry = InMemoryOCIRegistry(src.domain) + dst_registry = InMemoryOCIRegistry(dst.domain) + + urlopen = create_opener(src_registry, dst_registry).open + + # TODO: make it a bit easier to create bunch of blobs + config + manifest? + + # Create a few blobs and a config file + blobs = [tmpdir.join(f"blob{i}") for i in range(3)] + + for i, blob in enumerate(blobs): + blob.write(f"Blob {i}") + + digests = [ + Digest.from_sha256(hashlib.sha256(blob.read_binary()).hexdigest()) for blob in blobs + ] + + config = default_config(architecture="amd64", os="linux") + configfile = tmpdir.join("config.json") + configfile.write(json.dumps(config)) + config_digest = Digest.from_sha256(hashlib.sha256(configfile.read_binary()).hexdigest()) + + for blob, digest in zip(blobs, digests): + upload_blob(src, blob.strpath, digest, _urlopen=urlopen) + upload_blob(src, configfile.strpath, config_digest, _urlopen=urlopen) + + # Then create a manifest referencing them + manifest = default_manifest() + + for blob, digest in zip(blobs, digests): + manifest["layers"].append( + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": str(digest), + "size": blob.size(), + } + ) + + manifest["config"] = { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": str(config_digest), + "size": configfile.size(), + } + + upload_manifest(src, manifest, _urlopen=urlopen) + + # Finally, copy the image from src to dst + copy_missing_layers(src, dst, architecture="amd64", _urlopen=urlopen) + + # Check that all layers (not config) were copied and identical + assert len(dst_registry.blobs) == len(blobs) + for blob, digest in zip(blobs, digests): + assert dst_registry.blobs.get(str(digest)) == blob.read_binary() + + is_upload = lambda method, path: method == "POST" and path == "/v2/image/blobs/uploads/" + is_exists = lambda method, path: method == "HEAD" and path.startswith("/v2/image/blobs/") + + # Check that exactly 3 uploads were initiated, and that we don't do + # double existence checks when uploading. + assert sum(is_upload(method, path) for method, path in dst_registry.requests) == 3 + assert sum(is_exists(method, path) for method, path in dst_registry.requests) == 3 + + # Check that re-uploading skips existing layers. + dst_registry.clear_log() + copy_missing_layers(src, dst, architecture="amd64", _urlopen=urlopen) + + # Check that no uploads were initiated, only existence checks were done. + assert sum(is_upload(method, path) for method, path in dst_registry.requests) == 0 + assert sum(is_exists(method, path) for method, path in dst_registry.requests) == 3 + + +def test_image_from_mirror(): + mirror = spack.mirror.Mirror("oci://example.com/image") + assert image_from_mirror(mirror) == ImageReference.from_string("example.com/image") + + +def test_image_reference_str(): + """Test that with_digest() works with Digest and str.""" + digest_str = f"sha256:{1234:064x}" + digest = Digest.from_string(digest_str) + + img = ImageReference.from_string("example.com/image") + + assert str(img.with_digest(digest)) == f"example.com/image:latest@{digest}" + assert str(img.with_digest(digest_str)) == f"example.com/image:latest@{digest}" + assert str(img.with_tag("hello")) == "example.com/image:hello" + assert str(img.with_tag("hello").with_digest(digest)) == f"example.com/image:hello@{digest}" + + +@pytest.mark.parametrize( + "image", + [ + # white space issue + " example.com/image", + # not alpha-numeric + "hello#world:latest", + ], +) +def test_image_reference_invalid(image): + with pytest.raises(ValueError, match="Invalid image reference"): + ImageReference.from_string(image) + + +def test_default_credentials_provider(): + """The default credentials provider uses a collection of configured + mirrors.""" + + mirrors = [ + # OCI mirror with push credentials + spack.mirror.Mirror( + {"url": "oci://a.example.com/image", "push": {"access_pair": ["user.a", "pass.a"]}} + ), + # Not an OCI mirror + spack.mirror.Mirror( + {"url": "https://b.example.com/image", "access_pair": ["user.b", "pass.b"]} + ), + # No credentials + spack.mirror.Mirror("oci://c.example.com/image"), + # Top-level credentials + spack.mirror.Mirror( + {"url": "oci://d.example.com/image", "access_pair": ["user.d", "pass.d"]} + ), + # Dockerhub short reference + spack.mirror.Mirror( + {"url": "oci://user/image", "access_pair": ["dockerhub_user", "dockerhub_pass"]} + ), + # Localhost (not a dockerhub short reference) + spack.mirror.Mirror( + {"url": "oci://localhost/image", "access_pair": ["user.localhost", "pass.localhost"]} + ), + ] + + assert credentials_from_mirrors("a.example.com", mirrors=mirrors) == UsernamePassword( + "user.a", "pass.a" + ) + assert credentials_from_mirrors("b.example.com", mirrors=mirrors) is None + assert credentials_from_mirrors("c.example.com", mirrors=mirrors) is None + assert credentials_from_mirrors("d.example.com", mirrors=mirrors) == UsernamePassword( + "user.d", "pass.d" + ) + assert credentials_from_mirrors("index.docker.io", mirrors=mirrors) == UsernamePassword( + "dockerhub_user", "dockerhub_pass" + ) + assert credentials_from_mirrors("localhost", mirrors=mirrors) == UsernamePassword( + "user.localhost", "pass.localhost" + ) + + +def test_manifest_index(tmpdir): + """Test obtaining manifest + config from a registry + that has an index""" + urlopen = create_opener(InMemoryOCIRegistry("registry.example.com")).open + + img = ImageReference.from_string("registry.example.com/image") + + # Create two config files and manifests, for different architectures + manifest_descriptors = [] + manifest_and_config = {} + for arch in ("amd64", "arm64"): + file = tmpdir.join(f"config_{arch}.json") + config = default_config(architecture=arch, os="linux") + file.write(json.dumps(config)) + config_digest = Digest.from_sha256(hashlib.sha256(file.read_binary()).hexdigest()) + assert upload_blob(img, file, config_digest, _urlopen=urlopen) + manifest = { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": str(config_digest), + "size": file.size(), + }, + "layers": [], + } + manifest_digest, manifest_size = upload_manifest( + img, manifest, tag=False, _urlopen=urlopen + ) + + manifest_descriptors.append( + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "platform": {"architecture": arch, "os": "linux"}, + "digest": str(manifest_digest), + "size": manifest_size, + } + ) + + manifest_and_config[arch] = (manifest, config) + + # And a single index. + index = { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": manifest_descriptors, + } + + upload_manifest(img, index, tag=True, _urlopen=urlopen) + + # Check that we fetcht the correct manifest and config for each architecture + for arch in ("amd64", "arm64"): + assert ( + get_manifest_and_config(img, architecture=arch, _urlopen=urlopen) + == manifest_and_config[arch] + ) + + # Also test max recursion + with pytest.raises(Exception, match="Maximum recursion depth reached"): + get_manifest_and_config(img, architecture="amd64", recurse=0, _urlopen=urlopen) + + +class BrokenServer(DummyServer): + """Dummy server that returns 500 and 429 errors twice before succeeding""" + + def __init__(self, domain: str) -> None: + super().__init__(domain) + self.router.register("GET", r"/internal-server-error/", self.internal_server_error_twice) + self.router.register("GET", r"/rate-limit/", self.rate_limit_twice) + self.router.register("GET", r"/not-found/", self.not_found) + self.count_500 = 0 + self.count_429 = 0 + + def internal_server_error_twice(self, request: Request): + self.count_500 += 1 + if self.count_500 < 3: + return MockHTTPResponse(500, "Internal Server Error") + else: + return MockHTTPResponse(200, "OK") + + def rate_limit_twice(self, request: Request): + self.count_429 += 1 + if self.count_429 < 3: + return MockHTTPResponse(429, "Rate Limit Exceeded") + else: + return MockHTTPResponse(200, "OK") + + def not_found(self, request: Request): + return MockHTTPResponse(404, "Not Found") + + +@pytest.mark.parametrize( + "url,max_retries,expect_failure,expect_requests", + [ + # 500s should be retried + ("https://example.com/internal-server-error/", 2, True, 2), + ("https://example.com/internal-server-error/", 5, False, 3), + # 429s should be retried + ("https://example.com/rate-limit/", 2, True, 2), + ("https://example.com/rate-limit/", 5, False, 3), + # 404s shouldn't be retried + ("https://example.com/not-found/", 3, True, 1), + ], +) +def test_retry(url, max_retries, expect_failure, expect_requests): + server = BrokenServer("example.com") + urlopen = create_opener(server).open + sleep_time = [] + dont_sleep = lambda t: sleep_time.append(t) # keep track of sleep times + + try: + response = default_retry(urlopen, retries=max_retries, sleep=dont_sleep)(url) + except urllib.error.HTTPError as e: + if not expect_failure: + assert False, f"Unexpected HTTPError: {e}" + else: + if expect_failure: + assert False, "Expected HTTPError, but none was raised" + assert response.status == 200 + + assert len(server.requests) == expect_requests + assert sleep_time == [2**i for i in range(expect_requests - 1)] diff --git a/lib/spack/spack/test/package_class.py b/lib/spack/spack/test/package_class.py index a9df2750d4b117..279693a529b81b 100644 --- a/lib/spack/spack/test/package_class.py +++ b/lib/spack/spack/test/package_class.py @@ -17,6 +17,7 @@ import llnl.util.filesystem as fs +import spack.deptypes as dt import spack.install_test import spack.package_base import spack.repo @@ -36,6 +37,7 @@ def mpileaks_possible_deps(mock_packages, mpi_names): "low-priority-provider": set(), "dyninst": set(["libdwarf", "libelf"]), "fake": set(), + "intel-parallel-studio": set(), "libdwarf": set(["libelf"]), "libelf": set(), "mpich": set(), @@ -48,7 +50,7 @@ def mpileaks_possible_deps(mock_packages, mpi_names): def test_possible_dependencies(mock_packages, mpileaks_possible_deps): - pkg_cls = spack.repo.path.get_pkg_class("mpileaks") + pkg_cls = spack.repo.PATH.get_pkg_class("mpileaks") expanded_possible_deps = pkg_cls.possible_dependencies(expand_virtuals=True) assert mpileaks_possible_deps == expanded_possible_deps assert { @@ -62,14 +64,14 @@ def test_possible_dependencies(mock_packages, mpileaks_possible_deps): def test_possible_direct_dependencies(mock_packages, mpileaks_possible_deps): - pkg_cls = spack.repo.path.get_pkg_class("mpileaks") + pkg_cls = spack.repo.PATH.get_pkg_class("mpileaks") deps = pkg_cls.possible_dependencies(transitive=False, expand_virtuals=False) assert {"callpath": set(), "mpi": set(), "mpileaks": {"callpath", "mpi"}} == deps def test_possible_dependencies_virtual(mock_packages, mpi_names): expected = dict( - (name, set(spack.repo.path.get_pkg_class(name).dependencies)) for name in mpi_names + (name, set(spack.repo.PATH.get_pkg_class(name).dependencies)) for name in mpi_names ) # only one mock MPI has a dependency @@ -79,29 +81,29 @@ def test_possible_dependencies_virtual(mock_packages, mpi_names): def test_possible_dependencies_missing(mock_packages): - pkg_cls = spack.repo.path.get_pkg_class("missing-dependency") + pkg_cls = spack.repo.PATH.get_pkg_class("missing-dependency") missing = {} pkg_cls.possible_dependencies(transitive=True, missing=missing) assert {"this-is-a-missing-dependency"} == missing["missing-dependency"] def test_possible_dependencies_with_deptypes(mock_packages): - dtbuild1 = spack.repo.path.get_pkg_class("dtbuild1") + dtbuild1 = spack.repo.PATH.get_pkg_class("dtbuild1") assert { "dtbuild1": {"dtrun2", "dtlink2"}, "dtlink2": set(), "dtrun2": set(), - } == dtbuild1.possible_dependencies(deptype=("link", "run")) + } == dtbuild1.possible_dependencies(depflag=dt.LINK | dt.RUN) assert { "dtbuild1": {"dtbuild2", "dtlink2"}, "dtbuild2": set(), "dtlink2": set(), - } == dtbuild1.possible_dependencies(deptype=("build")) + } == dtbuild1.possible_dependencies(depflag=dt.BUILD) assert {"dtbuild1": {"dtlink2"}, "dtlink2": set()} == dtbuild1.possible_dependencies( - deptype=("link") + depflag=dt.LINK ) diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 3fb4a169f545e5..4a5831dce707fc 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -18,17 +18,17 @@ def pkg_factory(name): """Return a package object tied to an abstract spec""" - pkg_cls = spack.repo.path.get_pkg_class(name) + pkg_cls = spack.repo.PATH.get_pkg_class(name) return pkg_cls(Spec(name)) @pytest.mark.usefixtures("config", "mock_packages") class TestPackage: def test_load_package(self): - spack.repo.path.get_pkg_class("mpich") + spack.repo.PATH.get_pkg_class("mpich") def test_package_name(self): - pkg_cls = spack.repo.path.get_pkg_class("mpich") + pkg_cls = spack.repo.PATH.get_pkg_class("mpich") assert pkg_cls.name == "mpich" def test_package_filename(self): @@ -62,7 +62,7 @@ def test_import_package_as(self): from spack.pkg.builtin import mock # noqa: F401 def test_inheritance_of_diretives(self): - pkg_cls = spack.repo.path.get_pkg_class("simple-inheritance") + pkg_cls = spack.repo.PATH.get_pkg_class("simple-inheritance") # Check dictionaries that should have been filled by directives assert len(pkg_cls.dependencies) == 3 @@ -125,7 +125,7 @@ def test_urls_for_versions(mock_packages, config): def test_url_for_version_with_no_urls(mock_packages, config): spec = Spec("git-test") - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) with pytest.raises(spack.package_base.NoURLError): pkg_cls(spec).url_for_version("1.0") @@ -314,7 +314,7 @@ def test_fetch_options(version_str, digest_end, extra_options): def test_package_deprecated_version(mock_packages, mock_fetch, mock_stage): spec = Spec("deprecated-versions") - pkg_cls = spack.repo.path.get_pkg_class(spec.name) + pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) assert spack.package_base.deprecated_version(pkg_cls, "1.1.0") assert not spack.package_base.deprecated_version(pkg_cls, "1.0.0") diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index 43193b578c52d0..bd49cf94aee863 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -8,28 +8,27 @@ """ import argparse import os +import pathlib import platform import shutil -import stat -import sys from collections import OrderedDict import pytest -from llnl.util.filesystem import mkdirp +from llnl.util import filesystem as fs from llnl.util.symlink import symlink import spack.binary_distribution as bindist import spack.cmd.buildcache as buildcache +import spack.error import spack.package_base import spack.repo import spack.store import spack.util.gpg import spack.util.url as url_util -from spack.fetch_strategy import FetchStrategyComposite, URLFetchStrategy +from spack.fetch_strategy import URLFetchStrategy from spack.paths import mock_gpg_keys_path from spack.relocate import ( - ensure_binary_is_relocatable, macho_find_paths, macho_make_paths_normal, macho_make_paths_relative, @@ -40,147 +39,108 @@ ) from spack.spec import Spec -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") - - -def fake_fetchify(url, pkg): - """Fake the URL for a package so it downloads from a file.""" - fetcher = FetchStrategyComposite() - fetcher.append(URLFetchStrategy(url)) - pkg.fetcher = fetcher +pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.mark.usefixtures("install_mockery", "mock_gnupghome") -def test_buildcache(mock_archive, tmpdir): - # tweak patchelf to only do a download - pspec = Spec("patchelf").concretized() - pkg = pspec.package - fake_fetchify(pkg.fetcher, pkg) - mkdirp(os.path.join(pkg.prefix, "bin")) - patchelfscr = os.path.join(pkg.prefix, "bin", "patchelf") - f = open(patchelfscr, "w") - body = """#!/bin/bash -echo $PATH""" - f.write(body) - f.close() - st = os.stat(patchelfscr) - os.chmod(patchelfscr, st.st_mode | stat.S_IEXEC) - - # Install the test package - spec = Spec("trivial-install-test-package") - spec.concretize() - assert spec.concrete - pkg = spec.package - fake_fetchify(mock_archive.url, pkg) - pkg.do_install() +def test_buildcache(mock_archive, tmp_path, monkeypatch, mutable_config): + # Install a test package + spec = Spec("trivial-install-test-package").concretized() + monkeypatch.setattr(spec.package, "fetcher", URLFetchStrategy(mock_archive.url)) + spec.package.do_install() pkghash = "/" + str(spec.dag_hash(7)) # Put some non-relocatable file in there - filename = os.path.join(spec.prefix, "dummy.txt") - with open(filename, "w") as script: - script.write(spec.prefix) + dummy_txt = pathlib.Path(spec.prefix) / "dummy.txt" + dummy_txt.write_text(spec.prefix) # Create an absolute symlink linkname = os.path.join(spec.prefix, "link_to_dummy.txt") - symlink(filename, linkname) + symlink(dummy_txt, linkname) - # Create the build cache and - # put it directly into the mirror - mirror_path = os.path.join(str(tmpdir), "test-mirror") + # Create the build cache and put it directly into the mirror + mirror_path = str(tmp_path / "test-mirror") spack.mirror.create(mirror_path, specs=[]) # register mirror with spack config mirrors = {"spack-mirror-test": url_util.path_to_file_url(mirror_path)} spack.config.set("mirrors", mirrors) - stage = spack.stage.Stage(mirrors["spack-mirror-test"], name="build_cache", keep=True) - stage.create() - - # setup argument parser - parser = argparse.ArgumentParser() - buildcache.setup_parser(parser) - - create_args = ["create", "-a", "-f", mirror_path, pkghash] - # Create a private key to sign package with if gpg2 available - spack.util.gpg.create( - name="test key 1", expires="0", email="spack@googlegroups.com", comment="Spack test key" - ) + with spack.stage.Stage(mirrors["spack-mirror-test"], name="build_cache", keep=True): + parser = argparse.ArgumentParser() + buildcache.setup_parser(parser) + + create_args = ["create", "-f", "--rebuild-index", mirror_path, pkghash] + # Create a private key to sign package with if gpg2 available + spack.util.gpg.create( + name="test key 1", + expires="0", + email="spack@googlegroups.com", + comment="Spack test key", + ) - create_args.insert(create_args.index("-a"), "--rebuild-index") + args = parser.parse_args(create_args) + buildcache.buildcache(parser, args) + # trigger overwrite warning + buildcache.buildcache(parser, args) - args = parser.parse_args(create_args) - buildcache.buildcache(parser, args) - # trigger overwrite warning - buildcache.buildcache(parser, args) + # Uninstall the package + spec.package.do_uninstall(force=True) - # Uninstall the package - pkg.do_uninstall(force=True) + install_args = ["install", "-f", pkghash] + args = parser.parse_args(install_args) + # Test install + buildcache.buildcache(parser, args) - install_args = ["install", "-f", pkghash] - args = parser.parse_args(install_args) - # Test install - buildcache.buildcache(parser, args) + files = os.listdir(spec.prefix) - files = os.listdir(spec.prefix) + assert "link_to_dummy.txt" in files + assert "dummy.txt" in files - assert "link_to_dummy.txt" in files - assert "dummy.txt" in files + # Validate the relocation information + buildinfo = bindist.read_buildinfo_file(spec.prefix) + assert buildinfo["relocate_textfiles"] == ["dummy.txt"] + assert buildinfo["relocate_links"] == ["link_to_dummy.txt"] - # Validate the relocation information - buildinfo = bindist.read_buildinfo_file(spec.prefix) - assert buildinfo["relocate_textfiles"] == ["dummy.txt"] - assert buildinfo["relocate_links"] == ["link_to_dummy.txt"] + args = parser.parse_args(["keys"]) + buildcache.buildcache(parser, args) - args = parser.parse_args(["keys"]) - buildcache.buildcache(parser, args) + args = parser.parse_args(["list"]) + buildcache.buildcache(parser, args) - args = parser.parse_args(["list"]) - buildcache.buildcache(parser, args) + args = parser.parse_args(["list"]) + buildcache.buildcache(parser, args) - args = parser.parse_args(["list"]) - buildcache.buildcache(parser, args) + args = parser.parse_args(["list", "trivial"]) + buildcache.buildcache(parser, args) - args = parser.parse_args(["list", "trivial"]) - buildcache.buildcache(parser, args) + # Copy a key to the mirror to have something to download + shutil.copyfile(mock_gpg_keys_path + "/external.key", mirror_path + "/external.key") - # Copy a key to the mirror to have something to download - shutil.copyfile(mock_gpg_keys_path + "/external.key", mirror_path + "/external.key") + args = parser.parse_args(["keys"]) + buildcache.buildcache(parser, args) - args = parser.parse_args(["keys"]) - buildcache.buildcache(parser, args) + args = parser.parse_args(["keys", "-f"]) + buildcache.buildcache(parser, args) - args = parser.parse_args(["keys", "-f"]) - buildcache.buildcache(parser, args) + args = parser.parse_args(["keys", "-i", "-t"]) + buildcache.buildcache(parser, args) - args = parser.parse_args(["keys", "-i", "-t"]) - buildcache.buildcache(parser, args) - # unregister mirror with spack config - mirrors = {} - spack.config.set("mirrors", mirrors) - shutil.rmtree(mirror_path) - stage.destroy() +def test_relocate_text(tmp_path): + """Tests that a text file containing the original directory of an installation, can be + relocated to a target directory. + """ + original_dir = "/home/spack/opt/spack" + relocation_dir = "/opt/rh/devtoolset/" + dummy_txt = tmp_path / "dummy.txt" + dummy_txt.write_text(original_dir) + relocate_text([str(dummy_txt)], {original_dir: relocation_dir}) + text = dummy_txt.read_text() -@pytest.mark.usefixtures("install_mockery") -def test_relocate_text(tmpdir): - spec = Spec("trivial-install-test-package") - spec.concretize() - with tmpdir.as_cwd(): - # Validate the text path replacement - old_dir = "/home/spack/opt/spack" - filename = "dummy.txt" - with open(filename, "w") as script: - script.write(old_dir) - script.close() - filenames = [filename] - new_dir = "/opt/rh/devtoolset/" - # Singleton dict doesn't matter if Ordered - relocate_text(filenames, {old_dir: new_dir}) - with open(filename, "r") as script: - for line in script: - assert new_dir in line - ensure_binary_is_relocatable(os.path.realpath(filename)) + assert relocation_dir in text + assert original_dir not in text def test_relocate_links(tmpdir): @@ -188,8 +148,15 @@ def test_relocate_links(tmpdir): own_prefix_path = str(tmpdir.join("prefix_a", "file")) dep_prefix_path = str(tmpdir.join("prefix_b", "file")) + new_own_prefix_path = str(tmpdir.join("new_prefix_a", "file")) + new_dep_prefix_path = str(tmpdir.join("new_prefix_b", "file")) system_path = os.path.join(os.path.sep, "system", "path") + fs.touchp(own_prefix_path) + fs.touchp(new_own_prefix_path) + fs.touchp(dep_prefix_path) + fs.touchp(new_dep_prefix_path) + # Old prefixes to new prefixes prefix_to_prefix = OrderedDict( [ @@ -229,7 +196,6 @@ def test_needs_relocation(): assert not needs_binary_relocation("text", "x-") assert needs_text_relocation("text", "x-") assert not needs_text_relocation("symbolic link to", "x-") - assert needs_binary_relocation("application", "x-mach-binary") @@ -242,41 +208,41 @@ def test_replace_paths(tmpdir): hash_d = "hukkosc7ahff7o65h6cdhvcoxm57d4bw" hash_loco = "zy4oigsc4eovn5yhr2lk4aukwzoespob" - prefix2hash = dict() + prefix2hash = {} - old_spack_dir = os.path.join("%s" % tmpdir, "Users", "developer", "spack") - mkdirp(old_spack_dir) + old_spack_dir = os.path.join(f"{tmpdir}", "Users", "developer", "spack") + fs.mkdirp(old_spack_dir) - oldprefix_a = os.path.join("%s" % old_spack_dir, "pkgA-%s" % hash_a) - oldlibdir_a = os.path.join("%s" % oldprefix_a, "lib") - mkdirp(oldlibdir_a) + oldprefix_a = os.path.join(f"{old_spack_dir}", f"pkgA-{hash_a}") + oldlibdir_a = os.path.join(f"{oldprefix_a}", "lib") + fs.mkdirp(oldlibdir_a) prefix2hash[str(oldprefix_a)] = hash_a - oldprefix_b = os.path.join("%s" % old_spack_dir, "pkgB-%s" % hash_b) - oldlibdir_b = os.path.join("%s" % oldprefix_b, "lib") - mkdirp(oldlibdir_b) + oldprefix_b = os.path.join(f"{old_spack_dir}", f"pkgB-{hash_b}") + oldlibdir_b = os.path.join(f"{oldprefix_b}", "lib") + fs.mkdirp(oldlibdir_b) prefix2hash[str(oldprefix_b)] = hash_b - oldprefix_c = os.path.join("%s" % old_spack_dir, "pkgC-%s" % hash_c) - oldlibdir_c = os.path.join("%s" % oldprefix_c, "lib") - oldlibdir_cc = os.path.join("%s" % oldlibdir_c, "C") - mkdirp(oldlibdir_c) + oldprefix_c = os.path.join(f"{old_spack_dir}", f"pkgC-{hash_c}") + oldlibdir_c = os.path.join(f"{oldprefix_c}", "lib") + oldlibdir_cc = os.path.join(f"{oldlibdir_c}", "C") + fs.mkdirp(oldlibdir_c) prefix2hash[str(oldprefix_c)] = hash_c - oldprefix_d = os.path.join("%s" % old_spack_dir, "pkgD-%s" % hash_d) - oldlibdir_d = os.path.join("%s" % oldprefix_d, "lib") - mkdirp(oldlibdir_d) + oldprefix_d = os.path.join(f"{old_spack_dir}", f"pkgD-{hash_d}") + oldlibdir_d = os.path.join(f"{oldprefix_d}", "lib") + fs.mkdirp(oldlibdir_d) prefix2hash[str(oldprefix_d)] = hash_d - oldprefix_local = os.path.join("%s" % tmpdir, "usr", "local") - oldlibdir_local = os.path.join("%s" % oldprefix_local, "lib") - mkdirp(oldlibdir_local) + oldprefix_local = os.path.join(f"{tmpdir}", "usr", "local") + oldlibdir_local = os.path.join(f"{oldprefix_local}", "lib") + fs.mkdirp(oldlibdir_local) prefix2hash[str(oldprefix_local)] = hash_loco - libfile_a = "libA.%s" % suffix - libfile_b = "libB.%s" % suffix - libfile_c = "libC.%s" % suffix - libfile_d = "libD.%s" % suffix - libfile_loco = "libloco.%s" % suffix + libfile_a = f"libA.{suffix}" + libfile_b = f"libB.{suffix}" + libfile_c = f"libC.{suffix}" + libfile_d = f"libD.{suffix}" + libfile_loco = f"libloco.{suffix}" old_libnames = [ os.path.join(oldlibdir_a, libfile_a), os.path.join(oldlibdir_b, libfile_b), @@ -291,33 +257,33 @@ def test_replace_paths(tmpdir): hash2prefix = dict() - new_spack_dir = os.path.join("%s" % tmpdir, "Users", "Shared", "spack") - mkdirp(new_spack_dir) + new_spack_dir = os.path.join(f"{tmpdir}", "Users", "Shared", "spack") + fs.mkdirp(new_spack_dir) - prefix_a = os.path.join(new_spack_dir, "pkgA-%s" % hash_a) + prefix_a = os.path.join(new_spack_dir, f"pkgA-{hash_a}") libdir_a = os.path.join(prefix_a, "lib") - mkdirp(libdir_a) + fs.mkdirp(libdir_a) hash2prefix[hash_a] = str(prefix_a) - prefix_b = os.path.join(new_spack_dir, "pkgB-%s" % hash_b) + prefix_b = os.path.join(new_spack_dir, f"pkgB-{hash_b}") libdir_b = os.path.join(prefix_b, "lib") - mkdirp(libdir_b) + fs.mkdirp(libdir_b) hash2prefix[hash_b] = str(prefix_b) - prefix_c = os.path.join(new_spack_dir, "pkgC-%s" % hash_c) + prefix_c = os.path.join(new_spack_dir, f"pkgC-{hash_c}") libdir_c = os.path.join(prefix_c, "lib") libdir_cc = os.path.join(libdir_c, "C") - mkdirp(libdir_cc) + fs.mkdirp(libdir_cc) hash2prefix[hash_c] = str(prefix_c) - prefix_d = os.path.join(new_spack_dir, "pkgD-%s" % hash_d) + prefix_d = os.path.join(new_spack_dir, f"pkgD-{hash_d}") libdir_d = os.path.join(prefix_d, "lib") - mkdirp(libdir_d) + fs.mkdirp(libdir_d) hash2prefix[hash_d] = str(prefix_d) - prefix_local = os.path.join("%s" % tmpdir, "usr", "local") + prefix_local = os.path.join(f"{tmpdir}", "usr", "local") libdir_local = os.path.join(prefix_local, "lib") - mkdirp(libdir_local) + fs.mkdirp(libdir_local) hash2prefix[hash_loco] = str(prefix_local) new_libnames = [ @@ -386,10 +352,10 @@ def test_replace_paths(tmpdir): out_dict = macho_find_paths( [oldlibdir_a, oldlibdir_b, oldlibdir_c, oldlibdir_cc, oldlibdir_local], [ - "@rpath/%s" % libfile_a, - "@rpath/%s" % libfile_b, - "@rpath/%s" % libfile_c, - "@rpath/%s" % libfile_loco, + f"@rpath/{libfile_a}", + f"@rpath/{libfile_b}", + f"@rpath/{libfile_c}", + f"@rpath/{libfile_loco}", ], None, old_spack_dir, @@ -397,10 +363,10 @@ def test_replace_paths(tmpdir): ) assert out_dict == { - "@rpath/%s" % libfile_a: "@rpath/%s" % libfile_a, - "@rpath/%s" % libfile_b: "@rpath/%s" % libfile_b, - "@rpath/%s" % libfile_c: "@rpath/%s" % libfile_c, - "@rpath/%s" % libfile_loco: "@rpath/%s" % libfile_loco, + f"@rpath/{libfile_a}": f"@rpath/{libfile_a}", + f"@rpath/{libfile_b}": f"@rpath/{libfile_b}", + f"@rpath/{libfile_c}": f"@rpath/{libfile_c}", + f"@rpath/{libfile_loco}": f"@rpath/{libfile_loco}", oldlibdir_a: libdir_a, oldlibdir_b: libdir_b, oldlibdir_c: libdir_c, @@ -410,15 +376,15 @@ def test_replace_paths(tmpdir): out_dict = macho_find_paths( [oldlibdir_a, oldlibdir_b, oldlibdir_d, oldlibdir_local], - ["@rpath/%s" % libfile_a, "@rpath/%s" % libfile_b, "@rpath/%s" % libfile_loco], + [f"@rpath/{libfile_a}", f"@rpath/{libfile_b}", f"@rpath/{libfile_loco}"], None, old_spack_dir, prefix2prefix, ) assert out_dict == { - "@rpath/%s" % libfile_a: "@rpath/%s" % libfile_a, - "@rpath/%s" % libfile_b: "@rpath/%s" % libfile_b, - "@rpath/%s" % libfile_loco: "@rpath/%s" % libfile_loco, + f"@rpath/{libfile_a}": f"@rpath/{libfile_a}", + f"@rpath/{libfile_b}": f"@rpath/{libfile_b}", + f"@rpath/{libfile_loco}": f"@rpath/{libfile_loco}", oldlibdir_a: libdir_a, oldlibdir_b: libdir_b, oldlibdir_d: libdir_d, @@ -524,8 +490,7 @@ def fetch(self): "", "This FetchStrategy always fails" ) - fetcher = FetchStrategyComposite() - fetcher.append(FailedDownloadStrategy()) + fetcher = FailedDownloadStrategy() @property def fake_fn(self): @@ -550,18 +515,16 @@ def test_manual_download( @property def _instr(pkg): - return "Download instructions for {0}".format(pkg.spec.name) + return f"Download instructions for {pkg.spec.name}" spec = default_mock_concretization("a") - pkg = spec.package - - pkg.manual_download = manual + spec.package.manual_download = manual if instr: monkeypatch.setattr(spack.package_base.PackageBase, "download_instr", _instr) - expected = pkg.download_instr if manual else "All fetchers failed" - with pytest.raises(spack.util.web.FetchError, match=expected): - pkg.do_fetch() + expected = spec.package.download_instr if manual else "All fetchers failed" + with pytest.raises(spack.error.FetchError, match=expected): + spec.package.do_fetch() @pytest.fixture() @@ -573,9 +536,7 @@ def mirror_id(self): def fetch(self): raise Exception("Sources are fetched but shouldn't have been") - fetcher = FetchStrategyComposite() - fetcher.append(FetchingNotAllowed()) - monkeypatch.setattr(spack.package_base.PackageBase, "fetcher", fetcher) + monkeypatch.setattr(spack.package_base.PackageBase, "fetcher", FetchingNotAllowed()) def test_fetch_without_code_is_noop( diff --git a/lib/spack/spack/test/patch.py b/lib/spack/spack/test/patch.py index 1be3f44b18dc87..bfd918035b6ee2 100644 --- a/lib/spack/spack/test/patch.py +++ b/lib/spack/spack/test/patch.py @@ -71,7 +71,7 @@ def mock_patch_stage(tmpdir_factory, monkeypatch): data_path = os.path.join(spack.paths.test_path, "data", "patch") -@pytest.mark.skipif(sys.platform == "win32", reason="Line ending conflict on Windows") +@pytest.mark.not_on_windows("Line ending conflict on Windows") @pytest.mark.parametrize( "filename, sha256, archive_sha256", [ @@ -115,9 +115,11 @@ def test_url_patch(mock_patch_stage, filename, sha256, archive_sha256, config): """ ) # apply the patch and compare files - patch.fetch() - patch.apply(stage) - patch.clean() + with patch.stage: + patch.stage.create() + patch.stage.fetch() + patch.stage.expand_archive() + patch.apply(stage) with working_dir(stage.source_path): assert filecmp.cmp("foo.txt", "foo-expected.txt") @@ -189,7 +191,7 @@ def test_nested_directives(mock_packages): """Ensure pkg data structures are set up properly by nested directives.""" # this ensures that the patch() directive results were removed # properly from the DirectiveMeta._directives_to_be_executed list - patcher = spack.repo.path.get_pkg_class("patch-several-dependencies") + patcher = spack.repo.PATH.get_pkg_class("patch-several-dependencies") assert len(patcher.patches) == 0 # this ensures that results of dependency patches were properly added @@ -208,7 +210,7 @@ def test_nested_directives(mock_packages): assert len(fake_dep.patches[Spec()]) == 2 -@pytest.mark.skipif(sys.platform == "win32", reason="Test requires Autotools") +@pytest.mark.not_on_windows("Test requires Autotools") def test_patched_dependency(mock_packages, config, install_mockery, mock_fetch): """Test whether patched dependencies work.""" spec = Spec("patch-a-dependency") diff --git a/lib/spack/spack/test/permissions.py b/lib/spack/spack/test/permissions.py index 5e6050245e67b0..841595c516410b 100644 --- a/lib/spack/spack/test/permissions.py +++ b/lib/spack/spack/test/permissions.py @@ -5,7 +5,6 @@ import os import stat -import sys import pytest @@ -13,7 +12,7 @@ from spack.util.file_permissions import InvalidPermissionsError, set_permissions -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="chmod unsupported on Windows") +pytestmark = pytest.mark.not_on_windows("chmod unsupported on Windows") def ensure_known_group(path): diff --git a/lib/spack/spack/test/provider_index.py b/lib/spack/spack/test/provider_index.py index e073fb9c0044ab..d11c87f2c22233 100644 --- a/lib/spack/spack/test/provider_index.py +++ b/lib/spack/spack/test/provider_index.py @@ -26,19 +26,19 @@ def test_provider_index_round_trip(mock_packages): - p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.path) + p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) ostream = io.StringIO() p.to_json(ostream) istream = io.StringIO(ostream.getvalue()) - q = ProviderIndex.from_json(istream, repository=spack.repo.path) + q = ProviderIndex.from_json(istream, repository=spack.repo.PATH) assert p == q def test_providers_for_simple(mock_packages): - p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.path) + p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) blas_providers = p.providers_for("blas") assert Spec("netlib-blas") in blas_providers @@ -51,7 +51,7 @@ def test_providers_for_simple(mock_packages): def test_mpi_providers(mock_packages): - p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.path) + p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) mpi_2_providers = p.providers_for("mpi@2") assert Spec("mpich2") in mpi_2_providers @@ -64,12 +64,12 @@ def test_mpi_providers(mock_packages): def test_equal(mock_packages): - p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.path) - q = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.path) + p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) + q = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) assert p == q def test_copy(mock_packages): - p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.path) + p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) q = p.copy() assert p == q diff --git a/lib/spack/spack/test/relocate.py b/lib/spack/spack/test/relocate.py index a0553d4422cb42..cc09730a13c51c 100644 --- a/lib/spack/spack/test/relocate.py +++ b/lib/spack/spack/test/relocate.py @@ -6,12 +6,9 @@ import os.path import re import shutil -import sys import pytest -import llnl.util.filesystem - import spack.concretize import spack.paths import spack.platforms @@ -22,7 +19,7 @@ import spack.tengine import spack.util.executable -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="Tests fail on Windows") +pytestmark = pytest.mark.not_on_windows("Tests fail on Windows") def skip_unless_linux(f): @@ -49,30 +46,6 @@ def text_in_bin(text, binary): return True -@pytest.fixture(params=[True, False]) -def is_relocatable(request): - return request.param - - -@pytest.fixture() -def source_file(tmpdir, is_relocatable): - """Returns the path to a source file of a relocatable executable.""" - if is_relocatable: - template_src = os.path.join(spack.paths.test_path, "data", "templates", "relocatable.c") - src = tmpdir.join("relocatable.c") - shutil.copy(template_src, str(src)) - else: - template_dirs = (os.path.join(spack.paths.test_path, "data", "templates"),) - env = spack.tengine.make_environment(template_dirs) - template = env.get_template("non_relocatable.c") - text = template.render({"prefix": spack.store.layout.root}) - - src = tmpdir.join("non_relocatable.c") - src.write(text) - - return src - - @pytest.fixture() def mock_patchelf(tmpdir, mock_executable): def _factory(output): @@ -154,42 +127,6 @@ def _copy_somewhere(orig_binary): return _copy_somewhere -@pytest.mark.requires_executables("/usr/bin/gcc", "patchelf", "strings", "file") -@skip_unless_linux -def test_ensure_binary_is_relocatable(source_file, is_relocatable): - compiler = spack.util.executable.Executable("/usr/bin/gcc") - executable = str(source_file).replace(".c", ".x") - compiler_env = {"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"} - compiler(str(source_file), "-o", executable, env=compiler_env) - - assert spack.relocate.is_binary(executable) - - try: - spack.relocate.ensure_binary_is_relocatable(executable) - relocatable = True - except spack.relocate.InstallRootStringError: - relocatable = False - - assert relocatable == is_relocatable - - -@skip_unless_linux -def test_ensure_binary_is_relocatable_errors(tmpdir): - # The file passed in as argument must exist... - with pytest.raises(ValueError) as exc_info: - spack.relocate.ensure_binary_is_relocatable("/usr/bin/does_not_exist") - assert "does not exist" in str(exc_info.value) - - # ...and the argument must be an absolute path to it - file = tmpdir.join("delete.me") - file.write("foo") - - with llnl.util.filesystem.working_dir(str(tmpdir)): - with pytest.raises(ValueError) as exc_info: - spack.relocate.ensure_binary_is_relocatable("delete.me") - assert "is not an absolute path" in str(exc_info.value) - - @pytest.mark.parametrize( "start_path,path_root,paths,expected", [ @@ -233,7 +170,7 @@ def test_normalize_relative_paths(start_path, relative_paths, expected): assert normalized == expected -@pytest.mark.requires_executables("patchelf", "strings", "file", "gcc") +@pytest.mark.requires_executables("patchelf", "file", "gcc") @skip_unless_linux def test_relocate_text_bin(binary_with_rpaths, prefix_like): prefix = "/usr/" + prefix_like @@ -250,7 +187,7 @@ def test_relocate_text_bin(binary_with_rpaths, prefix_like): assert "%s/lib:%s/lib64" % (new_prefix, new_prefix) in rpaths_for(executable) -@pytest.mark.requires_executables("patchelf", "strings", "file", "gcc") +@pytest.mark.requires_executables("patchelf", "file", "gcc") @skip_unless_linux def test_relocate_elf_binaries_absolute_paths(binary_with_rpaths, copy_binary, prefix_tmpdir): # Create an executable, set some RPATHs, copy it to another location @@ -272,7 +209,7 @@ def test_relocate_elf_binaries_absolute_paths(binary_with_rpaths, copy_binary, p assert "/foo/lib:/usr/lib64" in rpaths_for(new_binary) -@pytest.mark.requires_executables("patchelf", "strings", "file", "gcc") +@pytest.mark.requires_executables("patchelf", "file", "gcc") @skip_unless_linux def test_relocate_elf_binaries_relative_paths(binary_with_rpaths, copy_binary): # Create an executable, set some RPATHs, copy it to another location @@ -293,7 +230,7 @@ def test_relocate_elf_binaries_relative_paths(binary_with_rpaths, copy_binary): assert "/foo/lib:/foo/lib64:/opt/local/lib" in rpaths_for(new_binary) -@pytest.mark.requires_executables("patchelf", "strings", "file", "gcc") +@pytest.mark.requires_executables("patchelf", "file", "gcc") @skip_unless_linux def test_make_elf_binaries_relative(binary_with_rpaths, copy_binary, prefix_tmpdir): orig_binary = binary_with_rpaths( @@ -313,7 +250,7 @@ def test_make_elf_binaries_relative(binary_with_rpaths, copy_binary, prefix_tmpd assert "$ORIGIN/lib:$ORIGIN/lib64:/opt/local/lib" in rpaths_for(new_binary) -@pytest.mark.requires_executables("patchelf", "strings", "file", "gcc") +@pytest.mark.requires_executables("patchelf", "file", "gcc") @skip_unless_linux def test_relocate_text_bin_with_message(binary_with_rpaths, copy_binary, prefix_tmpdir): orig_binary = binary_with_rpaths( diff --git a/lib/spack/spack/test/repo.py b/lib/spack/spack/test/repo.py index e5c5b006939b28..eb6b12391625aa 100644 --- a/lib/spack/spack/test/repo.py +++ b/lib/spack/spack/test/repo.py @@ -59,9 +59,9 @@ def test_repo_unknown_pkg(mutable_mock_repo): @pytest.mark.maybeslow def test_repo_last_mtime(): latest_mtime = max( - os.path.getmtime(p.module.__file__) for p in spack.repo.path.all_package_classes() + os.path.getmtime(p.module.__file__) for p in spack.repo.PATH.all_package_classes() ) - assert spack.repo.path.last_mtime() == latest_mtime + assert spack.repo.PATH.last_mtime() == latest_mtime def test_repo_invisibles(mutable_mock_repo, extra_repo): @@ -90,10 +90,10 @@ def test_use_repositories_doesnt_change_class(): """Test that we don't create the same package module and class multiple times when swapping repositories. """ - zlib_cls_outer = spack.repo.path.get_pkg_class("zlib") - current_paths = [r.root for r in spack.repo.path.repos] + zlib_cls_outer = spack.repo.PATH.get_pkg_class("zlib") + current_paths = [r.root for r in spack.repo.PATH.repos] with spack.repo.use_repositories(*current_paths): - zlib_cls_inner = spack.repo.path.get_pkg_class("zlib") + zlib_cls_inner = spack.repo.PATH.get_pkg_class("zlib") assert id(zlib_cls_inner) == id(zlib_cls_outer) @@ -123,12 +123,11 @@ def test_relative_import_spack_packages_as_python_modules(mock_packages): def test_all_virtual_packages_have_default_providers(): """All virtual packages must have a default provider explicitly set.""" - defaults = spack.config.get("packages", scope="defaults") + configuration = spack.config.create() + defaults = configuration.get("packages", scope="defaults") default_providers = defaults["all"]["providers"] - providers = spack.repo.path.provider_index.providers - default_providers_filename = spack.config.config.scopes["defaults"].get_section_filename( - "packages" - ) + providers = spack.repo.PATH.provider_index.providers + default_providers_filename = configuration.scopes["defaults"].get_section_filename("packages") for provider in providers: assert provider in default_providers, ( "all providers must have a default in %s" % default_providers_filename @@ -167,3 +166,30 @@ def test_repo_dump_virtuals(tmpdir, mutable_mock_repo, mock_packages, ensure_deb captured = capsys.readouterr()[1] assert "Installing" in captured assert "package.py" in os.listdir(tmpdir), "Expected the virtual's package to be copied" + + +@pytest.mark.parametrize( + "repo_paths,namespaces", + [ + ([spack.paths.packages_path], ["builtin"]), + ([spack.paths.mock_packages_path], ["builtin.mock"]), + ([spack.paths.packages_path, spack.paths.mock_packages_path], ["builtin", "builtin.mock"]), + ([spack.paths.mock_packages_path, spack.paths.packages_path], ["builtin.mock", "builtin"]), + ], +) +def test_repository_construction_doesnt_use_globals(nullify_globals, repo_paths, namespaces): + repo_path = spack.repo.RepoPath(*repo_paths) + assert len(repo_path.repos) == len(namespaces) + assert [x.namespace for x in repo_path.repos] == namespaces + + +@pytest.mark.parametrize("method_name", ["dirname_for_package_name", "filename_for_package_name"]) +def test_path_computation_with_names(method_name, mock_repo_path): + """Tests that repositories can compute the correct paths when using both fully qualified + names and unqualified names. + """ + repo_path = spack.repo.RepoPath(mock_repo_path) + method = getattr(repo_path, method_name) + unqualified = method("mpileaks") + qualified = method("builtin.mock.mpileaks") + assert qualified == unqualified diff --git a/lib/spack/spack/test/rewiring.py b/lib/spack/spack/test/rewiring.py index 3aed08a72060ba..8c0286a2436416 100644 --- a/lib/spack/spack/test/rewiring.py +++ b/lib/spack/spack/test/rewiring.py @@ -14,7 +14,7 @@ from spack.spec import Spec from spack.test.relocate import text_in_bin -args = ["strings", "file"] +args = ["file"] if sys.platform == "darwin": args.extend(["/usr/bin/clang++", "install_name_tool"]) else: @@ -38,7 +38,7 @@ def test_rewire_db(mock_fetch, install_mockery, transitive): assert os.path.exists(spliced_spec.prefix) # test that it made it into the database - rec = spack.store.db.get_record(spliced_spec) + rec = spack.store.STORE.db.get_record(spliced_spec) installed_in_db = rec.installed if rec else False assert installed_in_db @@ -68,7 +68,7 @@ def test_rewire_bin(mock_fetch, install_mockery, transitive): assert os.path.exists(spliced_spec.prefix) # test that it made it into the database - rec = spack.store.db.get_record(spliced_spec) + rec = spack.store.STORE.db.get_record(spliced_spec) installed_in_db = rec.installed if rec else False assert installed_in_db @@ -93,25 +93,31 @@ def test_rewire_writes_new_metadata(mock_fetch, install_mockery): # test install manifests for node in spliced_spec.traverse(root=True): - spack.store.layout.ensure_installed(node) + spack.store.STORE.layout.ensure_installed(node) manifest_file_path = os.path.join( - node.prefix, spack.store.layout.metadata_dir, spack.store.layout.manifest_file_name + node.prefix, + spack.store.STORE.layout.metadata_dir, + spack.store.STORE.layout.manifest_file_name, ) assert os.path.exists(manifest_file_path) orig_node = spec[node.name] orig_manifest_file_path = os.path.join( orig_node.prefix, - spack.store.layout.metadata_dir, - spack.store.layout.manifest_file_name, + spack.store.STORE.layout.metadata_dir, + spack.store.STORE.layout.manifest_file_name, ) assert os.path.exists(orig_manifest_file_path) assert not filecmp.cmp(orig_manifest_file_path, manifest_file_path, shallow=False) specfile_path = os.path.join( - node.prefix, spack.store.layout.metadata_dir, spack.store.layout.spec_file_name + node.prefix, + spack.store.STORE.layout.metadata_dir, + spack.store.STORE.layout.spec_file_name, ) assert os.path.exists(specfile_path) orig_specfile_path = os.path.join( - orig_node.prefix, spack.store.layout.metadata_dir, spack.store.layout.spec_file_name + orig_node.prefix, + spack.store.STORE.layout.metadata_dir, + spack.store.STORE.layout.spec_file_name, ) assert os.path.exists(orig_specfile_path) assert not filecmp.cmp(orig_specfile_path, specfile_path, shallow=False) @@ -128,7 +134,7 @@ def test_uninstall_rewired_spec(mock_fetch, install_mockery, transitive): spliced_spec = spec.splice(dep, transitive=transitive) spack.rewiring.rewire(spliced_spec) spliced_spec.package.do_uninstall() - assert len(spack.store.db.query(spliced_spec)) == 0 + assert len(spack.store.STORE.db.query(spliced_spec)) == 0 assert not os.path.exists(spliced_spec.prefix) diff --git a/lib/spack/spack/test/s3_fetch.py b/lib/spack/spack/test/s3_fetch.py index a495e4d5e8da19..241d2648b504a2 100644 --- a/lib/spack/spack/test/s3_fetch.py +++ b/lib/spack/spack/test/s3_fetch.py @@ -8,9 +8,9 @@ import pytest import spack.config as spack_config +import spack.error import spack.fetch_strategy as spack_fs import spack.stage as spack_stage -from spack.util.web import FetchError @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) @@ -33,7 +33,7 @@ def test_s3fetchstrategy_bad_url(tmpdir, _fetch_method): with spack_stage.Stage(fetcher, path=testpath) as stage: assert stage is not None assert fetcher.archive_file is None - with pytest.raises(FetchError): + with pytest.raises(spack.error.FetchError): fetcher.fetch() diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py index 7a1645d7fe4b0a..8f3396bb46180d 100644 --- a/lib/spack/spack/test/sbang.py +++ b/lib/spack/spack/test/sbang.py @@ -27,7 +27,7 @@ import grp -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +pytestmark = pytest.mark.not_on_windows("does not run on windows") too_long = sbang.system_shebang_limit + 1 @@ -53,13 +53,12 @@ php_line_patched = "\n" -sbang_line = "#!/bin/sh %s/bin/sbang\n" % spack.store.store.unpadded_root last_line = "last!\n" @pytest.fixture # type: ignore[no-redef] def sbang_line(): - yield "#!/bin/sh %s/bin/sbang\n" % spack.store.layout.root + yield "#!/bin/sh %s/bin/sbang\n" % spack.store.STORE.layout.root class ScriptDirectory: @@ -309,7 +308,7 @@ def configure_user_perms(): def check_sbang_installation(group=False): sbang_path = sbang.sbang_install_path() sbang_bin_dir = os.path.dirname(sbang_path) - assert sbang_path.startswith(spack.store.store.unpadded_root) + assert sbang_path.startswith(spack.store.STORE.unpadded_root) assert os.path.exists(sbang_path) assert fs.is_exe(sbang_path) @@ -333,7 +332,7 @@ def run_test_install_sbang(group): sbang_path = sbang.sbang_install_path() sbang_bin_dir = os.path.dirname(sbang_path) - assert sbang_path.startswith(spack.store.store.unpadded_root) + assert sbang_path.startswith(spack.store.STORE.unpadded_root) assert not os.path.exists(sbang_bin_dir) sbang.install_sbang() @@ -368,7 +367,7 @@ def test_install_sbang_too_long(tmpdir): add = min(num_extend, 255) long_path = os.path.join(long_path, "e" * add) num_extend -= add - with spack.store.use_store(spack.store.Store(long_path)): + with spack.store.use_store(long_path): with pytest.raises(sbang.SbangPathError) as exc_info: sbang.sbang_install_path() diff --git a/lib/spack/spack/test/schema.py b/lib/spack/spack/test/schema.py index d7f4e524ffadfc..916e61cf26c821 100644 --- a/lib/spack/spack/test/schema.py +++ b/lib/spack/spack/test/schema.py @@ -80,7 +80,17 @@ def test_module_suffixes(module_suffixes_schema): @pytest.mark.regression("10246") @pytest.mark.parametrize( "config_name", - ["compilers", "config", "env", "merged", "mirrors", "modules", "packages", "repos"], + [ + "compilers", + "config", + "definitions", + "env", + "merged", + "mirrors", + "modules", + "packages", + "repos", + ], ) def test_schema_validation(meta_schema, config_name): import importlib diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 38c42cf380b6a6..3a9c0350ae48a9 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -7,12 +7,13 @@ """ import pytest +import spack.deptypes as dt import spack.error import spack.package_base import spack.parser import spack.repo import spack.util.hash as hashutil -from spack.dependency import Dependency, all_deptypes, canonical_deptype +from spack.dependency import Dependency from spack.spec import Spec @@ -37,19 +38,19 @@ def set_dependency(saved_deps, monkeypatch): for a package in the ``saved_deps`` fixture. """ - def _mock(pkg_name, spec, deptypes=all_deptypes): + def _mock(pkg_name, spec): """Alters dependence information for a package. Adds a dependency on to pkg. Use this to mock up constraints. """ spec = Spec(spec) # Save original dependencies before making any changes. - pkg_cls = spack.repo.path.get_pkg_class(pkg_name) + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) if pkg_name not in saved_deps: saved_deps[pkg_name] = (pkg_cls, pkg_cls.dependencies.copy()) cond = Spec(pkg_cls.name) - dependency = Dependency(pkg_cls, spec, type=deptypes) + dependency = Dependency(pkg_cls, spec) monkeypatch.setitem(pkg_cls.dependencies, spec.name, {cond: dependency}) return _mock @@ -81,6 +82,7 @@ def test_test_deptype(tmpdir): @pytest.mark.usefixtures("config") +@pytest.mark.only_clingo("fails with the original concretizer and full hashes") def test_installed_deps(monkeypatch, mock_packages): """Ensure that concrete specs and their build deps don't constrain solves. @@ -93,9 +95,6 @@ def test_installed_deps(monkeypatch, mock_packages): constrain ``a``'s dependency on ``d``. """ - if spack.config.get("config:concretizer") == "original": - pytest.xfail("fails with the original concretizer and full hashes") - # see installed-deps-[abcde] test packages. # a # / \ @@ -125,7 +124,7 @@ def _mock_installed(self): # use the installed C. It should *not* force A to use the installed D # *if* we're doing a fresh installation. a_spec = Spec(a) - a_spec._add_dependency(c_spec, deptypes=("build", "link"), virtuals=()) + a_spec._add_dependency(c_spec, depflag=dt.BUILD | dt.LINK, virtuals=()) a_spec.concretize() assert spack.version.Version("2") == a_spec[c][d].version assert spack.version.Version("2") == a_spec[e].version @@ -148,7 +147,7 @@ def test_specify_preinstalled_dep(tmpdir, monkeypatch): monkeypatch.setattr(Spec, "installed", property(lambda x: x.name != "a")) a_spec = Spec("a") - a_spec._add_dependency(b_spec, deptypes=("build", "link"), virtuals=()) + a_spec._add_dependency(b_spec, depflag=dt.BUILD | dt.LINK, virtuals=()) a_spec.concretize() assert set(x.name for x in a_spec.traverse()) == set(["a", "b", "c"]) @@ -533,6 +532,7 @@ def test_normalize_mpileaks(self): assert not spec.eq_dag(expected_normalized, deptypes=True) assert not spec.eq_dag(non_unique_nodes, deptypes=True) + @pytest.mark.xfail(reason="String representation changed") def test_normalize_with_virtual_package(self): spec = Spec("mpileaks ^mpi ^libelf@1.8.11 ^libdwarf") spec.normalize() @@ -790,13 +790,13 @@ def test_construct_spec_with_deptypes(self): {"a": {"b": {"c:build": None}, "d": {"e:build,link": {"f:run": None}}}} ) - assert s["b"].edges_to_dependencies(name="c")[0].deptypes == ("build",) - assert s["d"].edges_to_dependencies(name="e")[0].deptypes == ("build", "link") - assert s["e"].edges_to_dependencies(name="f")[0].deptypes == ("run",) + assert s["b"].edges_to_dependencies(name="c")[0].depflag == dt.BUILD + assert s["d"].edges_to_dependencies(name="e")[0].depflag == dt.BUILD | dt.LINK + assert s["e"].edges_to_dependencies(name="f")[0].depflag == dt.RUN - assert s["c"].edges_from_dependents(name="b")[0].deptypes == ("build",) - assert s["e"].edges_from_dependents(name="d")[0].deptypes == ("build", "link") - assert s["f"].edges_from_dependents(name="e")[0].deptypes == ("run",) + assert s["c"].edges_from_dependents(name="b")[0].depflag == dt.BUILD + assert s["e"].edges_from_dependents(name="d")[0].depflag == dt.BUILD | dt.LINK + assert s["f"].edges_from_dependents(name="e")[0].depflag == dt.RUN def check_diamond_deptypes(self, spec): """Validate deptypes in dt-diamond spec. @@ -805,23 +805,22 @@ def check_diamond_deptypes(self, spec): depend on the same dependency in different ways. """ - assert spec["dt-diamond"].edges_to_dependencies(name="dt-diamond-left")[0].deptypes == ( - "build", - "link", + assert ( + spec["dt-diamond"].edges_to_dependencies(name="dt-diamond-left")[0].depflag + == dt.BUILD | dt.LINK ) - - assert spec["dt-diamond"].edges_to_dependencies(name="dt-diamond-right")[0].deptypes == ( - "build", - "link", + assert ( + spec["dt-diamond"].edges_to_dependencies(name="dt-diamond-right")[0].depflag + == dt.BUILD | dt.LINK + ) + assert ( + spec["dt-diamond-left"].edges_to_dependencies(name="dt-diamond-bottom")[0].depflag + == dt.BUILD + ) + assert ( + spec["dt-diamond-right"].edges_to_dependencies(name="dt-diamond-bottom")[0].depflag + == dt.BUILD | dt.LINK | dt.RUN ) - - assert spec["dt-diamond-left"].edges_to_dependencies(name="dt-diamond-bottom")[ - 0 - ].deptypes == ("build",) - - assert spec["dt-diamond-right"].edges_to_dependencies(name="dt-diamond-bottom")[ - 0 - ].deptypes == ("build", "link", "run") def check_diamond_normalized_dag(self, spec): dag = Spec.from_literal( @@ -914,48 +913,52 @@ def test_getitem_exceptional_paths(self): def test_canonical_deptype(self): # special values - assert canonical_deptype(all) == all_deptypes - assert canonical_deptype("all") == all_deptypes + assert dt.canonicalize(all) == dt.ALL + assert dt.canonicalize("all") == dt.ALL with pytest.raises(ValueError): - canonical_deptype(None) + dt.canonicalize(None) with pytest.raises(ValueError): - canonical_deptype([None]) + dt.canonicalize([None]) - # everything in all_deptypes is canonical - for v in all_deptypes: - assert canonical_deptype(v) == (v,) + # everything in all_types is canonical + for v in dt.ALL_TYPES: + assert dt.canonicalize(v) == dt.flag_from_string(v) # tuples - assert canonical_deptype(("build",)) == ("build",) - assert canonical_deptype(("build", "link", "run")) == ("build", "link", "run") - assert canonical_deptype(("build", "link")) == ("build", "link") - assert canonical_deptype(("build", "run")) == ("build", "run") + assert dt.canonicalize(("build",)) == dt.BUILD + assert dt.canonicalize(("build", "link", "run")) == dt.BUILD | dt.LINK | dt.RUN + assert dt.canonicalize(("build", "link")) == dt.BUILD | dt.LINK + assert dt.canonicalize(("build", "run")) == dt.BUILD | dt.RUN # lists - assert canonical_deptype(["build", "link", "run"]) == ("build", "link", "run") - assert canonical_deptype(["build", "link"]) == ("build", "link") - assert canonical_deptype(["build", "run"]) == ("build", "run") + assert dt.canonicalize(["build", "link", "run"]) == dt.BUILD | dt.LINK | dt.RUN + assert dt.canonicalize(["build", "link"]) == dt.BUILD | dt.LINK + assert dt.canonicalize(["build", "run"]) == dt.BUILD | dt.RUN # sorting - assert canonical_deptype(("run", "build", "link")) == ("build", "link", "run") - assert canonical_deptype(("run", "link", "build")) == ("build", "link", "run") - assert canonical_deptype(("run", "link")) == ("link", "run") - assert canonical_deptype(("link", "build")) == ("build", "link") + assert dt.canonicalize(("run", "build", "link")) == dt.BUILD | dt.LINK | dt.RUN + assert dt.canonicalize(("run", "link", "build")) == dt.BUILD | dt.LINK | dt.RUN + assert dt.canonicalize(("run", "link")) == dt.LINK | dt.RUN + assert dt.canonicalize(("link", "build")) == dt.BUILD | dt.LINK + + # deduplication + assert dt.canonicalize(("run", "run", "link")) == dt.RUN | dt.LINK + assert dt.canonicalize(("run", "link", "link")) == dt.RUN | dt.LINK # can't put 'all' in tuple or list with pytest.raises(ValueError): - canonical_deptype(["all"]) + dt.canonicalize(["all"]) with pytest.raises(ValueError): - canonical_deptype(("all",)) + dt.canonicalize(("all",)) # invalid values with pytest.raises(ValueError): - canonical_deptype("foo") + dt.canonicalize("foo") with pytest.raises(ValueError): - canonical_deptype(("foo", "bar")) + dt.canonicalize(("foo", "bar")) with pytest.raises(ValueError): - canonical_deptype(("foo",)) + dt.canonicalize(("foo",)) def test_invalid_literal_spec(self): # Can't give type 'build' to a top-level spec @@ -989,16 +992,16 @@ def test_synthetic_construction_of_split_dependencies_from_same_package(mock_pac link_run_spec = Spec("c@=1.0").concretized() build_spec = Spec("c@=2.0").concretized() - root.add_dependency_edge(link_run_spec, deptypes="link", virtuals=()) - root.add_dependency_edge(link_run_spec, deptypes="run", virtuals=()) - root.add_dependency_edge(build_spec, deptypes="build", virtuals=()) + root.add_dependency_edge(link_run_spec, depflag=dt.LINK, virtuals=()) + root.add_dependency_edge(link_run_spec, depflag=dt.RUN, virtuals=()) + root.add_dependency_edge(build_spec, depflag=dt.BUILD, virtuals=()) # Check dependencies from the perspective of root assert len(root.dependencies()) == 2 assert all(x.name == "c" for x in root.dependencies()) - assert "@2.0" in root.dependencies(name="c", deptype="build")[0] - assert "@1.0" in root.dependencies(name="c", deptype=("link", "run"))[0] + assert "@2.0" in root.dependencies(name="c", deptype=dt.BUILD)[0] + assert "@1.0" in root.dependencies(name="c", deptype=dt.LINK | dt.RUN)[0] # Check parent from the perspective of the dependencies assert len(build_spec.dependents()) == 1 @@ -1017,7 +1020,7 @@ def test_synthetic_construction_bootstrapping(mock_packages, config): root = Spec("b@=2.0").concretized() bootstrap = Spec("b@=1.0").concretized() - root.add_dependency_edge(bootstrap, deptypes="build", virtuals=()) + root.add_dependency_edge(bootstrap, depflag=dt.BUILD, virtuals=()) assert len(root.dependencies()) == 1 assert root.dependencies()[0].name == "b" @@ -1035,37 +1038,38 @@ def test_addition_of_different_deptypes_in_multiple_calls(mock_packages, config) root = Spec("b@=2.0").concretized() bootstrap = Spec("b@=1.0").concretized() - for current_deptype in ("build", "link", "run"): - root.add_dependency_edge(bootstrap, deptypes=current_deptype, virtuals=()) + for current_depflag in (dt.BUILD, dt.LINK, dt.RUN): + root.add_dependency_edge(bootstrap, depflag=current_depflag, virtuals=()) # Check edges in dependencies assert len(root.edges_to_dependencies()) == 1 - forward_edge = root.edges_to_dependencies(deptype=current_deptype)[0] - assert current_deptype in forward_edge.deptypes + forward_edge = root.edges_to_dependencies(depflag=current_depflag)[0] + assert current_depflag & forward_edge.depflag assert id(forward_edge.parent) == id(root) assert id(forward_edge.spec) == id(bootstrap) # Check edges from dependents assert len(bootstrap.edges_from_dependents()) == 1 - backward_edge = bootstrap.edges_from_dependents(deptype=current_deptype)[0] - assert current_deptype in backward_edge.deptypes + backward_edge = bootstrap.edges_from_dependents(depflag=current_depflag)[0] + assert current_depflag & backward_edge.depflag assert id(backward_edge.parent) == id(root) assert id(backward_edge.spec) == id(bootstrap) @pytest.mark.parametrize( - "c1_deptypes,c2_deptypes", [("link", ("build", "link")), (("link", "run"), ("build", "link"))] + "c1_depflag,c2_depflag", + [(dt.LINK, dt.BUILD | dt.LINK), (dt.LINK | dt.RUN, dt.BUILD | dt.LINK)], ) def test_adding_same_deptype_with_the_same_name_raises( - mock_packages, config, c1_deptypes, c2_deptypes + mock_packages, config, c1_depflag, c2_depflag ): p = Spec("b@=2.0").concretized() c1 = Spec("b@=1.0").concretized() c2 = Spec("b@=2.0").concretized() - p.add_dependency_edge(c1, deptypes=c1_deptypes, virtuals=()) + p.add_dependency_edge(c1, depflag=c1_depflag, virtuals=()) with pytest.raises(spack.error.SpackError): - p.add_dependency_edge(c2, deptypes=c2_deptypes, virtuals=()) + p.add_dependency_edge(c2, depflag=c2_depflag, virtuals=()) @pytest.mark.regression("33499") @@ -1084,16 +1088,16 @@ def test_indexing_prefers_direct_or_transitive_link_deps(): z3_flavor_1 = Spec("z3 +through_a1") z3_flavor_2 = Spec("z3 +through_z1") - root.add_dependency_edge(a1, deptypes=("build", "run", "test"), virtuals=()) + root.add_dependency_edge(a1, depflag=dt.BUILD | dt.RUN | dt.TEST, virtuals=()) # unique package as a dep of a build/run/test type dep. - a1.add_dependency_edge(a2, deptypes="all", virtuals=()) - a1.add_dependency_edge(z3_flavor_1, deptypes="all", virtuals=()) + a1.add_dependency_edge(a2, depflag=dt.ALL, virtuals=()) + a1.add_dependency_edge(z3_flavor_1, depflag=dt.ALL, virtuals=()) # chain of link type deps root -> z1 -> z2 -> z3 - root.add_dependency_edge(z1, deptypes="link", virtuals=()) - z1.add_dependency_edge(z2, deptypes="link", virtuals=()) - z2.add_dependency_edge(z3_flavor_2, deptypes="link", virtuals=()) + root.add_dependency_edge(z1, depflag=dt.LINK, virtuals=()) + z1.add_dependency_edge(z2, depflag=dt.LINK, virtuals=()) + z2.add_dependency_edge(z3_flavor_2, depflag=dt.LINK, virtuals=()) # Indexing should prefer the link-type dep. assert "through_z1" in root["z3"].variants diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 5c080e56c3981d..87ed1e4b3f994e 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -3,6 +3,8 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pathlib + import pytest import spack.directives @@ -11,6 +13,7 @@ from spack.spec import ( ArchSpec, CompilerSpec, + DependencySpec, Spec, SpecFormatSigilError, SpecFormatStringError, @@ -291,13 +294,10 @@ def test_concrete_specs_which_satisfies_abstract(self, lhs, rhs, default_mock_co ("foo@4.0%pgi@4.5", "@1:3%pgi@4.4:4.6"), ("builtin.mock.mpich", "builtin.mpich"), ("mpileaks ^builtin.mock.mpich", "^builtin.mpich"), - ("mpileaks^mpich", "^zmpi"), - ("mpileaks^zmpi", "^mpich"), ("mpileaks^mpich@1.2", "^mpich@2.0"), ("mpileaks^mpich@4.0^callpath@1.5", "^mpich@1:3^callpath@1.4:1.6"), ("mpileaks^mpich@2.0^callpath@1.7", "^mpich@1:3^callpath@1.4:1.6"), ("mpileaks^mpich@4.0^callpath@1.7", "^mpich@1:3^callpath@1.4:1.6"), - ("mpileaks^mpich", "^zmpi"), ("mpileaks^mpi@3", "^mpi@1.2:1.6"), ("mpileaks^mpi@3:", "^mpich2@1.4"), ("mpileaks^mpi@3:", "^mpich2"), @@ -335,30 +335,30 @@ def test_constraining_abstract_specs_with_empty_intersection(self, lhs, rhs): rhs.constrain(lhs) @pytest.mark.parametrize( - "lhs,rhs,intersection_expected", + "lhs,rhs", [ - ("mpich", "mpich +foo", True), - ("mpich", "mpich~foo", True), - ("mpich", "mpich foo=1", True), - ("mpich", "mpich++foo", True), - ("mpich", "mpich~~foo", True), - ("mpich", "mpich foo==1", True), + ("mpich", "mpich +foo"), + ("mpich", "mpich~foo"), + ("mpich", "mpich foo=1"), + ("mpich", "mpich++foo"), + ("mpich", "mpich~~foo"), + ("mpich", "mpich foo==1"), # Flags semantics is currently different from other variant - ("mpich", 'mpich cflags="-O3"', True), - ("mpich cflags=-O3", 'mpich cflags="-O3 -Ofast"', False), - ("mpich cflags=-O2", 'mpich cflags="-O3"', False), - ("multivalue-variant foo=bar", "multivalue-variant +foo", False), - ("multivalue-variant foo=bar", "multivalue-variant ~foo", False), - ("multivalue-variant fee=bar", "multivalue-variant fee=baz", False), + ("mpich", 'mpich cflags="-O3"'), + ("mpich cflags=-O3", 'mpich cflags="-O3 -Ofast"'), + ("mpich cflags=-O2", 'mpich cflags="-O3"'), + ("multivalue-variant foo=bar", "multivalue-variant +foo"), + ("multivalue-variant foo=bar", "multivalue-variant ~foo"), + ("multivalue-variant fee=bar", "multivalue-variant fee=baz"), ], ) def test_concrete_specs_which_do_not_satisfy_abstract( - self, lhs, rhs, intersection_expected, default_mock_concretization + self, lhs, rhs, default_mock_concretization ): lhs, rhs = default_mock_concretization(lhs), Spec(rhs) - assert lhs.intersects(rhs) is intersection_expected - assert rhs.intersects(lhs) is intersection_expected + assert lhs.intersects(rhs) is False + assert rhs.intersects(lhs) is False assert not lhs.satisfies(rhs) assert not rhs.satisfies(lhs) @@ -480,10 +480,14 @@ def test_intersects_virtual(self): assert Spec("mpich2").intersects(Spec("mpi")) assert Spec("zmpi").intersects(Spec("mpi")) - def test_intersects_virtual_dep_with_virtual_constraint(self): + def test_intersects_virtual_providers(self): + """Tests that we can always intersect virtual providers from abstract specs. + Concretization will give meaning to virtuals, and eventually forbid certain + configurations. + """ assert Spec("netlib-lapack ^openblas").intersects("netlib-lapack ^openblas") - assert not Spec("netlib-lapack ^netlib-blas").intersects("netlib-lapack ^openblas") - assert not Spec("netlib-lapack ^openblas").intersects("netlib-lapack ^netlib-blas") + assert Spec("netlib-lapack ^netlib-blas").intersects("netlib-lapack ^openblas") + assert Spec("netlib-lapack ^openblas").intersects("netlib-lapack ^netlib-blas") assert Spec("netlib-lapack ^netlib-blas").intersects("netlib-lapack ^netlib-blas") def test_intersectable_concrete_specs_must_have_the_same_hash(self): @@ -515,10 +519,10 @@ def test_dep_index(self): s.normalize() assert s["callpath"] == s - assert type(s["dyninst"]) == Spec - assert type(s["libdwarf"]) == Spec - assert type(s["libelf"]) == Spec - assert type(s["mpi"]) == Spec + assert isinstance(s["dyninst"], Spec) + assert isinstance(s["libdwarf"], Spec) + assert isinstance(s["libelf"], Spec) + assert isinstance(s["mpi"], Spec) assert s["dyninst"].name == "dyninst" assert s["libdwarf"].name == "libdwarf" @@ -670,7 +674,7 @@ def test_spec_formatting(self, default_mock_concretization): other_segments = [ ("{spack_root}", spack.paths.spack_root), - ("{spack_install}", spack.store.layout.root), + ("{spack_install}", spack.store.STORE.layout.root), ] def depify(depname, fmt_str, sigil): @@ -971,7 +975,7 @@ def test_error_message_unknown_variant(self): def test_satisfies_dependencies_ordered(self): d = Spec("zmpi ^fake") s = Spec("mpileaks") - s._add_dependency(d, deptypes=(), virtuals=()) + s._add_dependency(d, depflag=0, virtuals=()) assert s.satisfies("mpileaks ^zmpi ^fake") @pytest.mark.parametrize("transitive", [True, False]) @@ -1003,6 +1007,181 @@ def test_spec_override(self): assert new_spec.compiler_flags["cflags"] == ["-O2"] assert new_spec.compiler_flags["cxxflags"] == ["-O1"] + @pytest.mark.parametrize( + "spec_str,specs_in_dag", + [ + ("hdf5 ^[virtuals=mpi] mpich", [("mpich", "mpich"), ("mpi", "mpich")]), + # Try different combinations with packages that provides a + # disjoint set of virtual dependencies + ( + "netlib-scalapack ^mpich ^openblas-with-lapack", + [ + ("mpi", "mpich"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + ( + "netlib-scalapack ^[virtuals=mpi] mpich ^openblas-with-lapack", + [ + ("mpi", "mpich"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + ( + "netlib-scalapack ^mpich ^[virtuals=lapack] openblas-with-lapack", + [ + ("mpi", "mpich"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + ( + "netlib-scalapack ^[virtuals=mpi] mpich ^[virtuals=lapack] openblas-with-lapack", + [ + ("mpi", "mpich"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + # Test that we can mix dependencies that provide an overlapping + # sets of virtual dependencies + ( + "netlib-scalapack ^[virtuals=mpi] intel-parallel-studio " + "^[virtuals=lapack] openblas-with-lapack", + [ + ("mpi", "intel-parallel-studio"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + ( + "netlib-scalapack ^[virtuals=mpi] intel-parallel-studio ^openblas-with-lapack", + [ + ("mpi", "intel-parallel-studio"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + ( + "netlib-scalapack ^intel-parallel-studio ^[virtuals=lapack] openblas-with-lapack", + [ + ("mpi", "intel-parallel-studio"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + # Test that we can bind more than one virtual to the same provider + ( + "netlib-scalapack ^[virtuals=lapack,blas] openblas-with-lapack", + [("lapack", "openblas-with-lapack"), ("blas", "openblas-with-lapack")], + ), + ], + ) + def test_virtual_deps_bindings(self, default_mock_concretization, spec_str, specs_in_dag): + if spack.config.get("config:concretizer") == "original": + pytest.skip("Use case not supported by the original concretizer") + + s = default_mock_concretization(spec_str) + for label, expected in specs_in_dag: + assert label in s + assert s[label].satisfies(expected), label + + @pytest.mark.parametrize( + "spec_str", + [ + # openblas-with-lapack needs to provide blas and lapack together + "netlib-scalapack ^[virtuals=blas] intel-parallel-studio ^openblas-with-lapack", + # intel-* provides blas and lapack together, openblas can provide blas only + "netlib-scalapack ^[virtuals=lapack] intel-parallel-studio ^openblas", + ], + ) + def test_unsatisfiable_virtual_deps_bindings(self, spec_str): + if spack.config.get("config:concretizer") == "original": + pytest.skip("Use case not supported by the original concretizer") + + with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): + Spec(spec_str).concretized() + + +@pytest.mark.parametrize( + "spec_str,format_str,expected", + [ + ("zlib@git.foo/bar", "{name}-{version}", str(pathlib.Path("zlib-git.foo_bar"))), + ("zlib@git.foo/bar", "{name}-{version}-{/hash}", None), + ("zlib@git.foo/bar", "{name}/{version}", str(pathlib.Path("zlib", "git.foo_bar"))), + ( + "zlib@{0}=1.0%gcc".format("a" * 40), + "{name}/{version}/{compiler}", + str(pathlib.Path("zlib", "{0}_1.0".format("a" * 40), "gcc")), + ), + ( + "zlib@git.foo/bar=1.0%gcc", + "{name}/{version}/{compiler}", + str(pathlib.Path("zlib", "git.foo_bar_1.0", "gcc")), + ), + ], +) +def test_spec_format_path(spec_str, format_str, expected): + _check_spec_format_path(spec_str, format_str, expected) + + +def _check_spec_format_path(spec_str, format_str, expected, path_ctor=None): + spec = Spec(spec_str) + if not expected: + with pytest.raises((spack.spec.SpecFormatPathError, spack.spec.SpecFormatStringError)): + spec.format_path(format_str, _path_ctor=path_ctor) + else: + formatted = spec.format_path(format_str, _path_ctor=path_ctor) + assert formatted == expected + + +@pytest.mark.parametrize( + "spec_str,format_str,expected", + [ + ( + "zlib@git.foo/bar", + r"C:\\installroot\{name}\{version}", + r"C:\installroot\zlib\git.foo_bar", + ), + ( + "zlib@git.foo/bar", + r"\\hostname\sharename\{name}\{version}", + r"\\hostname\sharename\zlib\git.foo_bar", + ), + # Windows doesn't attribute any significance to a leading + # "/" so it is discarded + ("zlib@git.foo/bar", r"/installroot/{name}/{version}", r"installroot\zlib\git.foo_bar"), + ], +) +def test_spec_format_path_windows(spec_str, format_str, expected): + _check_spec_format_path(spec_str, format_str, expected, path_ctor=pathlib.PureWindowsPath) + + +@pytest.mark.parametrize( + "spec_str,format_str,expected", + [ + ("zlib@git.foo/bar", r"/installroot/{name}/{version}", "/installroot/zlib/git.foo_bar"), + ("zlib@git.foo/bar", r"//installroot/{name}/{version}", "//installroot/zlib/git.foo_bar"), + # This is likely unintentional on Linux: Firstly, "\" is not a + # path separator for POSIX, so this is treated as a single path + # component (containing literal "\" characters); secondly, + # Spec.format treats "\" as an escape character, so is + # discarded (unless directly following another "\") + ( + "zlib@git.foo/bar", + r"C:\\installroot\package-{name}-{version}", + r"C__installrootpackage-zlib-git.foo_bar", + ), + # "\" is not a POSIX separator, and Spec.format treats "\{" as a literal + # "{", which means that the resulting format string is invalid + ("zlib@git.foo/bar", r"package\{name}\{version}", None), + ], +) +def test_spec_format_path_posix(spec_str, format_str, expected): + _check_spec_format_path(spec_str, format_str, expected, path_ctor=pathlib.PurePosixPath) + @pytest.mark.regression("3887") @pytest.mark.parametrize("spec_str", ["py-extension2", "extension1", "perl-extension"]) @@ -1120,7 +1299,7 @@ def test_concretize_partial_old_dag_hash_spec(mock_packages, config): # add it to an abstract spec as a dependency top = Spec("dt-diamond") - top.add_dependency_edge(bottom, deptypes=(), virtuals=()) + top.add_dependency_edge(bottom, depflag=0, virtuals=()) # concretize with the already-concrete dependency top.concretize() @@ -1291,3 +1470,50 @@ def test_constrain(factory, lhs_str, rhs_str, result, constrained_str): rhs = factory(rhs_str) rhs.constrain(lhs) assert rhs == factory(constrained_str) + + +def test_abstract_hash_intersects_and_satisfies(default_mock_concretization): + concrete: Spec = default_mock_concretization("a") + hash = concrete.dag_hash() + hash_5 = hash[:5] + hash_6 = hash[:6] + # abstract hash that doesn't have a common prefix with the others. + hash_other = f"{'a' if hash_5[0] == 'b' else 'b'}{hash_5[1:]}" + + abstract_5 = Spec(f"a/{hash_5}") + abstract_6 = Spec(f"a/{hash_6}") + abstract_none = Spec(f"a/{hash_other}") + abstract = Spec("a") + + def assert_subset(a: Spec, b: Spec): + assert a.intersects(b) and b.intersects(a) and a.satisfies(b) and not b.satisfies(a) + + def assert_disjoint(a: Spec, b: Spec): + assert ( + not a.intersects(b) + and not b.intersects(a) + and not a.satisfies(b) + and not b.satisfies(a) + ) + + # left-hand side is more constrained, so its + # concretization space is a subset of the right-hand side's + assert_subset(concrete, abstract_5) + assert_subset(abstract_6, abstract_5) + assert_subset(abstract_5, abstract) + + # disjoint concretization space + assert_disjoint(abstract_none, concrete) + assert_disjoint(abstract_none, abstract_5) + + +def test_edge_equality_does_not_depend_on_virtual_order(): + """Tests that two edges that are constructed with just a different order of the virtuals in + the input parameters are equal to each other. + """ + parent, child = Spec("parent"), Spec("child") + edge1 = DependencySpec(parent, child, depflag=0, virtuals=("mpi", "lapack")) + edge2 = DependencySpec(parent, child, depflag=0, virtuals=("lapack", "mpi")) + assert edge1 == edge2 + assert tuple(sorted(edge1.virtuals)) == edge1.virtuals + assert tuple(sorted(edge2.virtuals)) == edge1.virtuals diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index b026bf01016dd8..3cbb59e69f0af2 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -472,33 +472,46 @@ def _specfile_for(spec_str, filename): [Token(TokenType.PROPAGATED_KEY_VALUE_PAIR, value='cflags=="-O3 -g"')], 'cflags=="-O3 -g"', ), - # Way too many spaces + # Whitespace is allowed in version lists + ("@1.2:1.4 , 1.6 ", [Token(TokenType.VERSION, value="@1.2:1.4 , 1.6")], "@1.2:1.4,1.6"), + # But not in ranges. `a@1:` and `b` are separate specs, not a single `a@1:b`. ( - "@1.2 : 1.4 , 1.6 ", - [Token(TokenType.VERSION, value="@1.2 : 1.4 , 1.6")], - "@1.2:1.4,1.6", + "a@1: b", + [ + Token(TokenType.UNQUALIFIED_PACKAGE_NAME, value="a"), + Token(TokenType.VERSION, value="@1:"), + Token(TokenType.UNQUALIFIED_PACKAGE_NAME, value="b"), + ], + "a@1:", + ), + ( + "@1.2: develop = foo", + [ + Token(TokenType.VERSION, value="@1.2:"), + Token(TokenType.KEY_VALUE_PAIR, value="develop = foo"), + ], + "@1.2: develop=foo", ), - ("@1.2 : develop", [Token(TokenType.VERSION, value="@1.2 : develop")], "@1.2:develop"), ( - "@1.2 : develop = foo", + "@1.2:develop = foo", [ - Token(TokenType.VERSION, value="@1.2 :"), + Token(TokenType.VERSION, value="@1.2:"), Token(TokenType.KEY_VALUE_PAIR, value="develop = foo"), ], "@1.2: develop=foo", ), ( - "% intel @ 12.1 : 12.6 + debug", + "% intel @ 12.1:12.6 + debug", [ - Token(TokenType.COMPILER_AND_VERSION, value="% intel @ 12.1 : 12.6"), + Token(TokenType.COMPILER_AND_VERSION, value="% intel @ 12.1:12.6"), Token(TokenType.BOOL_VARIANT, value="+ debug"), ], "%intel@12.1:12.6+debug", ), ( - "@ 12.1 : 12.6 + debug - qt_4", + "@ 12.1:12.6 + debug - qt_4", [ - Token(TokenType.VERSION, value="@ 12.1 : 12.6"), + Token(TokenType.VERSION, value="@ 12.1:12.6"), Token(TokenType.BOOL_VARIANT, value="+ debug"), Token(TokenType.BOOL_VARIANT, value="- qt_4"), ], @@ -517,6 +530,59 @@ def _specfile_for(spec_str, filename): [Token(TokenType.VERSION, value="@:0.4"), Token(TokenType.COMPILER, value="% nvhpc")], "@:0.4%nvhpc", ), + ( + "^[virtuals=mpi] openmpi", + [ + Token(TokenType.START_EDGE_PROPERTIES, value="^["), + Token(TokenType.KEY_VALUE_PAIR, value="virtuals=mpi"), + Token(TokenType.END_EDGE_PROPERTIES, value="]"), + Token(TokenType.UNQUALIFIED_PACKAGE_NAME, value="openmpi"), + ], + "^[virtuals=mpi] openmpi", + ), + ( + "^[deptypes=link,build] zlib", + [ + Token(TokenType.START_EDGE_PROPERTIES, value="^["), + Token(TokenType.KEY_VALUE_PAIR, value="deptypes=link,build"), + Token(TokenType.END_EDGE_PROPERTIES, value="]"), + Token(TokenType.UNQUALIFIED_PACKAGE_NAME, value="zlib"), + ], + "^[deptypes=build,link] zlib", + ), + ( + "zlib@git.foo/bar", + [ + Token(TokenType.UNQUALIFIED_PACKAGE_NAME, "zlib"), + Token(TokenType.GIT_VERSION, "@git.foo/bar"), + ], + "zlib@git.foo/bar", + ), + # Variant propagation + ( + "zlib ++foo", + [ + Token(TokenType.UNQUALIFIED_PACKAGE_NAME, "zlib"), + Token(TokenType.PROPAGATED_BOOL_VARIANT, "++foo"), + ], + "zlib++foo", + ), + ( + "zlib ~~foo", + [ + Token(TokenType.UNQUALIFIED_PACKAGE_NAME, "zlib"), + Token(TokenType.PROPAGATED_BOOL_VARIANT, "~~foo"), + ], + "zlib~~foo", + ), + ( + "zlib foo==bar", + [ + Token(TokenType.UNQUALIFIED_PACKAGE_NAME, "zlib"), + Token(TokenType.PROPAGATED_KEY_VALUE_PAIR, "foo==bar"), + ], + "zlib foo==bar", + ), ], ) def test_parse_single_spec(spec_str, tokens, expected_roundtrip): @@ -726,22 +792,31 @@ def test_multiple_specs_with_hash(database, config): @pytest.mark.db def test_ambiguous_hash(mutable_database, default_mock_concretization, config): + """Test that abstract hash ambiguity is delayed until concretization. + In the past this ambiguity error would happen during parse time.""" + + # This is a very sketchy as manually setting hashes easily breaks invariants x1 = default_mock_concretization("a") x2 = x1.copy() x1._hash = "xyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" + x1._process_hash = "xyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" x2._hash = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - mutable_database.add(x1, spack.store.layout) - mutable_database.add(x2, spack.store.layout) + x2._process_hash = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + assert x1 != x2 # doesn't hold when only the dag hash is modified. + + mutable_database.add(x1, directory_layout=None) + mutable_database.add(x2, directory_layout=None) # ambiguity in first hash character + s1 = SpecParser("/x").next_spec() with pytest.raises(spack.spec.AmbiguousHashError): - parsed_spec = SpecParser("/x").next_spec() - parsed_spec.replace_hash() + s1.lookup_hash() # ambiguity in first hash character AND spec name + s2 = SpecParser("a/x").next_spec() with pytest.raises(spack.spec.AmbiguousHashError): - parsed_spec = SpecParser("a/x").next_spec() - parsed_spec.replace_hash() + s2.lookup_hash() @pytest.mark.db @@ -868,6 +943,9 @@ def test_disambiguate_hash_by_spec(spec1, spec2, constraint, mock_packages, monk ("x platform=test platform=test", spack.spec.DuplicateArchitectureError), ("x os=fe platform=test target=fe os=fe", spack.spec.DuplicateArchitectureError), ("x target=be platform=test os=be os=fe", spack.spec.DuplicateArchitectureError), + ("^[@foo] zlib", spack.parser.SpecParsingError), + # TODO: Remove this as soon as use variants are added and we can parse custom attributes + ("^[foo=bar] zlib", spack.parser.SpecParsingError), ], ) def test_error_conditions(text, exc_cls): diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index 594072ac4fe360..b19bb4308f9489 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -198,7 +198,7 @@ def test_ordered_read_not_required_for_consistent_dag_hash(config, mock_packages round_trip_reversed_json_spec = Spec.from_yaml(reversed_json_string) # Strip spec if we stripped the yaml - spec = spec.copy(deps=ht.dag_hash.deptype) + spec = spec.copy(deps=ht.dag_hash.depflag) # specs are equal to the original assert spec == round_trip_yaml_spec @@ -326,9 +326,8 @@ def test_save_dependency_spec_jsons_subset(tmpdir, config): spec_a = Spec("a").concretized() b_spec = spec_a["b"] c_spec = spec_a["c"] - spec_a_json = spec_a.to_json() - save_dependency_specfiles(spec_a_json, output_path, ["b", "c"]) + save_dependency_specfiles(spec_a, output_path, [Spec("b"), Spec("c")]) assert check_specs_equal(b_spec, os.path.join(output_path, "b.json")) assert check_specs_equal(c_spec, os.path.join(output_path, "c.json")) @@ -501,3 +500,8 @@ def test_load_json_specfiles(specfile, expected_hash, reader_cls): openmpi_edges = s2.edges_to_dependencies(name="openmpi") assert len(openmpi_edges) == 1 + + # The virtuals attribute must be a tuple, when read from a + # JSON or YAML file, not a list + for edge in s2.traverse_edges(): + assert isinstance(edge.virtuals, tuple), edge diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index 1220a86904ad9c..8b2b53dd05054a 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -16,6 +16,7 @@ from llnl.util.filesystem import getuid, mkdirp, partition_path, touch, working_dir +import spack.error import spack.paths import spack.stage import spack.util.executable @@ -23,7 +24,6 @@ from spack.resource import Resource from spack.stage import DIYStage, ResourceStage, Stage, StageComposite from spack.util.path import canonicalize_path -from spack.util.web import FetchError # The following values are used for common fetch and stage mocking fixtures: _archive_base = "test-files" @@ -522,7 +522,7 @@ def test_no_search_mirror_only(self, failing_fetch_strategy, failing_search_fn): with stage: try: stage.fetch(mirror_only=True) - except FetchError: + except spack.error.FetchError: pass check_destroy(stage, self.stage_name) @@ -537,7 +537,7 @@ def test_search_if_default_fails(self, failing_fetch_strategy, search_fn, err_ms stage = Stage(failing_fetch_strategy, name=self.stage_name, search_fn=search_fn) with stage: - with pytest.raises(FetchError, match=expected): + with pytest.raises(spack.error.FetchError, match=expected): stage.fetch(mirror_only=False, err_msg=err_msg) check_destroy(stage, self.stage_name) @@ -659,7 +659,7 @@ def test_source_path_available(self, mock_stage_archive): assert source_path.endswith(spack.stage._source_path_subdir) assert not os.path.exists(source_path) - @pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") + @pytest.mark.not_on_windows("Windows file permission erroring is not yet supported") @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_first_accessible_path(self, tmpdir): """Test _first_accessible_path names.""" @@ -691,7 +691,6 @@ def test_first_accessible_path(self, tmpdir): # Cleanup shutil.rmtree(str(name)) - @pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") def test_create_stage_root(self, tmpdir, no_path_access): """Test create_stage_root permissions.""" test_dir = tmpdir.join("path") @@ -755,7 +754,7 @@ def test_resolve_paths(self): assert spack.stage._resolve_paths(paths) == res_paths - @pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") + @pytest.mark.not_on_windows("Windows file permission erroring is not yet supported") @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_get_stage_root_bad_path(self, clear_stage_root): """Ensure an invalid stage path root raises a StageError.""" @@ -864,7 +863,6 @@ def test_diystage_preserve_file(self, tmpdir): _file.read() == _readme_contents -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") def test_stage_create_replace_path(tmp_build_stage_dir): """Ensure stage creation replaces a non-directory path.""" _, test_stage_path = tmp_build_stage_dir @@ -872,16 +870,15 @@ def test_stage_create_replace_path(tmp_build_stage_dir): nondir = os.path.join(test_stage_path, "afile") touch(nondir) - path = str(nondir) + path = url_util.path_to_file_url(str(nondir)) - stage = Stage(path, name="") + stage = Stage(path, name="afile") stage.create() # Ensure the stage path is "converted" to a directory - assert os.path.isdir(stage.path) + assert os.path.isdir(nondir) -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") def test_cannot_access(capsys): """Ensure can_access dies with the expected error.""" with pytest.raises(SystemExit): @@ -890,3 +887,24 @@ def test_cannot_access(capsys): captured = capsys.readouterr() assert "Insufficient permissions" in str(captured) + + +def test_override_keep_in_composite_stage(): + stage_1 = Stage("file:///does-not-exist", keep=True) + stage_2 = Stage("file:///does-not-exist", keep=False) + stage_3 = Stage("file:///does-not-exist", keep=True) + stages = spack.stage.StageComposite.from_iterable((stage_1, stage_2, stage_3)) + + # The getter for the composite stage just returns the value of the first stage + # its just there so we have a setter too. + assert stages.keep + assert stage_1.keep + assert not stage_2.keep + assert stage_3.keep + + # This should override all stages + stages.keep = False + assert not stages.keep + assert not stage_1.keep + assert not stage_2.keep + assert not stage_3.keep diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 523b4b012cb711..e6b1fbba8354d8 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import sys import pytest @@ -22,7 +21,7 @@ pytest.mark.skipif( not which("svn") or not which("svnadmin"), reason="requires subversion to be installed" ), - pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows"), + pytest.mark.not_on_windows("does not run on windows"), ] diff --git a/lib/spack/spack/test/tag.py b/lib/spack/spack/test/tag.py index d8d9048d74d1b3..ae950002588a2e 100644 --- a/lib/spack/spack/test/tag.py +++ b/lib/spack/spack/test/tag.py @@ -97,7 +97,7 @@ def test_tag_get_installed_packages(mock_packages, mock_archive, mock_fetch, ins def test_tag_index_round_trip(mock_packages): # Assumes at least two packages -- mpich and mpich2 -- have tags - mock_index = spack.repo.path.tag_index + mock_index = spack.repo.PATH.tag_index assert mock_index.tags ostream = io.StringIO() @@ -153,7 +153,7 @@ def test_tag_no_tags(mock_packages): def test_tag_update_package(mock_packages): - mock_index = spack.repo.path.tag_index + mock_index = spack.repo.PATH.tag_index index = spack.tag.TagIndex(repository=mock_packages) for name in spack.repo.all_package_names(): index.update_package(name) diff --git a/lib/spack/spack/test/test_suite.py b/lib/spack/spack/test/test_suite.py index a082ff652808b5..cf7d32322ee07a 100644 --- a/lib/spack/spack/test/test_suite.py +++ b/lib/spack/spack/test/test_suite.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import os -import sys import pytest @@ -148,7 +147,7 @@ def test_test_spec_run_once(mock_packages, install_mockery, mock_test_stage): test_suite() -@pytest.mark.skipif(sys.platform == "win32", reason="Cannot find echo executable") +@pytest.mark.not_on_windows("Cannot find echo executable") def test_test_spec_passes(mock_packages, install_mockery, mock_test_stage, monkeypatch): spec = spack.spec.Spec("simple-standalone-test").concretized() monkeypatch.setattr(spack.spec.Spec, "installed", _true) diff --git a/lib/spack/spack/test/traverse.py b/lib/spack/spack/test/traverse.py index 2d9679d6ce7e03..482103e83c092c 100644 --- a/lib/spack/spack/test/traverse.py +++ b/lib/spack/spack/test/traverse.py @@ -5,6 +5,7 @@ import pytest +import spack.deptypes as dt import spack.traverse as traverse from spack.spec import Spec @@ -19,7 +20,9 @@ def create_dag(nodes, edges): """ specs = {name: Spec(name) for name in nodes} for parent, child, deptypes in edges: - specs[parent].add_dependency_edge(specs[child], deptypes=deptypes, virtuals=()) + specs[parent].add_dependency_edge( + specs[child], depflag=dt.canonicalize(deptypes), virtuals=() + ) return specs diff --git a/lib/spack/spack/test/url_fetch.py b/lib/spack/spack/test/url_fetch.py index 2cfea6b7b4dfa7..a3c0f7c10b5ede 100644 --- a/lib/spack/spack/test/url_fetch.py +++ b/lib/spack/spack/test/url_fetch.py @@ -13,6 +13,7 @@ from llnl.util.filesystem import is_exe, working_dir import spack.config +import spack.error import spack.fetch_strategy as fs import spack.repo import spack.util.crypto as crypto @@ -173,7 +174,7 @@ def test_fetch( # TODO-27021 -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.parametrize( "spec,url,digest", [ @@ -204,7 +205,7 @@ def test_from_list_url(mock_packages, config, spec, url, digest, _fetch_method): assert fetch_strategy.extra_options == {"timeout": 60} -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) @pytest.mark.parametrize( "requested_version,tarball,digest", @@ -221,12 +222,10 @@ def test_from_list_url(mock_packages, config, spec, url, digest, _fetch_method): ("2.0.0", "foo-2.0.0b2.tar.gz", "000000000000000000000000000200b2"), ], ) +@pytest.mark.only_clingo("Original concretizer doesn't resolve concrete versions to known ones") def test_new_version_from_list_url( mock_packages, config, _fetch_method, requested_version, tarball, digest ): - if spack.config.get("config:concretizer") == "original": - pytest.skip("Original concretizer doesn't resolve concrete versions to known ones") - """Test non-specific URLs from the url-list-test package.""" with spack.config.override("config:url_fetch_method", _fetch_method): s = Spec("url-list-test @%s" % requested_version).concretized() @@ -351,7 +350,7 @@ def _which(*args, **kwargs): def test_url_fetch_text_without_url(tmpdir): - with pytest.raises(web_util.FetchError, match="URL is required"): + with pytest.raises(spack.error.FetchError, match="URL is required"): web_util.fetch_url_text(None) @@ -368,18 +367,18 @@ def _which(*args, **kwargs): monkeypatch.setattr(spack.util.web, "which", _which) with spack.config.override("config:url_fetch_method", "curl"): - with pytest.raises(web_util.FetchError, match="Missing required curl"): + with pytest.raises(spack.error.FetchError, match="Missing required curl"): web_util.fetch_url_text("https://github.com/") def test_url_check_curl_errors(): """Check that standard curl error returncodes raise expected errors.""" # Check returncode 22 (i.e., 404) - with pytest.raises(web_util.FetchError, match="not found"): + with pytest.raises(spack.error.FetchError, match="not found"): web_util.check_curl_code(22) # Check returncode 60 (certificate error) - with pytest.raises(web_util.FetchError, match="invalid certificate"): + with pytest.raises(spack.error.FetchError, match="invalid certificate"): web_util.check_curl_code(60) @@ -396,7 +395,7 @@ def _which(*args, **kwargs): monkeypatch.setattr(spack.util.web, "which", _which) with spack.config.override("config:url_fetch_method", "curl"): - with pytest.raises(web_util.FetchError, match="Missing required curl"): + with pytest.raises(spack.error.FetchError, match="Missing required curl"): web_util.url_exists("https://github.com/") @@ -411,7 +410,7 @@ def _read_from_url(*args, **kwargs): monkeypatch.setattr(spack.util.web, "read_from_url", _read_from_url) with spack.config.override("config:url_fetch_method", "urllib"): - with pytest.raises(web_util.FetchError, match="failed with error code"): + with pytest.raises(spack.error.FetchError, match="failed with error code"): web_util.fetch_url_text("https://github.com/") @@ -422,5 +421,5 @@ def _raise_web_error(*args, **kwargs): monkeypatch.setattr(spack.util.web, "read_from_url", _raise_web_error) with spack.config.override("config:url_fetch_method", "urllib"): - with pytest.raises(web_util.FetchError, match="fetch failed to verify"): + with pytest.raises(spack.error.FetchError, match="fetch failed to verify"): web_util.fetch_url_text("https://github.com/") diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index 86ebf84fa78bef..dd094ed230b908 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -17,124 +17,11 @@ parse_name_offset, parse_version_offset, strip_name_suffixes, - strip_version_suffixes, substitute_version, ) from spack.version import Version -@pytest.mark.parametrize( - "url,expected", - [ - # No suffix - ("rgb-1.0.6", "rgb-1.0.6"), - # Misleading prefix - ("jpegsrc.v9b", "jpegsrc.v9b"), - ("turbolinux702", "turbolinux702"), - ("converge_install_2.3.16", "converge_install_2.3.16"), - # Download type - code, source - ("cistem-1.0.0-beta-source-code", "cistem-1.0.0-beta"), - # Download type - src - ("apache-ant-1.9.7-src", "apache-ant-1.9.7"), - ("go1.7.4.src", "go1.7.4"), - # Download type - source - ("bowtie2-2.2.5-source", "bowtie2-2.2.5"), - ("grib_api-1.17.0-Source", "grib_api-1.17.0"), - # Download type - full - ("julia-0.4.3-full", "julia-0.4.3"), - # Download type - bin - ("apache-maven-3.3.9-bin", "apache-maven-3.3.9"), - # Download type - binary - ("Jmol-14.8.0-binary", "Jmol-14.8.0"), - # Download type - gem - ("rubysl-date-2.0.9.gem", "rubysl-date-2.0.9"), - # Download type - tar - ("gromacs-4.6.1-tar", "gromacs-4.6.1"), - # Download type - sh - ("Miniconda2-4.3.11-Linux-x86_64.sh", "Miniconda2-4.3.11"), - # Download version - release - ("v1.0.4-release", "v1.0.4"), - # Download version - stable - ("libevent-2.0.21-stable", "libevent-2.0.21"), - # Download version - final - ("2.6.7-final", "2.6.7"), - # Download version - rel - ("v1.9.5.1rel", "v1.9.5.1"), - # Download version - orig - ("dash_0.5.5.1.orig", "dash_0.5.5.1"), - # Download version - plus - ("ncbi-blast-2.6.0+-src", "ncbi-blast-2.6.0"), - # License - ("cppad-20170114.gpl", "cppad-20170114"), - # Arch - ("pcraster-4.1.0_x86-64", "pcraster-4.1.0"), - ("dislin-11.0.linux.i586_64", "dislin-11.0"), - ("PAGIT.V1.01.64bit", "PAGIT.V1.01"), - # OS - linux - ("astyle_2.04_linux", "astyle_2.04"), - # OS - unix - ("install-tl-unx", "install-tl"), - # OS - macos - ("astyle_1.23_macosx", "astyle_1.23"), - ("haxe-2.08-osx", "haxe-2.08"), - # PyPI - wheel - ("entrypoints-0.2.2-py2.py3-none-any.whl", "entrypoints-0.2.2"), - ( - "numpy-1.12.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel." - "macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", - "numpy-1.12.0", - ), - # PyPI - exe - ("PyYAML-3.12.win-amd64-py3.5.exe", "PyYAML-3.12"), - # Combinations of multiple patterns - bin, release - ("rocketmq-all-4.5.2-bin-release", "rocketmq-all-4.5.2"), - # Combinations of multiple patterns - all - ("p7zip_9.04_src_all", "p7zip_9.04"), - # Combinations of multiple patterns - run - ("cuda_8.0.44_linux.run", "cuda_8.0.44"), - # Combinations of multiple patterns - file - ("ack-2.14-single-file", "ack-2.14"), - # Combinations of multiple patterns - jar - ("antlr-3.4-complete.jar", "antlr-3.4"), - # Combinations of multiple patterns - oss - ("tbb44_20160128oss_src_0", "tbb44_20160128"), - # Combinations of multiple patterns - darwin - ("ghc-7.0.4-x86_64-apple-darwin", "ghc-7.0.4"), - ("ghc-7.0.4-i386-apple-darwin", "ghc-7.0.4"), - # Combinations of multiple patterns - centos - ("sratoolkit.2.8.2-1-centos_linux64", "sratoolkit.2.8.2-1"), - # Combinations of multiple patterns - arch - ( - "VizGlow_v2.2alpha17-R21November2016-Linux-x86_64-Install", - "VizGlow_v2.2alpha17-R21November2016", - ), - ("jdk-8u92-linux-x64", "jdk-8u92"), - ("cuda_6.5.14_linux_64.run", "cuda_6.5.14"), - ("Mathematica_12.0.0_LINUX.sh", "Mathematica_12.0.0"), - ("trf407b.linux64", "trf407b"), - # Combinations of multiple patterns - with - ("mafft-7.221-with-extensions-src", "mafft-7.221"), - ("spark-2.0.0-bin-without-hadoop", "spark-2.0.0"), - ("conduit-v0.3.0-src-with-blt", "conduit-v0.3.0"), - # Combinations of multiple patterns - rock - ("bitlib-23-2.src.rock", "bitlib-23-2"), - # Combinations of multiple patterns - public - ("dakota-6.3-public.src", "dakota-6.3"), - # Combinations of multiple patterns - universal - ("synergy-1.3.6p2-MacOSX-Universal", "synergy-1.3.6p2"), - # Combinations of multiple patterns - dynamic - ("snptest_v2.5.2_linux_x86_64_dynamic", "snptest_v2.5.2"), - # Combinations of multiple patterns - other - ("alglib-3.11.0.cpp.gpl", "alglib-3.11.0"), - ("hpcviewer-2019.08-linux.gtk.x86_64", "hpcviewer-2019.08"), - ("apache-mxnet-src-1.3.0-incubating", "apache-mxnet-src-1.3.0"), - ], -) -def test_url_strip_version_suffixes(url, expected): - stripped = strip_version_suffixes(url) - assert stripped == expected - - @pytest.mark.parametrize( "url,version,expected", [ diff --git a/lib/spack/spack/test/util/compression.py b/lib/spack/spack/test/util/compression.py index 7b0f8d45fbcddd..29007a7e333ff5 100644 --- a/lib/spack/spack/test/util/compression.py +++ b/lib/spack/spack/test/util/compression.py @@ -3,12 +3,14 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) + import os import shutil -import sys +from itertools import product import pytest +import llnl.url from llnl.util.filesystem import working_dir from spack.paths import spack_root @@ -20,7 +22,7 @@ ext_archive = {} [ ext_archive.update({ext: ".".join(["Foo", ext])}) - for ext in scomp.ALLOWED_ARCHIVE_TYPES + for ext in llnl.url.ALLOWED_ARCHIVE_TYPES if "TAR" not in ext ] # Spack does not use Python native handling for tarballs or zip @@ -40,18 +42,24 @@ def compr_support_check(monkeypatch): @pytest.fixture -def archive_file(tmpdir_factory, request): - """Copy example archive to temp directory for test""" +def archive_file_and_extension(tmpdir_factory, request): + """Copy example archive to temp directory into an extension-less file for test""" archive_file_stub = os.path.join(datadir, "Foo") - extension = request.param + extension, add_extension = request.param tmpdir = tmpdir_factory.mktemp("compression") - shutil.copy(archive_file_stub + "." + extension, str(tmpdir)) - return os.path.join(str(tmpdir), "Foo.%s" % extension) - - -@pytest.mark.parametrize("archive_file", native_archive_list, indirect=True) -def test_native_unpacking(tmpdir_factory, archive_file): - util = scomp.decompressor_for(archive_file) + tmp_archive_file = os.path.join( + str(tmpdir), "Foo" + (("." + extension) if add_extension else "") + ) + shutil.copy(archive_file_stub + "." + extension, tmp_archive_file) + return (tmp_archive_file, extension) + + +@pytest.mark.parametrize( + "archive_file_and_extension", product(native_archive_list, [True, False]), indirect=True +) +def test_native_unpacking(tmpdir_factory, archive_file_and_extension): + archive_file, extension = archive_file_and_extension + util = scomp.decompressor_for(archive_file, extension) tmpdir = tmpdir_factory.mktemp("comp_test") with working_dir(str(tmpdir)): assert not os.listdir(os.getcwd()) @@ -63,10 +71,13 @@ def test_native_unpacking(tmpdir_factory, archive_file): assert "TEST" in contents -@pytest.mark.skipif(sys.platform == "win32", reason="Only Python unpacking available on Windows") -@pytest.mark.parametrize("archive_file", ext_archive.keys(), indirect=True) -def test_system_unpacking(tmpdir_factory, archive_file, compr_support_check): +@pytest.mark.not_on_windows("Only Python unpacking available on Windows") +@pytest.mark.parametrize( + "archive_file_and_extension", [(ext, True) for ext in ext_archive.keys()], indirect=True +) +def test_system_unpacking(tmpdir_factory, archive_file_and_extension, compr_support_check): # actually run test + archive_file, _ = archive_file_and_extension util = scomp.decompressor_for(archive_file) tmpdir = tmpdir_factory.mktemp("system_comp_test") with working_dir(str(tmpdir)): @@ -85,20 +96,3 @@ def test_unallowed_extension(): bad_ext_archive = "Foo.cxx" with pytest.raises(CommandNotFoundError): scomp.decompressor_for(bad_ext_archive) - - -@pytest.mark.parametrize("archive", ext_archive.values()) -def test_get_extension(archive): - ext = scomp.extension_from_path(archive) - assert ext_archive[ext] == archive - - -def test_get_bad_extension(): - archive = "Foo.cxx" - ext = scomp.extension_from_path(archive) - assert ext is None - - -@pytest.mark.parametrize("path", ext_archive.values()) -def test_allowed_archive(path): - assert scomp.allowed_archive(path) diff --git a/lib/spack/spack/test/util/editor.py b/lib/spack/spack/test/util/editor.py index 745c40ba8542b5..7365bd4e4d6295 100644 --- a/lib/spack/spack/test/util/editor.py +++ b/lib/spack/spack/test/util/editor.py @@ -14,7 +14,7 @@ pytestmark = [ pytest.mark.usefixtures("working_env"), - pytest.mark.skipif(sys.platform == "win32", reason="editor not implemented on windows"), + pytest.mark.not_on_windows("editor not implemented on windows"), ] @@ -33,7 +33,6 @@ def clean_env_vars(): @pytest.fixture(autouse=True) def working_editor_test_env(working_env): """Don't leak environent variables between functions here.""" - pass # parameterized fixture for editor var names diff --git a/lib/spack/spack/test/util/elf.py b/lib/spack/spack/test/util/elf.py index b80072d015c0c0..db826df1730fab 100644 --- a/lib/spack/spack/test/util/elf.py +++ b/lib/spack/spack/test/util/elf.py @@ -14,6 +14,7 @@ import spack.platforms import spack.util.elf as elf import spack.util.executable +from spack.hooks.drop_redundant_rpaths import drop_redundant_rpaths # note that our elf parser is platform independent... but I guess creating an elf file @@ -119,6 +120,21 @@ def test_parser_doesnt_deal_with_nonzero_offset(): elf.parse_elf(elf_at_offset_one) +def test_only_header(): + # When passing only_header=True parsing a file that is literally just a header + # without any sections/segments should not error. + + # 32 bit + elf_32 = elf.parse_elf(io.BytesIO(b"\x7fELF\x01\x01" + b"\x00" * 46), only_header=True) + assert not elf_32.is_64_bit + assert elf_32.is_little_endian + + # 64 bit + elf_64 = elf.parse_elf(io.BytesIO(b"\x7fELF\x02\x01" + b"\x00" * 58), only_header=True) + assert elf_64.is_64_bit + assert elf_64.is_little_endian + + @pytest.mark.requires_executables("gcc") @skip_unless_linux def test_elf_get_and_replace_rpaths(binary_with_rpaths): @@ -159,3 +175,30 @@ def test_elf_get_and_replace_rpaths(binary_with_rpaths): [(b"/short-a", b"/very/long/prefix-a"), (b"/short-b", b"/very/long/prefix-b")] ), ) + + +@pytest.mark.requires_executables("gcc") +@skip_unless_linux +def test_drop_redundant_rpath(tmpdir, binary_with_rpaths): + """Test the post install hook that drops redundant rpath entries""" + + # Use existing and non-existing dirs in tmpdir + non_existing_dirs = [str(tmpdir.join("a")), str(tmpdir.join("b"))] + existing_dirs = [str(tmpdir.join("c")), str(tmpdir.join("d"))] + all_dirs = non_existing_dirs + existing_dirs + + tmpdir.ensure("c", dir=True) + tmpdir.ensure("d", dir=True) + + # Create a binary with rpaths to both existing and non-existing dirs + binary = binary_with_rpaths(rpaths=all_dirs) + + # Verify that the binary has all the rpaths + # sometimes compilers add extra rpaths, so we test for a subset + assert set(all_dirs).issubset(elf.get_rpaths(binary)) + + # Test whether the right rpaths are dropped + drop_redundant_rpaths(binary) + new_rpaths = elf.get_rpaths(binary) + assert set(existing_dirs).issubset(new_rpaths) + assert set(non_existing_dirs).isdisjoint(new_rpaths) diff --git a/lib/spack/spack/test/util/environment.py b/lib/spack/spack/test/util/environment.py index 481a58db472331..b797331b779cb9 100644 --- a/lib/spack/spack/test/util/environment.py +++ b/lib/spack/spack/test/util/environment.py @@ -172,8 +172,8 @@ def test_escape_double_quotes_in_shell_modifications(): assert r'set "VAR=$PATH;$ANOTHER_PATH"' in cmds assert r'set "QUOTED_VAR="MY_VAL"' in cmds cmds = to_validate.shell_modifications(shell="pwsh") - assert r"$Env:VAR=$PATH;$ANOTHER_PATH" in cmds - assert r'$Env:QUOTED_VAR="MY_VAL"' in cmds + assert "$Env:VAR='$PATH;$ANOTHER_PATH'" in cmds + assert "$Env:QUOTED_VAR='\"MY_VAL\"'" in cmds else: cmds = to_validate.shell_modifications() assert 'export VAR="$PATH:$ANOTHER_PATH"' in cmds diff --git a/lib/spack/spack/test/util/executable.py b/lib/spack/spack/test/util/executable.py index b4e4b4888ec511..839cf04bfba6f5 100644 --- a/lib/spack/spack/test/util/executable.py +++ b/lib/spack/spack/test/util/executable.py @@ -5,6 +5,7 @@ import os import sys +from pathlib import PurePath import pytest @@ -16,13 +17,16 @@ def test_read_unicode(tmpdir, working_env): - script_name = "print_unicode.py" - # read the unicode back in and see whether things work - if sys.platform == "win32": - script = ex.Executable("%s %s" % (sys.executable, script_name)) - else: - script = ex.Executable("./%s" % script_name) with tmpdir.as_cwd(): + script_name = "print_unicode.py" + # read the unicode back in and see whether things work + if sys.platform == "win32": + script = ex.Executable("%s" % (sys.executable)) + script_args = [script_name] + else: + script = ex.Executable("./%s" % script_name) + script_args = [] + os.environ["LD_LIBRARY_PATH"] = spack.main.spack_ld_library_path # make a script that prints some unicode with open(script_name, "w") as f: @@ -38,7 +42,7 @@ def test_read_unicode(tmpdir, working_env): fs.set_executable(script_name) filter_shebangs_in_directory(".", [script_name]) - assert "\xc3" == script(output=str).strip() + assert "\xc3" == script(*script_args, output=str).strip() def test_which_relative_path_with_slash(tmpdir, working_env): @@ -68,7 +72,7 @@ def test_which_with_slash_ignores_path(tmpdir, working_env): path = str(tmpdir.join("exe")) wrong_path = str(tmpdir.join("bin", "exe")) - os.environ["PATH"] = os.path.dirname(wrong_path) + os.environ["PATH"] = str(PurePath(wrong_path).parent) with tmpdir.as_cwd(): if sys.platform == "win32": diff --git a/lib/spack/spack/test/util/file_cache.py b/lib/spack/spack/test/util/file_cache.py index 43e729433971bc..a26d688728b8f2 100644 --- a/lib/spack/spack/test/util/file_cache.py +++ b/lib/spack/spack/test/util/file_cache.py @@ -5,7 +5,6 @@ """Test Spack's FileCache.""" import os -import sys import pytest @@ -32,7 +31,7 @@ def test_write_and_read_cache_file(file_cache): assert text == "foobar\n" -@pytest.mark.skipif(sys.platform == "win32", reason="Locks not supported on Windows") +@pytest.mark.not_on_windows("Locks not supported on Windows") def test_failed_write_and_read_cache_file(file_cache): """Test failing to write then attempting to read a cached file.""" with pytest.raises(RuntimeError, match=r"^foobar$"): @@ -84,7 +83,7 @@ def test_write_and_remove_cache_file(file_cache): # assert os.path.exists(file_cache._lock_path('test.yaml')) -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_cache_init_entry_fails(file_cache): """Test init_entry failures.""" relpath = fs.join_path("test-dir", "read-only-file.txt") diff --git a/lib/spack/spack/test/util/package_hash.py b/lib/spack/spack/test/util/package_hash.py index 03735f1e26ac20..047f5df09e0f75 100644 --- a/lib/spack/spack/test/util/package_hash.py +++ b/lib/spack/spack/test/util/package_hash.py @@ -20,9 +20,9 @@ def compare_sans_name(eq, spec1, spec2): content1 = ph.canonical_source(spec1) - content1 = content1.replace(spack.repo.path.get_pkg_class(spec1.name).__name__, "TestPackage") + content1 = content1.replace(spack.repo.PATH.get_pkg_class(spec1.name).__name__, "TestPackage") content2 = ph.canonical_source(spec2) - content2 = content2.replace(spack.repo.path.get_pkg_class(spec2.name).__name__, "TestPackage") + content2 = content2.replace(spack.repo.PATH.get_pkg_class(spec2.name).__name__, "TestPackage") if eq: assert content1 == content2 else: @@ -31,12 +31,12 @@ def compare_sans_name(eq, spec1, spec2): def compare_hash_sans_name(eq, spec1, spec2): content1 = ph.canonical_source(spec1) - pkg_cls1 = spack.repo.path.get_pkg_class(spec1.name) + pkg_cls1 = spack.repo.PATH.get_pkg_class(spec1.name) content1 = content1.replace(pkg_cls1.__name__, "TestPackage") hash1 = pkg_cls1(spec1).content_hash(content=content1) content2 = ph.canonical_source(spec2) - pkg_cls2 = spack.repo.path.get_pkg_class(spec2.name) + pkg_cls2 = spack.repo.PATH.get_pkg_class(spec2.name) content2 = content2.replace(pkg_cls2.__name__, "TestPackage") hash2 = pkg_cls2(spec2).content_hash(content=content2) diff --git a/lib/spack/spack/test/util/path.py b/lib/spack/spack/test/util/path.py index 3949620e1ac323..db9fcce53fcb41 100644 --- a/lib/spack/spack/test/util/path.py +++ b/lib/spack/spack/test/util/path.py @@ -29,21 +29,19 @@ ] -def test_sanitze_file_path(tmpdir): - """Test filtering illegal characters out of potential file paths""" - # *nix illegal files characters are '/' and none others - illegal_file_path = str(tmpdir) + "//" + "abcdefghi.txt" +def test_sanitize_filename(): + """Test filtering illegal characters out of potential filenames""" + sanitized = sup.sanitize_filename("""acd/?e:f"g|h*i.\0txt""") if sys.platform == "win32": - # Windows has a larger set of illegal characters - illegal_file_path = os.path.join(tmpdir, 'acd?e:f"g|h*i.txt') - real_path = sup.sanitize_file_path(illegal_file_path) - assert real_path == os.path.join(str(tmpdir), "abcdefghi.txt") + assert sanitized == "a_b_cd__e_f_g_h_i._txt" + else: + assert sanitized == """acd_?e:f"g|h*i._txt""" # This class pertains to path string padding manipulation specifically # which is used for binary caching. This functionality is not supported # on Windows as of yet. -@pytest.mark.skipif(sys.platform == "win32", reason="Padding funtionality unsupported on Windows") +@pytest.mark.not_on_windows("Padding funtionality unsupported on Windows") class TestPathPadding: @pytest.mark.parametrize("padded,fixed", zip(padded_lines, fixed_lines)) def test_padding_substitution(self, padded, fixed): diff --git a/lib/spack/spack/test/util/spack_lock_wrapper.py b/lib/spack/spack/test/util/spack_lock_wrapper.py index cee8b2923286a0..0909ce06c339c8 100644 --- a/lib/spack/spack/test/util/spack_lock_wrapper.py +++ b/lib/spack/spack/test/util/spack_lock_wrapper.py @@ -17,25 +17,19 @@ def test_disable_locking(tmpdir): """Ensure that locks do no real locking when disabled.""" lock_path = str(tmpdir.join("lockfile")) + lock = lk.Lock(lock_path, enable=False) - old_value = spack.config.get("config:locks") + lock.acquire_read() + assert not os.path.exists(lock_path) - with spack.config.override("config:locks", False): - lock = lk.Lock(lock_path) + lock.acquire_write() + assert not os.path.exists(lock_path) - lock.acquire_read() - assert not os.path.exists(lock_path) + lock.release_write() + assert not os.path.exists(lock_path) - lock.acquire_write() - assert not os.path.exists(lock_path) - - lock.release_write() - assert not os.path.exists(lock_path) - - lock.release_read() - assert not os.path.exists(lock_path) - - assert old_value == spack.config.get("config:locks") + lock.release_read() + assert not os.path.exists(lock_path) # "Disable" mock_stage fixture to avoid subdir permissions issues on cleanup. diff --git a/lib/spack/spack/test/util/spack_yaml.py b/lib/spack/spack/test/util/spack_yaml.py index 9beccdec712b55..f7c8851d27194c 100644 --- a/lib/spack/spack/test/util/spack_yaml.py +++ b/lib/spack/spack/test/util/spack_yaml.py @@ -86,11 +86,13 @@ def get_file_lines(filename): if match: filename, line, key, val = match.groups() line = int(line) - val = val.strip("'\"") + lines = get_file_lines(filename) + assert key in lines[line] + val = val.strip("'\"") + printed_line = lines[line] if val.lower() in ("true", "false"): val = val.lower() + printed_line = printed_line.lower() - lines = get_file_lines(filename) - assert key in lines[line], filename - assert val in lines[line] + assert val in printed_line, filename diff --git a/lib/spack/spack/test/util/timer.py b/lib/spack/spack/test/util/timer.py index 5604c4d5249fe0..199b9355a7f4a7 100644 --- a/lib/spack/spack/test/util/timer.py +++ b/lib/spack/spack/test/util/timer.py @@ -126,8 +126,8 @@ def test_timer_write(): deserialized = json.loads(json_buffer.getvalue()) assert deserialized == { - "phases": [{"name": "timer", "seconds": 1.0}], - "total": {"seconds": 3.0}, + "phases": [{"name": "timer", "path": "timer", "seconds": 1.0, "count": 1}], + "total": 3.0, } diff --git a/lib/spack/spack/test/util/unparse/unparse.py b/lib/spack/spack/test/util/unparse/unparse.py index 986ba7e262ce62..4a8b624894c836 100644 --- a/lib/spack/spack/test/util/unparse/unparse.py +++ b/lib/spack/spack/test/util/unparse/unparse.py @@ -12,9 +12,7 @@ import spack.util.unparse -pytestmark = pytest.mark.skipif( - sys.platform == "win32", reason="Test module unsupported on Windows" -) +pytestmark = pytest.mark.not_on_windows("Test module unsupported on Windows") def read_pyfile(filename): diff --git a/lib/spack/spack/test/util/util_string.py b/lib/spack/spack/test/util/util_string.py deleted file mode 100644 index f4de3738598029..00000000000000 --- a/lib/spack/spack/test/util/util_string.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack.util.string import plural - - -def test_plural(): - assert plural(0, "thing") == "0 things" - assert plural(1, "thing") == "1 thing" - assert plural(2, "thing") == "2 things" - assert plural(1, "thing", "wombats") == "1 thing" - assert plural(2, "thing", "wombats") == "2 wombats" diff --git a/lib/spack/spack/test/util/util_url.py b/lib/spack/spack/test/util/util_url.py index 582393a02293d3..ec8f60627ab881 100644 --- a/lib/spack/spack/test/util/util_url.py +++ b/lib/spack/spack/test/util/util_url.py @@ -8,8 +8,7 @@ import os.path import urllib.parse -import pytest - +import spack.util.path import spack.util.url as url_util @@ -197,84 +196,14 @@ def test_url_join_absolute_paths(): assert url_util.join(*args, resolve_href=False) == "http://example.com/path/resource" -@pytest.mark.parametrize( - "url,parts", - [ - ( - "ssh://user@host.xz:500/path/to/repo.git/", - ("ssh", "user", "host.xz", 500, "/path/to/repo.git"), - ), - ( - "ssh://user@host.xz/path/to/repo.git/", - ("ssh", "user", "host.xz", None, "/path/to/repo.git"), - ), - ( - "ssh://host.xz:500/path/to/repo.git/", - ("ssh", None, "host.xz", 500, "/path/to/repo.git"), - ), - ("ssh://host.xz/path/to/repo.git/", ("ssh", None, "host.xz", None, "/path/to/repo.git")), - ( - "ssh://user@host.xz/path/to/repo.git/", - ("ssh", "user", "host.xz", None, "/path/to/repo.git"), - ), - ("ssh://host.xz/path/to/repo.git/", ("ssh", None, "host.xz", None, "/path/to/repo.git")), - ( - "ssh://user@host.xz/~user/path/to/repo.git/", - ("ssh", "user", "host.xz", None, "~user/path/to/repo.git"), - ), - ( - "ssh://host.xz/~user/path/to/repo.git/", - ("ssh", None, "host.xz", None, "~user/path/to/repo.git"), - ), - ( - "ssh://user@host.xz/~/path/to/repo.git", - ("ssh", "user", "host.xz", None, "~/path/to/repo.git"), - ), - ("ssh://host.xz/~/path/to/repo.git", ("ssh", None, "host.xz", None, "~/path/to/repo.git")), - ("git@github.com:spack/spack.git", (None, "git", "github.com", None, "spack/spack.git")), - ("user@host.xz:/path/to/repo.git/", (None, "user", "host.xz", None, "/path/to/repo.git")), - ("host.xz:/path/to/repo.git/", (None, None, "host.xz", None, "/path/to/repo.git")), - ( - "user@host.xz:~user/path/to/repo.git/", - (None, "user", "host.xz", None, "~user/path/to/repo.git"), - ), - ( - "host.xz:~user/path/to/repo.git/", - (None, None, "host.xz", None, "~user/path/to/repo.git"), - ), - ("user@host.xz:path/to/repo.git", (None, "user", "host.xz", None, "path/to/repo.git")), - ("host.xz:path/to/repo.git", (None, None, "host.xz", None, "path/to/repo.git")), - ( - "rsync://host.xz/path/to/repo.git/", - ("rsync", None, "host.xz", None, "/path/to/repo.git"), - ), - ("git://host.xz/path/to/repo.git/", ("git", None, "host.xz", None, "/path/to/repo.git")), - ( - "git://host.xz/~user/path/to/repo.git/", - ("git", None, "host.xz", None, "~user/path/to/repo.git"), - ), - ("http://host.xz/path/to/repo.git/", ("http", None, "host.xz", None, "/path/to/repo.git")), - ( - "https://host.xz/path/to/repo.git/", - ("https", None, "host.xz", None, "/path/to/repo.git"), - ), - ("https://github.com/spack/spack", ("https", None, "github.com", None, "/spack/spack")), - ("https://github.com/spack/spack/", ("https", None, "github.com", None, "/spack/spack")), - ("file:///path/to/repo.git/", ("file", None, None, None, "/path/to/repo.git")), - ("file://~/path/to/repo.git/", ("file", None, None, None, "~/path/to/repo.git")), - # bad ports should give us None - ("ssh://host.xz:port/path/to/repo.git/", None), - # bad ports should give us None - ("ssh://host-foo.xz:port/path/to/repo.git/", None), - # regular file paths should give us None - ("/path/to/repo.git/", None), - ("path/to/repo.git/", None), - ("~/path/to/repo.git", None), - ], -) -def test_git_url_parse(url, parts): - if parts is None: - with pytest.raises(ValueError): - url_util.parse_git_url(url) - else: - assert parts == url_util.parse_git_url(url) +def test_default_download_name(): + url = "https://example.com:1234/path/to/file.txt;params?abc=def#file=blob.tar" + filename = url_util.default_download_filename(url) + assert filename == spack.util.path.sanitize_filename(filename) + + +def test_default_download_name_dot_dot(): + """Avoid that downloaded files get names computed as ., .. or any hidden file.""" + assert url_util.default_download_filename("https://example.com/.") == "_" + assert url_util.default_download_filename("https://example.com/..") == "_." + assert url_util.default_download_filename("https://example.com/.abcdef") == "_abcdef" diff --git a/lib/spack/spack/test/variant.py b/lib/spack/spack/test/variant.py index ac6525f7488d84..517197654cfef8 100644 --- a/lib/spack/spack/test/variant.py +++ b/lib/spack/spack/test/variant.py @@ -525,13 +525,13 @@ def test_yaml_entry(self): def test_from_node_dict(): a = MultiValuedVariant.from_node_dict("foo", ["bar"]) - assert type(a) == MultiValuedVariant + assert type(a) is MultiValuedVariant a = MultiValuedVariant.from_node_dict("foo", "bar") - assert type(a) == SingleValuedVariant + assert type(a) is SingleValuedVariant a = MultiValuedVariant.from_node_dict("foo", "true") - assert type(a) == BoolValuedVariant + assert type(a) is BoolValuedVariant class TestVariant: diff --git a/lib/spack/spack/test/verification.py b/lib/spack/spack/test/verification.py index d23a430ab8f206..9b5de54e4e8480 100644 --- a/lib/spack/spack/test/verification.py +++ b/lib/spack/spack/test/verification.py @@ -7,7 +7,6 @@ import os import shutil import stat -import sys import pytest @@ -19,7 +18,7 @@ import spack.util.spack_json as sjson import spack.verify -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="Tests fail on Win") +pytestmark = pytest.mark.not_on_windows("Tests fail on Win") def test_link_manifest_entry(tmpdir): @@ -174,7 +173,9 @@ def test_check_prefix_manifest(tmpdir): assert results.errors[malware] == ["added"] manifest_file = os.path.join( - spec.prefix, spack.store.layout.metadata_dir, spack.store.layout.manifest_file_name + spec.prefix, + spack.store.STORE.layout.metadata_dir, + spack.store.STORE.layout.manifest_file_name, ) with open(manifest_file, "w") as f: f.write("{This) string is not proper json") @@ -189,7 +190,7 @@ def test_single_file_verification(tmpdir): # to which it belongs filedir = os.path.join(str(tmpdir), "a", "b", "c", "d") filepath = os.path.join(filedir, "file") - metadir = os.path.join(str(tmpdir), spack.store.layout.metadata_dir) + metadir = os.path.join(str(tmpdir), spack.store.STORE.layout.metadata_dir) fs.mkdirp(filedir) fs.mkdirp(metadir) @@ -199,7 +200,7 @@ def test_single_file_verification(tmpdir): data = spack.verify.create_manifest_entry(filepath) - manifest_file = os.path.join(metadir, spack.store.layout.manifest_file_name) + manifest_file = os.path.join(metadir, spack.store.STORE.layout.manifest_file_name) with open(manifest_file, "w") as f: sjson.dump({filepath: data}, f) diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 7d329336aee679..50fcc197986353 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -8,7 +8,7 @@ where it makes sense. """ import os -import sys +import pathlib import pytest @@ -17,6 +17,7 @@ import spack.package_base import spack.spec from spack.version import ( + EmptyRangeError, GitVersion, StandardVersion, Version, @@ -26,6 +27,7 @@ is_git_version, ver, ) +from spack.version.git_ref_lookup import SEMVER_REGEX def assert_ver_lt(a, b): @@ -597,11 +599,10 @@ def test_invalid_versions(version_str): Version(version_str) -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") def test_versions_from_git(git, mock_git_version_info, monkeypatch, mock_packages): repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( - spack.package_base.PackageBase, "git", "file://%s" % repo_path, raising=False + spack.package_base.PackageBase, "git", pathlib.Path(repo_path).as_uri(), raising=False ) for commit in commits: @@ -618,7 +619,6 @@ def test_versions_from_git(git, mock_git_version_info, monkeypatch, mock_package assert str(comparator) == expected -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") @pytest.mark.parametrize( "commit_idx,expected_satisfies,expected_not_satisfies", [ @@ -642,7 +642,7 @@ def test_git_hash_comparisons( """Check that hashes compare properly to versions""" repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( - spack.package_base.PackageBase, "git", "file://%s" % repo_path, raising=False + spack.package_base.PackageBase, "git", pathlib.Path(repo_path).as_uri(), raising=False ) spec = spack.spec.Spec(f"git-test-commit@{commits[commit_idx]}").concretized() @@ -653,12 +653,11 @@ def test_git_hash_comparisons( assert not spec.satisfies(item) -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") def test_git_ref_comparisons(mock_git_version_info, install_mockery, mock_packages, monkeypatch): """Check that hashes compare properly to versions""" repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( - spack.package_base.PackageBase, "git", "file://%s" % repo_path, raising=False + spack.package_base.PackageBase, "git", pathlib.Path(repo_path).as_uri(), raising=False ) # Spec based on tag v1.0 @@ -676,6 +675,25 @@ def test_git_ref_comparisons(mock_git_version_info, install_mockery, mock_packag assert str(spec_branch.version) == "git.1.x=1.2" +def test_git_branch_with_slash(): + class MockLookup(object): + def get(self, ref): + assert ref == "feature/bar" + return "1.2", 0 + + v = spack.version.from_string("git.feature/bar") + assert isinstance(v, GitVersion) + v.attach_lookup(MockLookup()) + + # Create a version range + test_number_version = spack.version.from_string("1.2") + v.satisfies(test_number_version) + + serialized = VersionList([v]).to_dict() + v_deserialized = VersionList.from_dict(serialized) + assert v_deserialized[0].ref == "feature/bar" + + @pytest.mark.parametrize( "string,git", [ @@ -697,9 +715,9 @@ def test_version_range_nonempty(): def test_empty_version_range_raises(): - with pytest.raises(ValueError): + with pytest.raises(EmptyRangeError, match="2:1.0 is an empty range"): assert VersionRange("2", "1.0") - with pytest.raises(ValueError): + with pytest.raises(EmptyRangeError, match="2:1.0 is an empty range"): assert ver("2:1.0") @@ -787,7 +805,6 @@ def test_version_intersects_satisfies_semantic(lhs_str, rhs_str, expected): ), ], ) -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") def test_git_versions_without_explicit_reference( spec_str, tested_intersects, @@ -798,7 +815,7 @@ def test_git_versions_without_explicit_reference( ): repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( - spack.package_base.PackageBase, "git", "file://%s" % repo_path, raising=False + spack.package_base.PackageBase, "git", pathlib.Path(repo_path).as_uri(), raising=False ) spec = spack.spec.Spec(spec_str) @@ -933,7 +950,7 @@ def test_inclusion_upperbound(): assert is_specific.intersects(upperbound) and is_range.intersects(upperbound) -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_git_version_repo_attached_after_serialization( mock_git_version_info, mock_packages, config, monkeypatch ): @@ -953,7 +970,7 @@ def test_git_version_repo_attached_after_serialization( assert spack.spec.Spec.from_dict(spec.to_dict()).satisfies("@1.0") -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_resolved_git_version_is_shown_in_str( mock_git_version_info, mock_packages, config, monkeypatch ): @@ -976,3 +993,25 @@ def test_unresolvable_git_versions_error(config, mock_packages): # The package exists, but does not have a git property set. When dereferencing # the version, we should get VersionLookupError, not a generic AttributeError. spack.spec.Spec(f"git-test-commit@{'a' * 40}").version.ref_version + + +@pytest.mark.parametrize( + "tag,expected", + [ + ("v100.2.3", "100.2.3"), + ("v1.2.3", "1.2.3"), + ("v1.2.3-pre.release+build.1", "1.2.3-pre.release+build.1"), + ("v1.2.3+build.1", "1.2.3+build.1"), + ("v1.2.3+build_1", None), + ("v1.2.3-pre.release", "1.2.3-pre.release"), + ("v1.2.3-pre_release", None), + ("1.2.3", "1.2.3"), + ("1.2.3.", None), + ], +) +def test_semver_regex(tag, expected): + result = SEMVER_REGEX.search(tag) + if expected is None: + assert result is None + else: + assert result.group() == expected diff --git a/lib/spack/spack/test/views.py b/lib/spack/spack/test/views.py index 77dd86d4114fa0..4df1da7e4080fc 100644 --- a/lib/spack/spack/test/views.py +++ b/lib/spack/spack/test/views.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import sys import pytest @@ -13,7 +12,7 @@ from spack.spec import Spec -@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") +@pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_remove_extensions_ordered(install_mockery, mock_fetch, tmpdir): view_dir = str(tmpdir.join("view")) layout = DirectoryLayout(view_dir) diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py index b681b5b0a0f274..ed4f693c5a539f 100644 --- a/lib/spack/spack/test/web.py +++ b/lib/spack/spack/test/web.py @@ -3,7 +3,10 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections +import email.message import os +import pickle +import urllib.request import pytest @@ -12,6 +15,8 @@ import spack.config import spack.mirror import spack.paths +import spack.url +import spack.util.path import spack.util.s3 import spack.util.url as url_util import spack.util.web @@ -93,36 +98,36 @@ def test_spider(depth, expected_found, expected_not_found, expected_text): def test_spider_no_response(monkeypatch): # Mock the absence of a response monkeypatch.setattr(spack.util.web, "read_from_url", lambda x, y: (None, None, None)) - pages, links = spack.util.web.spider(root, depth=0) + pages, links, _, _ = spack.util.web._spider(root, collect_nested=False, _visited=set()) assert not pages and not links def test_find_versions_of_archive_0(): - versions = spack.util.web.find_versions_of_archive(root_tarball, root, list_depth=0) + versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=0) assert Version("0.0.0") in versions def test_find_versions_of_archive_1(): - versions = spack.util.web.find_versions_of_archive(root_tarball, root, list_depth=1) + versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=1) assert Version("0.0.0") in versions assert Version("1.0.0") in versions def test_find_versions_of_archive_2(): - versions = spack.util.web.find_versions_of_archive(root_tarball, root, list_depth=2) + versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=2) assert Version("0.0.0") in versions assert Version("1.0.0") in versions assert Version("2.0.0") in versions def test_find_exotic_versions_of_archive_2(): - versions = spack.util.web.find_versions_of_archive(root_tarball, root, list_depth=2) + versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=2) # up for grabs to make this better. assert Version("2.0.0b2") in versions def test_find_versions_of_archive_3(): - versions = spack.util.web.find_versions_of_archive(root_tarball, root, list_depth=3) + versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=3) assert Version("0.0.0") in versions assert Version("1.0.0") in versions assert Version("2.0.0") in versions @@ -131,16 +136,14 @@ def test_find_versions_of_archive_3(): def test_find_exotic_versions_of_archive_3(): - versions = spack.util.web.find_versions_of_archive(root_tarball, root, list_depth=3) + versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=3) assert Version("2.0.0b2") in versions assert Version("3.0a1") in versions assert Version("4.5-rc5") in versions def test_find_versions_of_archive_with_fragment(): - versions = spack.util.web.find_versions_of_archive( - root_tarball, root_with_fragment, list_depth=0 - ) + versions = spack.url.find_versions_of_archive(root_tarball, root_with_fragment, list_depth=0) assert Version("5.0.0") in versions @@ -267,7 +270,7 @@ def head_object(self, Bucket=None, Key=None): def test_gather_s3_information(monkeypatch, capfd): - mirror = spack.mirror.Mirror.from_dict( + mirror = spack.mirror.Mirror( { "fetch": { "access_token": "AAAAAAA", @@ -307,7 +310,7 @@ def test_remove_s3_url(monkeypatch, capfd): def get_s3_session(url, method="fetch"): return MockS3Client() - monkeypatch.setattr(spack.util.s3, "get_s3_session", get_s3_session) + monkeypatch.setattr(spack.util.web, "get_s3_session", get_s3_session) current_debug_level = tty.debug_level() tty.set_debug(1) @@ -338,3 +341,25 @@ def get_s3_session(url, method="fetch"): def test_s3_url_parsing(): assert spack.util.s3._parse_s3_endpoint_url("example.com") == "https://example.com" assert spack.util.s3._parse_s3_endpoint_url("http://example.com") == "http://example.com" + + +def test_detailed_http_error_pickle(tmpdir): + tmpdir.join("response").write("response") + + headers = email.message.Message() + headers.add_header("Content-Type", "text/plain") + + # Use a temporary file object as a response body + with open(str(tmpdir.join("response")), "rb") as f: + error = spack.util.web.DetailedHTTPError( + urllib.request.Request("http://example.com"), 404, "Not Found", headers, f + ) + + deserialized = pickle.loads(pickle.dumps(error)) + + assert isinstance(deserialized, spack.util.web.DetailedHTTPError) + assert deserialized.code == 404 + assert deserialized.filename == "http://example.com" + assert deserialized.reason == "Not Found" + assert str(deserialized.info()) == str(headers) + assert str(deserialized) == str(error) diff --git a/lib/spack/spack/traverse.py b/lib/spack/spack/traverse.py index c0d45df6625fdc..580e4877240569 100644 --- a/lib/spack/spack/traverse.py +++ b/lib/spack/spack/traverse.py @@ -4,7 +4,9 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) from collections import defaultdict, namedtuple +from typing import Union +import spack.deptypes as dt import spack.spec # Export only the high-level API. @@ -26,8 +28,8 @@ class BaseVisitor: """A simple visitor that accepts all edges unconditionally and follows all edges to dependencies of a given ``deptype``.""" - def __init__(self, deptype="all"): - self.deptype = deptype + def __init__(self, depflag: dt.DepFlag = dt.ALL): + self.depflag = depflag def accept(self, item): """ @@ -43,15 +45,15 @@ def accept(self, item): return True def neighbors(self, item): - return sort_edges(item.edge.spec.edges_to_dependencies(deptype=self.deptype)) + return sort_edges(item.edge.spec.edges_to_dependencies(depflag=self.depflag)) class ReverseVisitor: """A visitor that reverses the arrows in the DAG, following dependents.""" - def __init__(self, visitor, deptype="all"): + def __init__(self, visitor, depflag: dt.DepFlag = dt.ALL): self.visitor = visitor - self.deptype = deptype + self.depflag = depflag def accept(self, item): return self.visitor.accept(item) @@ -61,7 +63,7 @@ def neighbors(self, item): generic programming""" spec = item.edge.spec return sort_edges( - [edge.flip() for edge in spec.edges_from_dependents(deptype=self.deptype)] + [edge.flip() for edge in spec.edges_from_dependents(depflag=self.depflag)] ) @@ -174,7 +176,9 @@ def edges(self): return list(reversed(self.reverse_order)) -def get_visitor_from_args(cover, direction, deptype, key=id, visited=None, visitor=None): +def get_visitor_from_args( + cover, direction, depflag: Union[dt.DepFlag, dt.DepTypes], key=id, visited=None, visitor=None +): """ Create a visitor object from common keyword arguments. @@ -190,7 +194,7 @@ def get_visitor_from_args(cover, direction, deptype, key=id, visited=None, visit direction (str): ``children`` or ``parents``. If ``children``, does a traversal of this spec's children. If ``parents``, traverses upwards in the DAG towards the root. - deptype (str or tuple): allowed dependency types + deptype: allowed dependency types key: function that takes a spec and outputs a key for uniqueness test. visited (set or None): a set of nodes not to follow (when using cover=nodes/edges) visitor: An initial visitor that is used for composition. @@ -198,13 +202,15 @@ def get_visitor_from_args(cover, direction, deptype, key=id, visited=None, visit Returns: A visitor """ - visitor = visitor or BaseVisitor(deptype) + if not isinstance(depflag, dt.DepFlag): + depflag = dt.canonicalize(depflag) + visitor = visitor or BaseVisitor(depflag) if cover == "nodes": visitor = CoverNodesVisitor(visitor, key, visited) elif cover == "edges": visitor = CoverEdgesVisitor(visitor, key, visited) if direction == "parents": - visitor = ReverseVisitor(visitor, deptype) + visitor = ReverseVisitor(visitor, depflag) return visitor @@ -212,7 +218,7 @@ def with_artificial_edges(specs): """Initialize a list of edges from an imaginary root node to the root specs.""" return [ EdgeAndDepth( - edge=spack.spec.DependencySpec(parent=None, spec=s, deptypes=(), virtuals=()), depth=0 + edge=spack.spec.DependencySpec(parent=None, spec=s, depflag=0, virtuals=()), depth=0 ) for s in specs ] @@ -374,7 +380,12 @@ def traverse_breadth_first_tree_nodes(parent_id, edges, key=id, depth=0): # Topologic order def traverse_edges_topo( - specs, direction="children", deptype="all", key=id, root=True, all_edges=False + specs, + direction="children", + deptype: Union[dt.DepFlag, dt.DepTypes] = "all", + key=id, + root=True, + all_edges=False, ): """ Returns a list of edges in topological order, in the sense that all in-edges of a @@ -386,13 +397,15 @@ def traverse_edges_topo( specs (list): List of root specs (considered to be depth 0) direction (str): ``children`` (edges are directed from dependent to dependency) or ``parents`` (edges are flipped / directed from dependency to dependent) - deptype (str or tuple): allowed dependency types + deptype: allowed dependency types key: function that takes a spec and outputs a key for uniqueness test. root (bool): Yield the root nodes themselves all_edges (bool): When ``False`` only one in-edge per node is returned, when ``True`` all reachable edges are returned. """ - visitor = BaseVisitor(deptype) + if not isinstance(deptype, dt.DepFlag): + deptype = dt.canonicalize(deptype) + visitor: Union[BaseVisitor, ReverseVisitor, TopoVisitor] = BaseVisitor(deptype) if direction == "parents": visitor = ReverseVisitor(visitor, deptype) visitor = TopoVisitor(visitor, key=key, root=root, all_edges=all_edges) @@ -409,7 +422,7 @@ def traverse_edges( order="pre", cover="nodes", direction="children", - deptype="all", + deptype: Union[dt.DepFlag, dt.DepTypes] = "all", depth=False, key=id, visited=None, @@ -435,7 +448,7 @@ def traverse_edges( direction (str): ``children`` or ``parents``. If ``children``, does a traversal of this spec's children. If ``parents``, traverses upwards in the DAG towards the root. - deptype (str or tuple): allowed dependency types + deptype: allowed dependency types depth (bool): When ``False``, yield just edges. When ``True`` yield the tuple (depth, edge), where depth corresponds to the depth at which edge.spec was discovered. @@ -478,7 +491,7 @@ def traverse_nodes( order="pre", cover="nodes", direction="children", - deptype="all", + deptype: Union[dt.DepFlag, dt.DepTypes] = "all", depth=False, key=id, visited=None, @@ -502,7 +515,7 @@ def traverse_nodes( direction (str): ``children`` or ``parents``. If ``children``, does a traversal of this spec's children. If ``parents``, traverses upwards in the DAG towards the root. - deptype (str or tuple): allowed dependency types + deptype: allowed dependency types depth (bool): When ``False``, yield just edges. When ``True`` yield the tuple ``(depth, edge)``, where depth corresponds to the depth at which ``edge.spec`` was discovered. @@ -517,7 +530,9 @@ def traverse_nodes( yield (item[0], item[1].spec) if depth else item.spec -def traverse_tree(specs, cover="nodes", deptype="all", key=id, depth_first=True): +def traverse_tree( + specs, cover="nodes", deptype: Union[dt.DepFlag, dt.DepTypes] = "all", key=id, depth_first=True +): """ Generator that yields ``(depth, DependencySpec)`` tuples in the depth-first pre-order, so that a tree can be printed from it. @@ -533,7 +548,7 @@ def traverse_tree(specs, cover="nodes", deptype="all", key=id, depth_first=True) ``paths`` -- Explore every unique path reachable from the root. This descends into visited subtrees and will accept nodes multiple times if they're reachable by multiple paths. - deptype (str or tuple): allowed dependency types + deptype: allowed dependency types key: function that takes a spec and outputs a key for uniqueness test. depth_first (bool): Explore the tree in depth-first or breadth-first order. When setting ``depth_first=True`` and ``cover=nodes``, each spec only diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index bf2990f42fbab7..460c42a1af2add 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -27,246 +27,22 @@ """ import io import os +import pathlib import re -from urllib.parse import urlsplit, urlunsplit -import llnl.util.tty as tty +import llnl.url +from llnl.path import convert_to_posix_path from llnl.util.tty.color import cescape, colorize import spack.error -import spack.util.compression as comp -import spack.util.path as spath +import spack.util.web import spack.version - # # Note: We call the input to most of these functions a "path" but the functions # work on paths and URLs. There's not a good word for both of these, but # "path" seemed like the most generic term. # -def find_list_urls(url): - r"""Find good list URLs for the supplied URL. - - By default, returns the dirname of the archive path. - - Provides special treatment for the following websites, which have a - unique list URL different from the dirname of the download URL: - - ========= ======================================================= - GitHub https://github.com///releases - GitLab https://gitlab.\*///tags - BitBucket https://bitbucket.org///downloads/?tab=tags - CRAN https://\*.r-project.org/src/contrib/Archive/ - PyPI https://pypi.org/simple// - LuaRocks https://luarocks.org/modules// - ========= ======================================================= - - Note: this function is called by `spack versions`, `spack checksum`, - and `spack create`, but not by `spack fetch` or `spack install`. - - Parameters: - url (str): The download URL for the package - - Returns: - set: One or more list URLs for the package - """ - - url_types = [ - # GitHub - # e.g. https://github.com/llnl/callpath/archive/v1.0.1.tar.gz - (r"(.*github\.com/[^/]+/[^/]+)", lambda m: m.group(1) + "/releases"), - # GitLab API endpoint - # e.g. https://gitlab.dkrz.de/api/v4/projects/k202009%2Flibaec/repository/archive.tar.gz?sha=v1.0.2 - ( - r"(.*gitlab[^/]+)/api/v4/projects/([^/]+)%2F([^/]+)", - lambda m: m.group(1) + "/" + m.group(2) + "/" + m.group(3) + "/tags", - ), - # GitLab non-API endpoint - # e.g. https://gitlab.dkrz.de/k202009/libaec/uploads/631e85bcf877c2dcaca9b2e6d6526339/libaec-1.0.0.tar.gz - (r"(.*gitlab[^/]+/(?!api/v4/projects)[^/]+/[^/]+)", lambda m: m.group(1) + "/tags"), - # BitBucket - # e.g. https://bitbucket.org/eigen/eigen/get/3.3.3.tar.bz2 - (r"(.*bitbucket.org/[^/]+/[^/]+)", lambda m: m.group(1) + "/downloads/?tab=tags"), - # CRAN - # e.g. https://cran.r-project.org/src/contrib/Rcpp_0.12.9.tar.gz - # e.g. https://cloud.r-project.org/src/contrib/rgl_0.98.1.tar.gz - ( - r"(.*\.r-project\.org/src/contrib)/([^_]+)", - lambda m: m.group(1) + "/Archive/" + m.group(2), - ), - # PyPI - # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip - # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip - # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip - # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip - # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip - # e.g. https://pypi.io/packages/py2.py3/o/opencensus-context/opencensus_context-0.1.1-py2.py3-none-any.whl - ( - r"(?:pypi|pythonhosted)[^/]+/packages/[^/]+/./([^/]+)", - lambda m: "https://pypi.org/simple/" + m.group(1) + "/", - ), - # LuaRocks - # e.g. https://luarocks.org/manifests/gvvaughan/lpeg-1.0.2-1.src.rock - # e.g. https://luarocks.org/manifests/openresty/lua-cjson-2.1.0-1.src.rock - ( - r"luarocks[^/]+/(?:modules|manifests)/(?P[^/]+)/" - + r"(?P.+?)-[0-9.-]*\.src\.rock", - lambda m: "https://luarocks.org/modules/" - + m.group("org") - + "/" - + m.group("name") - + "/", - ), - ] - - list_urls = set([os.path.dirname(url)]) - - for pattern, fun in url_types: - match = re.search(pattern, url) - if match: - list_urls.add(fun(match)) - - return list_urls - - -def strip_query_and_fragment(path): - try: - components = urlsplit(path) - stripped = components[:3] + (None, None) - - query, frag = components[3:5] - suffix = "" - if query: - suffix += "?" + query - if frag: - suffix += "#" + frag - - return (urlunsplit(stripped), suffix) - - except ValueError: - tty.debug("Got error parsing path %s" % path) - return (path, "") # Ignore URL parse errors here - - -def strip_version_suffixes(path): - """Some tarballs contain extraneous information after the version: - - * ``bowtie2-2.2.5-source`` - * ``libevent-2.0.21-stable`` - * ``cuda_8.0.44_linux.run`` - - These strings are not part of the version number and should be ignored. - This function strips those suffixes off and returns the remaining string. - The goal is that the version is always the last thing in ``path``: - - * ``bowtie2-2.2.5`` - * ``libevent-2.0.21`` - * ``cuda_8.0.44`` - - Args: - path (str): The filename or URL for the package - - Returns: - str: The ``path`` with any extraneous suffixes removed - """ - # NOTE: This could be done with complicated regexes in parse_version_offset - # NOTE: The problem is that we would have to add these regexes to the end - # NOTE: of every single version regex. Easier to just strip them off - # NOTE: permanently - - suffix_regexes = [ - # Download type - r"[Ii]nstall", - r"all", - r"code", - r"[Ss]ources?", - r"file", - r"full", - r"single", - r"with[a-zA-Z_-]+", - r"rock", - r"src(_0)?", - r"public", - r"bin", - r"binary", - r"run", - r"[Uu]niversal", - r"jar", - r"complete", - r"dynamic", - r"oss", - r"gem", - r"tar", - r"sh", - # Download version - r"release", - r"bin", - r"stable", - r"[Ff]inal", - r"rel", - r"orig", - r"dist", - r"\+", - # License - r"gpl", - # Arch - # Needs to come before and after OS, appears in both orders - r"ia32", - r"intel", - r"amd64", - r"linux64", - r"x64", - r"64bit", - r"x86[_-]64", - r"i586_64", - r"x86", - r"i[36]86", - r"ppc64(le)?", - r"armv?(7l|6l|64)", - # Other - r"cpp", - r"gtk", - r"incubating", - # OS - r"[Ll]inux(_64)?", - r"LINUX", - r"[Uu]ni?x", - r"[Ss]un[Oo][Ss]", - r"[Mm]ac[Oo][Ss][Xx]?", - r"[Oo][Ss][Xx]", - r"[Dd]arwin(64)?", - r"[Aa]pple", - r"[Ww]indows", - r"[Ww]in(64|32)?", - r"[Cc]ygwin(64|32)?", - r"[Mm]ingw", - r"centos", - # Arch - # Needs to come before and after OS, appears in both orders - r"ia32", - r"intel", - r"amd64", - r"linux64", - r"x64", - r"64bit", - r"x86[_-]64", - r"i586_64", - r"x86", - r"i[36]86", - r"ppc64(le)?", - r"armv?(7l|6l|64)?", - # PyPI - r"[._-]py[23].*\.whl", - r"[._-]cp[23].*\.whl", - r"[._-]win.*\.exe", - ] - - for regex in suffix_regexes: - # Remove the suffix from the end of the path - # This may be done multiple times - path = re.sub(r"[._-]?" + regex + "$", "", path) - - return path def strip_name_suffixes(path, version): @@ -341,69 +117,6 @@ def strip_name_suffixes(path, version): return path -def split_url_extension(path): - """Some URLs have a query string, e.g.: - - 1. https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true - 2. http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz - 3. https://gitlab.kitware.com/vtk/vtk/repository/archive.tar.bz2?ref=v7.0.0 - - In (1), the query string needs to be stripped to get at the - extension, but in (2) & (3), the filename is IN a single final query - argument. - - This strips the URL into three pieces: ``prefix``, ``ext``, and ``suffix``. - The suffix contains anything that was stripped off the URL to - get at the file extension. In (1), it will be ``'?raw=true'``, but - in (2), it will be empty. In (3) the suffix is a parameter that follows - after the file extension, e.g.: - - 1. ``('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7', '.tgz', '?raw=true')`` - 2. ``('http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin', '.tar.gz', None)`` - 3. ``('https://gitlab.kitware.com/vtk/vtk/repository/archive', '.tar.bz2', '?ref=v7.0.0')`` - """ - prefix, ext, suffix = path, "", "" - - # Strip off sourceforge download suffix. - # e.g. https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download - prefix, suffix = spath.find_sourceforge_suffix(path) - - ext = comp.extension_from_path(prefix) - if ext is not None: - prefix = comp.strip_extension(prefix) - - else: - prefix, suf = strip_query_and_fragment(prefix) - ext = comp.extension_from_path(prefix) - prefix = comp.strip_extension(prefix) - suffix = suf + suffix - if ext is None: - ext = "" - - return prefix, ext, suffix - - -def determine_url_file_extension(path): - """This returns the type of archive a URL refers to. This is - sometimes confusing because of URLs like: - - (1) https://github.com/petdance/ack/tarball/1.93_02 - - Where the URL doesn't actually contain the filename. We need - to know what type it is so that we can appropriately name files - in mirrors. - """ - match = re.search(r"github.com/.+/(zip|tar)ball/", path) - if match: - if match.group(1) == "zip": - return "zip" - elif match.group(1) == "tar": - return "tar.gz" - - prefix, ext, suffix = split_url_extension(path) - return ext - - def parse_version_offset(path): """Try to extract a version string from a filename or URL. @@ -426,13 +139,13 @@ def parse_version_offset(path): # path: The prefix of the URL, everything before the ext and suffix # ext: The file extension # suffix: Any kind of query string that begins with a '?' - path, ext, suffix = split_url_extension(path) + path, ext, suffix = llnl.url.split_url_extension(path) # stem: Everything from path after the final '/' original_stem = os.path.basename(path) # Try to strip off anything after the version number - stem = strip_version_suffixes(original_stem) + stem = llnl.url.strip_version_suffixes(original_stem) # Assumptions: # @@ -620,7 +333,7 @@ def parse_name_offset(path, v=None): # path: The prefix of the URL, everything before the ext and suffix # ext: The file extension # suffix: Any kind of query string that begins with a '?' - path, ext, suffix = split_url_extension(path) + path, ext, suffix = llnl.url.split_url_extension(path) # stem: Everything from path after the final '/' original_stem = os.path.basename(path) @@ -735,28 +448,6 @@ def parse_name_and_version(path): return (name, ver) -def insensitize(string): - """Change upper and lowercase letters to be case insensitive in - the provided string. e.g., 'a' becomes '[Aa]', 'B' becomes - '[bB]', etc. Use for building regexes.""" - - def to_ins(match): - char = match.group(1) - return "[%s%s]" % (char.lower(), char.upper()) - - return re.sub(r"([a-zA-Z])", to_ins, string) - - -def cumsum(elts, init=0, fn=lambda x: x): - """Return cumulative sum of result of fn on each element in elts.""" - sums = [] - s = init - for i, e in enumerate(elts): - sums.append(s) - s += fn(e) - return sums - - def find_all(substring, string): """Returns a list containing the indices of every occurrence of substring in string.""" @@ -912,6 +603,122 @@ def color_url(path, **kwargs): return colorize(out.getvalue()) +def find_versions_of_archive( + archive_urls, list_url=None, list_depth=0, concurrency=32, reference_package=None +): + """Scrape web pages for new versions of a tarball. This function prefers URLs in the + following order: links found on the scraped page that match a url generated by the + reference package, found and in the archive_urls list, found and derived from those + in the archive_urls list, and if none are found for a version then the item in the + archive_urls list is included for the version. + + Args: + archive_urls (str or list or tuple): URL or sequence of URLs for + different versions of a package. Typically these are just the + tarballs from the package file itself. By default, this searches + the parent directories of archives. + list_url (str or None): URL for a listing of archives. + Spack will scrape these pages for download links that look + like the archive URL. + list_depth (int): max depth to follow links on list_url pages. + Defaults to 0. + concurrency (int): maximum number of concurrent requests + reference_package (spack.package_base.PackageBase or None): a spack package + used as a reference for url detection. Uses the url_for_version + method on the package to produce reference urls which, if found, + are preferred. + """ + if not isinstance(archive_urls, (list, tuple)): + archive_urls = [archive_urls] + + # Generate a list of list_urls based on archive urls and any + # explicitly listed list_url in the package + list_urls = set() + if list_url is not None: + list_urls.add(list_url) + for aurl in archive_urls: + list_urls |= llnl.url.find_list_urls(aurl) + + # Add '/' to the end of the URL. Some web servers require this. + additional_list_urls = set() + for lurl in list_urls: + if not lurl.endswith("/"): + additional_list_urls.add(lurl + "/") + list_urls |= additional_list_urls + + # Grab some web pages to scrape. + _, links = spack.util.web.spider(list_urls, depth=list_depth, concurrency=concurrency) + + # Scrape them for archive URLs + regexes = [] + for aurl in archive_urls: + # This creates a regex from the URL with a capture group for + # the version part of the URL. The capture group is converted + # to a generic wildcard, so we can use this to extract things + # on a page that look like archive URLs. + url_regex = wildcard_version(aurl) + + # We'll be a bit more liberal and just look for the archive + # part, not the full path. + # this is a URL so it is a posixpath even on Windows + url_regex = pathlib.PurePosixPath(url_regex).name + + # We need to add a / to the beginning of the regex to prevent + # Spack from picking up similarly named packages like: + # https://cran.r-project.org/src/contrib/pls_2.6-0.tar.gz + # https://cran.r-project.org/src/contrib/enpls_5.7.tar.gz + # https://cran.r-project.org/src/contrib/autopls_1.3.tar.gz + # https://cran.r-project.org/src/contrib/matrixpls_1.0.4.tar.gz + url_regex = "/" + url_regex + + # We need to add a $ anchor to the end of the regex to prevent + # Spack from picking up signature files like: + # .asc + # .md5 + # .sha256 + # .sig + # However, SourceForge downloads still need to end in '/download'. + url_regex += r"(\/download)?" + # PyPI adds #sha256=... to the end of the URL + url_regex += "(#sha256=.*)?" + url_regex += "$" + + regexes.append(url_regex) + + regexes = [re.compile(r) for r in regexes] + # Build a dict version -> URL from any links that match the wildcards. + # Walk through archive_url links first. + # Any conflicting versions will be overwritten by the list_url links. + versions = {} + matched = set() + for url in sorted(links): + url = convert_to_posix_path(url) + if any(r.search(url) for r in regexes): + try: + ver = parse_version(url) + if ver in matched: + continue + versions[ver] = url + # prevent this version from getting overwritten + if reference_package is not None: + if url == reference_package.url_for_version(ver): + matched.add(ver) + else: + extrapolated_urls = [substitute_version(u, ver) for u in archive_urls] + if url in extrapolated_urls: + matched.add(ver) + except UndetectableVersionError: + continue + + for url in archive_urls: + url = convert_to_posix_path(url) + ver = parse_version(url) + if ver not in versions: + versions[ver] = url + + return versions + + class UrlParseError(spack.error.SpackError): """Raised when the URL module can't parse something correctly.""" diff --git a/lib/spack/spack/user_environment.py b/lib/spack/spack/user_environment.py index 7ad1d408c79d36..6e1c798e51dc1c 100644 --- a/lib/spack/spack/user_environment.py +++ b/lib/spack/spack/user_environment.py @@ -4,11 +4,19 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import sys +from contextlib import contextmanager +from typing import Callable + +from llnl.util.lang import nullcontext import spack.build_environment import spack.config +import spack.error +import spack.spec import spack.util.environment as environment import spack.util.prefix as prefix +from spack import traverse +from spack.context import Context #: Environment variable name Spack uses to track individually loaded packages spack_loaded_hashes_var = "SPACK_LOADED_HASHES" @@ -26,8 +34,8 @@ def prefix_inspections(platform): A dictionary mapping subdirectory names to lists of environment variables to modify with that directory if it exists. """ - inspections = spack.config.get("modules:prefix_inspections", {}) - if inspections: + inspections = spack.config.get("modules:prefix_inspections") + if isinstance(inspections, dict): return inspections inspections = { @@ -62,40 +70,58 @@ def unconditional_environment_modifications(view): return env -def environment_modifications_for_spec(spec, view=None, set_package_py_globals=True): +@contextmanager +def projected_prefix(*specs: spack.spec.Spec, projection: Callable[[spack.spec.Spec], str]): + """Temporarily replace every Spec's prefix with projection(s)""" + prefixes = dict() + for s in traverse.traverse_nodes(specs, key=lambda s: s.dag_hash()): + if s.external: + continue + prefixes[s.dag_hash()] = s.prefix + s.prefix = prefix.Prefix(projection(s)) + + yield + + for s in traverse.traverse_nodes(specs, key=lambda s: s.dag_hash()): + s.prefix = prefixes.get(s.dag_hash(), s.prefix) + + +def environment_modifications_for_specs( + *specs: spack.spec.Spec, view=None, set_package_py_globals: bool = True +): """List of environment (shell) modifications to be processed for spec. This list is specific to the location of the spec or its projection in the view. Args: - spec (spack.spec.Spec): spec for which to list the environment modifications + specs: spec(s) for which to list the environment modifications view: view associated with the spec passed as first argument - set_package_py_globals (bool): whether or not to set the global variables in the + set_package_py_globals: whether or not to set the global variables in the package.py files (this may be problematic when using buildcaches that have been built on a different but compatible OS) """ - spec = spec.copy() - if view and not spec.external: - spec.prefix = prefix.Prefix(view.get_projection_for_spec(spec)) - - # generic environment modifications determined by inspecting the spec - # prefix - env = environment.inspect_path( - spec.prefix, prefix_inspections(spec.platform), exclude=environment.is_system_path - ) - - # Let the extendee/dependency modify their extensions/dependents - # before asking for package-specific modifications - env.extend( - spack.build_environment.modifications_from_dependencies( - spec, context="run", set_package_py_globals=set_package_py_globals - ) - ) - - if set_package_py_globals: - spack.build_environment.set_module_variables_for_package(spec.package) - - spec.package.setup_run_environment(env) + env = environment.EnvironmentModifications() + topo_ordered = traverse.traverse_nodes(specs, root=True, deptype=("run", "link"), order="topo") + + if view: + maybe_projected = projected_prefix(*specs, projection=view.get_projection_for_spec) + else: + maybe_projected = nullcontext() + + with maybe_projected: + # Static environment changes (prefix inspections) + for s in reversed(list(topo_ordered)): + static = environment.inspect_path( + s.prefix, prefix_inspections(s.platform), exclude=environment.is_system_path + ) + env.extend(static) + + # Dynamic environment changes (setup_run_environment etc) + setup_context = spack.build_environment.SetupContext(*specs, context=Context.RUN) + if set_package_py_globals: + setup_context.set_all_package_py_globals() + dynamic = setup_context.get_env_modifications() + env.extend(dynamic) return env diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index ab7313d47dc342..25ccfdf0bbcc05 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -9,26 +9,13 @@ import re import shutil import sys -from itertools import product +import llnl.url from llnl.util import tty -import spack.util.path as spath from spack.error import SpackError from spack.util.executable import CommandNotFoundError, which -# Supported archive extensions. -PRE_EXTS = ["tar", "TAR"] -EXTS = ["gz", "bz2", "xz", "Z"] -NOTAR_EXTS = ["zip", "tgz", "tbz2", "tbz", "txz"] - -# Add PRE_EXTS and EXTS last so that .tar.gz is matched *before* .tar or .gz -ALLOWED_ARCHIVE_TYPES = ( - [".".join(ext) for ext in product(PRE_EXTS, EXTS)] + PRE_EXTS + EXTS + NOTAR_EXTS -) - -ALLOWED_SINGLE_EXT_ARCHIVE_TYPES = PRE_EXTS + EXTS + NOTAR_EXTS - try: import bz2 # noqa @@ -65,11 +52,7 @@ def is_bz2_supported(): return _bz2_support -def allowed_archive(path): - return False if not path else any(path.endswith(t) for t in ALLOWED_ARCHIVE_TYPES) - - -def _system_untar(archive_file): +def _system_untar(archive_file, remove_archive_file=False): """Returns path to unarchived tar file. Untars archive via system tar. @@ -77,11 +60,22 @@ def _system_untar(archive_file): archive_file (str): absolute path to the archive to be extracted. Can be one of .tar(.[gz|bz2|xz|Z]) or .(tgz|tbz|tbz2|txz). """ - outfile = os.path.basename(strip_extension(archive_file, "tar")) - + archive_file_no_ext = llnl.url.strip_extension(archive_file) + outfile = os.path.basename(archive_file_no_ext) + if archive_file_no_ext == archive_file: + # the archive file has no extension. Tar on windows cannot untar onto itself + # archive_file can be a tar file (which causes the problem on windows) but it can + # also have other extensions (on Unix) such as tgz, tbz2, ... + archive_file = archive_file_no_ext + "-input" + shutil.move(archive_file_no_ext, archive_file) tar = which("tar", required=True) tar.add_default_arg("-oxf") tar(archive_file) + if remove_archive_file: + # remove input file to prevent two stage + # extractions from being treated as exploding + # archives by the fetcher + os.remove(archive_file) return outfile @@ -102,7 +96,7 @@ def _bunzip2(archive_file): def _py_bunzip(archive_file): """Returns path to decompressed file. Decompresses bz2 compressed archives/files via python's bz2 module""" - decompressed_file = os.path.basename(strip_compression_extension(archive_file, "bz2")) + decompressed_file = os.path.basename(llnl.url.strip_compression_extension(archive_file, "bz2")) working_dir = os.getcwd() archive_out = os.path.join(working_dir, decompressed_file) f_bz = bz2.BZ2File(archive_file, mode="rb") @@ -116,7 +110,7 @@ def _system_bunzip(archive_file): """Returns path to decompressed file. Decompresses bz2 compressed archives/files via system bzip2 utility""" compressed_file_name = os.path.basename(archive_file) - decompressed_file = os.path.basename(strip_compression_extension(archive_file, "bz2")) + decompressed_file = os.path.basename(llnl.url.strip_compression_extension(archive_file, "bz2")) working_dir = os.getcwd() archive_out = os.path.join(working_dir, decompressed_file) copy_path = os.path.join(working_dir, compressed_file_name) @@ -146,7 +140,7 @@ def _gunzip(archive_file): def _py_gunzip(archive_file): """Returns path to gunzip'd file Decompresses `.gz` compressed archvies via python gzip module""" - decompressed_file = os.path.basename(strip_compression_extension(archive_file, "gz")) + decompressed_file = os.path.basename(llnl.url.strip_compression_extension(archive_file, "gz")) working_dir = os.getcwd() destination_abspath = os.path.join(working_dir, decompressed_file) f_in = gzip.open(archive_file, "rb") @@ -159,7 +153,12 @@ def _py_gunzip(archive_file): def _system_gunzip(archive_file): """Returns path to gunzip'd file Decompresses `.gz` compressed files via system gzip""" - decompressed_file = os.path.basename(strip_compression_extension(archive_file, "gz")) + archive_file_no_ext = llnl.url.strip_compression_extension(archive_file) + if archive_file_no_ext == archive_file: + # the zip file has no extension. On Unix gunzip cannot unzip onto itself + archive_file = archive_file + ".gz" + shutil.move(archive_file_no_ext, archive_file) + decompressed_file = os.path.basename(archive_file_no_ext) working_dir = os.getcwd() destination_abspath = os.path.join(working_dir, decompressed_file) compressed_file = os.path.basename(archive_file) @@ -179,7 +178,7 @@ def _unzip(archive_file): Args: archive_file (str): absolute path of the file to be decompressed """ - extracted_file = os.path.basename(strip_extension(archive_file, "zip")) + extracted_file = os.path.basename(llnl.url.strip_extension(archive_file, extension="zip")) if sys.platform == "win32": return _system_untar(archive_file) else: @@ -231,16 +230,10 @@ def _win_compressed_tarball_handler(decompressor): def unarchive(archive_file): # perform intermediate extraction step # record name of new archive so we can extract - # and later clean up decomped_tarball = decompressor(archive_file) - if check_extension(decomped_tarball, "tar"): - # run tar on newly decomped archive - outfile = _system_untar(decomped_tarball) - # clean intermediate archive to mimic end result - # produced by one shot decomp/extraction - os.remove(decomped_tarball) - return outfile - return decomped_tarball + # run tar on newly decomped archive + outfile = _system_untar(decomped_tarball, remove_archive_file=True) + return outfile return unarchive @@ -248,7 +241,7 @@ def unarchive(archive_file): def _py_lzma(archive_file): """Returns path to decompressed .xz files Decompress lzma compressed .xz files via python lzma module""" - decompressed_file = os.path.basename(strip_extension(archive_file, "xz")) + decompressed_file = os.path.basename(llnl.url.strip_compression_extension(archive_file, "xz")) archive_out = os.path.join(os.getcwd(), decompressed_file) with open(archive_out, "wb") as ar: with lzma.open(archive_file) as lar: @@ -261,7 +254,7 @@ def _xz(archive_file): Decompress lzma compressed .xz files via xz command line tool. """ - decompressed_file = os.path.basename(strip_extension(archive_file, "xz")) + decompressed_file = os.path.basename(llnl.url.strip_extension(archive_file, extension="xz")) working_dir = os.getcwd() destination_abspath = os.path.join(working_dir, decompressed_file) compressed_file = os.path.basename(archive_file) @@ -286,13 +279,13 @@ def _system_7zip(archive_file): Args: archive_file (str): absolute path of file to be unarchived """ - outfile = os.path.basename(strip_compression_extension(archive_file)) + outfile = os.path.basename(llnl.url.strip_compression_extension(archive_file)) _7z = which("7z") if not _7z: raise CommandNotFoundError( "7z unavailable,\ unable to extract %s files. 7z can be installed via Spack" - % extension_from_path(archive_file) + % llnl.url.extension_from_path(archive_file) ) _7z.add_default_arg("e") _7z(archive_file) @@ -307,7 +300,7 @@ def decompressor_for(path, extension=None): if not extension: extension = extension_from_file(path, decompress=True) - if not allowed_archive(extension): + if not llnl.url.allowed_archive(extension): raise CommandNotFoundError( "Cannot extract archive, \ unrecognized file extension: '%s'" @@ -383,7 +376,7 @@ def decompressor_for_win(extension): path (str): path of the archive file requiring decompression extension (str): extension """ - extension = expand_contracted_extension(extension) + extension = llnl.url.expand_contracted_extension(extension) # Windows native tar can handle .zip extensions, use standard # unzip method if re.match(r"zip$", extension): @@ -404,7 +397,7 @@ def decompressor_for_win(extension): # python based decompression strategy # Expand extension from contracted extension i.e. tar.gz from .tgz # no-op on non contracted extensions - compression_extension = compression_ext_from_compressed_archive(extension) + compression_extension = llnl.url.compression_ext_from_compressed_archive(extension) decompressor = _determine_py_decomp_archive_strategy(compression_extension) if not decompressor: raise SpackError( @@ -646,7 +639,7 @@ def extension_from_stream(stream, decompress=False): "Cannot derive file extension from magic number;" " falling back to regex path parsing." ) - return extension_from_path(stream.name) + return llnl.url.extension_from_path(stream.name) resultant_ext = suffix_ext if not prefix_ext else ".".join([prefix_ext, suffix_ext]) tty.debug("File extension %s successfully derived by magic number." % resultant_ext) return resultant_ext @@ -682,112 +675,11 @@ def extension_from_file(file, decompress=False): if ext and ext.startswith("tar."): suf = ext.split(".")[1] abbr = "t" + suf - if check_extension(file, abbr): + if llnl.url.has_extension(file, abbr): return abbr if not ext: # If unable to parse extension from stream, # attempt to fall back to string parsing - ext = extension_from_path(file) + ext = llnl.url.extension_from_path(file) return ext return None - - -def extension_from_path(path): - """Returns the allowed archive extension for a path. - If path does not include a valid archive extension - (see`spack.util.compression.ALLOWED_ARCHIVE_TYPES`) return None - """ - if path is None: - raise ValueError("Can't call extension() on None") - - for t in ALLOWED_ARCHIVE_TYPES: - if check_extension(path, t): - return t - return None - - -def strip_compression_extension(path, ext=None): - """Returns path with last supported or provided archive extension stripped""" - path = expand_contracted_extension_in_path(path) - exts_to_check = EXTS - if ext: - exts_to_check = [ext] - for ext_check in exts_to_check: - mod_path = check_and_remove_ext(path, ext_check) - if mod_path != path: - return mod_path - return path - - -def strip_extension(path, ext=None): - """Returns the part of a path that does not include extension. - If ext is given, only attempts to remove that extension. If no - extension given, attempts to strip any valid extension from path""" - if ext: - return check_and_remove_ext(path, ext) - for t in ALLOWED_ARCHIVE_TYPES: - mod_path = check_and_remove_ext(path, t) - if mod_path != path: - return mod_path - return path - - -def check_extension(path, ext): - """Returns true if extension is present in path - false otherwise""" - # Strip sourceforge suffix. - prefix, _ = spath.find_sourceforge_suffix(path) - if not ext.startswith(r"\."): - ext = r"\.%s$" % ext - if re.search(ext, prefix): - return True - return False - - -def reg_remove_ext(path, ext): - """Returns path with ext remove via regex""" - if path and ext: - suffix = r"\.%s$" % ext - return re.sub(suffix, "", path) - return path - - -def check_and_remove_ext(path, ext): - """Returns path with extension removed if extension - is present in path. Otherwise just returns path""" - if check_extension(path, ext): - return reg_remove_ext(path, ext) - return path - - -def _substitute_extension(path, old_ext, new_ext): - """Returns path with old_ext replaced with new_ext. - old_ext and new_ext can be extension strings or regexs""" - return re.sub(rf"{old_ext}", rf"{new_ext}", path) - - -def expand_contracted_extension_in_path(path, ext=None): - """Returns path with any contraction extension (i.e. tgz) expanded - (i.e. tar.gz). If ext is specified, only attempt to expand that extension""" - if not ext: - ext = extension_from_path(path) - expanded_ext = expand_contracted_extension(ext) - if expanded_ext != ext: - return _substitute_extension(path, ext, expanded_ext) - return path - - -def expand_contracted_extension(extension): - """Return expanded version of contracted extension - i.e. .tgz -> .tar.gz, no op on non contracted extensions""" - extension = extension.strip(".") - contraction_map = {"tgz": "tar.gz", "txz": "tar.xz", "tbz": "tar.bz2", "tbz2": "tar.bz2"} - return contraction_map.get(extension, extension) - - -def compression_ext_from_compressed_archive(extension): - """Returns compression extension for a compressed archive""" - extension = expand_contracted_extension(extension) - for ext in [*EXTS]: - if ext in extension: - return ext diff --git a/lib/spack/spack/util/cpus.py b/lib/spack/spack/util/cpus.py index acb6569b1d1a3a..b8b371ebb088f2 100644 --- a/lib/spack/spack/util/cpus.py +++ b/lib/spack/spack/util/cpus.py @@ -5,6 +5,9 @@ import multiprocessing import os +from typing import Optional + +import spack.config def cpus_available(): @@ -18,3 +21,36 @@ def cpus_available(): return len(os.sched_getaffinity(0)) # novermin except Exception: return multiprocessing.cpu_count() + + +def determine_number_of_jobs( + *, + parallel: bool = False, + max_cpus: int = cpus_available(), + config: Optional["spack.config.Configuration"] = None, +) -> int: + """ + Packages that require sequential builds need 1 job. Otherwise we use the + number of jobs set on the command line. If not set, then we use the config + defaults (which is usually set through the builtin config scope), but we + cap to the number of CPUs available to avoid oversubscription. + + Parameters: + parallel: true when package supports parallel builds + max_cpus: maximum number of CPUs to use (defaults to cpus_available()) + config: configuration object (defaults to global config) + """ + if not parallel: + return 1 + + cfg = config or spack.config.CONFIG + + # Command line overrides all + try: + command_line = cfg.get("config:build_jobs", default=None, scope="command_line") + if command_line is not None: + return command_line + except ValueError: + pass + + return min(max_cpus, cfg.get("config:build_jobs", 16)) diff --git a/lib/spack/spack/util/crypto.py b/lib/spack/spack/util/crypto.py index df8102352e93f2..2765a6042e26f4 100644 --- a/lib/spack/spack/util/crypto.py +++ b/lib/spack/spack/util/crypto.py @@ -4,12 +4,15 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import hashlib -from typing import Any, Callable, Dict # novm +from typing import BinaryIO, Callable, Dict, Optional import llnl.util.tty as tty +HashFactory = Callable[[], "hashlib._Hash"] + #: Set of hash algorithms that Spack can use, mapped to digest size in bytes -hashes = {"md5": 16, "sha1": 20, "sha224": 28, "sha256": 32, "sha384": 48, "sha512": 64} +hashes = {"sha256": 32, "md5": 16, "sha1": 20, "sha224": 28, "sha384": 48, "sha512": 64} +# Note: keys are ordered by popularity for earliest return in ``hash_key in version_dict`` checks. #: size of hash digests in bytes, mapped to algoritm names @@ -22,7 +25,7 @@ #: cache of hash functions generated -_hash_functions: Dict[str, Callable[[], Any]] = {} +_hash_functions: Dict[str, HashFactory] = {} class DeprecatedHash: @@ -43,55 +46,57 @@ def __call__(self, disable_alert=False): return hashlib.new(self.hash_alg) -def hash_fun_for_algo(algo): +def hash_fun_for_algo(algo: str) -> HashFactory: """Get a function that can perform the specified hash algorithm.""" - hash_gen = _hash_functions.get(algo) - if hash_gen is None: - if algo in _deprecated_hash_algorithms: - try: - hash_gen = DeprecatedHash(algo, tty.debug, disable_security_check=False) - - # call once to get a ValueError if usedforsecurity is needed - hash_gen(disable_alert=True) - except ValueError: - # Some systems may support the 'usedforsecurity' option - # so try with that (but display a warning when it is used) - hash_gen = DeprecatedHash(algo, tty.warn, disable_security_check=True) - else: - hash_gen = getattr(hashlib, algo) - _hash_functions[algo] = hash_gen - - return hash_gen - - -def hash_algo_for_digest(hexdigest): + fun = _hash_functions.get(algo) + if fun: + return fun + elif algo not in _deprecated_hash_algorithms: + _hash_functions[algo] = getattr(hashlib, algo) + else: + try: + deprecated_fun = DeprecatedHash(algo, tty.debug, disable_security_check=False) + + # call once to get a ValueError if usedforsecurity is needed + deprecated_fun(disable_alert=True) + except ValueError: + # Some systems may support the 'usedforsecurity' option + # so try with that (but display a warning when it is used) + deprecated_fun = DeprecatedHash(algo, tty.warn, disable_security_check=True) + _hash_functions[algo] = deprecated_fun + return _hash_functions[algo] + + +def hash_algo_for_digest(hexdigest: str) -> str: """Gets name of the hash algorithm for a hex digest.""" - bytes = len(hexdigest) / 2 - if bytes not in _size_to_hash: - raise ValueError("Spack knows no hash algorithm for this digest: %s" % hexdigest) - return _size_to_hash[bytes] + algo = _size_to_hash.get(len(hexdigest) // 2) + if algo is None: + raise ValueError(f"Spack knows no hash algorithm for this digest: {hexdigest}") + return algo -def hash_fun_for_digest(hexdigest): +def hash_fun_for_digest(hexdigest: str) -> HashFactory: """Gets a hash function corresponding to a hex digest.""" return hash_fun_for_algo(hash_algo_for_digest(hexdigest)) -def checksum(hashlib_algo, filename, **kwargs): - """Returns a hex digest of the filename generated using an - algorithm from hashlib. - """ - block_size = kwargs.get("block_size", 2**20) +def checksum_stream(hashlib_algo: HashFactory, fp: BinaryIO, *, block_size: int = 2**20) -> str: + """Returns a hex digest of the stream generated using given algorithm from hashlib.""" hasher = hashlib_algo() - with open(filename, "rb") as file: - while True: - data = file.read(block_size) - if not data: - break - hasher.update(data) + while True: + data = fp.read(block_size) + if not data: + break + hasher.update(data) return hasher.hexdigest() +def checksum(hashlib_algo: HashFactory, filename: str, *, block_size: int = 2**20) -> str: + """Returns a hex digest of the filename generated using an algorithm from hashlib.""" + with open(filename, "rb") as f: + return checksum_stream(hashlib_algo, f, block_size=block_size) + + class Checker: """A checker checks files against one particular hex digest. It will automatically determine what hashing algorithm @@ -114,18 +119,18 @@ class Checker: a 1MB (2**20 bytes) buffer. """ - def __init__(self, hexdigest, **kwargs): + def __init__(self, hexdigest: str, **kwargs) -> None: self.block_size = kwargs.get("block_size", 2**20) self.hexdigest = hexdigest - self.sum = None + self.sum: Optional[str] = None self.hash_fun = hash_fun_for_digest(hexdigest) @property - def hash_name(self): + def hash_name(self) -> str: """Get the name of the hash function this Checker is using.""" return self.hash_fun().name.lower() - def check(self, filename): + def check(self, filename: str) -> bool: """Read the file with the specified name and check its checksum against self.hexdigest. Return True if they match, False otherwise. Actual checksum is stored in self.sum. diff --git a/lib/spack/spack/util/editor.py b/lib/spack/spack/util/editor.py index 50e6b272c2a9a0..eff896f87e0d83 100644 --- a/lib/spack/spack/util/editor.py +++ b/lib/spack/spack/util/editor.py @@ -61,7 +61,7 @@ def executable(exe: str, args: List[str]) -> int: return cmd.returncode -def editor(*args: List[str], exec_fn: Callable[[str, List[str]], int] = os.execv) -> bool: +def editor(*args: str, exec_fn: Callable[[str, List[str]], int] = os.execv) -> bool: """Invoke the user's editor. This will try to execute the following, in order: diff --git a/lib/spack/spack/util/elf.py b/lib/spack/spack/util/elf.py index 39d653aa24682d..6d0881f4946429 100644 --- a/lib/spack/spack/util/elf.py +++ b/lib/spack/spack/util/elf.py @@ -377,7 +377,7 @@ def parse_header(f, elf): elf.elf_hdr = ElfHeader._make(unpack(elf_header_fmt, data)) -def _do_parse_elf(f, interpreter=True, dynamic_section=True): +def _do_parse_elf(f, interpreter=True, dynamic_section=True, only_header=False): # We don't (yet?) allow parsing ELF files at a nonzero offset, we just # jump to absolute offsets as they are specified in the ELF file. if f.tell() != 0: @@ -386,6 +386,9 @@ def _do_parse_elf(f, interpreter=True, dynamic_section=True): elf = ElfFile() parse_header(f, elf) + if only_header: + return elf + # We don't handle anything but executables and shared libraries now. if elf.elf_hdr.e_type not in (ELF_CONSTANTS.ET_EXEC, ELF_CONSTANTS.ET_DYN): raise ElfParsingError("Not an ET_DYN or ET_EXEC type") @@ -403,11 +406,11 @@ def _do_parse_elf(f, interpreter=True, dynamic_section=True): return elf -def parse_elf(f, interpreter=False, dynamic_section=False): +def parse_elf(f, interpreter=False, dynamic_section=False, only_header=False): """Given a file handle f for an ELF file opened in binary mode, return an ElfFile object that is stores data about rpaths""" try: - return _do_parse_elf(f, interpreter, dynamic_section) + return _do_parse_elf(f, interpreter, dynamic_section, only_header) except (DeprecationWarning, struct.error): # According to the docs old versions of Python can throw DeprecationWarning # instead of struct.error. @@ -432,6 +435,47 @@ def get_rpaths(path): return rpath.split(":") +def delete_rpath(path): + """Modifies a binary to remove the rpath. It zeros out the rpath string + and also drops the DT_R(UN)PATH entry from the dynamic section, so it doesn't + show up in 'readelf -d file', nor in 'strings file'.""" + with open(path, "rb+") as f: + elf = parse_elf(f, interpreter=False, dynamic_section=True) + + if not elf.has_rpath: + return + + # Zero out the rpath *string* in the binary + new_rpath_string = b"\x00" * len(elf.dt_rpath_str) + rpath_offset = elf.pt_dynamic_strtab_offset + elf.rpath_strtab_offset + f.seek(rpath_offset) + f.write(new_rpath_string) + + # Next update the dynamic array + f.seek(elf.pt_dynamic_p_offset) + dynamic_array_fmt = elf.byte_order + ("qQ" if elf.is_64_bit else "lL") + dynamic_array_size = calcsize(dynamic_array_fmt) + new_offset = elf.pt_dynamic_p_offset # points to the new dynamic array + old_offset = elf.pt_dynamic_p_offset # points to the current dynamic array + for _ in range(elf.pt_dynamic_p_filesz // dynamic_array_size): + data = read_exactly(f, dynamic_array_size, "Malformed dynamic array entry") + tag, _ = unpack(dynamic_array_fmt, data) + + # Overwrite any entry that is not DT_RPATH or DT_RUNPATH, including DT_NULL + if tag != ELF_CONSTANTS.DT_RPATH and tag != ELF_CONSTANTS.DT_RUNPATH: + if new_offset != old_offset: + f.seek(new_offset) + f.write(data) + f.seek(old_offset + dynamic_array_size) + new_offset += dynamic_array_size + + # End of the dynamic array + if tag == ELF_CONSTANTS.DT_NULL: + break + + old_offset += dynamic_array_size + + def replace_rpath_in_place_or_raise(path, substitutions): regex = re.compile(b"|".join(re.escape(p) for p in substitutions.keys())) diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index da6832e4ddb295..d4c352b9935390 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -10,21 +10,16 @@ import os import os.path import pickle -import platform import re -import socket import sys from functools import wraps from typing import Any, Callable, Dict, List, MutableMapping, Optional, Tuple, Union +from llnl.path import path_to_os_path, system_path_filter from llnl.util import tty from llnl.util.lang import dedupe -import spack.platforms -import spack.spec - from .executable import Executable, which -from .path import path_to_os_path, system_path_filter if sys.platform == "win32": SYSTEM_PATHS = [ @@ -47,7 +42,7 @@ "csh": "setenv {0} {1};\n", "fish": "set -gx {0} {1};\n", "bat": 'set "{0}={1}"\n', - "pwsh": "$Env:{0}={1}\n", + "pwsh": "$Env:{0}='{1}'\n", } @@ -56,7 +51,7 @@ "csh": "unsetenv {0};\n", "fish": "set -e {0};\n", "bat": 'set "{0}="\n', - "pwsh": "Remove-Item Env:{0}\n", + "pwsh": "Set-Item -Path Env:{0}\n", } @@ -224,43 +219,6 @@ def pickle_environment(path: Path, environment: Optional[Dict[str, str]] = None) pickle.dump(dict(environment if environment else os.environ), pickle_file, protocol=2) -def get_host_environment_metadata() -> Dict[str, str]: - """Get the host environment, reduce to a subset that we can store in - the install directory, and add the spack version. - """ - import spack.main - - environ = get_host_environment() - return { - "host_os": environ["os"], - "platform": environ["platform"], - "host_target": environ["target"], - "hostname": environ["hostname"], - "spack_version": spack.main.get_version(), - "kernel_version": platform.version(), - } - - -def get_host_environment() -> Dict[str, Any]: - """Return a dictionary (lookup) with host information (not including the - os.environ). - """ - host_platform = spack.platforms.host() - host_target = host_platform.target("default_target") - host_os = host_platform.operating_system("default_os") - arch_fmt = "platform={0} os={1} target={2}" - arch_spec = spack.spec.Spec(arch_fmt.format(host_platform, host_os, host_target)) - return { - "target": str(host_target), - "os": str(host_os), - "platform": str(host_platform), - "arch": arch_spec, - "architecture": arch_spec, - "arch_str": str(arch_spec), - "hostname": socket.gethostname(), - } - - @contextlib.contextmanager def set_env(**kwargs): """Temporarily sets and restores environment variables. @@ -429,7 +387,7 @@ class RemovePath(NameValueModifier): def execute(self, env: MutableMapping[str, str]): tty.debug(f"RemovePath: {self.name}-{str(self.value)}", level=3) environment_value = env.get(self.name, "") - directories = environment_value.split(self.separator) if environment_value else [] + directories = environment_value.split(self.separator) directories = [ path_to_os_path(os.path.normpath(x)).pop() for x in directories @@ -638,6 +596,14 @@ def group_by_name(self) -> Dict[str, ModificationList]: modifications[item.name].append(item) return modifications + def drop(self, *name) -> bool: + """Drop all modifications to the variable with the given name.""" + old_mods = self.env_modifications + new_mods = [x for x in self.env_modifications if x.name not in name] + self.env_modifications = new_mods + + return len(old_mods) != len(new_mods) + def is_unset(self, variable_name: str) -> bool: """Returns True if the last modification to a variable is to unset it, False otherwise.""" modifications = self.group_by_name() @@ -724,11 +690,10 @@ def shell_modifications( cmds += _SHELL_UNSET_STRINGS[shell].format(name) else: if sys.platform != "win32": - cmd = _SHELL_SET_STRINGS[shell].format( - name, double_quote_escape(new_env[name]) - ) + new_env_name = double_quote_escape(new_env[name]) else: - cmd = _SHELL_SET_STRINGS[shell].format(name, new_env[name]) + new_env_name = new_env[name] + cmd = _SHELL_SET_STRINGS[shell].format(name, new_env_name) cmds += cmd return cmds @@ -1066,8 +1031,8 @@ def environment_after_sourcing_files( Keyword Args: env (dict): the initial environment (default: current environment) - shell (str): the shell to use (default: ``/bin/bash``) - shell_options (str): options passed to the shell (default: ``-c``) + shell (str): the shell to use (default: ``/bin/bash`` or ``cmd.exe`` (Windows)) + shell_options (str): options passed to the shell (default: ``-c`` or ``/C`` (Windows)) source_command (str): the command to run (default: ``source``) suppress_output (str): redirect used to suppress output of command (default: ``&> /dev/null``) @@ -1075,15 +1040,23 @@ def environment_after_sourcing_files( only when the previous command succeeds (default: ``&&``) """ # Set the shell executable that will be used to source files - shell_cmd = kwargs.get("shell", "/bin/bash") - shell_options = kwargs.get("shell_options", "-c") - source_command = kwargs.get("source_command", "source") - suppress_output = kwargs.get("suppress_output", "&> /dev/null") + if sys.platform == "win32": + shell_cmd = kwargs.get("shell", "cmd.exe") + shell_options = kwargs.get("shell_options", "/C") + suppress_output = kwargs.get("suppress_output", "") + source_command = kwargs.get("source_command", "") + else: + shell_cmd = kwargs.get("shell", "/bin/bash") + shell_options = kwargs.get("shell_options", "-c") + suppress_output = kwargs.get("suppress_output", "&> /dev/null") + source_command = kwargs.get("source_command", "source") concatenate_on_success = kwargs.get("concatenate_on_success", "&&") - shell = Executable(" ".join([shell_cmd, shell_options])) + shell = Executable(shell_cmd) def _source_single_file(file_and_args, environment): + shell_options_list = shell_options.split() + source_file = [source_command] source_file.extend(x for x in file_and_args) source_file = " ".join(source_file) @@ -1101,7 +1074,14 @@ def _source_single_file(file_and_args, environment): source_file_arguments = " ".join( [source_file, suppress_output, concatenate_on_success, dump_environment_cmd] ) - output = shell(source_file_arguments, output=str, env=environment, ignore_quotes=True) + output = shell( + *shell_options_list, + source_file_arguments, + output=str, + env=environment, + ignore_quotes=True, + ) + return json.loads(output) current_environment = kwargs.get("env", dict(os.environ)) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 788348879ab4ec..36c7e73e0638ae 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -5,14 +5,13 @@ import os import re -import shlex import subprocess import sys +from pathlib import Path, PurePath import llnl.util.tty as tty import spack.error -from spack.util.path import Path, format_os_path, path_to_os_path, system_path_filter __all__ = ["Executable", "which", "ProcessError"] @@ -21,11 +20,12 @@ class Executable: """Class representing a program that can be run on the command line.""" def __init__(self, name): - # necesary here for the shlex call to succeed - name = format_os_path(name, mode=Path.unix) - self.exe = shlex.split(str(name)) - # filter back to platform dependent path - self.exe = path_to_os_path(*self.exe) + file_path = str(Path(name)) + if sys.platform != "win32" and name.startswith("."): + # pathlib strips the ./ from relative paths so it must be added back + file_path = os.path.join(".", file_path) + self.exe = [file_path] + self.default_env = {} from spack.util.environment import EnvironmentModifications # no cycle @@ -35,12 +35,10 @@ def __init__(self, name): if not self.exe: raise ProcessError("Cannot construct executable for '%s'" % name) - @system_path_filter - def add_default_arg(self, arg): - """Add a default argument to the command.""" - self.exe.append(arg) + def add_default_arg(self, *args): + """Add default argument(s) to the command.""" + self.exe.extend(args) - @system_path_filter def add_default_env(self, key, value): """Set an environment variable when the command is run. @@ -70,7 +68,7 @@ def name(self): Returns: str: The basename of the executable """ - return os.path.basename(self.path) + return PurePath(self.path).name @property def path(self): @@ -79,7 +77,7 @@ def path(self): Returns: str: The path to the executable """ - return self.exe[0] + return str(PurePath(self.exe[0])) def __call__(self, *args, **kwargs): """Run this executable in a subprocess. @@ -102,6 +100,8 @@ def __call__(self, *args, **kwargs): an exception even if ``fail_on_error`` is set to ``True`` ignore_quotes (bool): If False, warn users that quotes are not needed as Spack does not use a shell. Defaults to False. + timeout (int or float): The number of seconds to wait before killing + the child process input: Where to read stdin from output: Where to send stdout error: Where to send stderr @@ -122,6 +122,29 @@ def __call__(self, *args, **kwargs): By default, the subprocess inherits the parent's file descriptors. """ + + def process_cmd_output(out, err): + result = None + if output in (str, str.split) or error in (str, str.split): + result = "" + if output in (str, str.split): + if sys.platform == "win32": + outstr = str(out.decode("ISO-8859-1")) + else: + outstr = str(out.decode("utf-8")) + result += outstr + if output is str.split: + sys.stdout.write(outstr) + if error in (str, str.split): + if sys.platform == "win32": + errstr = str(err.decode("ISO-8859-1")) + else: + errstr = str(err.decode("utf-8")) + result += errstr + if error is str.split: + sys.stderr.write(errstr) + return result + # Environment env_arg = kwargs.get("env", None) @@ -152,6 +175,7 @@ def __call__(self, *args, **kwargs): fail_on_error = kwargs.pop("fail_on_error", True) ignore_errors = kwargs.pop("ignore_errors", ()) ignore_quotes = kwargs.pop("ignore_quotes", False) + timeout = kwargs.pop("timeout", None) # If they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): @@ -198,28 +222,9 @@ def streamify(arg, mode): proc = subprocess.Popen( cmd, stdin=istream, stderr=estream, stdout=ostream, env=env, close_fds=False ) - out, err = proc.communicate() - - result = None - if output in (str, str.split) or error in (str, str.split): - result = "" - if output in (str, str.split): - if sys.platform == "win32": - outstr = str(out.decode("ISO-8859-1")) - else: - outstr = str(out.decode("utf-8")) - result += outstr - if output is str.split: - sys.stdout.write(outstr) - if error in (str, str.split): - if sys.platform == "win32": - errstr = str(err.decode("ISO-8859-1")) - else: - errstr = str(err.decode("utf-8")) - result += errstr - if error is str.split: - sys.stderr.write(errstr) + out, err = proc.communicate(timeout=timeout) + result = process_cmd_output(out, err) rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): long_msg = cmd_line_string @@ -235,7 +240,11 @@ def streamify(arg, mode): return result except OSError as e: - raise ProcessError("%s: %s" % (self.exe[0], e.strerror), "Command: " + cmd_line_string) + message = "Command: " + cmd_line_string + if " " in self.exe[0]: + message += "\nDid you mean to add a space to the command?" + + raise ProcessError("%s: %s" % (self.exe[0], e.strerror), message) except subprocess.CalledProcessError as e: if fail_on_error: @@ -244,6 +253,18 @@ def streamify(arg, mode): "\nExit status %d when invoking command: %s" % (proc.returncode, cmd_line_string), ) + except subprocess.TimeoutExpired as te: + proc.kill() + out, err = proc.communicate() + result = process_cmd_output(out, err) + long_msg = cmd_line_string + f"\n{result}" + if fail_on_error: + raise ProcessTimeoutError( + f"\nProcess timed out after {timeout}s" + f"We expected the following command to run quickly but\ +it did not, please report this as an issue: {long_msg}", + long_message=long_msg, + ) from te finally: if close_ostream: @@ -269,39 +290,51 @@ def __str__(self): return " ".join(self.exe) -@system_path_filter def which_string(*args, **kwargs): """Like ``which()``, but return a string instead of an ``Executable``.""" path = kwargs.get("path", os.environ.get("PATH", "")) required = kwargs.get("required", False) + if isinstance(path, list): + paths = [Path(str(x)) for x in path] + if isinstance(path, str): - path = path.split(os.pathsep) + paths = [Path(x) for x in path.split(os.pathsep)] + + def get_candidate_items(search_item): + if sys.platform == "win32" and not search_item.suffix: + return [search_item.parent / (search_item.name + ext) for ext in [".exe", ".bat"]] - for name in args: - win_candidates = [] - if sys.platform == "win32" and (not name.endswith(".exe") and not name.endswith(".bat")): - win_candidates = [name + ext for ext in [".exe", ".bat"]] - candidate_names = [name] if not win_candidates else win_candidates + return [Path(search_item)] + def add_extra_search_paths(paths): + with_parents = [] + with_parents.extend(paths) if sys.platform == "win32": - new_path = path[:] - for p in path: - if os.path.basename(p) == "bin": - new_path.append(os.path.dirname(p)) - path = new_path - - for candidate_name in candidate_names: - if os.path.sep in candidate_name: - exe = os.path.abspath(candidate_name) - if os.path.isfile(exe) and os.access(exe, os.X_OK): - return exe - else: - for directory in path: - directory = path_to_os_path(directory).pop() - exe = os.path.join(directory, candidate_name) - if os.path.isfile(exe) and os.access(exe, os.X_OK): - return exe + for p in paths: + if p.name == "bin": + with_parents.append(p.parent) + return with_parents + + for search_item in args: + search_paths = [] + search_paths.extend(paths) + if search_item.startswith("."): + # we do this because pathlib will strip any leading ./ + search_paths.insert(0, Path.cwd()) + search_paths = add_extra_search_paths(search_paths) + + search_item = Path(search_item) + candidate_items = get_candidate_items(Path(search_item)) + + for candidate_item in candidate_items: + for directory in search_paths: + exe = directory / candidate_item + try: + if exe.is_file() and os.access(str(exe), os.X_OK): + return str(exe) + except OSError: + pass if required: raise CommandNotFoundError("spack requires '%s'. Make sure it is in your path." % args[0]) @@ -326,12 +359,17 @@ def which(*args, **kwargs): Executable: The first executable that is found in the path """ exe = which_string(*args, **kwargs) - return Executable(shlex.quote(exe)) if exe else None + return Executable(exe) if exe else None class ProcessError(spack.error.SpackError): """ProcessErrors are raised when Executables exit with an error code.""" +class ProcessTimeoutError(ProcessError): + """ProcessTimeoutErrors are raised when Executable calls with a + specified timeout exceed that time""" + + class CommandNotFoundError(spack.error.SpackError): """Raised when ``which()`` can't find a required executable.""" diff --git a/lib/spack/spack/util/file_cache.py b/lib/spack/spack/util/file_cache.py index 9c3b09c2169f18..2b98f291996ee8 100644 --- a/lib/spack/spack/util/file_cache.py +++ b/lib/spack/spack/util/file_cache.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import errno +import math import os import shutil @@ -151,18 +152,17 @@ def __exit__(cm, type, value, traceback): return WriteTransaction(self._get_lock(key), acquire=WriteContextManager) - def mtime(self, key): - """Return modification time of cache file, or 0 if it does not exist. + def mtime(self, key) -> float: + """Return modification time of cache file, or -inf if it does not exist. Time is in units returned by os.stat in the mtime field, which is platform-dependent. """ if not self.init_entry(key): - return 0 + return -math.inf else: - sinfo = os.stat(self.cache_path(key)) - return sinfo.st_mtime + return os.stat(self.cache_path(key)).st_mtime def remove(self, key): file = self.cache_path(key) diff --git a/lib/spack/spack/util/format.py b/lib/spack/spack/util/format.py new file mode 100644 index 00000000000000..68fe8733cc6136 --- /dev/null +++ b/lib/spack/spack/util/format.py @@ -0,0 +1,36 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from typing import Optional + + +def get_version_lines(version_hashes_dict: dict, url_dict: Optional[dict] = None) -> str: + """ + Renders out a set of versions like those found in a package's + package.py file for a given set of versions and hashes. + + Args: + version_hashes_dict (dict): A dictionary of the form: version -> checksum. + url_dict (dict): A dictionary of the form: version -> URL. + + Returns: + (str): Rendered version lines. + """ + version_lines = [] + + for v, h in version_hashes_dict.items(): + expand_arg = "" + + # Extract the url for a version if url_dict is provided. + url = "" + if url_dict is not None and v in url_dict: + url = url_dict[v] + + # Add expand_arg since wheels should not be expanded during stanging + if url.endswith(".whl") or ".whl#" in url: + expand_arg = ", expand=False" + version_lines.append(f' version("{v}", sha256="{h}"{expand_arg})') + + return "\n".join(version_lines) diff --git a/lib/spack/spack/util/gcs.py b/lib/spack/spack/util/gcs.py index 856fe7300188b8..4e997df52bd662 100644 --- a/lib/spack/spack/util/gcs.py +++ b/lib/spack/spack/util/gcs.py @@ -10,6 +10,10 @@ import os import sys +import urllib.parse +import urllib.response +from urllib.error import URLError +from urllib.request import BaseHandler import llnl.util.tty as tty @@ -222,3 +226,21 @@ def get_blob_headers(self): } return headers + + +def gcs_open(req, *args, **kwargs): + """Open a reader stream to a blob object on GCS""" + url = urllib.parse.urlparse(req.get_full_url()) + gcsblob = GCSBlob(url) + + if not gcsblob.exists(): + raise URLError("GCS blob {0} does not exist".format(gcsblob.blob_path)) + stream = gcsblob.get_blob_byte_stream() + headers = gcsblob.get_blob_headers() + + return urllib.response.addinfourl(stream, headers, url) + + +class GCSHandler(BaseHandler): + def gs_open(self, req): + return gcs_open(req) diff --git a/lib/spack/spack/util/git.py b/lib/spack/spack/util/git.py index ceb0013412028c..39efdda9c3c3bb 100644 --- a/lib/spack/spack/util/git.py +++ b/lib/spack/spack/util/git.py @@ -24,7 +24,6 @@ def git(required: bool = False): # If we're running under pytest, add this to ignore the fix for CVE-2022-39253 in # git 2.38.1+. Do this in one place; we need git to do this in all parts of Spack. if git and "pytest" in sys.modules: - git.add_default_arg("-c") - git.add_default_arg("protocol.file.allow=always") + git.add_default_arg("-c", "protocol.file.allow=always") return git diff --git a/lib/spack/spack/util/lock.py b/lib/spack/spack/util/lock.py index 7d8036bb359d17..0174dad0570d29 100644 --- a/lib/spack/spack/util/lock.py +++ b/lib/spack/spack/util/lock.py @@ -7,6 +7,7 @@ import os import stat import sys +from typing import Optional, Tuple import llnl.util.lock @@ -17,7 +18,6 @@ from llnl.util.lock import ReadTransaction # noqa: F401 from llnl.util.lock import WriteTransaction # noqa: F401 -import spack.config import spack.error import spack.paths @@ -30,31 +30,48 @@ class Lock(llnl.util.lock.Lock): the actual locking mechanism can be disabled via ``_enable_locks``. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._enable = spack.config.get("config:locks", sys.platform != "win32") - - def _lock(self, op, timeout=0): + def __init__( + self, + path: str, + *, + start: int = 0, + length: int = 0, + default_timeout: Optional[float] = None, + debug: bool = False, + desc: str = "", + enable: Optional[bool] = None, + ) -> None: + enable_lock = enable + if sys.platform == "win32": + enable_lock = False + elif sys.platform != "win32" and enable_lock is None: + enable_lock = True + self._enable = enable_lock + super().__init__( + path, + start=start, + length=length, + default_timeout=default_timeout, + debug=debug, + desc=desc, + ) + + def _lock(self, op: int, timeout: Optional[float] = 0.0) -> Tuple[float, int]: if self._enable: return super()._lock(op, timeout) - else: - return 0, 0 + return 0.0, 0 - def _unlock(self): + def _unlock(self) -> None: """Unlock call that always succeeds.""" if self._enable: super()._unlock() - def _debug(self, *args): - if self._enable: - super()._debug(*args) - - def cleanup(self, *args): + def cleanup(self, *args) -> None: if self._enable: super().cleanup(*args) -def check_lock_safety(path): +def check_lock_safety(path: str) -> None: """Do some extra checks to ensure disabling locks is safe. This will raise an error if ``path`` can is group- or world-writable @@ -78,9 +95,9 @@ def check_lock_safety(path): writable = "world" if writable: - msg = "Refusing to disable locks: spack is {0}-writable.".format(writable) + msg = f"Refusing to disable locks: spack is {writable}-writable." long_msg = ( - "Running a shared spack without locks is unsafe. You must " - "restrict permissions on {0} or enable locks." - ).format(path) + f"Running a shared spack without locks is unsafe. You must " + f"restrict permissions on {path} or enable locks." + ) raise spack.error.SpackError(msg, long_msg) diff --git a/lib/spack/spack/util/package_hash.py b/lib/spack/spack/util/package_hash.py index 9343b1f6899bbb..db2c28528f6a70 100644 --- a/lib/spack/spack/util/package_hash.py +++ b/lib/spack/spack/util/package_hash.py @@ -337,7 +337,7 @@ def package_ast(spec, filter_multimethods=True, source=None): spec = spack.spec.Spec(spec) if source is None: - filename = spack.repo.path.filename_for_package_name(spec.name) + filename = spack.repo.PATH.filename_for_package_name(spec.name) with open(filename) as f: source = f.read() diff --git a/lib/spack/spack/util/parallel.py b/lib/spack/spack/util/parallel.py index 06e9ed52256828..c8e6ef7907f584 100644 --- a/lib/spack/spack/util/parallel.py +++ b/lib/spack/spack/util/parallel.py @@ -2,13 +2,11 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import contextlib import multiprocessing import os import sys import traceback - -from .cpus import cpus_available +from typing import Optional class ErrorFromWorker: @@ -56,79 +54,29 @@ def __call__(self, *args, **kwargs): return value -def raise_if_errors(*results, **kwargs): - """Analyze results from worker Processes to search for ErrorFromWorker - objects. If found print all of them and raise an exception. - - Args: - *results: results from worker processes - debug: if True show complete stacktraces - - Raise: - RuntimeError: if ErrorFromWorker objects are in the results - """ - debug = kwargs.get("debug", False) # This can be a keyword only arg in Python 3 - errors = [x for x in results if isinstance(x, ErrorFromWorker)] - if not errors: - return - - msg = "\n".join([error.stacktrace if debug else str(error) for error in errors]) - - error_fmt = "{0}" - if len(errors) > 1 and not debug: - error_fmt = "errors occurred during concretization of the environment:\n{0}" - - raise RuntimeError(error_fmt.format(msg)) - - -@contextlib.contextmanager -def pool(*args, **kwargs): - """Context manager to start and terminate a pool of processes, similar to the - default one provided in Python 3.X - - Arguments are forwarded to the multiprocessing.Pool.__init__ method. - """ - try: - p = multiprocessing.Pool(*args, **kwargs) - yield p - finally: - p.terminate() - p.join() - - -def num_processes(max_processes=None): - """Return the number of processes in a pool. - - Currently the function return the minimum between the maximum number - of processes and the cpus available. - - When a maximum number of processes is not specified return the cpus available. - - Args: - max_processes (int or None): maximum number of processes allowed - """ - max_processes or cpus_available() - return min(cpus_available(), max_processes) - - -def parallel_map(func, arguments, max_processes=None, debug=False): - """Map a task object to the list of arguments, return the list of results. +def imap_unordered( + f, list_of_args, *, processes: int, maxtaskperchild: Optional[int] = None, debug=False +): + """Wrapper around multiprocessing.Pool.imap_unordered. Args: - func (Task): user defined task object - arguments (list): list of arguments for the task - max_processes (int or None): maximum number of processes allowed - debug (bool): if False, raise an exception containing just the error messages + f: function to apply + list_of_args: list of tuples of args for the task + processes: maximum number of processes allowed + debug: if False, raise an exception containing just the error messages from workers, if True an exception with complete stacktraces + maxtaskperchild: number of tasks to be executed by a child before being + killed and substituted Raises: RuntimeError: if any error occurred in the worker processes """ - task_wrapper = Task(func) - if sys.platform != "darwin" and sys.platform != "win32": - with pool(processes=num_processes(max_processes=max_processes)) as p: - results = p.map(task_wrapper, arguments) - else: - results = list(map(task_wrapper, arguments)) - raise_if_errors(*results, debug=debug) - return results + if sys.platform in ("darwin", "win32") or len(list_of_args) == 1: + yield from map(f, list_of_args) + return + + with multiprocessing.Pool(processes, maxtasksperchild=maxtaskperchild) as p: + for result in p.imap_unordered(Task(f), list_of_args): + if isinstance(result, ErrorFromWorker): + raise RuntimeError(result.stacktrace if debug else str(result)) + yield result diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index f894f6f3d8ff2d..e2aee48df1e2cd 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -15,7 +15,6 @@ import sys import tempfile from datetime import date -from urllib.parse import urlparse import llnl.util.tty as tty from llnl.util.lang import memoized @@ -98,110 +97,30 @@ def replacements(): SPACK_PATH_PADDING_CHARS = "__spack_path_placeholder__" -def is_path_url(path): - if "\\" in path: - return False - url_tuple = urlparse(path) - return bool(url_tuple.scheme) and len(url_tuple.scheme) > 1 - - def win_exe_ext(): - return ".exe" - - -def find_sourceforge_suffix(path): - """find and match sourceforge filepath components - Return match object""" - match = re.search(r"(.*(?:sourceforge\.net|sf\.net)/.*)(/download)$", path) - if match: - return match.groups() - return path, "" - - -def path_to_os_path(*pths): - """ - Takes an arbitrary number of positional parameters - converts each arguemnt of type string to use a normalized - filepath separator, and returns a list of all values - """ - ret_pths = [] - for pth in pths: - if isinstance(pth, str) and not is_path_url(pth): - pth = convert_to_platform_path(pth) - ret_pths.append(pth) - return ret_pths + return r"(?:\.bat|\.exe)" -def sanitize_file_path(pth): +def sanitize_filename(filename: str) -> str: """ - Formats strings to contain only characters that can - be used to generate legal file paths. + Replaces unsupported characters (for the host) in a filename with underscores. Criteria for legal files based on https://en.wikipedia.org/wiki/Filename#Comparison_of_filename_limitations Args: - pth: string containing path to be created - on the host filesystem + filename: string containing filename to be created on the host filesystem Return: - sanitized string that can legally be made into a path - """ - # on unix, splitting path by seperators will remove - # instances of illegal characters on join - pth_cmpnts = pth.split(os.path.sep) - - if sys.platform == "win32": - drive_match = r"[a-zA-Z]:" - is_abs = bool(re.match(drive_match, pth_cmpnts[0])) - drive = pth_cmpnts[0] + os.path.sep if is_abs else "" - pth_cmpnts = pth_cmpnts[1:] if drive else pth_cmpnts - illegal_chars = r'[<>?:"|*\\]' - else: - drive = "/" if not pth_cmpnts[0] else "" - illegal_chars = r"[/]" - - pth = [] - for cmp in pth_cmpnts: - san_cmp = re.sub(illegal_chars, "", cmp) - pth.append(san_cmp) - return drive + os.path.join(*pth) - - -def system_path_filter(_func=None, arg_slice=None): + filename that can be created on the host filesystem """ - Filters function arguments to account for platform path separators. - Optional slicing range can be specified to select specific arguments - - This decorator takes all (or a slice) of a method's positional arguments - and normalizes usage of filepath separators on a per platform basis. - - Note: **kwargs, urls, and any type that is not a string are ignored - so in such cases where path normalization is required, that should be - handled by calling path_to_os_path directly as needed. - - Parameters: - arg_slice (slice): a slice object specifying the slice of arguments - in the decorated method over which filepath separators are - normalized - """ - from functools import wraps - - def holder_func(func): - @wraps(func) - def path_filter_caller(*args, **kwargs): - args = list(args) - if arg_slice: - args[arg_slice] = path_to_os_path(*args[arg_slice]) - else: - args = path_to_os_path(*args) - return func(*args, **kwargs) + if sys.platform != "win32": + # Only disallow null bytes and directory separators. + return re.sub("[\0/]", "_", filename) - return path_filter_caller - - if _func: - return holder_func(_func) - return holder_func + # On Windows, things are more involved. + # NOTE: this is incomplete, missing reserved names + return re.sub(r'[\x00-\x1F\x7F"*/:<>?\\|]', "_", filename) @memoized @@ -225,54 +144,6 @@ def get_system_path_max(): return sys_max_path_length -class Path: - """ - Describes the filepath separator types - in an enum style - with a helper attribute - exposing the path type of - the current platform. - """ - - unix = 0 - windows = 1 - platform_path = windows if sys.platform == "win32" else unix - - -def format_os_path(path, mode=Path.unix): - """ - Format path to use consistent, platform specific - separators. Absolute paths are converted between - drive letters and a prepended '/' as per platform - requirement. - - Parameters: - path (str): the path to be normalized, must be a string - or expose the replace method. - mode (Path): the path filesperator style to normalize the - passed path to. Default is unix style, i.e. '/' - """ - if not path: - return path - if mode == Path.windows: - path = path.replace("/", "\\") - else: - path = path.replace("\\", "/") - return path - - -def convert_to_posix_path(path): - return format_os_path(path, mode=Path.unix) - - -def convert_to_windows_path(path): - return format_os_path(path, mode=Path.windows) - - -def convert_to_platform_path(path): - return format_os_path(path, mode=Path.platform_path) - - def substitute_config_variables(path): """Substitute placeholders into paths. diff --git a/lib/spack/spack/util/s3.py b/lib/spack/spack/util/s3.py index c4d53d86b63bc1..796c49a8c8e6cf 100644 --- a/lib/spack/spack/util/s3.py +++ b/lib/spack/spack/util/s3.py @@ -3,10 +3,13 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os +import urllib.error import urllib.parse +import urllib.request +import urllib.response +from io import BufferedReader, BytesIO, IOBase from typing import Any, Dict, Tuple -import spack import spack.config #: Map (mirror name, method) tuples to s3 client instances. @@ -114,4 +117,72 @@ def get_mirror_s3_connection_info(mirror, method): if endpoint_url: s3_client_args["endpoint_url"] = _parse_s3_endpoint_url(endpoint_url) - return (s3_connection, s3_client_args) + return s3_connection, s3_client_args + + +# NOTE(opadron): Workaround issue in boto where its StreamingBody +# implementation is missing several APIs expected from IOBase. These missing +# APIs prevent the streams returned by boto from being passed as-are along to +# urllib. +# +# https://github.com/boto/botocore/issues/879 +# https://github.com/python/cpython/pull/3249 +class WrapStream(BufferedReader): + def __init__(self, raw): + # In botocore >=1.23.47, StreamingBody inherits from IOBase, so we + # only add missing attributes in older versions. + # https://github.com/boto/botocore/commit/a624815eabac50442ed7404f3c4f2664cd0aa784 + if not isinstance(raw, IOBase): + raw.readable = lambda: True + raw.writable = lambda: False + raw.seekable = lambda: False + raw.closed = False + raw.flush = lambda: None + super().__init__(raw) + + def detach(self): + self.raw = None + + def read(self, *args, **kwargs): + return self.raw.read(*args, **kwargs) + + def __getattr__(self, key): + return getattr(self.raw, key) + + +def _s3_open(url, method="GET"): + parsed = urllib.parse.urlparse(url) + s3 = get_s3_session(url, method="fetch") + + bucket = parsed.netloc + key = parsed.path + + if key.startswith("/"): + key = key[1:] + + if method not in ("GET", "HEAD"): + raise urllib.error.URLError( + "Only GET and HEAD verbs are currently supported for the s3:// scheme" + ) + + try: + if method == "GET": + obj = s3.get_object(Bucket=bucket, Key=key) + # NOTE(opadron): Apply workaround here (see above) + stream = WrapStream(obj["Body"]) + elif method == "HEAD": + obj = s3.head_object(Bucket=bucket, Key=key) + stream = BytesIO() + except s3.ClientError as e: + raise urllib.error.URLError(e) from e + + headers = obj["ResponseMetadata"]["HTTPHeaders"] + + return url, headers, stream + + +class UrllibS3Handler(urllib.request.BaseHandler): + def s3_open(self, req): + orig_url = req.get_full_url() + url, headers, stream = _s3_open(orig_url, method=req.get_method()) + return urllib.response.addinfourl(stream, headers, url) diff --git a/lib/spack/spack/util/spack_json.py b/lib/spack/spack/util/spack_json.py index 232852fd229859..579d046985133b 100644 --- a/lib/spack/spack/util/spack_json.py +++ b/lib/spack/spack/util/spack_json.py @@ -11,7 +11,7 @@ __all__ = ["load", "dump", "SpackJSONError"] -_json_dump_args = {"indent": 2, "separators": (",", ": ")} +_json_dump_args = {"indent": None, "separators": (",", ":")} def load(stream: Any) -> Dict: diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py index f0cb05ab0430dc..2ca9acbed73e59 100644 --- a/lib/spack/spack/util/spack_yaml.py +++ b/lib/spack/spack/util/spack_yaml.py @@ -68,7 +68,7 @@ def syaml_type(obj): """ for syaml_t, t in syaml_types.items(): if type(obj) is not bool and isinstance(obj, t): - return syaml_t(obj) if type(obj) != syaml_t else obj + return syaml_t(obj) if type(obj) is not syaml_t else obj return obj diff --git a/lib/spack/spack/util/string.py b/lib/spack/spack/util/string.py deleted file mode 100644 index 1c88c17777724d..00000000000000 --- a/lib/spack/spack/util/string.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - - -def comma_list(sequence, article=""): - if type(sequence) != list: - sequence = list(sequence) - - if not sequence: - return - elif len(sequence) == 1: - return sequence[0] - else: - out = ", ".join(str(s) for s in sequence[:-1]) - if len(sequence) != 2: - out += "," # oxford comma - out += " " - if article: - out += article + " " - out += str(sequence[-1]) - return out - - -def comma_or(sequence): - return comma_list(sequence, "or") - - -def comma_and(sequence): - return comma_list(sequence, "and") - - -def quote(sequence, q="'"): - return ["%s%s%s" % (q, e, q) for e in sequence] - - -def plural(n, singular, plural=None, show_n=True): - """Pluralize word by adding an s if n != 1. - - Arguments: - n (int): number of things there are - singular (str): singular form of word - plural (str or None): optional plural form, for when it's not just - singular + 's' - show_n (bool): whether to include n in the result string (default True) - - Returns: - (str): "1 thing" if n == 1 or "n things" if n != 1 - """ - number = "%s " % n if show_n else "" - if n == 1: - return "%s%s" % (number, singular) - elif plural is not None: - return "%s%s" % (number, plural) - else: - return "%s%ss" % (number, singular) diff --git a/lib/spack/spack/util/timer.py b/lib/spack/spack/util/timer.py index 31c756540d153d..ef15c7001fe7ea 100644 --- a/lib/spack/spack/util/timer.py +++ b/lib/spack/spack/util/timer.py @@ -13,34 +13,32 @@ import sys import time from contextlib import contextmanager -from typing import Dict +from typing import Callable, Dict, List from llnl.util.lang import pretty_seconds_formatter import spack.util.spack_json as sjson -Interval = collections.namedtuple("Interval", ("begin", "end")) +TimerEvent = collections.namedtuple("TimerEvent", ("time", "running", "label")) +TimeTracker = collections.namedtuple("TimeTracker", ("total", "start", "count", "path")) #: name for the global timer (used in start(), stop(), duration() without arguments) global_timer_name = "_global" -class NullTimer: - """Timer interface that does nothing, useful in for "tell - don't ask" style code when timers are optional.""" - - def start(self, name=global_timer_name): +class BaseTimer: + def start(self, name=None): pass - def stop(self, name=global_timer_name): + def stop(self, name=None): pass - def duration(self, name=global_timer_name): + def duration(self, name=None): return 0.0 @contextmanager def measure(self, name): - yield + yield self @property def phases(self): @@ -53,23 +51,28 @@ def write_tty(self, out=sys.stdout): pass -#: instance of a do-nothing timer -NULL_TIMER = NullTimer() +class NullTimer(BaseTimer): + """Timer interface that does nothing, useful in for "tell + don't ask" style code when timers are optional.""" + + pass -class Timer: +class Timer(BaseTimer): """Simple interval timer""" - def __init__(self, now=time.time): + def __init__(self, now: Callable[[], float] = time.time): """ Arguments: now: function that gives the seconds since e.g. epoch """ self._now = now - self._timers: Dict[str, Interval] = collections.OrderedDict() + self._timers: Dict[str, TimeTracker] = {} + self._timer_stack: List[str] = [] - # _global is the overal timer since the instance was created - self._timers[global_timer_name] = Interval(self._now(), end=None) + self._events: List[TimerEvent] = [] + # Push start event + self._events.append(TimerEvent(self._now(), True, global_timer_name)) def start(self, name=global_timer_name): """ @@ -79,7 +82,7 @@ def start(self, name=global_timer_name): name (str): Optional name of the timer. When no name is passed, the global timer is started. """ - self._timers[name] = Interval(self._now(), None) + self._events.append(TimerEvent(self._now(), True, name)) def stop(self, name=global_timer_name): """ @@ -90,10 +93,7 @@ def stop(self, name=global_timer_name): name (str): Optional name of the timer. When no name is passed, all timers are stopped. """ - interval = self._timers.get(name, None) - if not interval: - return - self._timers[name] = Interval(interval.begin, self._now()) + self._events.append(TimerEvent(self._now(), False, name)) def duration(self, name=global_timer_name): """ @@ -107,13 +107,13 @@ def duration(self, name=global_timer_name): Returns: float: duration of timer. """ - try: - interval = self._timers[name] - except KeyError: + self._flatten() + if name in self._timers: + if name in self._timer_stack: + return self._timers[name].total + (self._now() - self._timers[name].start) + return self._timers[name].total + else: return 0.0 - # Take either the interval end, the global timer, or now. - end = interval.end or self._timers[global_timer_name].end or self._now() - return end - interval.begin @contextmanager def measure(self, name): @@ -123,23 +123,72 @@ def measure(self, name): Arguments: name (str): Name of the timer """ - begin = self._now() - yield - self._timers[name] = Interval(begin, self._now()) + self.start(name) + yield self + self.stop(name) @property def phases(self): """Get all named timers (excluding the global/total timer)""" - return [k for k in self._timers.keys() if k != global_timer_name] - - def write_json(self, out=sys.stdout): + self._flatten() + return [k for k in self._timers.keys() if not k == global_timer_name] + + def _flatten(self): + for event in self._events: + if event.running: + if event.label not in self._timer_stack: + self._timer_stack.append(event.label) + # Only start the timer if it is on top of the stack + # restart doesn't work after a subtimer is started + if event.label == self._timer_stack[-1]: + timer_path = "/".join(self._timer_stack[1:]) + tracker = self._timers.get( + event.label, TimeTracker(0.0, event.time, 0, timer_path) + ) + assert tracker.path == timer_path + self._timers[event.label] = TimeTracker( + tracker.total, event.time, tracker.count, tracker.path + ) + else: # if not event.running: + if event.label in self._timer_stack: + index = self._timer_stack.index(event.label) + for label in self._timer_stack[index:]: + tracker = self._timers[label] + self._timers[label] = TimeTracker( + tracker.total + (event.time - tracker.start), + None, + tracker.count + 1, + tracker.path, + ) + self._timer_stack = self._timer_stack[: max(0, index)] + # clear events + self._events = [] + + def write_json(self, out=sys.stdout, extra_attributes={}): """Write a json object with times to file""" - phases = [{"name": p, "seconds": self.duration(p)} for p in self.phases] - times = {"phases": phases, "total": {"seconds": self.duration()}} - out.write(sjson.dump(times)) + self._flatten() + data = { + "total": self._timers[global_timer_name].total, + "phases": [ + { + "name": phase, + "path": self._timers[phase].path, + "seconds": self._timers[phase].total, + "count": self._timers[phase].count, + } + for phase in self.phases + ], + } + if extra_attributes: + data.update(extra_attributes) + if out: + out.write(sjson.dump(data)) + else: + return data def write_tty(self, out=sys.stdout): - """Write a human-readable summary of timings""" + """Write a human-readable summary of timings (depth is 1)""" + self._flatten() times = [self.duration(p) for p in self.phases] @@ -153,3 +202,7 @@ def write_tty(self, out=sys.stdout): # Write to out for name, duration in formatted: out.write(f" {name:10s} {pretty_seconds(duration):>10s}\n") + + +#: instance of a do-nothing timer +NULL_TIMER = NullTimer() diff --git a/lib/spack/spack/util/unparse/unparser.py b/lib/spack/spack/util/unparse/unparser.py index 75f61e53f90aa6..8ca4cd57a3eaa8 100644 --- a/lib/spack/spack/util/unparse/unparser.py +++ b/lib/spack/spack/util/unparse/unparser.py @@ -270,16 +270,6 @@ def visit_Assert(self, node): self.write(", ") self.dispatch(node.msg) - def visit_Exec(self, node): - self.fill("exec ") - self.dispatch(node.body) - if node.globals: - self.write(" in ") - self.dispatch(node.globals) - if node.locals: - self.write(", ") - self.dispatch(node.locals) - def visit_Global(self, node): self.fill("global ") interleave(lambda: self.write(", "), self.write, node.names) @@ -338,31 +328,6 @@ def visit_Try(self, node): with self.block(): self.dispatch(node.finalbody) - def visit_TryExcept(self, node): - self.fill("try") - with self.block(): - self.dispatch(node.body) - - for ex in node.handlers: - self.dispatch(ex) - if node.orelse: - self.fill("else") - with self.block(): - self.dispatch(node.orelse) - - def visit_TryFinally(self, node): - if len(node.body) == 1 and isinstance(node.body[0], ast.TryExcept): - # try-except-finally - self.dispatch(node.body) - else: - self.fill("try") - with self.block(): - self.dispatch(node.body) - - self.fill("finally") - with self.block(): - self.dispatch(node.finalbody) - def visit_ExceptHandler(self, node): self.fill("except") if node.type: @@ -380,6 +345,10 @@ def visit_ClassDef(self, node): self.fill("@") self.dispatch(deco) self.fill("class " + node.name) + if getattr(node, "type_params", False): + self.write("[") + interleave(lambda: self.write(", "), self.dispatch, node.type_params) + self.write("]") with self.delimit_if("(", ")", condition=node.bases or node.keywords): comma = False for e in node.bases: @@ -394,21 +363,6 @@ def visit_ClassDef(self, node): else: comma = True self.dispatch(e) - if sys.version_info[:2] < (3, 5): - if node.starargs: - if comma: - self.write(", ") - else: - comma = True - self.write("*") - self.dispatch(node.starargs) - if node.kwargs: - if comma: - self.write(", ") - else: - comma = True - self.write("**") - self.dispatch(node.kwargs) with self.block(): self.dispatch(node.body) @@ -425,6 +379,10 @@ def __FunctionDef_helper(self, node, fill_suffix): self.dispatch(deco) def_str = fill_suffix + " " + node.name self.fill(def_str) + if getattr(node, "type_params", False): + self.write("[") + interleave(lambda: self.write(", "), self.dispatch, node.type_params) + self.write("]") with self.delimit("(", ")"): self.dispatch(node.args) if getattr(node, "returns", False): @@ -640,11 +598,6 @@ def visit_Name(self, node): def visit_NameConstant(self, node): self.write(repr(node.value)) - def visit_Repr(self, node): - self.write("`") - self.dispatch(node.value) - self.write("`") - def _write_constant(self, value): if isinstance(value, (float, complex)): # Substitute overflowing decimal literal for AST infinities. @@ -985,16 +938,10 @@ def visit_arguments(self, node): self.write(", ") self.write("*") if node.vararg: - if hasattr(node.vararg, "arg"): - self.write(node.vararg.arg) - if node.vararg.annotation: - self.write(": ") - self.dispatch(node.vararg.annotation) - else: - self.write(node.vararg) - if getattr(node, "varargannotation", None): - self.write(": ") - self.dispatch(node.varargannotation) + self.write(node.vararg.arg) + if node.vararg.annotation: + self.write(": ") + self.dispatch(node.vararg.annotation) # keyword-only arguments if getattr(node, "kwonlyargs", False): @@ -1014,16 +961,10 @@ def visit_arguments(self, node): first = False else: self.write(", ") - if hasattr(node.kwarg, "arg"): - self.write("**" + node.kwarg.arg) - if node.kwarg.annotation: - self.write(": ") - self.dispatch(node.kwarg.annotation) - else: - self.write("**" + node.kwarg) - if getattr(node, "kwargannotation", None): - self.write(": ") - self.dispatch(node.kwargannotation) + self.write("**" + node.kwarg.arg) + if node.kwarg.annotation: + self.write(": ") + self.dispatch(node.kwarg.annotation) def visit_keyword(self, node): if node.arg is None: @@ -1138,3 +1079,27 @@ def visit_MatchOr(self, node): with self.require_parens(_Precedence.BOR, node): self.set_precedence(pnext(_Precedence.BOR), *node.patterns) interleave(lambda: self.write(" | "), self.dispatch, node.patterns) + + def visit_TypeAlias(self, node): + self.fill("type ") + self.dispatch(node.name) + if node.type_params: + self.write("[") + interleave(lambda: self.write(", "), self.dispatch, node.type_params) + self.write("]") + self.write(" = ") + self.dispatch(node.value) + + def visit_TypeVar(self, node): + self.write(node.name) + if node.bound: + self.write(": ") + self.dispatch(node.bound) + + def visit_TypeVarTuple(self, node): + self.write("*") + self.write(node.name) + + def visit_ParamSpec(self, node): + self.write("**") + self.write(node.name) diff --git a/lib/spack/spack/util/url.py b/lib/spack/spack/util/url.py index cacee8f1fb3fdd..0644a17adac580 100644 --- a/lib/spack/spack/util/url.py +++ b/lib/spack/spack/util/url.py @@ -10,12 +10,13 @@ import itertools import os import posixpath -import re import sys import urllib.parse import urllib.request -from spack.util.path import convert_to_posix_path +from llnl.path import convert_to_posix_path + +from spack.util.path import sanitize_filename def validate_scheme(scheme): @@ -243,54 +244,20 @@ def _join(base_url, path, *extra, **kwargs): ) -git_re = ( - r"^(?:([a-z]+)://)?" # 1. optional scheme - r"(?:([^@]+)@)?" # 2. optional user - r"([^:/~]+)?" # 3. optional hostname - r"(?(1)(?::([^:/]+))?|:)" # 4. : if scheme else : - r"(.*[^/])/?$" # 5. path -) - - -def parse_git_url(url): - """Parse git URL into components. - - This parses URLs that look like: - - * ``https://host.com:443/path/to/repo.git``, or - * ``git@host.com:path/to/repo.git`` - - Anything not matching those patterns is likely a local - file or invalid. - - Returned components are as follows (optional values can be ``None``): - - 1. ``scheme`` (optional): git, ssh, http, https - 2. ``user`` (optional): ``git@`` for github, username for http or ssh - 3. ``hostname``: domain of server - 4. ``port`` (optional): port on server - 5. ``path``: path on the server, e.g. spack/spack - - Returns: - (tuple): tuple containing URL components as above - - Raises ``ValueError`` for invalid URLs. - """ - match = re.match(git_re, url) - if not match: - raise ValueError("bad git URL: %s" % url) - - # initial parse - scheme, user, hostname, port, path = match.groups() - - # special handling for ~ paths (they're never absolute) - if path.startswith("/~"): - path = path[1:] +def default_download_filename(url: str) -> str: + """This method computes a default file name for a given URL. + Note that it makes no request, so this is not the same as the + option curl -O, which uses the remote file name from the response + header.""" + parsed_url = urllib.parse.urlparse(url) + # Only use the last path component + params + query + fragment + name = urllib.parse.urlunparse( + parsed_url._replace(scheme="", netloc="", path=posixpath.basename(parsed_url.path)) + ) + valid_name = sanitize_filename(name) - if port is not None: - try: - port = int(port) - except ValueError: - raise ValueError("bad port in git url: %s" % url) + # Don't download to hidden files please + if valid_name[0] == ".": + valid_name = "_" + valid_name[1:] - return (scheme, user, hostname, port, path) + return valid_name diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index b4218a7b31e030..57158db950e917 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -4,8 +4,9 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import codecs +import concurrent.futures +import email.message import errno -import multiprocessing.pool import os import os.path import re @@ -16,37 +17,62 @@ import urllib.parse from html.parser import HTMLParser from pathlib import Path, PurePosixPath -from urllib.error import URLError +from typing import IO, Dict, List, Optional, Set, Union +from urllib.error import HTTPError, URLError from urllib.request import HTTPSHandler, Request, build_opener -import llnl.util.lang -import llnl.util.tty as tty +import llnl.url +from llnl.util import lang, tty from llnl.util.filesystem import mkdirp, rename, working_dir -import spack import spack.config import spack.error -import spack.gcs_handler -import spack.s3_handler -import spack.url -import spack.util.crypto -import spack.util.gcs as gcs_util -import spack.util.s3 as s3_util import spack.util.url as url_util -from spack.util.compression import ALLOWED_ARCHIVE_TYPES -from spack.util.executable import CommandNotFoundError, which -from spack.util.path import convert_to_posix_path + +from .executable import CommandNotFoundError, which +from .gcs import GCSBlob, GCSBucket, GCSHandler +from .s3 import UrllibS3Handler, get_s3_session + + +class DetailedHTTPError(HTTPError): + def __init__( + self, req: Request, code: int, msg: str, hdrs: email.message.Message, fp: Optional[IO] + ) -> None: + self.req = req + super().__init__(req.get_full_url(), code, msg, hdrs, fp) + + def __str__(self): + # Note: HTTPError, is actually a kind of non-seekable response object, so + # best not to read the response body here (even if it may include a human-readable + # error message). + # Note: use self.filename, not self.url, because the latter requires fp to be an + # IO object, which is not the case after unpickling. + return f"{self.req.get_method()} {self.filename} returned {self.code}: {self.msg}" + + def __reduce__(self): + # fp is an IO object and not picklable, the rest should be. + return DetailedHTTPError, (self.req, self.code, self.msg, self.hdrs, None) + + +class SpackHTTPDefaultErrorHandler(urllib.request.HTTPDefaultErrorHandler): + def http_error_default(self, req, fp, code, msg, hdrs): + raise DetailedHTTPError(req, code, msg, hdrs, fp) def _urlopen(): - s3 = spack.s3_handler.UrllibS3Handler() - gcs = spack.gcs_handler.GCSHandler() + s3 = UrllibS3Handler() + gcs = GCSHandler() + error_handler = SpackHTTPDefaultErrorHandler() # One opener with HTTPS ssl enabled - with_ssl = build_opener(s3, gcs, HTTPSHandler(context=ssl.create_default_context())) + with_ssl = build_opener( + s3, gcs, HTTPSHandler(context=ssl.create_default_context()), error_handler + ) # One opener with HTTPS ssl disabled - without_ssl = build_opener(s3, gcs, HTTPSHandler(context=ssl._create_unverified_context())) + without_ssl = build_opener( + s3, gcs, HTTPSHandler(context=ssl._create_unverified_context()), error_handler + ) # And dynamically dispatch based on the config:verify_ssl. def dispatch_open(fullurl, data=None, timeout=None): @@ -58,7 +84,7 @@ def dispatch_open(fullurl, data=None, timeout=None): #: Dispatches to the correct OpenerDirector.open, based on Spack configuration. -urlopen = llnl.util.lang.Singleton(_urlopen) +urlopen = lang.Singleton(_urlopen) #: User-Agent used in Request objects SPACK_USER_AGENT = "Spackbot/{0}".format(spack.spack_version) @@ -84,19 +110,28 @@ def handle_starttag(self, tag, attrs): self.links.append(val) -class IncludeFragmentParser(HTMLParser): +class ExtractMetadataParser(HTMLParser): """This parser takes an HTML page and selects the include-fragments, - used on GitHub, https://github.github.io/include-fragment-element.""" + used on GitHub, https://github.github.io/include-fragment-element, + as well as a possible base url.""" def __init__(self): super().__init__() - self.links = [] + self.fragments = [] + self.base_url = None def handle_starttag(self, tag, attrs): + # if tag == "include-fragment": for attr, val in attrs: if attr == "src": - self.links.append(val) + self.fragments.append(val) + + # + elif tag == "base": + for attr, val in attrs: + if attr == "href": + self.base_url = val def read_from_url(url, accept_content_type=None): @@ -158,14 +193,14 @@ def push_to_url(local_file_path, remote_path, keep_original=True, extra_args=Non while remote_path.startswith("/"): remote_path = remote_path[1:] - s3 = s3_util.get_s3_session(remote_url, method="push") + s3 = get_s3_session(remote_url, method="push") s3.upload_file(local_file_path, remote_url.netloc, remote_path, ExtraArgs=extra_args) if not keep_original: os.remove(local_file_path) elif remote_url.scheme == "gs": - gcs = gcs_util.GCSBlob(remote_url) + gcs = GCSBlob(remote_url) gcs.upload_to_blob(local_file_path) if not keep_original: os.remove(local_file_path) @@ -231,11 +266,11 @@ def check_curl_code(returncode): if returncode != 0: if returncode == 22: # This is a 404. Curl will print the error. - raise FetchError("URL was not found!") + raise spack.error.FetchError("URL was not found!") if returncode == 60: # This is a certificate error. Suggest spack -k - raise FetchError( + raise spack.error.FetchError( "Curl was unable to fetch due to invalid certificate. " "This is either an attack, or your cluster's SSL " "configuration is bad. If you believe your SSL " @@ -244,7 +279,7 @@ def check_curl_code(returncode): "Use this at your own risk." ) - raise FetchError("Curl failed with error {0}".format(returncode)) + raise spack.error.FetchError("Curl failed with error {0}".format(returncode)) def _curl(curl=None): @@ -253,7 +288,7 @@ def _curl(curl=None): curl = which("curl", required=True) except CommandNotFoundError as exc: tty.error(str(exc)) - raise FetchError("Missing required curl fetch method") + raise spack.error.FetchError("Missing required curl fetch method") return curl @@ -281,7 +316,7 @@ def fetch_url_text(url, curl=None, dest_dir="."): Raises FetchError if the curl returncode indicates failure """ if not url: - raise FetchError("A URL is required to fetch its text") + raise spack.error.FetchError("A URL is required to fetch its text") tty.debug("Fetching text at {0}".format(url)) @@ -293,7 +328,7 @@ def fetch_url_text(url, curl=None, dest_dir="."): if fetch_method == "curl": curl_exe = _curl(curl) if not curl_exe: - raise FetchError("Missing required fetch method (curl)") + raise spack.error.FetchError("Missing required fetch method (curl)") curl_args = ["-O"] curl_args.extend(base_curl_fetch_args(url)) @@ -311,7 +346,9 @@ def fetch_url_text(url, curl=None, dest_dir="."): returncode = response.getcode() if returncode and returncode != 200: - raise FetchError("Urllib failed with error code {0}".format(returncode)) + raise spack.error.FetchError( + "Urllib failed with error code {0}".format(returncode) + ) output = codecs.getreader("utf-8")(response).read() if output: @@ -322,7 +359,7 @@ def fetch_url_text(url, curl=None, dest_dir="."): return path except SpackWebError as err: - raise FetchError("Urllib fetch failed to verify url: {0}".format(str(err))) + raise spack.error.FetchError("Urllib fetch failed to verify url: {0}".format(str(err))) return None @@ -395,7 +432,7 @@ def remove_url(url, recursive=False): if url.scheme == "s3": # Try to find a mirror for potential connection information - s3 = s3_util.get_s3_session(url, method="push") + s3 = get_s3_session(url, method="push") bucket = url.netloc if recursive: # Because list_objects_v2 can only return up to 1000 items @@ -428,10 +465,10 @@ def remove_url(url, recursive=False): elif url.scheme == "gs": if recursive: - bucket = gcs_util.GCSBucket(url) + bucket = GCSBucket(url) bucket.destroy(recursive=recursive) else: - blob = gcs_util.GCSBlob(url) + blob = GCSBlob(url) blob.delete_blob() return @@ -506,296 +543,172 @@ def list_url(url, recursive=False): ] if url.scheme == "s3": - s3 = s3_util.get_s3_session(url, method="fetch") + s3 = get_s3_session(url, method="fetch") if recursive: return list(_iter_s3_prefix(s3, url)) return list(set(key.split("/", 1)[0] for key in _iter_s3_prefix(s3, url))) elif url.scheme == "gs": - gcs = gcs_util.GCSBucket(url) + gcs = GCSBucket(url) return gcs.get_all_blobs(recursive=recursive) -def spider(root_urls, depth=0, concurrency=32): +def spider(root_urls: Union[str, List[str]], depth: int = 0, concurrency: Optional[int] = None): """Get web pages from root URLs. - If depth is specified (e.g., depth=2), then this will also follow - up to levels of links from each root. + If depth is specified (e.g., depth=2), then this will also follow up to levels + of links from each root. Args: - root_urls (str or list): root urls used as a starting point - for spidering - depth (int): level of recursion into links - concurrency (int): number of simultaneous requests that can be sent + root_urls: root urls used as a starting point for spidering + depth: level of recursion into links + concurrency: number of simultaneous requests that can be sent Returns: - A dict of pages visited (URL) mapped to their full text and the - set of visited links. + A dict of pages visited (URL) mapped to their full text and the set of visited links. """ - # Cache of visited links, meant to be captured by the closure below - _visited = set() - - def _spider(url, collect_nested): - """Fetches URL and any pages it links to. - - Prints out a warning only if the root can't be fetched; it ignores - errors with pages that the root links to. - - Args: - url (str): url being fetched and searched for links - collect_nested (bool): whether we want to collect arguments - for nested spidering on the links found in this url - - Returns: - A tuple of: - - pages: dict of pages visited (URL) mapped to their full text. - - links: set of links encountered while visiting the pages. - - spider_args: argument for subsequent call to spider - """ - pages = {} # dict from page URL -> text content. - links = set() # set of all links seen on visited pages. - subcalls = [] - - try: - response_url, _, response = read_from_url(url, "text/html") - if not response_url or not response: - return pages, links, subcalls - - page = codecs.getreader("utf-8")(response).read() - pages[response_url] = page - - # Parse out the include-fragments in the page - # https://github.github.io/include-fragment-element - include_fragment_parser = IncludeFragmentParser() - include_fragment_parser.feed(page) - - fragments = set() - while include_fragment_parser.links: - raw_link = include_fragment_parser.links.pop() - abs_link = url_util.join(response_url, raw_link.strip(), resolve_href=True) - - try: - # This seems to be text/html, though text/fragment+html is also used - fragment_response_url, _, fragment_response = read_from_url( - abs_link, "text/html" - ) - except Exception as e: - msg = f"Error reading fragment: {(type(e), str(e))}:{traceback.format_exc()}" - tty.debug(msg) - - if not fragment_response_url or not fragment_response: - continue - - fragment = codecs.getreader("utf-8")(fragment_response).read() - fragments.add(fragment) - - pages[fragment_response_url] = fragment - - # Parse out the links in the page and all fragments - link_parser = LinkParser() - link_parser.feed(page) - for fragment in fragments: - link_parser.feed(fragment) - - while link_parser.links: - raw_link = link_parser.links.pop() - abs_link = url_util.join(response_url, raw_link.strip(), resolve_href=True) - links.add(abs_link) - - # Skip stuff that looks like an archive - if any(raw_link.endswith(s) for s in ALLOWED_ARCHIVE_TYPES): - continue - - # Skip already-visited links - if abs_link in _visited: - continue - - # If we're not at max depth, follow links. - if collect_nested: - subcalls.append((abs_link,)) - _visited.add(abs_link) - - except URLError as e: - tty.debug(str(e)) - - if hasattr(e, "reason") and isinstance(e.reason, ssl.SSLError): - tty.warn( - "Spack was unable to fetch url list due to a " - "certificate verification problem. You can try " - "running spack -k, which will not check SSL " - "certificates. Use this at your own risk." - ) - - except HTMLParseError as e: - # This error indicates that Python's HTML parser sucks. - msg = "Got an error parsing HTML." - tty.warn(msg, url, "HTMLParseError: " + str(e)) - - except Exception as e: - # Other types of errors are completely ignored, - # except in debug mode - tty.debug("Error in _spider: %s:%s" % (type(e), str(e)), traceback.format_exc()) - - finally: - tty.debug("SPIDER: [url={0}]".format(url)) - - return pages, links, subcalls - if isinstance(root_urls, str): root_urls = [root_urls] - # Clear the local cache of visited pages before starting the search - _visited.clear() - current_depth = 0 pages, links, spider_args = {}, set(), [] - collect = current_depth < depth - for root in root_urls: - root = urllib.parse.urlparse(root) - spider_args.append((root, collect)) + _visited: Set[str] = set() + go_deeper = current_depth < depth + for root_str in root_urls: + root = urllib.parse.urlparse(root_str) + spider_args.append((root, go_deeper, _visited)) - tp = multiprocessing.pool.ThreadPool(processes=concurrency) - try: + with concurrent.futures.ProcessPoolExecutor(max_workers=concurrency) as tp: while current_depth <= depth: tty.debug( - "SPIDER: [depth={0}, max_depth={1}, urls={2}]".format( - current_depth, depth, len(spider_args) - ) + f"SPIDER: [depth={current_depth}, max_depth={depth}, urls={len(spider_args)}]" ) - results = tp.map(llnl.util.lang.star(_spider), spider_args) + results = [tp.submit(_spider, *one_search_args) for one_search_args in spider_args] spider_args = [] - collect = current_depth < depth - for sub_pages, sub_links, sub_spider_args in results: - sub_spider_args = [x + (collect,) for x in sub_spider_args] + go_deeper = current_depth < depth + for future in results: + sub_pages, sub_links, sub_spider_args, sub_visited = future.result() + _visited.update(sub_visited) + sub_spider_args = [(x, go_deeper, _visited) for x in sub_spider_args] pages.update(sub_pages) links.update(sub_links) spider_args.extend(sub_spider_args) current_depth += 1 - finally: - tp.terminate() - tp.join() return pages, links -def find_versions_of_archive( - archive_urls, list_url=None, list_depth=0, concurrency=32, reference_package=None -): - """Scrape web pages for new versions of a tarball. This function prefers URLs in the - following order: links found on the scraped page that match a url generated by the - reference package, found and in the archive_urls list, found and derived from those - in the archive_urls list, and if none are found for a version then the item in the - archive_urls list is included for the version. +def _spider(url: urllib.parse.ParseResult, collect_nested: bool, _visited: Set[str]): + """Fetches URL and any pages it links to. + + Prints out a warning only if the root can't be fetched; it ignores errors with pages + that the root links to. Args: - archive_urls (str or list or tuple): URL or sequence of URLs for - different versions of a package. Typically these are just the - tarballs from the package file itself. By default, this searches - the parent directories of archives. - list_url (str or None): URL for a listing of archives. - Spack will scrape these pages for download links that look - like the archive URL. - list_depth (int): max depth to follow links on list_url pages. - Defaults to 0. - concurrency (int): maximum number of concurrent requests - reference_package (spack.package_base.PackageBase or None): a spack package - used as a reference for url detection. Uses the url_for_version - method on the package to produce reference urls which, if found, - are preferred. + url: url being fetched and searched for links + collect_nested: whether we want to collect arguments for nested spidering on the + links found in this url + _visited: links already visited + + Returns: + A tuple of: + - pages: dict of pages visited (URL) mapped to their full text. + - links: set of links encountered while visiting the pages. + - spider_args: argument for subsequent call to spider + - visited: updated set of visited urls """ - if not isinstance(archive_urls, (list, tuple)): - archive_urls = [archive_urls] - - # Generate a list of list_urls based on archive urls and any - # explicitly listed list_url in the package - list_urls = set() - if list_url is not None: - list_urls.add(list_url) - for aurl in archive_urls: - list_urls |= spack.url.find_list_urls(aurl) - - # Add '/' to the end of the URL. Some web servers require this. - additional_list_urls = set() - for lurl in list_urls: - if not lurl.endswith("/"): - additional_list_urls.add(lurl + "/") - list_urls |= additional_list_urls - - # Grab some web pages to scrape. - pages, links = spider(list_urls, depth=list_depth, concurrency=concurrency) - - # Scrape them for archive URLs - regexes = [] - for aurl in archive_urls: - # This creates a regex from the URL with a capture group for - # the version part of the URL. The capture group is converted - # to a generic wildcard, so we can use this to extract things - # on a page that look like archive URLs. - url_regex = spack.url.wildcard_version(aurl) - - # We'll be a bit more liberal and just look for the archive - # part, not the full path. - # this is a URL so it is a posixpath even on Windows - url_regex = PurePosixPath(url_regex).name - - # We need to add a / to the beginning of the regex to prevent - # Spack from picking up similarly named packages like: - # https://cran.r-project.org/src/contrib/pls_2.6-0.tar.gz - # https://cran.r-project.org/src/contrib/enpls_5.7.tar.gz - # https://cran.r-project.org/src/contrib/autopls_1.3.tar.gz - # https://cran.r-project.org/src/contrib/matrixpls_1.0.4.tar.gz - url_regex = "/" + url_regex - - # We need to add a $ anchor to the end of the regex to prevent - # Spack from picking up signature files like: - # .asc - # .md5 - # .sha256 - # .sig - # However, SourceForge downloads still need to end in '/download'. - url_regex += r"(\/download)?" - # PyPI adds #sha256=... to the end of the URL - url_regex += "(#sha256=.*)?" - url_regex += "$" - - regexes.append(url_regex) - - # Build a dict version -> URL from any links that match the wildcards. - # Walk through archive_url links first. - # Any conflicting versions will be overwritten by the list_url links. - versions = {} - matched = set() - for url in sorted(links): - url = convert_to_posix_path(url) - if any(re.search(r, url) for r in regexes): + pages: Dict[str, str] = {} # dict from page URL -> text content. + links: Set[str] = set() # set of all links seen on visited pages. + subcalls: List[str] = [] + + try: + response_url, _, response = read_from_url(url, "text/html") + if not response_url or not response: + return pages, links, subcalls, _visited + + page = codecs.getreader("utf-8")(response).read() + pages[response_url] = page + + # Parse out the include-fragments in the page + # https://github.github.io/include-fragment-element + metadata_parser = ExtractMetadataParser() + metadata_parser.feed(page) + + # Change of base URL due to tag + response_url = metadata_parser.base_url or response_url + + fragments = set() + while metadata_parser.fragments: + raw_link = metadata_parser.fragments.pop() + abs_link = url_util.join(response_url, raw_link.strip(), resolve_href=True) + try: - ver = spack.url.parse_version(url) - if ver in matched: - continue - versions[ver] = url - # prevent this version from getting overwritten - if reference_package is not None: - if url == reference_package.url_for_version(ver): - matched.add(ver) - else: - extrapolated_urls = [ - spack.url.substitute_version(u, ver) for u in archive_urls - ] - if url in extrapolated_urls: - matched.add(ver) - except spack.url.UndetectableVersionError: + # This seems to be text/html, though text/fragment+html is also used + fragment_response_url, _, fragment_response = read_from_url(abs_link, "text/html") + except Exception as e: + msg = f"Error reading fragment: {(type(e), str(e))}:{traceback.format_exc()}" + tty.debug(msg) + + if not fragment_response_url or not fragment_response: continue - for url in archive_urls: - url = convert_to_posix_path(url) - ver = spack.url.parse_version(url) - if ver not in versions: - versions[ver] = url + fragment = codecs.getreader("utf-8")(fragment_response).read() + fragments.add(fragment) + + pages[fragment_response_url] = fragment - return versions + # Parse out the links in the page and all fragments + link_parser = LinkParser() + link_parser.feed(page) + for fragment in fragments: + link_parser.feed(fragment) + + while link_parser.links: + raw_link = link_parser.links.pop() + abs_link = url_util.join(response_url, raw_link.strip(), resolve_href=True) + links.add(abs_link) + + # Skip stuff that looks like an archive + if any(raw_link.endswith(s) for s in llnl.url.ALLOWED_ARCHIVE_TYPES): + continue + + # Skip already-visited links + if abs_link in _visited: + continue + + # If we're not at max depth, follow links. + if collect_nested: + subcalls.append(abs_link) + _visited.add(abs_link) + + except URLError as e: + tty.debug(f"[SPIDER] Unable to read: {url}") + tty.debug(str(e), level=2) + if hasattr(e, "reason") and isinstance(e.reason, ssl.SSLError): + tty.warn( + "Spack was unable to fetch url list due to a " + "certificate verification problem. You can try " + "running spack -k, which will not check SSL " + "certificates. Use this at your own risk." + ) + + except HTMLParseError as e: + # This error indicates that Python's HTML parser sucks. + msg = "Got an error parsing HTML." + tty.warn(msg, url, "HTMLParseError: " + str(e)) + + except Exception as e: + # Other types of errors are completely ignored, + # except in debug mode + tty.debug(f"Error in _spider: {type(e)}:{str(e)}", traceback.format_exc()) + + finally: + tty.debug(f"SPIDER: [url={url}]") + + return pages, links, subcalls, _visited def get_header(headers, header_name): @@ -858,10 +771,6 @@ def parse_etag(header_value): return valid.group(1) if valid else None -class FetchError(spack.error.SpackError): - """Superclass for fetch-related errors.""" - - class SpackWebError(spack.error.SpackError): """Superclass for Spack web spidering errors.""" diff --git a/lib/spack/spack/util/windows_registry.py b/lib/spack/spack/util/windows_registry.py index 5cc0edd8bf5271..cfc16724563287 100644 --- a/lib/spack/spack/util/windows_registry.py +++ b/lib/spack/spack/util/windows_registry.py @@ -8,6 +8,7 @@ """ import os +import re import sys from contextlib import contextmanager @@ -68,8 +69,19 @@ def _gather_subkey_info(self): sub_keys, _, _ = winreg.QueryInfoKey(self.hkey) for i in range(sub_keys): sub_name = winreg.EnumKey(self.hkey, i) - sub_handle = winreg.OpenKeyEx(self.hkey, sub_name, access=winreg.KEY_READ) - self._keys.append(RegistryKey(os.path.join(self.path, sub_name), sub_handle)) + try: + sub_handle = winreg.OpenKeyEx(self.hkey, sub_name, access=winreg.KEY_READ) + self._keys.append(RegistryKey(os.path.join(self.path, sub_name), sub_handle)) + except OSError as e: + if hasattr(e, "winerror"): + if e.winerror == 5: + # This is a permission error, we can't read this key + # move on + pass + else: + raise + else: + raise def _gather_value_info(self): """Compose all values for this key into a dict of form value name: RegistryValue Object""" @@ -161,6 +173,15 @@ def __init__(self, key, root_key=HKEY.HKEY_CURRENT_USER): self.root = root_key self._reg = None + class KeyMatchConditions: + @staticmethod + def regex_matcher(subkey_name): + return lambda x: re.match(subkey_name, x.name) + + @staticmethod + def name_matcher(subkey_name): + return lambda x: subkey_name == x.name + @contextmanager def invalid_reg_ref_error_handler(self): try: @@ -193,6 +214,10 @@ def _valid_reg_check(self): return False return True + def _regex_match_subkeys(self, subkey): + r_subkey = re.compile(subkey) + return [key for key in self.get_subkeys() if r_subkey.match(key.name)] + @property def reg(self): if not self._reg: @@ -218,51 +243,106 @@ def get_subkeys(self): with self.invalid_reg_ref_error_handler(): return self.reg.subkeys + def get_matching_subkeys(self, subkey_name): + """Returns all subkeys regex matching subkey name + + Note: this method obtains only direct subkeys of the given key and does not + desced to transtitve subkeys. For this behavior, see `find_matching_subkeys`""" + self._regex_match_subkeys(subkey_name) + def get_values(self): if not self._valid_reg_check(): raise RegistryError("Cannot query values from invalid key %s" % self.key) with self.invalid_reg_ref_error_handler(): return self.reg.values - def _traverse_subkeys(self, stop_condition): + def _traverse_subkeys(self, stop_condition, collect_all_matching=False): """Perform simple BFS of subkeys, returning the key that successfully triggers the stop condition. Args: stop_condition: lambda or function pointer that takes a single argument a key and returns a boolean value based on that key + collect_all_matching: boolean value, if True, the traversal collects and returns + all keys meeting stop condition. If false, once stop + condition is met, the key that triggered the condition ' + is returned. Return: the key if stop_condition is triggered, or None if not """ + collection = [] if not self._valid_reg_check(): raise RegistryError("Cannot query values from invalid key %s" % self.key) with self.invalid_reg_ref_error_handler(): queue = self.reg.subkeys for key in queue: if stop_condition(key): - return key + if collect_all_matching: + collection.append(key) + else: + return key queue.extend(key.subkeys) - return None + return collection if collection else None + + def _find_subkey_s(self, search_key, collect_all_matching=False): + """Retrieve one or more keys regex matching `search_key`. + One key will be returned unless `collect_all_matching` is enabled, + in which case call matches are returned. + + Args: + search_key (str): regex string represeting a subkey name structure + to be matched against. + Cannot be provided alongside `direct_subkey` + collect_all_matching (bool): No-op if `direct_subkey` is specified + Return: + the desired subkey as a RegistryKey object, or none + """ + return self._traverse_subkeys(search_key, collect_all_matching=collect_all_matching) - def find_subkey(self, subkey_name, recursive=True): - """If non recursive, this method is the same as get subkey with error handling - Otherwise perform a BFS of subkeys until desired key is found + def find_subkey(self, subkey_name): + """Perform a BFS of subkeys until desired key is found Returns None or RegistryKey object corresponding to requested key name Args: - subkey_name (str): string representing subkey to be searched for - recursive (bool): optional argument, if True, subkey need not be a direct - sub key of this registry entry, and this method will - search all subkeys recursively. - Default is True + subkey_name (str) Return: the desired subkey as a RegistryKey object, or none + + For more details, see the WindowsRegistryView._find_subkey_s method docstring """ + return self._find_subkey_s( + WindowsRegistryView.KeyMatchConditions.name_matcher(subkey_name) + ) - if not recursive: - return self.get_subkey(subkey_name) + def find_matching_subkey(self, subkey_name): + """Perform a BFS of subkeys until a key matching subkey name regex is found + Returns None or the first RegistryKey object corresponding to requested key name - else: - return self._traverse_subkeys(lambda x: x.name == subkey_name) + Args: + subkey_name (str) + Return: + the desired subkey as a RegistryKey object, or none + + For more details, see the WindowsRegistryView._find_subkey_s method docstring + """ + return self._find_subkey_s( + WindowsRegistryView.KeyMatchConditions.regex_matcher(subkey_name) + ) + + def find_subkeys(self, subkey_name): + """Exactly the same as find_subkey, except this function tries to match + a regex to multiple keys + + Args: + subkey_name (str) + Return: + the desired subkeys as a list of RegistryKey object, or none + + For more details, see the WindowsRegistryView._find_subkey_s method docstring + """ + kwargs = {"collect_all_matching": True} + return self._find_subkey_s( + WindowsRegistryView.KeyMatchConditions.regex_matcher(subkey_name), **kwargs + ) def find_value(self, val_name, recursive=True): """ diff --git a/lib/spack/spack/variant.py b/lib/spack/spack/variant.py index e0b9a5540b9b8c..7b045d62628022 100644 --- a/lib/spack/spack/variant.py +++ b/lib/spack/spack/variant.py @@ -15,10 +15,10 @@ import llnl.util.lang as lang import llnl.util.tty.color +from llnl.string import comma_or import spack.directives import spack.error as error -from spack.util.string import comma_or special_variant_values = [None, "none", "*"] diff --git a/lib/spack/spack/verify.py b/lib/spack/spack/verify.py index 6abd1c9f66ddfb..01ca96fc7a4997 100644 --- a/lib/spack/spack/verify.py +++ b/lib/spack/spack/verify.py @@ -50,7 +50,9 @@ def create_manifest_entry(path: str) -> Dict[str, Any]: def write_manifest(spec): manifest_file = os.path.join( - spec.prefix, spack.store.layout.metadata_dir, spack.store.layout.manifest_file_name + spec.prefix, + spack.store.STORE.layout.metadata_dir, + spack.store.STORE.layout.manifest_file_name, ) if not os.path.exists(manifest_file): @@ -107,14 +109,14 @@ def check_file_manifest(filename): dirname = os.path.dirname(filename) results = VerificationResults() - while spack.store.layout.metadata_dir not in os.listdir(dirname): + while spack.store.STORE.layout.metadata_dir not in os.listdir(dirname): if dirname == os.path.sep: results.add_error(filename, "not owned by any package") return results dirname = os.path.dirname(dirname) manifest_file = os.path.join( - dirname, spack.store.layout.metadata_dir, spack.store.layout.manifest_file_name + dirname, spack.store.STORE.layout.metadata_dir, spack.store.STORE.layout.manifest_file_name ) if not os.path.exists(manifest_file): @@ -140,7 +142,7 @@ def check_spec_manifest(spec): results = VerificationResults() manifest_file = os.path.join( - prefix, spack.store.layout.metadata_dir, spack.store.layout.manifest_file_name + prefix, spack.store.STORE.layout.metadata_dir, spack.store.STORE.layout.manifest_file_name ) if not os.path.exists(manifest_file): diff --git a/lib/spack/spack/version/__init__.py b/lib/spack/spack/version/__init__.py new file mode 100644 index 00000000000000..b25048f0523f9b --- /dev/null +++ b/lib/spack/spack/version/__init__.py @@ -0,0 +1,60 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +""" +This module implements Version and version-ish objects. These are: + +StandardVersion: A single version of a package. +ClosedOpenRange: A range of versions of a package. +VersionList: A ordered list of Version and VersionRange elements. + +The set of Version and ClosedOpenRange is totally ordered wiht < +defined as Version(x) < VersionRange(Version(y), Version(x)) +if Version(x) <= Version(y). +""" + +from .common import ( + EmptyRangeError, + VersionChecksumError, + VersionError, + VersionLookupError, + infinity_versions, + is_git_version, +) +from .version_types import ( + ClosedOpenRange, + GitVersion, + StandardVersion, + Version, + VersionList, + VersionRange, + from_string, + next_version, + prev_version, + ver, +) + +#: This version contains all possible versions. +any_version: VersionList = VersionList([":"]) + +__all__ = [ + "Version", + "VersionRange", + "ver", + "from_string", + "is_git_version", + "infinity_versions", + "prev_version", + "next_version", + "VersionList", + "ClosedOpenRange", + "StandardVersion", + "GitVersion", + "VersionError", + "VersionChecksumError", + "VersionLookupError", + "EmptyRangeError", + "any_version", +] diff --git a/lib/spack/spack/version/common.py b/lib/spack/spack/version/common.py new file mode 100644 index 00000000000000..28dced815c3329 --- /dev/null +++ b/lib/spack/spack/version/common.py @@ -0,0 +1,41 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import re + +import spack.error + +# regex for a commit version +COMMIT_VERSION = re.compile(r"^[a-f0-9]{40}$") + +# Infinity-like versions. The order in the list implies the comparison rules +infinity_versions = ["stable", "trunk", "head", "master", "main", "develop"] + +iv_min_len = min(len(s) for s in infinity_versions) + + +def is_git_version(string: str) -> bool: + return ( + string.startswith("git.") + or len(string) == 40 + and bool(COMMIT_VERSION.match(string)) + or "=" in string[1:] + ) + + +class VersionError(spack.error.SpackError): + """This is raised when something is wrong with a version.""" + + +class VersionChecksumError(VersionError): + """Raised for version checksum errors.""" + + +class VersionLookupError(VersionError): + """Raised for errors looking up git commits as versions.""" + + +class EmptyRangeError(VersionError): + """Raised when constructing an empty version range.""" diff --git a/lib/spack/spack/version/git_ref_lookup.py b/lib/spack/spack/version/git_ref_lookup.py new file mode 100644 index 00000000000000..946cd03a187f04 --- /dev/null +++ b/lib/spack/spack/version/git_ref_lookup.py @@ -0,0 +1,217 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import re +from pathlib import Path +from typing import Dict, Optional, Tuple + +from llnl.util.filesystem import mkdirp, working_dir + +import spack.caches +import spack.fetch_strategy +import spack.paths +import spack.repo +import spack.util.executable +import spack.util.hash +import spack.util.spack_json as sjson +import spack.version + +from .common import VersionLookupError +from .lookup import AbstractRefLookup + +# regular expression for semantic versioning +_VERSION_CORE = r"\d+\.\d+\.\d+" +_IDENT = r"[0-9A-Za-z-]+" +_SEPARATED_IDENT = rf"{_IDENT}(?:\.{_IDENT})*" +_PRERELEASE = rf"\-{_SEPARATED_IDENT}" +_BUILD = rf"\+{_SEPARATED_IDENT}" +_SEMVER = rf"{_VERSION_CORE}(?:{_PRERELEASE})?(?:{_BUILD})?" + +# clamp on the end, so versions like v1.2.3-rc1 will match +# without the leading 'v'. +SEMVER_REGEX = re.compile(rf"{_SEMVER}$") + + +class GitRefLookup(AbstractRefLookup): + """An object for cached lookups of git refs + + GitRefLookup objects delegate to the MISC_CACHE for locking. GitRefLookup objects may + be attached to a GitVersion to allow for comparisons between git refs and versions as + represented by tags in the git repository. + """ + + def __init__(self, pkg_name): + self.pkg_name = pkg_name + + self.data: Dict[str, Tuple[Optional[str], int]] = {} + + self._pkg = None + self._fetcher = None + self._cache_key = None + self._cache_path = None + + # The following properties are used as part of a lazy reference scheme + # to avoid querying the package repository until it is necessary (and + # in particular to wait until after the configuration has been + # assembled) + @property + def cache_key(self): + if not self._cache_key: + key_base = "git_metadata" + self._cache_key = (Path(key_base) / self.repository_uri).as_posix() + + # Cache data in MISC_CACHE + # If this is the first lazy access, initialize the cache as well + spack.caches.MISC_CACHE.init_entry(self.cache_key) + return self._cache_key + + @property + def cache_path(self): + if not self._cache_path: + self._cache_path = spack.caches.MISC_CACHE.cache_path(self.cache_key) + return self._cache_path + + @property + def pkg(self): + if not self._pkg: + try: + pkg = spack.repo.PATH.get_pkg_class(self.pkg_name) + pkg.git + except (spack.repo.RepoError, AttributeError) as e: + raise VersionLookupError(f"Couldn't get the git repo for {self.pkg_name}") from e + self._pkg = pkg + return self._pkg + + @property + def fetcher(self): + if not self._fetcher: + # We require the full git repository history + fetcher = spack.fetch_strategy.GitFetchStrategy(git=self.pkg.git) + fetcher.get_full_repo = True + self._fetcher = fetcher + return self._fetcher + + @property + def repository_uri(self): + """Identifier for git repos used within the repo and metadata caches.""" + return Path(spack.util.hash.b32_hash(self.pkg.git)[-7:]) + + def save(self): + """Save the data to file""" + with spack.caches.MISC_CACHE.write_transaction(self.cache_key) as (old, new): + sjson.dump(self.data, new) + + def load_data(self): + """Load data if the path already exists.""" + if os.path.isfile(self.cache_path): + with spack.caches.MISC_CACHE.read_transaction(self.cache_key) as cache_file: + self.data = sjson.load(cache_file) + + def get(self, ref) -> Tuple[Optional[str], int]: + if not self.data: + self.load_data() + + if ref not in self.data: + self.data[ref] = self.lookup_ref(ref) + self.save() + + return self.data[ref] + + def lookup_ref(self, ref) -> Tuple[Optional[str], int]: + """Lookup the previous version and distance for a given commit. + + We use git to compare the known versions from package to the git tags, + as well as any git tags that are SEMVER versions, and find the latest + known version prior to the commit, as well as the distance from that version + to the commit in the git repo. Those values are used to compare Version objects. + """ + pathlib_dest = Path(spack.paths.user_repos_cache_path) / self.repository_uri + dest = str(pathlib_dest) + + # prepare a cache for the repository + dest_parent = os.path.dirname(dest) + if not os.path.exists(dest_parent): + mkdirp(dest_parent) + + # Only clone if we don't have it! + if not os.path.exists(dest): + self.fetcher.clone(dest, bare=True) + + # Lookup commit info + with working_dir(dest): + # TODO: we need to update the local tags if they changed on the + # remote instance, simply adding '-f' may not be sufficient + # (if commits are deleted on the remote, this command alone + # won't properly update the local rev-list) + self.fetcher.git("fetch", "--tags", output=os.devnull, error=os.devnull) + + # Ensure ref is a commit object known to git + # Note the brackets are literals, the ref replaces the format string + try: + self.fetcher.git( + "cat-file", "-e", "%s^{commit}" % ref, output=os.devnull, error=os.devnull + ) + except spack.util.executable.ProcessError: + raise VersionLookupError("%s is not a valid git ref for %s" % (ref, self.pkg_name)) + + # List tags (refs) by date, so last reference of a tag is newest + tag_info = self.fetcher.git( + "for-each-ref", + "--sort=creatordate", + "--format", + "%(objectname) %(refname)", + "refs/tags", + output=str, + ).split("\n") + + # Lookup of commits to spack versions + commit_to_version = {} + + for entry in tag_info: + if not entry: + continue + tag_commit, tag = entry.split() + tag = tag.replace("refs/tags/", "", 1) + + # For each tag, try to match to a version + for v in [v.string for v in self.pkg.versions]: + if v == tag or "v" + v == tag: + commit_to_version[tag_commit] = v + break + else: + # try to parse tag to compare versions spack does not know + match = SEMVER_REGEX.search(tag) + if match: + commit_to_version[tag_commit] = match.group() + + ancestor_commits = [] + for tag_commit in commit_to_version: + self.fetcher.git("merge-base", "--is-ancestor", tag_commit, ref, ignore_errors=[1]) + if self.fetcher.git.returncode == 0: + distance = self.fetcher.git( + "rev-list", "%s..%s" % (tag_commit, ref), "--count", output=str, error=str + ).strip() + ancestor_commits.append((tag_commit, int(distance))) + + if ancestor_commits: + # Get nearest ancestor that is a known version + prev_version_commit, distance = min(ancestor_commits, key=lambda x: x[1]) + prev_version = commit_to_version[prev_version_commit] + else: + # Get list of all commits, this is in reverse order + # We use this to get the first commit below + ref_info = self.fetcher.git("log", "--all", "--pretty=format:%H", output=str) + commits = [c for c in ref_info.split("\n") if c] + + # No previous version and distance from first commit + prev_version = None + distance = int( + self.fetcher.git( + "rev-list", "%s..%s" % (commits[-1], ref), "--count", output=str, error=str + ).strip() + ) + + return prev_version, distance diff --git a/lib/spack/spack/version/lookup.py b/lib/spack/spack/version/lookup.py new file mode 100644 index 00000000000000..e33b5c217691ac --- /dev/null +++ b/lib/spack/spack/version/lookup.py @@ -0,0 +1,17 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from typing import Optional, Tuple + + +class AbstractRefLookup: + def get(self, ref) -> Tuple[Optional[str], int]: + """Get the version string and distance for a given git ref. + + Args: + ref (str): git ref to lookup + + Returns: optional version string and distance""" + return None, 0 diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version/version_types.py similarity index 79% rename from lib/spack/spack/version.py rename to lib/spack/spack/version/version_types.py index 4be1afd90c5efe..87f4d26308cfff 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version/version_types.py @@ -3,54 +3,29 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -""" -This module implements Version and version-ish objects. These are: - -StandardVersion: A single version of a package. -ClosedOpenRange: A range of versions of a package. -VersionList: A ordered list of Version and VersionRange elements. - -The set of Version and ClosedOpenRange is totally ordered wiht < -defined as Version(x) < VersionRange(Version(y), Version(x)) -if Version(x) <= Version(y). -""" import numbers -import os import re from bisect import bisect_left -from typing import Dict, List, Optional, Tuple, Union - -from llnl.util.filesystem import mkdirp, working_dir +from typing import List, Optional, Tuple, Union -import spack.caches -import spack.error -import spack.paths -import spack.util.executable -import spack.util.spack_json as sjson -import spack.util.url from spack.util.spack_yaml import syaml_dict +from .common import ( + COMMIT_VERSION, + EmptyRangeError, + VersionLookupError, + infinity_versions, + is_git_version, + iv_min_len, +) +from .lookup import AbstractRefLookup + # Valid version characters VALID_VERSION = re.compile(r"^[A-Za-z0-9_.-][=A-Za-z0-9_.-]*$") -# regex for a commit version -COMMIT_VERSION = re.compile(r"^[a-f0-9]{40}$") - # regex for version segments SEGMENT_REGEX = re.compile(r"(?:(?P[0-9]+)|(?P[a-zA-Z]+))(?P[_.-]*)") -# regular expression for semantic versioning -SEMVER_REGEX = re.compile( - ".+(?P([0-9]+)[.]([0-9]+)[.]([0-9]+)" - "(?:-([0-9A-Za-z-]+(?:[.][0-9A-Za-z-]+)*))?" - "(?:[+][0-9A-Za-z-]+)?)" -) - -# Infinity-like versions. The order in the list implies the comparison rules -infinity_versions = ["stable", "trunk", "head", "master", "main", "develop"] - -iv_min_len = min(len(s) for s in infinity_versions) - class VersionStrComponent: __slots__ = ["data"] @@ -407,7 +382,7 @@ class GitVersion(ConcreteVersion): def __init__(self, string: str): # An object that can lookup git refs to compare them to versions - self._ref_lookup: Optional[CommitLookup] = None + self._ref_lookup: Optional[AbstractRefLookup] = None # This is the effective version. self._ref_version: Optional[StandardVersion] @@ -440,8 +415,7 @@ def ref_version(self) -> StandardVersion: if self.ref_lookup is None: raise VersionLookupError( - f"git ref '{self.ref}' cannot be looked up: " - "call attach_git_lookup_from_package first" + f"git ref '{self.ref}' cannot be looked up: " "call attach_lookup first" ) version_string, distance = self.ref_lookup.get(self.ref) @@ -573,7 +547,7 @@ def ref_lookup(self): self._ref_lookup.get(self.ref) return self._ref_lookup - def attach_git_lookup_from_package(self, pkg_name): + def attach_lookup(self, lookup: AbstractRefLookup): """ Use the git fetcher to look up a version for a commit. @@ -585,7 +559,7 @@ def attach_git_lookup_from_package(self, pkg_name): alongside the GitFetcher because eventually the git repos cache will be one and the same with the source cache. """ - self._ref_lookup = self._ref_lookup or CommitLookup(pkg_name) + self._ref_lookup = lookup def __iter__(self): return self.ref_version.__iter__() @@ -622,14 +596,17 @@ def up_to(self, index) -> StandardVersion: class ClosedOpenRange: def __init__(self, lo: StandardVersion, hi: StandardVersion): if hi < lo: - raise ValueError(f"{lo}:{hi} is an empty range") + raise EmptyRangeError(f"{lo}..{hi} is an empty range") self.lo: StandardVersion = lo self.hi: StandardVersion = hi @classmethod def from_version_range(cls, lo: StandardVersion, hi: StandardVersion): """Construct ClosedOpenRange from lo:hi range.""" - return ClosedOpenRange(lo, next_version(hi)) + try: + return ClosedOpenRange(lo, next_version(hi)) + except EmptyRangeError as e: + raise EmptyRangeError(f"{lo}:{hi} is an empty range") from e def __str__(self): # This simplifies 3.1:<3.2 to 3.1:3.1 to 3.1 @@ -758,7 +735,7 @@ def __init__(self, vlist=None): if vlist is not None: if isinstance(vlist, str): vlist = from_string(vlist) - if type(vlist) == VersionList: + if isinstance(vlist, VersionList): self.versions = vlist.versions else: self.versions = [vlist] @@ -792,7 +769,7 @@ def add(self, item): self.versions.insert(i, item) - elif type(item) == VersionList: + elif isinstance(item, VersionList): for v in item: self.add(v) @@ -1073,15 +1050,6 @@ def prev_version(v: StandardVersion) -> StandardVersion: return StandardVersion("".join(string_components), v.version[:-1] + (prev,), v.separators) -def is_git_version(string: str) -> bool: - return ( - string.startswith("git.") - or len(string) == 40 - and bool(COMMIT_VERSION.match(string)) - or "=" in string[1:] - ) - - def Version(string: Union[str, int]) -> Union[GitVersion, StandardVersion]: if not isinstance(string, (str, int)): raise ValueError(f"Cannot construct a version from {type(string)}") @@ -1140,216 +1108,3 @@ def ver(obj) -> Union[VersionList, ClosedOpenRange, StandardVersion, GitVersion] return obj else: raise TypeError("ver() can't convert %s to version!" % type(obj)) - - -#: This version contains all possible versions. -any_version: VersionList = VersionList([":"]) - - -class VersionError(spack.error.SpackError): - """This is raised when something is wrong with a version.""" - - -class VersionChecksumError(VersionError): - """Raised for version checksum errors.""" - - -class VersionLookupError(VersionError): - """Raised for errors looking up git commits as versions.""" - - -class CommitLookup: - """An object for cached lookups of git commits - - CommitLookup objects delegate to the misc_cache for locking. CommitLookup objects may - be attached to a GitVersion to allow for comparisons between git refs and versions as - represented by tags in the git repository. - """ - - def __init__(self, pkg_name): - self.pkg_name = pkg_name - - self.data: Dict[str, Tuple[Optional[str], int]] = {} - - self._pkg = None - self._fetcher = None - self._cache_key = None - self._cache_path = None - - # The following properties are used as part of a lazy reference scheme - # to avoid querying the package repository until it is necessary (and - # in particular to wait until after the configuration has been - # assembled) - @property - def cache_key(self): - if not self._cache_key: - key_base = "git_metadata" - if not self.repository_uri.startswith("/"): - key_base += "/" - self._cache_key = key_base + self.repository_uri - - # Cache data in misc_cache - # If this is the first lazy access, initialize the cache as well - spack.caches.misc_cache.init_entry(self.cache_key) - return self._cache_key - - @property - def cache_path(self): - if not self._cache_path: - self._cache_path = spack.caches.misc_cache.cache_path(self.cache_key) - return self._cache_path - - @property - def pkg(self): - if not self._pkg: - import spack.repo # break cycle - - try: - pkg = spack.repo.path.get_pkg_class(self.pkg_name) - pkg.git - except (spack.repo.RepoError, AttributeError) as e: - raise VersionLookupError(f"Couldn't get the git repo for {self.pkg_name}") from e - self._pkg = pkg - return self._pkg - - @property - def fetcher(self): - if not self._fetcher: - # We require the full git repository history - import spack.fetch_strategy # break cycle - - fetcher = spack.fetch_strategy.GitFetchStrategy(git=self.pkg.git) - fetcher.get_full_repo = True - self._fetcher = fetcher - return self._fetcher - - @property - def repository_uri(self): - """Identifier for git repos used within the repo and metadata caches.""" - try: - components = [ - str(c).lstrip("/") for c in spack.util.url.parse_git_url(self.pkg.git) if c - ] - return os.path.join(*components) - except ValueError: - # If it's not a git url, it's a local path - return os.path.abspath(self.pkg.git) - - def save(self): - """Save the data to file""" - with spack.caches.misc_cache.write_transaction(self.cache_key) as (old, new): - sjson.dump(self.data, new) - - def load_data(self): - """Load data if the path already exists.""" - if os.path.isfile(self.cache_path): - with spack.caches.misc_cache.read_transaction(self.cache_key) as cache_file: - self.data = sjson.load(cache_file) - - def get(self, ref) -> Tuple[Optional[str], int]: - if not self.data: - self.load_data() - - if ref not in self.data: - self.data[ref] = self.lookup_ref(ref) - self.save() - - return self.data[ref] - - def lookup_ref(self, ref) -> Tuple[Optional[str], int]: - """Lookup the previous version and distance for a given commit. - - We use git to compare the known versions from package to the git tags, - as well as any git tags that are SEMVER versions, and find the latest - known version prior to the commit, as well as the distance from that version - to the commit in the git repo. Those values are used to compare Version objects. - """ - dest = os.path.join(spack.paths.user_repos_cache_path, self.repository_uri) - if dest.endswith(".git"): - dest = dest[:-4] - - # prepare a cache for the repository - dest_parent = os.path.dirname(dest) - if not os.path.exists(dest_parent): - mkdirp(dest_parent) - - # Only clone if we don't have it! - if not os.path.exists(dest): - self.fetcher.clone(dest, bare=True) - - # Lookup commit info - with working_dir(dest): - # TODO: we need to update the local tags if they changed on the - # remote instance, simply adding '-f' may not be sufficient - # (if commits are deleted on the remote, this command alone - # won't properly update the local rev-list) - self.fetcher.git("fetch", "--tags", output=os.devnull, error=os.devnull) - - # Ensure ref is a commit object known to git - # Note the brackets are literals, the ref replaces the format string - try: - self.fetcher.git( - "cat-file", "-e", "%s^{commit}" % ref, output=os.devnull, error=os.devnull - ) - except spack.util.executable.ProcessError: - raise VersionLookupError("%s is not a valid git ref for %s" % (ref, self.pkg_name)) - - # List tags (refs) by date, so last reference of a tag is newest - tag_info = self.fetcher.git( - "for-each-ref", - "--sort=creatordate", - "--format", - "%(objectname) %(refname)", - "refs/tags", - output=str, - ).split("\n") - - # Lookup of commits to spack versions - commit_to_version = {} - - for entry in tag_info: - if not entry: - continue - tag_commit, tag = entry.split() - tag = tag.replace("refs/tags/", "", 1) - - # For each tag, try to match to a version - for v in [v.string for v in self.pkg.versions]: - if v == tag or "v" + v == tag: - commit_to_version[tag_commit] = v - break - else: - # try to parse tag to copare versions spack does not know - match = SEMVER_REGEX.match(tag) - if match: - semver = match.groupdict()["semver"] - commit_to_version[tag_commit] = semver - - ancestor_commits = [] - for tag_commit in commit_to_version: - self.fetcher.git("merge-base", "--is-ancestor", tag_commit, ref, ignore_errors=[1]) - if self.fetcher.git.returncode == 0: - distance = self.fetcher.git( - "rev-list", "%s..%s" % (tag_commit, ref), "--count", output=str, error=str - ).strip() - ancestor_commits.append((tag_commit, int(distance))) - - if ancestor_commits: - # Get nearest ancestor that is a known version - prev_version_commit, distance = min(ancestor_commits, key=lambda x: x[1]) - prev_version = commit_to_version[prev_version_commit] - else: - # Get list of all commits, this is in reverse order - # We use this to get the first commit below - ref_info = self.fetcher.git("log", "--all", "--pretty=format:%H", output=str) - commits = [c for c in ref_info.split("\n") if c] - - # No previous version and distance from first commit - prev_version = None - distance = int( - self.fetcher.git( - "rev-list", "%s..%s" % (commits[-1], ref), "--count", output=str, error=str - ).strip() - ) - - return prev_version, distance diff --git a/pyproject.toml b/pyproject.toml index a89a206666f980..d33b27a99c20ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,12 +141,29 @@ ignore_missing_imports = true ignore_errors = true ignore_missing_imports = true - # pytest (which we depend on) optionally imports numpy, which requires Python 3.8 in - # recent versions. mypy still imports its .pyi file, which has positional-only - # arguments, which don't work in 3.7, which causes mypy to bail out early if you have - # numpy installed. + # Spack imports a number of external packages, and they *may* require Python 3.8 or + # higher in recent versions. This can cause mypy to fail because we check for 3.7 + # compatibility. We could restrict mypy to run for the oldest supported version (3.7), + # but that means most developers won't be able to run mypy, which means it'll fail + # more in CI. Instead, we exclude these imported packages from mypy checking. [[tool.mypy.overrides]] - module = 'numpy' + module = [ + 'IPython', + 'altgraph', + 'attr', + 'boto3', + 'botocore', + 'distro', + 'jinja2', + 'jsonschema', + 'macholib', + 'markupsafe', + 'numpy', + 'pyristent', + 'pytest', + 'ruamel.yaml', + 'six', + ] follow_imports = 'skip' follow_imports_for_stubs = true diff --git a/pytest.ini b/pytest.ini index 9f6157c9b81dd6..617881d77b905d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -14,3 +14,6 @@ markers = enable_compiler_verification: enable compiler verification within unit tests enable_compiler_link_paths: verifies compiler link paths within unit tests disable_clean_stage_check: avoid failing tests if there are leftover files in the stage area + only_clingo: mark unit tests that run only with clingo + only_original: mark unit tests that are specific to the original concretizer + not_on_windows: mark tests that are skipped on Windows diff --git a/share/spack/bash/spack-completion.in b/share/spack/bash/spack-completion.bash similarity index 83% rename from share/spack/bash/spack-completion.in rename to share/spack/bash/spack-completion.bash index 048c8656494619..9a5b367be7a49a 100755 --- a/share/spack/bash/spack-completion.in +++ b/share/spack/bash/spack-completion.bash @@ -7,7 +7,7 @@ # NOTE: spack-completion.bash is auto-generated by: # # $ spack commands --aliases --format=bash -# --header=bash/spack-completion.in --update=spack-completion.bash +# --header=bash/spack-completion.bash --update=spack-completion.bash # # Please do not manually modify this file. @@ -52,6 +52,20 @@ if test -n "${ZSH_VERSION:-}" ; then fi fi +# compgen -W doesn't work in some versions of zsh, so use this instead. +# see https://www.zsh.org/mla/workers/2011/msg00582.html +_compgen_w() { + if test -n "${ZSH_VERSION:-}" ; then + typeset -a words + words=( ${~=1} ) + local find="$2" + results=(${(M)words[@]:#$find*}) + echo "${results[@]}" + else + compgen -W "$1" -- "$2" + fi +} + # Bash programmable completion for Spack _bash_completion_spack() { # In all following examples, let the cursor be denoted by brackets, i.e. [] @@ -137,8 +151,11 @@ _bash_completion_spack() { if [[ "$(LC_ALL=C type $subfunction 2>&1)" =~ $rgx ]] then $subfunction - COMPREPLY=($(compgen -W "$SPACK_COMPREPLY" -- "$cur")) + COMPREPLY=($(_compgen_w "$SPACK_COMPREPLY" "$cur")) fi + + # if every completion is an alias for the same thing, just return that thing. + _spack_compress_aliases } # Helper functions for subcommands @@ -328,6 +345,51 @@ _spacktivate() { _spack_env_activate } +# Simple function to get the spack alias for a command +_spack_get_alias() { + local possible_alias="${1-}" + local IFS=";" + + # spack aliases are a ;-separated list of :-separated pairs + for item in $SPACK_ALIASES; do + # maps a possible alias to its command + eval "local real_command=\"\${item#*${possible_alias}:}\"" + if [ "$real_command" != "$item" ]; then + SPACK_ALIAS="$real_command" + return + fi + done + + # no alias found -- just return $1 + SPACK_ALIAS="$possible_alias" +} + +# If all commands in COMPREPLY alias to the same thing, set COMPREPLY to +# just the real command, not the aliases. +_spack_compress_aliases() { + # If there are zero or one completions, don't do anything + # If this isn't the first argument, bail because aliases currently only apply + # to top-level commands. + if [ "${#COMPREPLY[@]}" -le "1" ] || [ "$COMP_CWORD_NO_FLAGS" != "1" ]; then + return + fi + + # get the alias of the first thing in the list of completions + _spack_get_alias "${COMPREPLY[@]:0:1}" + local first_alias="$SPACK_ALIAS" + + # if anything in the list would alias to something different, stop + for comp in "${COMPREPLY[@]:1}"; do + _spack_get_alias "$comp" + if [ "$SPACK_ALIAS" != "$first_alias" ]; then + return + fi + done + + # all commands alias to first alias; just return that + COMPREPLY=("$first_alias") +} + # Spack commands # # Everything below here is auto-generated. diff --git a/share/spack/bootstrap/github-actions-v0.3/clingo.json b/share/spack/bootstrap/github-actions-v0.3/clingo.json deleted file mode 100644 index 60e771221df3dd..00000000000000 --- a/share/spack/bootstrap/github-actions-v0.3/clingo.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "verified": [ - { - "binaries": [ - [ - "clingo-bootstrap", - "i5rx6vbyw7cyg3snajcpnuozo7l3lcab", - "c55d1c76adb82ac9fbe67725641ef7e4fe1ae11e2e8da0dc93a3efe362549127" - ] - ], - "python": "python@3.10", - "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "xoxkdgo3n332ewhbh7pz2zuevrjxkrke", - "b50e2fba026e85af3f99b3c412b4f0c88ec2fbce15b48eeb75072f1d3737f3cc" - ] - ], - "python": "python@3.5", - "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "sgmirxbu3bpn4rdpfs6jlyycfrkfxl5i", - "b0a574df6f5d59491a685a31a8ed99fb345c850a91df62ef232fbe0cca716ed1" - ] - ], - "python": "python@3.6", - "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "5hn7hszlizeqq3leqi6lrdmyy5ssv6zs", - "36e24bc3bd27b125fdeb30d51d2554e44288877c0ce6df5a878bb4e8a1d5847a" - ] - ], - "python": "python@3.7", - "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "qk3ecxakadq4naakng6mhdfkwauef3dn", - "9d974c0d2b546d18f0ec35e08d5ba114bf2867f7ff7c7ea990b79d019ece6380" - ] - ], - "python": "python@3.8", - "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "2omdsvzshkn2u3l5vwvwoey4es5cowfu", - "cbf72eb932ac847f87b1640f8e70e26f5261967288f7d6db19206ef352e36a88" - ] - ], - "python": "python@3.9", - "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "ifgzrctoh2ibrmitp6ushrvrnaeqtkr7", - "1c609df7351286fe09aa3452fa7ed7fedf903e9fa12cde89b916a0fc4c022949" - ] - ], - "python": "python@3.10", - "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "esfzjhodgh5be22hvh3trg2ojzrmhzwt", - "8d070cdb2a5103cde3e6f873b1eb11d25f60464f3059d8643f943e5c9a9ec76c" - ] - ], - "python": "python@3.6", - "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "5b4uhkhrvtvdmsnctjx2isrxciy6v2o2", - "336b8b1202a8a28a0e34a98e5780ae0e2b2370b342ce67434551009b1a7c8db9" - ] - ], - "python": "python@3.7", - "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "czapgrrey6llnsu2m4qaamv3so2lybxm", - "16bdfe4b08ee8da38f3e2c7d5cc44a38d87696cc2b6de0971a4de25efb8ad8e4" - ] - ], - "python": "python@3.8", - "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "7za6vsetahbghs4d2qe4ajtf2iyiacwx", - "730ae7e6096ec8b83a0fc9464dda62bd6c2fec1f8565bb291f4d1ffe7746703b" - ] - ], - "python": "python@3.9", - "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "zulnxrmchldtasffqw6qacmgg4y2qumj", - "8988325db53c0c650f64372c21571ac85e9ba4577975d14ae7dba8ab7728b5fc" - ] - ], - "python": "python@3.10", - "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "lx54ebqzwtjpfgch7kagoxkmul56z7fa", - "81d64229299e76f9dc81f88d286bc94725e7cbcbb29ad0d66aaeaff73dd6473a" - ] - ], - "python": "python@3.6", - "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "isu2rjoicl4xzmbl3k2c4bg35gvejkgz", - "fcc4b052832cfd327d11f657c2b7715d981b0894ed03bbce18b23a842c7d706d" - ] - ], - "python": "python@3.7", - "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "ob3k3g2wjy7cw33lfobjar44sqmojyth", - "f51fd6256bfd3afc8470614d87df61e5c9dd582fcc70f707ca66ba2b7255da12" - ] - ], - "python": "python@3.8", - "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "norpsmparkl5dfuzdqj4537o77vjbgsl", - "477c041857b60f29ff9d6c7d2982b7eb49a2e02ebbc98af11488c32e2fb24081" - ] - ], - "python": "python@3.9", - "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "gypv5loj2ml73duq6sr76yg5rj25te2m", - "c855d7d32aadec37c41e51f19b83558b32bc0b946a9565dba0e659c6820bd6c3" - ] - ], - "python": "python@2.7+ucs4", - "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "rjopyx7hum3hqhgsdyw3st7frdfgrv3p", - "0e555f9bc99b4e4152939b30b2257f4f353941d152659e716bf6123c0ce11a60" - ] - ], - "python": "python@2.7~ucs4", - "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "2l45t4kw3cqqwj6vbxhfwhzlo6b3q2p4", - "6cb90de5a3d123b7408cfef693a9a78bb69c66abbfed746c1e85aa0acb848d03" - ] - ], - "python": "python@3.10", - "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "4psiezojm7dexequtbnav77wvgcajigq", - "b3fc33b5482357613294becb54968bd74de638abeae69e27c6c4319046a7e352" - ] - ], - "python": "python@3.5", - "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "dzhvhynye4z7oalowdcy5zt25lej3m2n", - "61c5f3e80bcc7acfc65e335f1910762df2cc5ded9d7e1e5977380a24de553dd7" - ] - ], - "python": "python@3.6", - "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "dtwevigmwgke4g6ee5byktpmzmrp2kvx", - "636937244b58611ec2eedb4422a1076fcaf09f3998593befb5a6ff1a74e1d5f7" - ] - ], - "python": "python@3.7", - "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "shqedxgvjnhiwdcdrvjhbd73jaevv7wt", - "b3615b2a94a8a15fddaa74cf4d9f9b3a516467a843cdeab597f72dcf6be5e31d" - ] - ], - "python": "python@3.8", - "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" - }, - { - "binaries": [ - [ - "clingo-bootstrap", - "z6v6zvc6awioeompbvo735b4flr3yuyz", - "1389192bd74c1f7059d95c4a41500201cbc2905cbba553678613e0b7e3b96c71" - ] - ], - "python": "python@3.9", - "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" - } - ] -} \ No newline at end of file diff --git a/share/spack/bootstrap/github-actions-v0.3/gnupg.json b/share/spack/bootstrap/github-actions-v0.3/gnupg.json deleted file mode 100644 index 2f568297892227..00000000000000 --- a/share/spack/bootstrap/github-actions-v0.3/gnupg.json +++ /dev/null @@ -1,204 +0,0 @@ -{ - "verified": [ - { - "binaries": [ - [ - "libiconv", - "d6dhoguolmllbzy2h6pnvjm3tti6uy6f", - "7fe765a87945991d4e57782ed67c4bf42a10f95582eecd6f57de80a545bde821" - ], - [ - "npth", - "x6fb7zx6n7mos5knvi6wlnaadd7r2szx", - "fd1e5a62107339f45219c32ba20b5e82aa0880c31ac86d1b245d388ca4546990" - ], - [ - "zlib", - "c5wm3jilx6zsers3sfgdisjqusoza4wr", - "7500a717c62736872aa65df4599f797ef67b21086dd6236b4c7712cfffac9bf3" - ], - [ - "libassuan", - "3qv4bprobfwes37clg764cfipdzjdbto", - "d85cd9d2c63a296300d4dcbd667421956df241109daef5e12d3ca63fa241cb14" - ], - [ - "libgcrypt", - "3y4ubdgxvgpvhxr3bk4l5mkw4gv42n7e", - "9dad7c2635344957c4db68378964d3af84ea052d45dbe8ded9a6e6e47211daa8" - ], - [ - "libgpg-error", - "doido34kfwsvwpj4c4jcocahjb5ltebw", - "20e5c238bee91d2a841f0b4bd0358ded59a0bd665d7f251fd9cd42f83e0b283b" - ], - [ - "libksba", - "mttecm7gzdv544lbzcoahchnboxysrvi", - "1c0ae64e828a597e4cf15dd997c66cd677e41f68c63db09b9551480a197052a2" - ], - [ - "pinentry", - "se7xgv7yf4ywpjnbv7voxgeuuvs77ahb", - "2fd13fbee7ca2361dc5dd09708c72d0489611301b60635cb0206bc5b94add884" - ], - [ - "gnupg", - "yannph34bpaqkhsv5mz2icwhy3epiqxd", - "1de8b4e119fa3455d0170466fa0fb8e04957fab740aec32535b4667279312b3f" - ] - ], - "spec": "gnupg@2.3: %apple-clang platform=darwin target=x86_64" - }, - { - "binaries": [ - [ - "zlib", - "t2hjzsyf3txkg64e4bq3nihe26rzzdws", - "171e720840a28af50b62141be77bc525e666cffd1fbbe2ee62673214e8b0280f" - ], - [ - "libiconv", - "yjdji2wj4njz72fyrg46jlz5f5wfbhfr", - "94c773c3d0294cf248ec1f3e9862669dfa743fe1a76de580d9425c14c8f7dcd2" - ], - [ - "npth", - "kx3vzmpysee7jxwsudarthrmyop6hzgc", - "f8cc6204fa449ce576d450396ec2cad40a75d5712c1381a61ed1681a54f9c79f" - ], - [ - "libassuan", - "e5n5l5ftzwxs4ego5furrdbegphb6hxp", - "ef0428874aa81bcb9944deed88e1fc639f629fe3d522cab3c281235ae2a53db9" - ], - [ - "libgcrypt", - "wyncpahrpqsmpk4b7nlhg5ekkjzyjdzs", - "2309548c51a17f580f036445b701feb85d2bc552b9c4404418c2f223666cfe3b" - ], - [ - "libgpg-error", - "vhcdd6jkbiday2seg3rlkbzpf6jzfdx7", - "79dd719538d9223d6287c0bba07b981944ab6d3ab11e5060274f1b7c727daf55" - ], - [ - "libksba", - "azcgpgncynoox3dce45hkz46bp2tb5rr", - "15d301f201a5162234261fcfccd579b0ff484131444a0b6f5c0006224bb155d6" - ], - [ - "pinentry", - "e3z5ekbv4jlsie4qooubcfvsk2sb6t7l", - "5fd27b8e47934b06554e84f1374a90a93e71e60a14dbde672a8da414b27b97f4" - ], - [ - "gnupg", - "i5agfvsmzdokuooaqhlh6vro5giwei2t", - "f1bde7a1f0c84c1bbcde5757a96cf7a3e9157c2cfa9907fde799aa8e04c0d51f" - ] - ], - "spec": "gnupg@2.3: %gcc platform=linux target=aarch64" - }, - { - "binaries": [ - [ - "zlib", - "v5rr6ba37tudzfuv2jszwikgcl4wd3cd", - "371ad4b277af7b97c7871b9931f2764c97362620c7990c5ad8fdb5c42a1d30dc" - ], - [ - "libiconv", - "bvcnx2e4bumjcgya4dczdhjb3fhqyass", - "65a00b717b3a4ee1b5ab9f84163722bdfea8eb20a2eecc9cf657c0eaac0227e9" - ], - [ - "npth", - "dkb6ez6a4c3iyrv67llwf5mzmynqdmtj", - "4d77351661d0e0130b1c89fb6c6a944aee41d701ef80d056d3fc0178a7f36075" - ], - [ - "libassuan", - "tuydcxdbb5jfvw3gri7y24b233kgotgd", - "d8775e7c1dd252437c6fa0781675b1d2202cfc0c8190e60d248928b6fca8bc9f" - ], - [ - "libgcrypt", - "kgxmg4eukwx6nn3bdera3j7cf7hxfy6n", - "6046523f10ed54be50b0211c27191b3422886984fc0c00aed1a85d1f121c42e6" - ], - [ - "libgpg-error", - "ewhrwnltlrzkpqyix2vbkf4ruq6b6ea3", - "3f3bbbf1a3cb82d39313e39bcbe3dad94a176130fc0e9a8045417d6223fb4f31" - ], - [ - "libksba", - "onxt5ry2fotgwiognwmhxlgnekuvtviq", - "3a4df13f8b880441d1df4b234a4ca01de7601d84a6627185c2b3191a34445d40" - ], - [ - "pinentry", - "fm3m4rsszzxxakcpssd34jbbe4ihrhac", - "73afa46176a7ec8f02d01a2caad3e400dc18c3c8a53f92b88a9aa9e3653db3e6" - ], - [ - "gnupg", - "gwr65ovh4wbxjgniaoqlbt3yla6rdikj", - "7a3f7afe69ca67797a339c04028ca45a9630933020b57cb56e28453197fe8a57" - ] - ], - "spec": "gnupg@2.3: %gcc platform=linux target=ppc64le" - }, - { - "binaries": [ - [ - "libiconv", - "vec3ac6t4ag3lb7ycvisafthqmpci74b", - "35d184218e525d8aaea60082fd2d0f1e80449ec32746cceda2ea0ca106e9a095" - ], - [ - "npth", - "jx3kmy3ilc66rgg5mqtbed5z6qwt3vrd", - "74c2c1b087667661da3e24ac83bcecf1bc2d10d69e7678d1fd232875fe295135" - ], - [ - "zlib", - "wnpbp4pu7xca24goggcy773d2y4pobbd", - "bcbd5310e8c5e75cbf33d8155448b212486dc543469d6df7e56dcecb6112ee88" - ], - [ - "libassuan", - "ynn33wutdtoo2lbjjoizgslintxst2zl", - "ac3b060690c6da0c64dcf35da047b84cc81793118fb9ff29b993f3fb9efdc258" - ], - [ - "libgcrypt", - "zzofcjer43vsxwj27c3rxapjxhsz4hlx", - "4b1977d815f657c2d6af540ea4b4ce80838cadcf4ada72a8ba142a7441e571ea" - ], - [ - "libgpg-error", - "gzr2ucybgks5jquvf4lv7iprxq5vx5le", - "a12ecb5cfd083a29d042fd309ebb5ab8fd4ace0b68b27f89b857e9a84d75b5be" - ], - [ - "libksba", - "hw4u4pam6mp3henpw476axtqaahfdy64", - "5424caf98a2d48e0ed0b9134353c242328ebeef6d2b31808d58969165e809b47" - ], - [ - "pinentry", - "hffsjitsewdgoijwgzvub6vpjwm33ywr", - "8ed7504b5b2d13ab7e1f4a0e27a882c33c5a6ebfcb43c51269333c0d6d5e1448" - ], - [ - "gnupg", - "lge4h2kjgvssyspnvutq6t3q2xual5oc", - "6080ce00fcc24185e4051a30f6d52982f86f46eee6d8a2dc4d83ab08d8195be8" - ] - ], - "spec": "gnupg@2.3: %gcc platform=linux target=x86_64" - } - ] -} \ No newline at end of file diff --git a/share/spack/bootstrap/github-actions-v0.5/clingo.json b/share/spack/bootstrap/github-actions-v0.5/clingo.json new file mode 100644 index 00000000000000..822875a6123338 --- /dev/null +++ b/share/spack/bootstrap/github-actions-v0.5/clingo.json @@ -0,0 +1,389 @@ +{ + "verified": [ + { + "binaries": [ + [ + "clingo-bootstrap", + "riu2vekwzrloc3fktlf6v7kwv6fja7lp", + "7527bc4d2d75671162fe0db3de04c5d3e1e6ab7991dfd85442c302c698febb45" + ] + ], + "python": "python@3.10.13", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "sgf6pgn4ihfcbxutxhevp36n3orfpdkw", + "958531adcb449094bca7703f8f08d0f55a18f9a4c0f10a175ae4190d20982891" + ] + ], + "python": "python@3.11.5", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "ie4wlhhnb4snroymbnjksajwvoid6omx", + "4af14c3375a211ead3d2b4a31b59683744adcb79b820cc0c6b168ab162a7d983" + ] + ], + "python": "python@3.12.0", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "5ke32podcipzxxwrj6uzm324bxegbwca", + "a4106c42ee68d07c3d954ab73fe305ca4204f44d90b58fd91a8f784d9b96e7e3" + ] + ], + "python": "python@3.6", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "scu4cnnf5axmjgozqc7cccpqnj5nc5tj", + "54de4ca141b92222c8f1729e9e336c8a71dad9efa641e76438fcfb79bb58fc7f" + ] + ], + "python": "python@3.7.17", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "ajbswc25irhmhbc4qibdcr6ohsvpcdku", + "8b9e7af163a4259256eca4b4a1a92b5d95463a5cf467be2a11c64ab536ca5b04" + ] + ], + "python": "python@3.8.18", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "vwkuxa5z4pj7vviwsmrpw2r6kbbqej2p", + "a3f10024ff859e15b79ccd06c970a5f0e6ba11b0eae423f096ec9a35863816d2" + ] + ], + "python": "python@3.9.18", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "attdjmyzpfnhoobadw55pgg4hwkyp7zk", + "f3258af3a648b47f12285dd3f048b685ed652b2b55b53861ac9913926de0f1c3" + ] + ], + "python": "python@3.10", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "w4vnbsxjgkhsmgwozudzcsqlvccjsec4", + "19322c2c951fc80234963ac068c78442df57ac63055325b24a39ab705d27a5b9" + ] + ], + "python": "python@3.11", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "dw7ez2xcx6e5dxo3n4jin7pdbo3ihwtw", + "c368edda4b3c8fd767f5f0f098ea416864b088c767dc43135df49cf5f6ef4c93" + ] + ], + "python": "python@3.12", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "audrlxaw3ny3kyjkf6kqywumhokcxh3p", + "db2f44966ec104ffe57c0911f0b1e0d3d052753f4c46c30c0890dfb26d547b09" + ] + ], + "python": "python@3.6", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "al7brxvvvhih5nlxvtfkavufqc3pe5t2", + "4e09b6d50d42c898e075fd20f7c7eddf91cb80edfd2d1326d26fd779e4d1ffed" + ] + ], + "python": "python@3.7", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "v3ctpkyogl542wjibng6m2h2426spjbb", + "d9ceb4f9ca23ef1dcc33872e5410ccfef6ea0360247d3e8faedf1751fb1ae4ca" + ] + ], + "python": "python@3.8", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "zxo5ih5ac6r7lj6miwyx36ot7s6a4dcw", + "f8f5e124d0e7bada34ff687a05e80b2fe207ce4d26205dab09b144edb148f05e" + ] + ], + "python": "python@3.9", + "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "wki4qcy3wzpoxav3auxt2u7yb4sk3xcc", + "f5b9251eb51c60a71f7a0359c252f48c1a1121c426e1e6f9181808c626cb5fef" + ] + ], + "python": "python@3.10.13", + "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "gun6hbksmsecau5wjyrmxodq4hxievzx", + "28839ec43db444d6725bde3fcff99adadf61a392d967041fb16f0ffc0afa2f9d" + ] + ], + "python": "python@3.11.5", + "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "er73owosuqfmmkxvuw3f7sqnvvj6s4xp", + "99264d48c290256bf16e202c155bf3f8c88fdbbe9894d901344d0db7258abce3" + ] + ], + "python": "python@3.12.0", + "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "kv6l7qttuzk7zxkxi5fhff52qso3pj7m", + "59aa052e89d3c698fdd35e30ac21a896c8e49bbcc2f589a8f777bd5dafff2af7" + ] + ], + "python": "python@3.6", + "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "uw5o2z753otspa3lmmy2bdodh5munkir", + "7a8b6359ce83463541ff68c221296fe9875adf28ea2b2c1416229750cf4935d2" + ] + ], + "python": "python@3.7.17", + "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "d63pp2l453bfygh6q7afwdj5mw7lhsns", + "425bef3a8605732b2fbe74cdd77ef6a359cbdb62800490bbd05620a57da35b0c" + ] + ], + "python": "python@3.8.18", + "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "nap44jiznzwlma6n75uxbpznppazs7av", + "316d940ca9af8c6b3bc50f8fdaadba02b0e955c4f24345a63a1a6715b01a752c" + ] + ], + "python": "python@3.9.18", + "spec": "clingo-bootstrap%gcc platform=linux target=aarch64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "qhvnw4yowmk2tofg3u7a4uomisktgzw5", + "d30ec81385377521dd2d1ac091546cc2dec6a852ad31f35c24c65919f94fbf64" + ] + ], + "python": "python@3.10.13", + "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "b3y37ryfuhjq6ljbkq7piglsafg5stgw", + "3c2f9cca3a6d37685fdf7d7dffb7a0505336c32562715069004631c446e46a7c" + ] + ], + "python": "python@3.11.5", + "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "dbloojtq5kcfd3pjmj4pislgpzrcvjpn", + "f8aeba80e6c106b769adba164702db94e077255fe1a22d6d265ccc3172b4ab1a" + ] + ], + "python": "python@3.12.0", + "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "gtlngzdb7iggcjmaottob54qi3b24blt", + "3efc534ba293ee51156971b8c19a597ebcb237b003c98e3c215a49a88064dfd1" + ] + ], + "python": "python@3.6", + "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "4ab4wobwa7bvhlkrmhdp2dwgtcq5rpzo", + "3dc6539a989701ec1d83d644a79953af912c11fe6046a8d720970faf8e477991" + ] + ], + "python": "python@3.7.17", + "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "fgout3h4mt4i64xaovqrpcsdy3ly2aml", + "ade67f0623e941b16f2dd531270b4863de8befd56a9a47bd87af85345bc8bed6" + ] + ], + "python": "python@3.8.18", + "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "5fv2q4agg4b4g53f4zhnymrbv6ogiwpy", + "18047d48538a770f014cce73756258c1a320d4ac143abef3c5d8bc09dd7a03cc" + ] + ], + "python": "python@3.9.18", + "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "smkmkb5xqz4v2f7tl22g4e2ghamglox5", + "a850c80c7a48dab506f807cc936b9e54e6f5640fe96543ff58281c046140f112" + ] + ], + "python": "python@3.10.13", + "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "55qeu52pkt5shpwd7ulugv7wzt5j7vqd", + "e5e1a10b3b2d543b1555f5caef9ac1a9ccdcddb36a1278d3bf68bf0e9f490626" + ] + ], + "python": "python@3.11.5", + "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "zcw5ieomfwwpzpzpabetix2plfqzpvwd", + "ed409165109488d13afe8ef12edd3b373ed08967903dc802889523b5d3bccd14" + ] + ], + "python": "python@3.12.0", + "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "t4yf34cuvquqp5xd66zybmcfyhwbdlsf", + "b14e26e86bcfdac98b3a55109996265683f32910d3452e034ddc0d328bf62d67" + ] + ], + "python": "python@3.6", + "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "grkrpj76lxsxa753uzndwfmrj3pwvyhp", + "11a535d4a8a9dbb18c2f995e10bc90b27b6ebc61f7ac2090f15db9b4f9be1a64" + ] + ], + "python": "python@3.7.17", + "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "zowwoarrf3hvo6i3iereolfujr42iyro", + "154d3a725f02c1775644d99a0b74f9e2cdf6736989a264ccfd5d9a8bce77a16b" + ] + ], + "python": "python@3.8.18", + "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" + }, + { + "binaries": [ + [ + "clingo-bootstrap", + "bhqgwuvef354fwuxq7heeighavunpber", + "399dec8cb6b8cd1b03737e68ea32e6ed69030b57e5f05d983e8856024143ea78" + ] + ], + "python": "python@3.9.18", + "spec": "clingo-bootstrap%gcc platform=linux target=x86_64" + } + ] +} \ No newline at end of file diff --git a/share/spack/bootstrap/github-actions-v0.5/gnupg.json b/share/spack/bootstrap/github-actions-v0.5/gnupg.json new file mode 100644 index 00000000000000..25d607b7a8c44e --- /dev/null +++ b/share/spack/bootstrap/github-actions-v0.5/gnupg.json @@ -0,0 +1,254 @@ +{ + "verified": [ + { + "binaries": [ + [ + "libgpg-error", + "stcmj3wdfxrohn2a53ecvsfsxe7rzrn4", + "942b0f0918798f0a5f007de0f104d71273e6988165c7a34a874e0846b1aa8977" + ], + [ + "libassuan", + "z27suzptvelnavipmldx6dcntiwqmguq", + "c703d6b534e89e383893913fb3b71b47322726c5e19f69178e4d1a3a42a76426" + ], + [ + "libgcrypt", + "if4uocx75kk6nc5vwvvuxq4dvaoljxkm", + "a2320f8cfc8201d15c0e9e244b824ce3d76542c148f4f0631648987957759f07" + ], + [ + "libiconv", + "nccvt7adwkq5anilrjspffdzl4hggon5", + "e23aa0184eb6661331bc850292fa22579005fd8ed62efd4c0c7a87489d8acaf6" + ], + [ + "libksba", + "lbfaarmpo2tupbezmqhfjvyspvwepv4r", + "96888ed37642a2425e2262a5904b82a38f9eecfb18a900493e32d4ab742f994b" + ], + [ + "npth", + "yc7h5c7cp7mupstvh5wlujp3xqet3xxq", + "3ac8e284878c5a556e38aab706e4303daf0a4d2bbb9fac2644495f8a362f9988" + ], + [ + "pinentry", + "rlo36pidutbjxxc3atooiwruaptfwmml", + "70114fe6c9e8723daa960f1a3dc36ed8b5a6c6f9cc828d43f79b8f59f7363605" + ], + [ + "zlib-ng", + "hewnrm76ju4qcjaezxole5htrulkij25", + "7babbe4d3d6e58631a944472356c07f0f4ad4a0759eaeefcf8584f33cce51ca6" + ], + [ + "gnupg", + "5cguax2vflgy2cwmt2ikvixtynommlmr", + "23fdd223493f441fa2e5f82d7e02837ecfad831fbfa4c27c175b3e294ed977d1" + ] + ], + "spec": "gnupg@2.3: %apple-clang platform=darwin target=aarch64" + }, + { + "binaries": [ + [ + "libgpg-error", + "7yjoei55i6wxycmzbopyrw7nrquc22ac", + "c29cfe32521a4a1e2108c711233964c27ca74ffc7505eea86cb8c047ace5715b" + ], + [ + "libassuan", + "b4pkkugfhdtitffvlh4o3dexmthr6rmk", + "27ee6fc272f011f9ad4f000dc54961cccd67b34d6f24f316ca7faf26673bf98b" + ], + [ + "libgcrypt", + "uqjmpmpeta3w7c66m4e5jojopngpibvp", + "d73fbb6e9327faec75af450d602b663ed6bb65ac9657bd795034a53f6acd32c8" + ], + [ + "libiconv", + "rfsiwcq6tlw6to42a3uxw7wcmcyk5m6r", + "1f0176395130ed8b919538fa4b1cbda9f0ff8b836e51097258efc8cf5e11f753" + ], + [ + "libksba", + "gsobopcvr2p7d7rpgrbk2ulrnhvrpt6u", + "0e404a8353f91918f385db8cf661f53f91ffd805798fcd83fb1168a1f1758fe8" + ], + [ + "npth", + "gib2edyujm2oymkvu2hllm2yeghttvn3", + "e04e579e514cd965baf71b7f160b063bff8b116e991e6931c6919cd5f3270e59" + ], + [ + "pinentry", + "5ndbckveeaywx77rqmujglfnqwpxu3t6", + "0ec02dca08ad2e8b3dd1c71195ed3fe3bb8856b746726708f5e5d450619e1285" + ], + [ + "zlib-ng", + "fg366ys6nx3hthuiix4xooi6xx4qe5d2", + "cc372a21608885182233c7800355c7c0bbaff47ea16e190827a9618b0c4703e2" + ], + [ + "gnupg", + "2x5ftl46zcnxk6knz5y3nuhyn7zcttk3", + "b9481e122e2cb26f69b70505830d0fcc0d200aadbb6c6572339825f17ad1e52d" + ] + ], + "spec": "gnupg@2.3: %apple-clang platform=darwin target=x86_64" + }, + { + "binaries": [ + [ + "libgpg-error", + "b7o5zrguyniw5362eey3peglzhlmig7l", + "b4373f2b0a2567b3b87e6bfc934135ce7790432aea58c802139bb5352f24b6a9" + ], + [ + "libassuan", + "6k2arop3mjwfhe4cwga6a775ud5m4scp", + "1e5143d35b0938a206ecf1ecb39b77e732629897d2b936cb8274239770055d90" + ], + [ + "libgcrypt", + "eh5h3zisjkupzr2pgqarvgs2fm7pun5r", + "b57eff265b48d0472243babfd1221c7c16189a4e324ea26e65d1a0a8c1391020" + ], + [ + "libiconv", + "vgk2zgjeflpnksj3lywuwdzs2nez63qv", + "d153953c40c630fd2bf271f3de901d7671f80e8161cf746cb54afbf28d934d03" + ], + [ + "libksba", + "au3xdl4oyfbxat6dknp3mldid7gupgt5", + "f1b1a1a02138109bc41b0b2ba54e689b43f35e2828f58b5de74280ce754fac0b" + ], + [ + "npth", + "ja7cauk7yhhyj7msnprlirue7cn3jpnj", + "cf6fd998a8f92ce1cf34c63db09c77b1891bf8f5915deef03c0cae5492bd691b" + ], + [ + "pinentry", + "6yo4flozla2tvw3ojkh2atvnfxuqx6ym", + "e78826a269109b3d67a54b1d01ff0a93be043dddcb4f52d329770ae1f75313f3" + ], + [ + "zlib-ng", + "4cgenrt3rcinueq6peyolxhegnryoeem", + "918a1e48f823806f1562c95569953a4658b2fbc54a2606a09bcd7e259b62f492" + ], + [ + "gnupg", + "lrmigjenpqj5fy4ojcs5jy6doktiu4qz", + "228ccb475932f7f40a64e9d87dec045931cc57f71b1dfd4b4c3926107222d96c" + ] + ], + "spec": "gnupg@2.3: %gcc platform=linux target=aarch64" + }, + { + "binaries": [ + [ + "libgpg-error", + "km6l24czfhnmlya74nu6cxwufgimyhzz", + "23c3b7b487b36b9b03eeebbcc484adc6c8190c1bbcaa458943847148c915c6b2" + ], + [ + "libassuan", + "crkk525xdgsn2k5s4xqdaxkudz6pjqbm", + "ae3048a8059c0709d3efe832de1a8f82594373ba853d4bc2dfa05fb9dbfbc782" + ], + [ + "libgcrypt", + "4s5lkowqilor35fscjwvtmg4wasdknkc", + "62d3d13278d60d0329af1a9649b06591153ff68de4584f57777d13d693c7012e" + ], + [ + "libiconv", + "kbijqx45l3n64dlhenbuwgqpmf434g2d", + "dddf581a14a35b85cb69a8c785dd8e250f41e6de7697e34bb0ab2a942e0c2128" + ], + [ + "libksba", + "jnll3rfuh6xhgqxbwfnpizammcwloxjc", + "6200f2b6150aaf6d0e69771dfd5621582bd99ed0024fe83e7bc777cb66cabb29" + ], + [ + "npth", + "6j6b4hbkhwkb5gfigysqgn5lpu3i4kw5", + "0be0c70f3d9d45c4fe7490d8fdb8d7584de6324c3bfac8d884072409799c9951" + ], + [ + "pinentry", + "cdpcdd4iah6jot4odehm3xmulw3t3e32", + "5b447c770d0f705fbc97564fccdfbb0dfff8b6f8e2b4abbea326a538bc1bff80" + ], + [ + "zlib-ng", + "ogchs3i5tosoqrtsp3czp2azxvm7icig", + "acfa12c4e73560416e1169b37adabfbec5ee9a580a684b23e75d7591d8e39a03" + ], + [ + "gnupg", + "jwpu2wrofbwylpztltmi257benj2wp6z", + "98e2bcb4064ec0830d896938bc1fe5264dac611da71ea546b9ca03349b752041" + ] + ], + "spec": "gnupg@2.3: %gcc platform=linux target=ppc64le" + }, + { + "binaries": [ + [ + "libgpg-error", + "dwcgnnqt364enpf5554dio7kklspmrko", + "bfe9b506ccba0cca619133a3d2e05aa23c929749428bf6eecbff0c6985447009" + ], + [ + "libassuan", + "yl5rfsfuxd6if36h7rap7zbbpbfztkpw", + "4343dabbeed0851885992acd7b63fd74cb9d1acc06501a8af934e7e103801a15" + ], + [ + "libgcrypt", + "ka3t3dq73bkz4bs5ilyz6kymkypgbzxl", + "ec1bcc324e9f9d660395e2c586094431361a02196da43fce91be41cca5da9636" + ], + [ + "libiconv", + "5tog27ephuzc4j6kdxavhjsjm2kd5nu6", + "928fab3c32a1ae09651bb8491ee3855ccaf3c57a146ee72a289a073accd3fc8f" + ], + [ + "libksba", + "4ezfhjkmfc4fr34ozzl5q6b4x6jqqmsw", + "3045841c50c19a41beb0f32b4e8a960901397b95e82af3a73817babf35d4cfca" + ], + [ + "npth", + "bn4zrugdajgpk5dssoeccbl7o2gfgmcp", + "ef90ef85a818456afbff709b4a0757a077d69fd3c07d1b7612e1d461d837c46f" + ], + [ + "pinentry", + "cdwqocmusjomjjavnz6nn764oo54j5xj", + "b251047c1cb4be1bb884a7843d4419fae40fdbe5e1d36904e35f5e3fef5e4ced" + ], + [ + "zlib-ng", + "ozawh46coczjwtlul27msr3swe6pl6l5", + "0a397b53d64ac8191a36de8b32c5ced28a4c7a6dbafe9396dd897c55bcf7a168" + ], + [ + "gnupg", + "jra2dbsvpr5c5gj3ittejusa2mjh2sf5", + "054fac6eaad7c862ea4661461d847fb069876eb114209416b015748266f7d166" + ] + ], + "spec": "gnupg@2.3: %gcc platform=linux target=x86_64" + } + ] +} \ No newline at end of file diff --git a/share/spack/bootstrap/github-actions-v0.3/metadata.yaml b/share/spack/bootstrap/github-actions-v0.5/metadata.yaml similarity index 83% rename from share/spack/bootstrap/github-actions-v0.3/metadata.yaml rename to share/spack/bootstrap/github-actions-v0.5/metadata.yaml index d27e261721ca07..0fd413a618a5fd 100644 --- a/share/spack/bootstrap/github-actions-v0.3/metadata.yaml +++ b/share/spack/bootstrap/github-actions-v0.5/metadata.yaml @@ -3,6 +3,6 @@ description: | Buildcache generated from a public workflow using Github Actions. The sha256 checksum of binaries is checked before installation. info: - url: https://mirror.spack.io/bootstrap/github-actions/v0.3 + url: https://mirror.spack.io/bootstrap/github-actions/v0.5 homepage: https://github.com/spack/spack-bootstrap-mirrors releases: https://github.com/spack/spack-bootstrap-mirrors/releases diff --git a/share/spack/bootstrap/github-actions-v0.3/patchelf.json b/share/spack/bootstrap/github-actions-v0.5/patchelf.json similarity index 56% rename from share/spack/bootstrap/github-actions-v0.3/patchelf.json rename to share/spack/bootstrap/github-actions-v0.5/patchelf.json index 699c51c8abff53..f26fd9ce86f088 100644 --- a/share/spack/bootstrap/github-actions-v0.3/patchelf.json +++ b/share/spack/bootstrap/github-actions-v0.5/patchelf.json @@ -4,8 +4,8 @@ "binaries": [ [ "patchelf", - "cn4gsqzdnnffk7ynvbcai6wrt5ehqqrl", - "8c6a28cbe8133d719be27ded11159f0aa2c97ed1d0881119ae0ebd71f8ccc755" + "4txke6ixd2zg2yzg33l3fqnjyassono7", + "102800775f789cc293e244899f39a22f0b7a19373305ef0497ca3189223123f3" ] ], "spec": "patchelf@0.13: %gcc platform=linux target=aarch64" @@ -14,8 +14,8 @@ "binaries": [ [ "patchelf", - "mgq6n2heyvcx2ebdpchkbknwwn3u63s6", - "1d4ea9167fb8345a178c1352e0377cc37ef2b421935cf2b48fb6fa03a94fca3d" + "tnbgxc22uebqsiwrhchf3nieatuqlsrr", + "91cf0a9d4750c04575c5ed3bcdefc4754e1cf9d1cd1bf197eb1fe20ccaa869f1" ] ], "spec": "patchelf@0.13: %gcc platform=linux target=ppc64le" @@ -24,8 +24,8 @@ "binaries": [ [ "patchelf", - "htk62k7efo2z22kh6kmhaselru7bfkuc", - "833df21b20eaa7999ac4c5779ae26aa90397d9027aebaa686a428589befda693" + "afv7arjarb7nzmlh7c5slkfxykybuqce", + "73f4bde46b843c96521e3f5c31ab94756491404c1ad6429c9f61dbafbbfa6470" ] ], "spec": "patchelf@0.13: %gcc platform=linux target=x86_64" diff --git a/share/spack/fish/spack-completion.fish b/share/spack/fish/spack-completion.fish new file mode 100644 index 00000000000000..ee06dade57340b --- /dev/null +++ b/share/spack/fish/spack-completion.fish @@ -0,0 +1,347 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# NOTE: spack-completion.fish is auto-generated by: +# +# $ spack commands --aliases --format=fish +# --header=fish/spack-completion.fish --update=spack-completion.fish +# +# Please do not manually modify this file. + +# Check fish version before proceeding +set -l fish_version (string split '.' $FISH_VERSION) +if test $fish_version[1] -lt 3 + if test $fish_version[1] -eq 3 + and test $fish_version[2] -lt 2 + echo 'Fish version is older than 3.2.0. Some completion features may not work' + set -g __fish_spack_force_files + else + echo 'This script requires fish version 3.0 or later' + exit 1 + end +else + set -g __fish_spack_force_files -F +end + +# The following global variables are used as a cache of `__fish_spack_argparse` + +# Cached command line +set -g __fish_spack_argparse_cache_line +# Parsed command +set -g __fish_spack_argparse_command +# Remaining arguments +set -g __fish_spack_argparse_argv +# Return value +set -g __fish_spack_argparse_return + +# Spack command generates an optspec variable $__fish_spack_optspecs_. +# We check if this command exists, and echo the optspec variable name. +function __fish_spack_get_optspecs -d 'Get optspecs of spack command' + # Convert arguments to replace ' ' and '-' by '_' + set -l cmd_var (string replace -ra -- '[ -]' '_' $argv | string join '_') + # Set optspec variable name + set -l optspecs_var __fish_spack_optspecs_$cmd_var + # Query if variable $$optspecs_var exists + set -q $optspecs_var; or return 1 + # If it exists, echo all optspecs line by line. + # String join returns 1 if no join was performed, so we return 0 in such case. + string join \n $$optspecs_var; or return 0 +end + +# Parse command-line arguments, save results to global variables, +# and add found flags to __fish_spack_flag_. +# Returns 1 if help flag is found. +function __fish_spack_argparse + # Figure out if the current invocation already has a command. + set -l args $argv + set -l commands + + # Return cached result if arguments haven't changed + if test "$__fish_spack_argparse_cache_line" = "$args" + return $__fish_spack_argparse_return + end + + # Clear all flags found in last run + set -g | string replace -rf -- '^(__fish_spack_flag_\w+)(.*?)$' 'set -ge $1' | source + + # Set default return value to 0, indicating success + set -g __fish_spack_argparse_return 0 + # Set command line to current arguments + set -g __fish_spack_argparse_cache_line $argv + + # Recursively check arguments for commands + while set -q args[1] + # Get optspecs of current command + set -l optspecs (__fish_spack_get_optspecs $commands $args[1]) + or break + + # If command exists, shift arguments + set -a commands $args[1] + set -e args[1] + + # If command has no arguments, continue + set -q optspecs[1]; or continue + + # Parse arguments. Set variable _flag_ if flag is found. + # We find all these variables and set them to the global variable __fish_spack_flag_. + argparse -i -s $optspecs -- $args 2>/dev/null; or break + set -l | string replace -rf -- '^(_flag_.*)$' 'set -g __fish_spack$1' | source + + # Set args to not parsed arguments + set args $argv + + # If command has help flag, we don't need to parse more so short circuit + if set -q _flag_help + set -g __fish_spack_argparse_return 1 + break + end + end + + # Set cached variables + set -g __fish_spack_argparse_command $commands + set -g __fish_spack_argparse_argv $args + + return $__fish_spack_argparse_return +end + +# Check if current commandline's command is "spack $argv" +function __fish_spack_using_command + set -l line (commandline -opc) + __fish_spack_argparse $line; or return 1 + + set -p argv spack + test "$__fish_spack_argparse_command" = "$argv" +end + +# Check if current commandline's command is "spack $argv[2..-1]", +# and cursor is at $argv[1]-th positional argument +function __fish_spack_using_command_pos + __fish_spack_using_command $argv[2..-1] + or return + + test (count $__fish_spack_argparse_argv) -eq $argv[1] +end + +function __fish_spack_using_command_pos_remainder + __fish_spack_using_command $argv[2..-1] + or return + + test (count $__fish_spack_argparse_argv) -ge $argv[1] +end + +# Helper functions for subcommands + +function __fish_spack_bootstrap_names + if set -q __fish_spack_flag_scope + spack bootstrap list --scope $__fish_spack_flag_scope | string replace -rf -- '^Name: (\w+).*?$' '$1' + else + spack bootstrap list | string replace -rf -- '^Name: (\w+).*?$' '$1' + end +end + +# Reference: sudo's fish completion +function __fish_spack_build_env_spec + set token (commandline -opt) + + set -l index (contains -- -- $__fish_spack_argparse_argv) + if set -q index[1] + __fish_complete_subcommand --commandline $__fish_spack_argparse_argv[(math $index + 1)..-1] + else if set -q __fish_spack_argparse_argv[1] + __fish_complete_subcommand --commandline "$__fish_spack_argparse_argv[2..-1] $token" + else + __fish_spack_specs + end +end + +function __fish_spack_commands + spack commands +end + +function __fish_spack_colon_path + set token (string split -rm1 ':' (commandline -opt)) + + if test (count $token) -lt 2 + __fish_complete_path $token[1] + else + __fish_complete_path $token[2] | string replace -r -- '^' "$token[1]:" + end +end + +function __fish_spack_config_sections + if set -q __fish_spack_flag_scope + spack config --scope $__fish_spack_flag_scope list | string split ' ' + else + spack config list | string split ' ' + end +end + +function __fish_spack_environments + string trim (spack env list) +end + +function __fish_spack_extensions + # Skip optional flags, or it will be really slow + string match -q -- '-*' (commandline -opt) + and return + + comm -1 -2 (spack extensions | string trim | psub) (__fish_spack_installed_packages | sort | psub) +end + +function __fish_spack_gpg_keys + spack gpg list +end + +function __fish_spack_installed_compilers + spack compilers | grep -v '^[=-]\|^$' +end + +function __fish_spack_installed_packages + spack find --no-groups --format '{name}' | uniq +end + +function __fish_spack_installed_specs + # Try match local hash first + __fish_spack_installed_specs_id + and return + + spack find --no-groups --format '{name}@{version}' +end + +function __fish_spack_installed_specs_id + set -l token (commandline -opt) + string match -q -- '/*' $token + or return 1 + + spack find --format '/{hash:7}'\t'{name}{@version}' +end + +function __fish_spack_git_rev + type -q __fish_git_ranges + and __fish_git_ranges +end + +function __fish_spack_mirrors + spack mirror list | awk {'printf ("%s\t%s", $1, $2)'} +end + +function __fish_spack_package_versions + string trim (spack versions $argv) +end + +function __fish_spack_packages + spack list +end + +function __fish_spack_pkg_packages + spack pkg list +end + +function __fish_spack_providers + string trim (spack providers | grep -v '^$') +end + +function __fish_spack_repos + spack repo list | awk {'printf ("%s\t%s", $1, $2)'} +end + +function __fish_spack_scopes + # TODO: how to list all scopes? + set -l scope system site user defaults + set -l platform cray darwin linux test + + string join \n $scope +end + +function __fish_spack_specs + set -l token (commandline -opt) + + # Complete compilers + if string match -rq -- '^(?
.*%)[\w-]*(@[\w\.+~-]*)?$' $token
+        __fish_spack_installed_compilers | string replace -r -- '^' "$pre"
+        return
+    end
+
+    # Try to complete spec version
+    # Currently we can only match '@' after a package name
+    set -l package
+
+    # Match ^ following package name
+    if string match -rq -- '^(?
.*?\^)[\w\.+~-]*$' $token
+        # Package name is the nearest, assuming first character is always a letter or digit
+        set packages (string match -ar -- '^[\w-]+' $__fish_spack_argparse_argv $token)
+        set package $packages[-1]
+
+        if test -n "$package"
+            spack dependencies $package | string replace -r -- '^' "$pre"
+            return
+        end
+    end
+
+    # Match @ following package name
+    if string match -rq -- '^(?
.*?\^?(?[\w\.+~-]*)@)[\w\.]*$' $token
+        set package $packages[-1]
+
+        # Matched @ starting at next token
+        if test -z "$package"
+            string match -arq -- '(^|\^)(?[\w\.+~-]*)$' $__fish_spack_argparse_argv[-1]
+            if test -n "$inners[1]"
+                set package $inners[-1]
+            end
+        end
+    end
+
+    # Complete version if package found
+    if test -n "$package"
+        # Only list safe versions for speed
+        string trim (spack versions --safe $package) | string replace -r -- '^' "$pre"
+        return
+    end
+
+    # Else complete package name
+    __fish_spack_installed_packages | string replace -r -- '$' \t"installed"
+    spack list
+end
+
+function __fish_spack_specs_or_id
+    # Try to match local hash first
+    __fish_spack_installed_specs_id
+    and return
+
+    __fish_spack_specs
+end
+
+function __fish_spack_tags
+    string trim (spack tags)
+end
+
+function __fish_spack_tests
+    spack test list | grep -v '^[=-]'
+end
+
+function __fish_spack_unit_tests
+    # Skip optional flags, or it will be really slow
+    string match -q -- '-*' (commandline -opt)
+    and return
+
+    spack unit-test -l
+end
+
+function __fish_spack_yamls
+    # Trim flag from current token
+    string match -rq -- '(?
-.)?(?.*)' (commandline -opt)
+
+    if test -n "$token"
+        find $token* -type f '(' -iname '*.yaml' -or -iname '*.yml' ')'
+    else
+        find -maxdepth 2 -type f '(' -iname '*.yaml' -or -iname '*.yml' ')' | cut -c 3-
+    end
+end
+
+# Reset existing completions
+complete -c spack --erase
+
+# Spack commands
+#
+# Everything below here is auto-generated.
diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index fa36ae453e114f..880aeb6811a1d0 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -1,12 +1,27 @@
 stages: [ "generate", "build", "publish" ]
 
+variables:
+  SPACK_DISABLE_LOCAL_CONFIG: "1"
+  SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/"
+  # PR_MIRROR_FETCH_DOMAIN: "https://binaries-prs.spack.io"
+  PR_MIRROR_FETCH_DOMAIN: "s3://spack-binaries-prs"
+  PR_MIRROR_PUSH_DOMAIN: "s3://spack-binaries-prs"
+  # PROTECTED_MIRROR_FETCH_DOMAIN: "https://binaries.spack.io"
+  PROTECTED_MIRROR_FETCH_DOMAIN: "s3://spack-binaries"
+  PROTECTED_MIRROR_PUSH_DOMAIN: "s3://spack-binaries"
+
 default:
   image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
 
 # CI Platform-Arch
-.cray_zen4:
+.cray_rhel_zen4:
   variables:
-    SPACK_TARGET_PLATFORM: "cray"
+    SPACK_TARGET_PLATFORM: "cray-rhel"
+    SPACK_TARGET_ARCH: "zen4"
+
+.cray_sles_zen4:
+  variables:
+    SPACK_TARGET_PLATFORM: "cray-sles"
     SPACK_TARGET_ARCH: "zen4"
 
 .darwin_x86_64:
@@ -14,6 +29,11 @@ default:
     SPACK_TARGET_PLATFORM: "darwin"
     SPACK_TARGET_ARCH: "x86_64"
 
+.darwin_aarch64:
+  variables:
+    SPACK_TARGET_PLATFORM: "darwin"
+    SPACK_TARGET_ARCH: "aarch64"
+
 .linux_x86_64_v3:
   variables:
     SPACK_TARGET_PLATFORM: "linux"
@@ -54,47 +74,63 @@ default:
 ########################################
 .base-job:
   variables:
-    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+    PIPELINE_MIRROR_TEMPLATE: "single-src-protected-mirrors.yaml.in"
+    # TODO: We can remove this when we drop the "deprecated" stack
+    PUSH_BUILDCACHE_DEPRECATED: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+
   rules:
     - if: $CI_COMMIT_REF_NAME == "develop"
       # Pipelines on develop only rebuild what is missing from the mirror
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_protected_branch"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
+        SPACK_REQUIRE_SIGNING: "True"
         AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
         AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
+        OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
     - if: $CI_COMMIT_REF_NAME =~ /^releases\/v.*/
       # Pipelines on release branches always rebuild everything
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_protected_branch"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
         SPACK_PRUNE_UNTOUCHED: "False"
         SPACK_PRUNE_UP_TO_DATE: "False"
+        SPACK_REQUIRE_SIGNING: "True"
         AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
         AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
+        OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
     - if: $CI_COMMIT_TAG =~ /^develop-[\d]{4}-[\d]{2}-[\d]{2}$/ || $CI_COMMIT_TAG =~ /^v.*/
       # Pipelines on tags (release or dev snapshots) only copy binaries from one mirror to another
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_copy_only"
-        SPACK_SOURCE_MIRROR: "s3://spack-binaries/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}"
-        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
+        PIPELINE_MIRROR_TEMPLATE: "copy-only-protected-mirrors.yaml.in"
         AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
         AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
+        OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
     - if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/
       # Pipelines on PR branches rebuild only what's missing, and do extra pruning
       when: always
       variables:
         SPACK_PIPELINE_TYPE: "spack_pull_request"
-        SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+        # TODO: We can remove this when we drop the "deprecated" stack
+        PUSH_BUILDCACHE_DEPRECATED: "${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
         SPACK_PRUNE_UNTOUCHED: "True"
         SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
+        # TODO: Change sync script to include target in branch name.  Then we could
+        # TODO: have multiple types of "PR" pipeline here.  It would be better if we could
+        # TODO: keep just this one and use a regex to capture the target branch, but so
+        # TODO: far gitlab doesn't support that.
+        PR_TARGET_REF_NAME: "develop"
+        PIPELINE_MIRROR_TEMPLATE: "multi-src-mirrors.yaml.in"
         AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
         AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
+        OIDC_TOKEN_AUDIENCE: "pr_binary_mirror"
 
-.generate-base:
+.generate-common:
   stage: generate
   script:
     - uname -a || true
@@ -106,12 +142,20 @@ default:
     - spack env activate --without-view .
     - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
     - spack
+      --config-scope "${SPACK_CI_CONFIG_ROOT}"
+      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
+      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
+      ${CI_STACK_CONFIG_SCOPES}
+      compiler find
+    - spack python -c "import os,sys; print(os.path.expandvars(sys.stdin.read()))"
+      < "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}" > "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
+    - spack config add -f "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
+    - spack -v  --color=always
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
       ${CI_STACK_CONFIG_SCOPES}
       ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
   after_script:
@@ -122,6 +166,9 @@ default:
   variables:
     KUBERNETES_CPU_REQUEST: 4000m
     KUBERNETES_MEMORY_REQUEST: 16G
+    # avoid moving targets like branches and tags
+    SPACK_CONCRETIZER_REQUIRE_CHECKSUM: 1
+    SPACK_BACKTRACE: 1
   interruptible: true
   timeout: 60 minutes
   retry:
@@ -129,41 +176,21 @@ default:
     when:
       - always
 
-.generate:
-  extends: [ ".base-job", ".generate-base" ]
-  tags: ["spack", "public", "medium", "x86_64"]
+# Generate without tags for cases using external runners
+.generate-base:
+  extends: [ ".base-job", ".generate-common" ]
 
-.darwin-generate-base:
-  stage: generate
-  script:
-  - export SPACK_DISABLE_LOCAL_CONFIG=1
-  - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
-  - uname -a || true
-  - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
-  - nproc || true
-  - . "./share/spack/setup-env.sh"
-  - spack --version
-  - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
-  - spack env activate --without-view .
-  - spack -d ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
-      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
-      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
-  after_script:
-  - cat /proc/loadavg || true
-  artifacts:
-    paths:
-    - "${CI_PROJECT_DIR}/jobs_scratch_dir"
-  interruptible: true
-  timeout: 60 minutes
-  retry:
-    max: 2
-    when:
-    - always
+.generate-x86_64:
+  extends: [ ".generate-base" ]
+  tags: ["spack", "public", "medium", "x86_64"]
 
-.darwin-generate:
-  extends: [ ".base-job", ".darwin-generate-base" ]
+.generate-aarch64:
+  extends: [ ".generate-base" ]
+  tags: ["spack", "public", "medium", "aarch64"]
 
+.generate-neoverse_v1:
+  extends: [ ".generate-base" ]
+  tags: ["spack", "public", "medium", "aarch64", "graviton3"]
 
 .generate-deprecated:
   extends: [ ".base-job" ]
@@ -176,9 +203,9 @@ default:
     - spack --version
     - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
     - spack env activate --without-view .
-    - spack
+    - spack -v --color=always
       ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
+      --buildcache-destination "${PUSH_BUILDCACHE_DEPRECATED}"
       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
   after_script:
@@ -197,10 +224,6 @@ default:
       - always
   tags: ["spack", "public", "medium", "x86_64"]
 
-.generate-aarch64:
-  extends: [ ".base-job", ".generate" ]
-  tags: ["spack", "public", "medium", "aarch64"]
-
 .build:
   extends: [ ".base-job" ]
   stage: build
@@ -219,8 +242,7 @@ protected-publish:
     max: 2
     when: ["runner_system_failure", "stuck_or_timeout_failure"]
   variables:
-    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
-    SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
+    SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
     SPACK_PIPELINE_TYPE: "spack_protected_branch"
     AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
     AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
@@ -234,6 +256,9 @@ protected-publish:
     - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
     - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_COPY_BUILDCACHE}/build_cache/_pgp/spack-public-binary-key.pub"
     - spack buildcache update-index --keys "${SPACK_COPY_BUILDCACHE}"
+  id_tokens:
+    GITLAB_OIDC_TOKEN:
+      aud: "protected_binary_mirror"
 
 ########################################
 # TEMPLATE FOR ADDING ANOTHER PIPELINE
@@ -282,8 +307,8 @@ protected-publish:
     SPACK_CI_STACK_NAME: e4s
 
 e4s-generate:
-  extends: [ ".e4s", ".generate"]
-  image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
+  extends: [ ".e4s", ".generate-x86_64"]
+  image: ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01
 
 e4s-build:
   extends: [ ".e4s", ".build" ]
@@ -296,6 +321,52 @@ e4s-build:
     - artifacts: True
       job: e4s-generate
 
+########################################
+# E4S Neoverse V1 pipeline
+########################################
+.e4s-neoverse_v1:
+  extends: [ ".linux_neoverse_v1" ]
+  variables:
+    SPACK_CI_STACK_NAME: e4s-neoverse_v1
+
+e4s-neoverse_v1-generate:
+  extends: [ ".e4s-neoverse_v1", ".generate-neoverse_v1" ]
+  image: ghcr.io/spack/ubuntu20.04-runner-arm64-gcc-11.4:2023.08.01
+
+e4s-neoverse_v1-build:
+  extends: [ ".e4s-neoverse_v1", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: e4s-neoverse_v1-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: e4s-neoverse_v1-generate
+
+########################################
+# E4S ROCm External pipeline
+########################################
+.e4s-rocm-external:
+  extends: [ ".linux_x86_64_v3" ]
+  variables:
+    SPACK_CI_STACK_NAME: e4s-rocm-external
+
+e4s-rocm-external-generate:
+  extends: [ ".e4s-rocm-external", ".generate-x86_64"]
+  image: ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4-rocm5.4.3:2023.08.01
+
+e4s-rocm-external-build:
+  extends: [ ".e4s-rocm-external", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: e4s-rocm-external-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: e4s-rocm-external-generate
+
 ########################################
 # GPU Testing Pipeline
 ########################################
@@ -305,8 +376,8 @@ e4s-build:
     SPACK_CI_STACK_NAME: gpu-tests
 
 gpu-tests-generate:
-  extends: [ ".gpu-tests", ".generate"]
-  image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
+  extends: [ ".gpu-tests", ".generate-x86_64"]
+  image: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01
 
 gpu-tests-build:
   extends: [ ".gpu-tests", ".build" ]
@@ -328,8 +399,8 @@ gpu-tests-build:
     SPACK_CI_STACK_NAME: e4s-oneapi
 
 e4s-oneapi-generate:
-  extends: [ ".e4s-oneapi", ".generate"]
-  image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023.06.01
+  extends: [ ".e4s-oneapi", ".generate-x86_64"]
+  image: ghcr.io/spack/ubuntu20.04-runner-amd64-oneapi-2023.2.1:2023.08.01
 
 e4s-oneapi-build:
   extends: [ ".e4s-oneapi", ".build" ]
@@ -346,7 +417,7 @@ e4s-oneapi-build:
 # E4S on Power
 ########################################
 .e4s-power-generate-tags-and-image:
-  image: { "name": "ecpe4s/ubuntu20.04-runner-ppc64le:2023-01-01", "entrypoint": [""] }
+  image: { "name": "ghcr.io/spack/ubuntu20.04-runner-ppc64-gcc-11.4:2023.08.01", "entrypoint": [""] }
   tags: ["spack", "public", "large", "ppc64le"]
 
 .e4s-power:
@@ -355,7 +426,7 @@ e4s-oneapi-build:
     SPACK_CI_STACK_NAME: e4s-power
 
 e4s-power-generate:
-  extends: [ ".e4s-power", ".generate", ".e4s-power-generate-tags-and-image"]
+  extends: [ ".e4s-power", ".generate-x86_64", ".e4s-power-generate-tags-and-image"]
 
 e4s-power-build:
   extends: [ ".e4s-power", ".build" ]
@@ -377,7 +448,7 @@ e4s-power-build:
     SPACK_CI_STACK_NAME: build_systems
 
 build_systems-generate:
-  extends: [ ".build_systems", ".generate"]
+  extends: [ ".build_systems", ".generate-x86_64"]
 
 build_systems-build:
   extends: [ ".build_systems", ".build" ]
@@ -399,7 +470,7 @@ build_systems-build:
     SPACK_CI_STACK_NAME: radiuss
 
 radiuss-generate:
-  extends: [ ".radiuss", ".generate" ]
+  extends: [ ".radiuss", ".generate-x86_64" ]
 
 radiuss-build:
   extends: [ ".radiuss", ".build" ]
@@ -431,7 +502,7 @@ radiuss-build:
     SPACK_CI_STACK_NAME: radiuss-aws
 
 radiuss-aws-generate:
-  extends: [ ".radiuss-aws", ".generate", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]
+  extends: [ ".radiuss-aws", ".generate-x86_64", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]
 
 radiuss-aws-build:
   extends: [ ".radiuss-aws", ".build" ]
@@ -475,8 +546,8 @@ radiuss-aws-aarch64-build:
     SPACK_CI_STACK_NAME: data-vis-sdk
 
 data-vis-sdk-generate:
-  extends: [ ".data-vis-sdk", ".generate"]
-  image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
+  extends: [ ".data-vis-sdk", ".generate-x86_64"]
+  image: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01
 
 data-vis-sdk-build:
   extends: [ ".data-vis-sdk", ".build" ]
@@ -489,55 +560,6 @@ data-vis-sdk-build:
     - artifacts: True
       job: data-vis-sdk-generate
 
-########################################
-# AWS AHUG Applications (x86_64)
-########################################
-
-# Include this AFTER .*-generate in "extends" lists
-.aws-ahug-overrides:
-  # This controls image for generate step; build step is controlled by spack.yaml
-  # Note that generator emits OS info for build so these should be the same.
-  image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09", "entrypoint": [""] }
-
-.aws-ahug:
-  extends: [ ".linux_x86_64_v3" ]
-  variables:
-    SPACK_CI_STACK_NAME: aws-ahug
-
-aws-ahug-generate:
-  extends: [ ".aws-ahug", ".generate", ".aws-ahug-overrides", ".tags-x86_64_v4" ]
-
-aws-ahug-build:
-  extends: [ ".aws-ahug", ".build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: aws-ahug-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: aws-ahug-generate
-
-# Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)
-.aws-ahug-aarch64:
-  extends: [ ".linux_aarch64" ]
-  variables:
-    SPACK_CI_STACK_NAME: aws-ahug-aarch64
-
-aws-ahug-aarch64-generate:
-  extends: [ ".aws-ahug-aarch64", ".generate-aarch64", ".aws-ahug-overrides" ]
-
-aws-ahug-aarch64-build:
-  extends: [ ".aws-ahug-aarch64", ".build" ]
-  trigger:
-    include:
-      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: aws-ahug-aarch64-generate
-    strategy: depend
-  needs:
-    - artifacts: True
-      job: aws-ahug-aarch64-generate
-
 ########################################
 # AWS ISC Applications (x86_64)
 ########################################
@@ -554,7 +576,7 @@ aws-ahug-aarch64-build:
     SPACK_CI_STACK_NAME: aws-isc
 
 aws-isc-generate:
-  extends: [ ".aws-isc", ".generate", ".aws-isc-overrides", ".tags-x86_64_v4" ]
+  extends: [ ".aws-isc", ".generate-x86_64", ".aws-isc-overrides", ".tags-x86_64_v4" ]
 
 aws-isc-build:
   extends: [ ".aws-isc", ".build" ]
@@ -598,8 +620,8 @@ aws-isc-aarch64-build:
     SPACK_CI_STACK_NAME: tutorial
 
 tutorial-generate:
-  extends: [ ".tutorial", ".generate"]
-  image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07
+  extends: [ ".tutorial", ".generate-x86_64"]
+  image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-10-30
 
 tutorial-build:
   extends: [ ".tutorial", ".build" ]
@@ -621,8 +643,8 @@ tutorial-build:
     SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu
 
 ml-linux-x86_64-cpu-generate:
-  extends: [ ".generate", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
-  image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
+  extends: [ ".generate-x86_64", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
+  image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2023-07-01
 
 ml-linux-x86_64-cpu-build:
   extends: [ ".build", ".ml-linux-x86_64-cpu" ]
@@ -644,8 +666,8 @@ ml-linux-x86_64-cpu-build:
     SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda
 
 ml-linux-x86_64-cuda-generate:
-  extends: [ ".generate", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
-  image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
+  extends: [ ".generate-x86_64", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
+  image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2023-07-01
 
 ml-linux-x86_64-cuda-build:
   extends: [ ".build", ".ml-linux-x86_64-cuda"  ]
@@ -667,8 +689,8 @@ ml-linux-x86_64-cuda-build:
     SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm
 
 ml-linux-x86_64-rocm-generate:
-  extends: [ ".generate", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
-  image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
+  extends: [ ".generate-x86_64", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
+  image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2023-07-01
 
 ml-linux-x86_64-rocm-build:
   extends: [ ".build", ".ml-linux-x86_64-rocm" ]
@@ -685,12 +707,13 @@ ml-linux-x86_64-rocm-build:
 # Machine Learning - Darwin aarch64 (MPS)
 ########################################
 .ml-darwin-aarch64-mps:
+  extends: [".darwin_aarch64"]
   variables:
     SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps
 
 ml-darwin-aarch64-mps-generate:
-  tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ]
-  extends: [ ".ml-darwin-aarch64-mps", ".darwin-generate"]
+  tags: [ "macos-ventura", "apple-clang-15", "aarch64-macos" ]
+  extends: [ ".ml-darwin-aarch64-mps", ".generate-base"]
 
 ml-darwin-aarch64-mps-build:
   extends: [ ".ml-darwin-aarch64-mps", ".build" ]
@@ -754,7 +777,7 @@ deprecated-ci-build:
     SPACK_CI_STACK_NAME: aws-pcluster-icelake
 
 # aws-pcluster-generate-icelake:
-#   extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
+#   extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
 
 # aws-pcluster-build-icelake:
 #   extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".build" ]
@@ -773,7 +796,7 @@ deprecated-ci-build:
     SPACK_CI_STACK_NAME: aws-pcluster-skylake
 
 # aws-pcluster-generate-skylake:
-#   extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
+#   extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
 
 # aws-pcluster-build-skylake:
 #   extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".build" ]
@@ -825,87 +848,62 @@ deprecated-ci-build:
 #       job: aws-pcluster-generate-neoverse_v1
 
 # Cray definitions
-.base-cray-job:
-  variables:
-    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
-    AWS_ACCESS_KEY_ID: ${CRAY_MIRRORS_AWS_ACCESS_KEY_ID}
-    AWS_SECRET_ACCESS_KEY: ${CRAY_MIRRORS_AWS_SECRET_ACCESS_KEY}
-  rules:
-    - if: $CI_COMMIT_REF_NAME == "develop"
-      # Pipelines on develop only rebuild what is missing from the mirror
-      when: always
-      variables:
-        SPACK_PIPELINE_TYPE: "spack_protected_branch"
-    - if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/
-      # Pipelines on PR branches rebuild only what's missing, and do extra pruning
-      when: always
-      variables:
-        SPACK_PIPELINE_TYPE: "spack_pull_request"
-        SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
-        SPACK_PRUNE_UNTOUCHED: "True"
-        SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
-
 .generate-cray:
-  tags: [ "cce@15.0.1", "cray-zen4", "public" ]
-  extends: [ ".base-cray-job" ]
-  stage: generate
-  script:
+  extends: [ ".generate-common", ".base-job" ]
+  before_script:
     - echo $PATH
     - module avail
     - module list
-    - export SPACK_DISABLE_LOCAL_CONFIG=1
-    - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
-    - uname -a || true
-    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
-    - nproc || true
-    - . "./share/spack/setup-env.sh"
-    - spack --version
-    - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
-    - spack env activate --without-view .
-    - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
-    - spack
-      --config-scope "${SPACK_CI_CONFIG_ROOT}"
-      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
-      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
-      ${CI_STACK_CONFIG_SCOPES}
-      ci generate --check-index-only
-      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
-      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
-      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
-  after_script:
-    - cat /proc/loadavg || true
-  artifacts:
-    paths:
-      - "${CI_PROJECT_DIR}/jobs_scratch_dir"
-  interruptible: true
-  timeout: 60 minutes
-  retry:
-    max: 2
-    when:
-      - always
 
-.build-cray:
-  extends: [ ".base-cray-job" ]
-  stage: build
+.generate-cray-rhel:
+  tags: [ "cray-rhel-zen4", "public" ]
+  extends: [ ".generate-cray" ]
+
+.generate-cray-sles:
+  tags: [ "cray-sles-zen4", "public" ]
+  extends: [ ".generate-cray" ]
+
 
 #######################################
-# E4S - Cray
+# E4S - Cray RHEL
 #######################################
-.e4s-cray:
-  extends: [ ".cray_zen4" ]
+.e4s-cray-rhel:
+  extends: [ ".cray_rhel_zen4" ]
   variables:
-    SPACK_CI_STACK_NAME: e4s-cray
+    SPACK_CI_STACK_NAME: e4s-cray-rhel
 
-e4s-cray-generate:
-  extends: [ ".generate-cray", ".e4s-cray" ]
+e4s-cray-rhel-generate:
+  extends: [ ".generate-cray-rhel", ".e4s-cray-rhel" ]
 
-e4s-cray-build:
-  extends: [ ".build-cray", ".e4s-cray" ]
+e4s-cray-rhel-build:
+  extends: [ ".build", ".e4s-cray-rhel" ]
   trigger:
     include:
       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-        job: e4s-cray-generate
+        job: e4s-cray-rhel-generate
     strategy: depend
   needs:
     - artifacts: True
-      job: e4s-cray-generate
\ No newline at end of file
+      job: e4s-cray-rhel-generate
+
+#######################################
+# E4S - Cray SLES
+#######################################
+.e4s-cray-sles:
+  extends: [ ".cray_sles_zen4" ]
+  variables:
+    SPACK_CI_STACK_NAME: e4s-cray-sles
+
+# e4s-cray-sles-generate:
+#   extends: [ ".generate-cray-sles", ".e4s-cray-sles" ]
+
+# e4s-cray-sles-build:
+#   extends: [ ".build", ".e4s-cray-sles" ]
+#   trigger:
+#     include:
+#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+#         job: e4s-cray-sles-generate
+#     strategy: depend
+#   needs:
+#     - artifacts: True
+#       job: e4s-cray-sles-generate
diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
index f65760c9235063..29dc993a15578c 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml
@@ -3,38 +3,51 @@ ci:
 
   broken-tests-packages:
     - gptune
+    - superlu-dist    # srun -n 4 hangs
 
   broken-specs-url: "https://dummy.io" # s3://spack-binaries/broken-specs"
 
   pipeline-gen:
   - build-job:
-      script::
-      - - spack compiler find
-        - cd ${SPACK_CONCRETE_ENV_DIR}
-        - spack env activate --without-view .
+      before_script-:
+      - - spack list --count  # ensure that spack's cache is populated
+      - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
+        - spack compiler list
         - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi
-        - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
-        - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
+      - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
         # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
-        - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi
+      - - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k
         # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
-        - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
-        - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
+        - k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k
+        - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
+      script::
+      - - spack config blame mirrors
         - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+      - - spack python ${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/scripts/common/aggregate_package_logs.spack.py
+          --prefix /home/software/spack:${CI_PROJECT_DIR}
+          --log install_times.json
+          ${SPACK_ARTIFACTS_ROOT}/user_data/install_times.json
       after_script:
       - - cat /proc/loadavg || true
       variables:
         CI_JOB_SIZE: "default"
+        CI_GPG_KEY_ROOT: /mnt/key
         # SPACK_VERBOSE_SCRIPT: "1"
+      id_tokens:
+        GITLAB_OIDC_TOKEN:
+          aud: "${OIDC_TOKEN_AUDIENCE}"
 
   - signing-job:
       image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
       tags: ["aws"]
       script:
-      - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
+      - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_BUILDCACHE_DESTINATION}/build_cache /tmp
         - /sign.sh
-        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
-        - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
+        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_BUILDCACHE_DESTINATION}/build_cache
+        - aws s3 cp /tmp/public_keys ${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
+      id_tokens:
+        GITLAB_OIDC_TOKEN:
+          aud: "${OIDC_TOKEN_AUDIENCE}"
 
   - copy-job:
       tags: ["service", "x86_64"]
@@ -42,15 +55,14 @@ ci:
       before_script:
       - - if [[ $CI_COMMIT_TAG == "v"* ]]; then export SPACK_REPLACE_VERSION=$(echo "$CI_COMMIT_TAG" | sed 's/\(v[[:digit:]]\+\.[[:digit:]]\+\).*/releases\/\1/'); fi
         - if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi
-        - export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
+        - export SPACK_COPY_ONLY_SOURCE=${SPACK_BUILDCACHE_SOURCE//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
       script:
-      - - cd ${SPACK_CONCRETE_ENV_DIR}
-        - spack env activate --without-view .
-        - echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION}
-        - spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}"
+      - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
+        - echo Copying environment specs from ${SPACK_COPY_ONLY_SOURCE} to ${SPACK_COPY_ONLY_DESTINATION}
+        - spack buildcache sync "${SPACK_COPY_ONLY_SOURCE}" "${SPACK_COPY_ONLY_DESTINATION}"
         - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
-        - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
-        - spack buildcache update-index --keys "${SPACK_BUILDCACHE_DESTINATION}"
+        - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_COPY_ONLY_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
+        - spack buildcache update-index --keys "${SPACK_COPY_ONLY_DESTINATION}"
       when: "always"
       retry:
         max: 2
@@ -63,20 +75,31 @@ ci:
         CI_JOB_SIZE: "medium"
         KUBERNETES_CPU_REQUEST: "4000m"
         KUBERNETES_MEMORY_REQUEST: "16G"
+      id_tokens:
+        GITLAB_OIDC_TOKEN:
+          aud: "${OIDC_TOKEN_AUDIENCE}"
 
   - reindex-job:
-      tags: ["service"]
+      tags: ["service", "x86_64"]
+      image: "ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01"
       variables:
         CI_JOB_SIZE: "medium"
         KUBERNETES_CPU_REQUEST: "4000m"
         KUBERNETES_MEMORY_REQUEST: "16G"
+      id_tokens:
+        GITLAB_OIDC_TOKEN:
+          aud: "${OIDC_TOKEN_AUDIENCE}"
 
+  # TODO: Remove this block in Spack 0.23
   - cleanup-job:
       tags: ["service"]
       variables:
         CI_JOB_SIZE: "small"
         KUBERNETES_CPU_REQUEST: "500m"
         KUBERNETES_MEMORY_REQUEST: "500M"
+      id_tokens:
+        GITLAB_OIDC_TOKEN:
+          aud: "${OIDC_TOKEN_AUDIENCE}"
 
   - noop-job:
       tags: ["service"]
@@ -86,12 +109,16 @@ ci:
         KUBERNETES_MEMORY_REQUEST: "500M"
 
   - any-job:
-      image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
       tags: ["spack"]
+      image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
+      variables:
+        # Disable local configs to avoid issues on shell runners
+        SPACK_DISABLE_LOCAL_CONFIG: "1"
       before_script:
+      - - export SPACK_USER_CACHE_PATH="${CI_PROJECT_DIR}/_user_cache/"
       - - uname -a || true
         - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
-        - nproc
+        - nproc || true
       - - . "./share/spack/setup-env.sh"
         - spack --version
         - spack arch
diff --git a/share/spack/gitlab/cloud_pipelines/configs/copy-only-protected-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/copy-only-protected-mirrors.yaml.in
new file mode 100644
index 00000000000000..39e5c733b236d2
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/copy-only-protected-mirrors.yaml.in
@@ -0,0 +1,11 @@
+mirrors:
+  buildcache-source:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-destination:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml
new file mode 100644
index 00000000000000..a74ef3f98f89e2
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml
@@ -0,0 +1,7 @@
+ci:
+  pipeline-gen:
+  - build-job-remove:
+      image: no-image
+  - build-job:
+      variables:
+        CI_GPG_KEY_ROOT: /etc/protected-runner
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/compilers.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/compilers.yaml
new file mode 100644
index 00000000000000..91ac37dfa58326
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/compilers.yaml
@@ -0,0 +1,31 @@
+compilers:
+- compiler:
+    spec: cce@15.0.1
+    paths:
+      cc: cc
+      cxx: CC
+      f77: ftn
+      fc: ftn
+    flags: {}
+    operating_system: rhel8
+    target: any
+    modules:
+    - PrgEnv-cray/8.3.3
+    - cce/15.0.1
+    environment:
+      set:
+        MACHTYPE: x86_64
+- compiler:
+    spec: gcc@11.2.0
+    paths:
+      cc: gcc
+      cxx: g++
+      f77: gfortran
+      fc: gfortran
+    flags: {}
+    operating_system: rhel8
+    target: any
+    modules:
+    - PrgEnv-gnu
+    - gcc/11.2.0
+    environment: {}
\ No newline at end of file
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/config.yaml
similarity index 100%
rename from share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml
rename to share/spack/gitlab/cloud_pipelines/configs/cray-rhel/config.yaml
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/packages.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/packages.yaml
new file mode 100644
index 00000000000000..a74d19ba626814
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/packages.yaml
@@ -0,0 +1,16 @@
+packages:
+  # EXTERNALS
+  cray-mpich:
+    buildable: false
+    externals:
+    - spec: cray-mpich@8.1.25 %cce@15.0.1
+      prefix: /opt/cray/pe/mpich/8.1.25/ofi/cray/10.0
+      modules:
+      - cray-mpich/8.1.25
+  cray-libsci:
+    buildable: false
+    externals:
+    - spec: cray-libsci@23.02.1.1 %cce@15.0.1
+      prefix: /opt/cray/pe/libsci/23.02.1.1/CRAY/9.0/x86_64/
+      modules:
+      - cray-libsci/23.02.1.1
\ No newline at end of file
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/zen4/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/zen4/ci.yaml
new file mode 100644
index 00000000000000..43bbbc9249fe28
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/zen4/ci.yaml
@@ -0,0 +1,4 @@
+ci:
+  pipeline-gen:
+  - build-job:
+      tags: ["cray-rhel-zen4"]
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml
new file mode 100644
index 00000000000000..a74ef3f98f89e2
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml
@@ -0,0 +1,7 @@
+ci:
+  pipeline-gen:
+  - build-job-remove:
+      image: no-image
+  - build-job:
+      variables:
+        CI_GPG_KEY_ROOT: /etc/protected-runner
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/compilers.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/compilers.yaml
new file mode 100644
index 00000000000000..0e5511ea057f9f
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/compilers.yaml
@@ -0,0 +1,14 @@
+compilers:
+- compiler:
+    spec: gcc@=10.3.0
+    paths:
+      cc: /opt/cray/pe/gcc/10.3.0/bin/gcc
+      cxx: /opt/cray/pe/gcc/10.3.0/bin/g++
+      f77: /opt/cray/pe/gcc/10.3.0/bin/gfortran
+      fc: /opt/cray/pe/gcc/10.3.0/bin/gfortran
+    flags: {}
+    operating_system: sle_hpc15
+    target: x86_64
+    modules: [PrgEnv-gnu-amd]
+    environment: {}
+    extra_rpaths: []
\ No newline at end of file
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/config.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/config.yaml
new file mode 100644
index 00000000000000..81cc5c04cc5b8e
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/config.yaml
@@ -0,0 +1,4 @@
+config:
+  install_tree:
+    root: $spack/opt/spack
+
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/packages.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/packages.yaml
new file mode 100644
index 00000000000000..18656517be3464
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/packages.yaml
@@ -0,0 +1,16 @@
+packages:
+  # EXTERNALS
+  cray-mpich:
+    buildable: false
+    externals:
+    - spec: cray-mpich@8.1.25 %gcc
+      prefix: /opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/
+      modules:
+      - cray-mpich/8.1.25
+  cray-libsci:
+    buildable: false
+    externals:
+    - spec: cray-libsci@23.02.1.1 %gcc
+      prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/
+      modules:
+      - cray-libsci/23.02.1.1
\ No newline at end of file
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/zen4/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/zen4/ci.yaml
new file mode 100644
index 00000000000000..e96413a14cc994
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/zen4/ci.yaml
@@ -0,0 +1,4 @@
+ci:
+  pipeline-gen:
+  - build-job:
+      tags: ["cray-sles-zen4"]
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml
deleted file mode 100644
index 114dcc81dcd1c4..00000000000000
--- a/share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml
+++ /dev/null
@@ -1,295 +0,0 @@
-ci:
-  pipeline-gen:
-  - build-job-remove:
-      image: no-image
-  - build-job:
-      script+:
-      # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
-      - if [[ -r /etc/protected-runner/e4s.gpg ]]; then spack gpg trust /etc/protected-runner/e4s.gpg; fi
-      # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
-      - if [[ -r /etc/protected-runner/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /etc/protected-runner/intermediate_ci_signing_key.gpg; fi
-      - if [[ -r /etc/protected-runner/spack_public_key.gpg ]]; then spack gpg trust /etc/protected-runner/spack_public_key.gpg; fi
-  - match_behavior: first
-    submapping:
-    - match:
-      - hipblas
-      - llvm
-      - llvm-amdgpu
-      - pango
-      - paraview
-      - py-tensorflow
-      - py-torch
-      - qt
-      - rocblas
-      - visit
-      build-job:
-        tags: [ "spack", "huge" ]
-        variables:
-          CI_JOB_SIZE: huge
-          SPACK_BUILD_JOBS: "12"
-
-    - match:
-      - ascent
-      - atk
-      - axom
-      - cistem
-      - cmake
-      - ctffind
-      - cuda
-      - dealii
-      - dray
-      - dyninst
-      - ecp-data-vis-sdk
-      - gcc
-      - ginkgo
-      - hdf5
-      - hpx
-      - kokkos-kernels
-      - kokkos-nvcc-wrapper
-      - lbann
-      - magma
-      - mesa
-      - mfem
-      - mpich
-      - netlib-lapack
-      - nvhpc
-      - oce
-      - openblas
-      - openfoam
-      - openturns
-      - parallelio
-      - plumed
-      - precice
-      #- py-tensorflow
-      #- qt
-      - raja
-      - relion
-      #- rocblas
-      - rocfft
-      - rocsolver
-      - rocsparse
-      - rust
-      - slate
-      - strumpack
-      - sundials
-      - trilinos
-      - umpire
-      #- visit
-      - vtk
-      - vtk-h
-      - vtk-m
-      - warpx
-      - wrf
-      - wxwidgets
-      build-job:
-        tags: [ "spack", "large" ]
-        variables:
-          CI_JOB_SIZE: large
-          SPACK_BUILD_JOBS: "8"
-
-    - match:
-      - adios2
-      - amrex
-      - archer
-      - ascent
-      - autoconf-archive
-      - axom
-      - binutils
-      - blaspp
-      - blt
-      - boost
-      - butterflypack
-      - cabana
-      - caliper
-      - camp
-      - chai
-      - conduit
-      - curl
-      - datatransferkit
-      - double-conversion
-      - dray
-      - eigen
-      - faodel
-      - ffmpeg
-      - fftw
-      - fortrilinos
-      - gettext
-      - gperftools
-      - gptune
-      - hdf5
-      - heffte
-      - hpctoolkit
-      - hwloc
-      - hydrogen
-      - hypre
-      - kokkos
-      - lammps
-      - lapackpp
-      - legion
-      - libtool
-      - libxml2
-      - libzmq
-      - llvm-openmp-ompt
-      - mbedtls
-      - mfem
-      - mpich
-      - mvapich2
-      - nasm
-      - netlib-scalapack
-      - omega-h
-      - openblas
-      - openjpeg
-      - openmpi
-      - openpmd-api
-      - pagmo2
-      - papyrus
-      - parsec
-      - pdt
-      - pegtl
-      - petsc
-      - pumi
-      - py-beniget
-      - py-cinemasci
-      - pygmo
-      - py-ipython-genutils
-      - py-packaging
-      - py-petsc4py
-      - py-scipy
-      - py-statsmodels
-      - py-warlock
-      - py-warpx
-      - raja
-      - samrai
-      - slepc
-      - slurm
-      - sqlite
-      - strumpack
-      - sundials
-      - superlu-dist
-      - tasmanian
-      - tau
-      - upcxx
-      - vtk
-      - vtk-h
-      - vtk-m
-      - zfp
-      build-job:
-        tags: [ "spack", "medium" ]
-        variables:
-          CI_JOB_SIZE: "medium"
-          SPACK_BUILD_JOBS: "2"
-
-    - match:
-      - alsa-lib
-      - ant
-      - antlr
-      - argobots
-      - autoconf-archive
-      - automake
-      - berkeley-db
-      - bison
-      - blt
-      - bzip2
-      - camp
-      - cmake
-      - curl
-      - czmq
-      - darshan-util
-      - diffutils
-      - docbook-xml
-      - exmcutils
-      - expat
-      - findutils
-      - flit
-      - freetype
-      - gawk
-      - gdbm
-      - gettext
-      - glib
-      - gmake
-      - gotcha
-      - hpcviewer
-      - hwloc
-      - jansson
-      - json-c
-      - libbsd
-      - libedit
-      - libevent
-      - libfabric
-      - libffi
-      - libgcrypt
-      - libiconv
-      - libidn2
-      - libjpeg-turbo
-      - libmd
-      - libnrm
-      - libpciaccess
-      - libpng
-      - libsigsegv
-      - libsodium
-      - libunistring
-      - libunwind
-      - libxml2
-      - libyaml
-      - libzmq
-      - lua
-      - lua-luaposix
-      - lz4
-      - m4
-      - meson
-      - metis
-      - mpfr
-      - ncurses
-      - ninja
-      - numactl
-      - openblas
-      - openjdk
-      - openssh
-      - openssl
-      - papi
-      - parallel-netcdf
-      - pcre
-      - pcre2
-      - pdsh
-      - perl
-      - perl-data-dumper
-      - pkgconf
-      - py-alembic
-      - py-cffi
-      - py-cycler
-      - py-decorator
-      - py-idna
-      - py-jsonschema
-      - py-kiwisolver
-      - py-mistune
-      - py-pycparser
-      - py-setuptools
-      - py-setuptools-scm
-      - py-six
-      - py-testpath
-      - py-wheel
-      - qhull
-      - readline
-      - sed
-      - slurm
-      - snappy
-      - sqlite
-      - superlu
-      - swig
-      - tar
-      - tcl
-      - texinfo
-      - tut
-      - unzip
-      - util-linux-uuid
-      - util-macros
-      - xz
-      - yaml-cpp
-      - zfp
-      - zlib
-      - zstd
-      build-job:
-        tags: [ "spack", "small" ]
-        variables:
-          CI_JOB_SIZE: "small"
-          SPACK_BUILD_JOBS: "1"
diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml
deleted file mode 100644
index 3bee4c4000b7ec..00000000000000
--- a/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-ci:
-  pipeline-gen:
-  - build-job:
-      tags: ["cce@15.0.1", "cray-zen4"]
diff --git a/share/spack/gitlab/cloud_pipelines/configs/darwin/aarch64/compilers.yaml b/share/spack/gitlab/cloud_pipelines/configs/darwin/aarch64/compilers.yaml
deleted file mode 100644
index d5a0130341e246..00000000000000
--- a/share/spack/gitlab/cloud_pipelines/configs/darwin/aarch64/compilers.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-compilers:
-- compiler:
-    spec: apple-clang@14.0.0
-    paths:
-      cc: /usr/bin/clang
-      cxx: /usr/bin/clang++
-      f77: /opt/homebrew/bin/gfortran
-      fc: /opt/homebrew/bin/gfortran
-    flags: {}
-    operating_system: ventura
-    target: aarch64
-    modules: []
-    environment: {}
-    extra_rpaths: []
-- compiler:
-    spec: gcc@12.2.0
-    paths:
-      cc: /opt/homebrew/bin/gcc-12
-      cxx: /opt/homebrew/bin/g++-12
-      f77: /opt/homebrew/bin/gfortran-12
-      fc: /opt/homebrew/bin/gfortran-12
-    flags: {}
-    operating_system: ventura
-    target: aarch64
-    modules: []
-    environment: {}
-    extra_rpaths: []
diff --git a/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml
new file mode 100644
index 00000000000000..8dfb169a3e83ea
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml
@@ -0,0 +1,4 @@
+ci:
+  pipeline-gen:
+  - build-job-remove:
+      image: macos-run-on-metal
diff --git a/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
index b7748793297259..86340421014601 100644
--- a/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
@@ -1,5 +1,10 @@
 ci:
   pipeline-gen:
+  - build-job:
+      before_script-:
+      # Test package relocation on linux using a modified prefix
+      # This is not well supported on MacOS (https://github.com/spack/spack/issues/37162)
+      - - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
   - match_behavior: first
     submapping:
     - match:
diff --git a/share/spack/gitlab/cloud_pipelines/configs/multi-src-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/multi-src-mirrors.yaml.in
new file mode 100644
index 00000000000000..0ad46d5fc9014f
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/multi-src-mirrors.yaml.in
@@ -0,0 +1,16 @@
+mirrors:
+  buildcache-source:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${PR_TARGET_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${PR_TARGET_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-destination:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
+  buildcache-shared:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/shared_pr_mirror/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/shared_pr_mirror/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/configs/packages.yaml b/share/spack/gitlab/cloud_pipelines/configs/packages.yaml
deleted file mode 100644
index 1771f770c44830..00000000000000
--- a/share/spack/gitlab/cloud_pipelines/configs/packages.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-packages: {}
-
-# CI should never build develop/main/master versions of packages. Current issues:
-# - e4s/dav
-#   - hdf5-vol-async => argobot@main
-# - aws-ahug-*
-#   - snap
-#   - tycho2
-#   - amg2013
-#   - cosp2
-#   - snbone
-#   - vpfft
-#   - vpic
-# - aws-isc-aarch64
-#   - sse2neon
-
-# packages:
-#   all:
-#     require: "@:999999999"
\ No newline at end of file
diff --git a/share/spack/gitlab/cloud_pipelines/configs/single-src-pr-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/single-src-pr-mirrors.yaml.in
new file mode 100644
index 00000000000000..0a2775a4a27def
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/single-src-pr-mirrors.yaml.in
@@ -0,0 +1,6 @@
+mirrors:
+  buildcache-destination:
+    fetch: ${PR_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PR_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/configs/single-src-protected-mirrors.yaml.in b/share/spack/gitlab/cloud_pipelines/configs/single-src-protected-mirrors.yaml.in
new file mode 100644
index 00000000000000..a55cd7273750ee
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/configs/single-src-protected-mirrors.yaml.in
@@ -0,0 +1,6 @@
+mirrors:
+  buildcache-destination:
+    fetch: ${PROTECTED_MIRROR_FETCH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    push: ${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}
+    source: False
+    binary: True
diff --git a/share/spack/gitlab/cloud_pipelines/scripts/common/aggregate_package_logs.spack.py b/share/spack/gitlab/cloud_pipelines/scripts/common/aggregate_package_logs.spack.py
new file mode 100644
index 00000000000000..9adab64e576a0c
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/scripts/common/aggregate_package_logs.spack.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env spack-python
+"""
+This script is meant to be run using:
+    `spack python aggregate_logs.spack.py`
+"""
+
+import os
+
+
+def find_logs(prefix, filename):
+    for root, _, files in os.walk(prefix):
+        if filename in files:
+            yield os.path.join(root, filename)
+
+
+if __name__ == "__main__":
+    import json
+    from argparse import ArgumentParser
+
+    parser = ArgumentParser("aggregate_logs")
+    parser.add_argument("output_file")
+    parser.add_argument("--log", default="install_times.json")
+    parser.add_argument("--prefix", required=True)
+
+    args = parser.parse_args()
+
+    prefixes = [p for p in args.prefix.split(":") if os.path.exists(p)]
+
+    # Aggregate the install timers into a single json
+    data = []
+    for prefix in prefixes:
+        time_logs = find_logs(prefix, args.log)
+        for log in time_logs:
+            with open(log) as fd:
+                data.append(json.load(fd))
+
+    with open(args.output_file, "w") as fd:
+        json.dump(data, fd)
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml
deleted file mode 100644
index 40917b600f7aae..00000000000000
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml
+++ /dev/null
@@ -1,238 +0,0 @@
-spack:
-  view: false
-  packages:
-    all:
-      providers:
-        blas:
-          - openblas
-        mkl:
-          - intel-oneapi-mkl
-        mpi:
-          - openmpi
-          - mpich
-      variants: +mpi
-    tbb:
-      require: "intel-tbb"
-    binutils:
-      variants: +ld +gold +headers +libiberty ~nls
-      version:
-        - 2.36.1
-    doxygen:
-      version:
-        - 1.8.20
-    elfutils:
-      variants: +bzip2 ~nls +xz
-    hdf5:
-      variants: +fortran +hl +shared
-    libfabric:
-      variants: fabrics=efa,tcp,udp,sockets,verbs,shm,mrail,rxd,rxm
-    libunwind:
-      variants: +pic +xz
-    #m4:
-    #  version:
-    #    - 1.4.18
-    mesa:
-      variants: ~llvm
-    mesa18:
-      variants: ~llvm
-    mpich:
-      #variants: ~wrapperrpath pmi=pmi netmod=ofi device=ch4
-      variants: ~wrapperrpath netmod=ofi device=ch4
-    #munge:
-    #  variants: localstatedir=/var
-    ncurses:
-      variants: +termlib
-    openblas:
-      variants: threads=openmp
-    openmpi:
-      #variants: +pmi +internal-hwloc fabrics=ofi schedulers=slurm
-      variants: fabrics=ofi
-    openturns:
-      version: [1.18]
-    #slurm:
-    #  variants: +pmix sysconfdir=/opt/slurm/etc
-    trilinos:
-      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
-    xz:
-      variants: +pic
-
-  definitions:
-
-  - compiler_specs:
-    - gcc
-    #- nvhpc@22.1
-
-
-  # - compilers:
-  #   - '%gcc@7.5.0'
-  #   - '%arm@21.0.0.879'
-  #   - '%nvhpc@21.2'
-    # - 'arm@21.0.0.879'
-    #- when: arch.satisfies('os=ubuntu18.04')
-    #  compilers: ['gcc@7.5.0', $bootstrap-compilers]
-    #- when: arch.satisfies('os=amzn2')
-    #  compilers: ['gcc@7.3.1', $bootstrap-compilers]
-
-    # Note skipping spot since no spack package for it
-  - ahug_miniapps:
-    - cloverleaf
-  # - coevp
-  # Bad code pushed to github; needs specific versioning
-  # - cohmm
-  # - examinimd
-  #  - exampm
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - exasp2
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - gamess-ri-mp2-miniapp
-    - hpcg
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - laghos
-    - lulesh
-  # - miniaero
-    - miniamr
-    - minife
-    - minighost
-  # fails due to x86 specific timer (asm instructions)
-  # - minigmg
-    - minimd
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - miniqmc
-    - minismac2d
-    - minitri
-    - minivite
-    - minixyce
-    - pennant
-    - picsarlite
-    - quicksilver
-  # - remhos
-    - rsbench
-    - simplemoc
-    - snap
-    - snappy
-    - tealeaf
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - thornado-mini
-    - tycho2
-    - xsbench
-
-  - ahug_fullapps:
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - abinit
-    - abyss
-  # conflicts on trilinos
-  # - albany
-  # - amber
-    - amg2013
-  # Bad variant fftw
-  # - aoflagger
-  # - athena
-  # - bowtie2
-    - branson
-  # - camx
-  # Bad variant gpu
-  # - candle-benchmarks
-    - cbench
-    - cgm
-    - chatterbug
-  # - cistem
-    - comd
-  # old version of openmpi
-  # - converge
-  # bad variant tensor ops mpi
-  # - cosmoflow-benchmark
-  # - cosmomc
-    - cosp2
-  # libxsmm not avail on arm
-  # - cp2k
-  # - dock
-  # - elk
-    - elmerfem
-  # - exabayes
-  # - examl
-  # - flecsph
-  # trilinos variant mumps
-  # - frontistr
-    - gatk
-    - graph500
-    - hpgmg
-    - lammps
-    - latte
-    - macsio
-  # - meep
-    - meme
-  # - modylas
-  # - mrbayes
-  # - mrchem
- # cudnn depednency
- #  - mxnet
- # trilinos variant mumps
- #  - nalu
-  # - nalu-wind
-  # - namd
-    - nek5000
-    - nekbone
-  # - nektar
-  # - nest
-    - nut
-  # - nwchem
-    - octopus
-    - openmm
-    - pathfinder
-  # - picsar
-    - pism
-  # meson version
-  # - qbox
-    - qmcpack
-    - quantum-espresso
-  # - relion
-  # - siesta
-    - snbone
-    - star
-    - su2
-    - swfft
-    - tinker
-  # gfortran lt 9 unsupported
-  # - vasp
-    - vpfft
-    - vpic
-    - warpx
-  # - yambo
-
-  - compiler:
-    - '%gcc@7.3.1'
-
-  - target:
-    - 'target=aarch64'
-    - 'target=neoverse_n1'
-
-
-  specs:
-
-  - matrix:
-    - - $ahug_miniapps
-    - - $compiler
-    - - $target
-
-  - matrix:
-    - - $ahug_fullapps
-    - - $compiler
-    - - $target
-
-  # Build compilers to stage in binary cache
-  - matrix:
-    - - $compiler_specs
-    - - $compiler
-    - - $target
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-ahug-aarch64" }
-
-  ci:
-    pipeline-gen:
-    - build-job:
-        image:
-          name: "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09"
-          entrypoint: [""]
-
-  cdash:
-    build-group: AHUG ARM HPC User Group
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml
deleted file mode 100644
index 6483e72b971ffe..00000000000000
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml
+++ /dev/null
@@ -1,235 +0,0 @@
-spack:
-  view: false
-  packages:
-    all:
-      providers:
-        blas:
-          - openblas
-        mkl:
-          - intel-oneapi-mkl
-        mpi:
-          - openmpi
-          - mpich
-      variants: +mpi
-    binutils:
-      variants: +ld +gold +headers +libiberty ~nls
-      version:
-        - 2.36.1
-    doxygen:
-      version:
-        - 1.8.20
-    elfutils:
-      variants: +bzip2 ~nls +xz
-    hdf5:
-      variants: +fortran +hl +shared
-    libfabric:
-      variants: fabrics=efa,tcp,udp,sockets,verbs,shm,mrail,rxd,rxm
-    libunwind:
-      variants: +pic +xz
-    #m4:
-    #  version:
-    #    - 1.4.18
-    mesa:
-      variants: ~llvm
-    mesa18:
-      variants: ~llvm
-    mpich:
-      #variants: ~wrapperrpath pmi=pmi netmod=ofi device=ch4
-      variants: ~wrapperrpath netmod=ofi device=ch4
-    #munge:
-    #  variants: localstatedir=/var
-    ncurses:
-      variants: +termlib
-    openblas:
-      variants: threads=openmp
-    openmpi:
-      #variants: +pmi +internal-hwloc fabrics=ofi schedulers=slurm
-      variants: fabrics=ofi
-    openturns:
-      version: [1.18]
-    #slurm:
-    #  variants: +pmix sysconfdir=/opt/slurm/etc
-    trilinos:
-      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
-    xz:
-      variants: +pic
-
-  definitions:
-
-  - compiler_specs:
-    - gcc
-    #- nvhpc@22.1
-
-
-  # - compilers:
-  #   - '%gcc@7.5.0'
-  #   - '%arm@21.0.0.879'
-  #   - '%nvhpc@21.2'
-    # - 'arm@21.0.0.879'
-    #- when: arch.satisfies('os=ubuntu18.04')
-    #  compilers: ['gcc@7.5.0', $bootstrap-compilers]
-    #- when: arch.satisfies('os=amzn2')
-    #  compilers: ['gcc@7.3.1', $bootstrap-compilers]
-
-    # Note skipping spot since no spack package for it
-  - ahug_miniapps:
-    - cloverleaf
-  # - coevp
-  # Bad code pushed to github; needs specific versioning
-  # - cohmm
-  # - examinimd
-  #  - exampm
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - exasp2
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - gamess-ri-mp2-miniapp
-    - hpcg
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - laghos
-    - lulesh
-  # - miniaero
-    - miniamr
-    - minife
-    - minighost
-  # fails due to x86 specific timer (asm instructions)
-  # - minigmg
-    - minimd
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - miniqmc
-    - minismac2d
-    - minitri
-    - minivite
-    - minixyce
-    - pennant
-    - picsarlite
-    - quicksilver
-  # - remhos
-    - rsbench
-    - simplemoc
-    - snap
-    - snappy
-    - tealeaf
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - thornado-mini
-    - tycho2
-    - xsbench
-
-  - ahug_fullapps:
-  # depends on openblas version conflicting on gcc@7.3.1
-  # - abinit
-    - abyss
-  # conflicts on trilinos
-  # - albany
-  # - amber
-    - amg2013
-  # Bad variant fftw
-  # - aoflagger
-  # - athena
-  # - bowtie2
-    - branson
-  # - camx
-  # Bad variant gpu
-  # - candle-benchmarks
-    - cbench
-    - cgm
-    - chatterbug
-  # - cistem
-    - comd
-  # old version of openmpi
-  # - converge
-  # bad variant tensor ops mpi
-  # - cosmoflow-benchmark
-  # - cosmomc
-    - cosp2
-  # libxsmm not avail on arm
-  # - cp2k
-  # - dock
-  # - elk
-    - elmerfem
-  # - exabayes
-  # - examl
-  # - flecsph
-  # trilinos variant mumps
-  # - frontistr
-    - gatk
-    - graph500
-    - hpgmg
-    - lammps
-    - latte
-    - macsio
-  # - meep
-    - meme
-  # - modylas
-  # - mrbayes
-  # - mrchem
- # cudnn depednency
- #  - mxnet
- # trilinos variant mumps
- #  - nalu
-  # - nalu-wind
-  # - namd
-    - nek5000
-    - nekbone
-  # - nektar
-  # - nest
-    - nut
-  # - nwchem
-    - octopus
-    - openmm
-    - pathfinder
-  # - picsar
-    - pism
-  # meson version
-  # - qbox
-    - qmcpack
-    - quantum-espresso
-  # - relion
-  # - siesta
-    - snbone
-    - star
-    - su2
-    - swfft
-    - tinker
-  # gfortran lt 9 unsupported
-  # - vasp
-    - vpfft
-    - vpic
-    - warpx
-  # - yambo
-
-  - compiler:
-    - '%gcc@7.3.1'
-
-  - target:
-    - 'target=x86_64_v3'
-
-
-  specs:
-
-  - matrix:
-    - - $ahug_miniapps
-    - - $compiler
-    - - $target
-
-  - matrix:
-    - - $ahug_fullapps
-    - - $compiler
-    - - $target
-
-  # Build compilers to stage in binary cache
-  - matrix:
-    - - $compiler_specs
-    - - $compiler
-    - - $target
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-ahug" }
-
-  ci:
-    pipeline-gen:
-    - build-job:
-        image:
-          name: "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09"
-          entrypoint: [""]
-
-  cdash:
-    build-group: AHUG ARM HPC User Group
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
index 1c4e2de308eea2..abd8f4d0242df2 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
@@ -131,9 +131,6 @@ spack:
     - - $compiler
     - - $target
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc-aarch64" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
index 0a898d1a752b37..038761ac1873fa 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
@@ -142,9 +142,6 @@ spack:
     - - $compiler
     - - $target
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-isc" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
index 5ce6d1c8692e2c..85cf7660686d90 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
@@ -30,8 +30,6 @@ spack:
   - $optimized_configs
   # - $optimized_libs
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-icelake" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
index 5708338a2b3efe..50ba40992a7bc9 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
@@ -30,9 +30,6 @@ spack:
   - $optimized_configs
   - $optimized_libs
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_n1" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
index f2df7696106aa9..50ba40992a7bc9 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
@@ -30,9 +30,6 @@ spack:
   - $optimized_configs
   - $optimized_libs
 
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_v1" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
index 029dd67351c1f6..85cf7660686d90 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
@@ -30,8 +30,6 @@ spack:
   - $optimized_configs
   # - $optimized_libs
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-skylake" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
index 78a3ea785c827d..d154894830c155 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
@@ -21,7 +21,5 @@ spack:
         - - $default_specs
         - - $arch
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/build_systems" }
-
   cdash:
     build-group: Build Systems
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
index 33c63f0a01eb82..bf298d606db0ea 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
@@ -3,6 +3,11 @@ spack:
   packages:
     cmake:
       variants: ~ownlibs
+    ecp-data-vis-sdk:
+      require: "+ascent +adios2 +cinema +darshan +faodel +hdf5 +pnetcdf +sensei +sz +unifyfs +veloc +vtkm +zfp"
+    hdf5:
+      require:
+      - one_of: ['@1.14', '@1.12']
     mesa:
       require: "+glx +osmesa +opengl ~opengles +llvm"
     libosmesa:
@@ -10,43 +15,32 @@ spack:
     libglx:
       require: "mesa +glx"
     ospray:
-      require: "@2.8.0 +denoiser +mpi"
+      require: '@2.8.0 +denoiser +mpi'
     llvm:
-      require: "@14:"
+      require: '@14:'
       # Minimize LLVM
-      variants: "~lldb~lld~polly~gold libunwind=none compiler-rt=none"
+      variants: ~lldb~lld~libomptarget~polly~gold libunwind=none compiler-rt=none
     all:
       require: target=x86_64_v3
 
   definitions:
   - paraview_specs:
     - matrix:
-      - - paraview
+      - - paraview +raytracing
       - - +qt~osmesa # GUI Support w/ GLX Rendering
         - ~qt~osmesa # GLX Rendering
         - ~qt+osmesa # OSMesa Rendering
   - visit_specs:
     - matrix:
       - - visit
-      - - +gui~osmesa # GUI Support w/ GLX Rendering
-        - ~gui~osmesa # GLX Rendering
+      - - ~gui~osmesa # GLX Rendering
         - ~gui+osmesa # OSMesa Rendering
+        # VisIt GUI does not work with Qt 5.14.2
+        # - +gui~osmesa # GUI Support w/ GLX Rendering
   - sdk_base_spec:
     - matrix:
-      - - ecp-data-vis-sdk
-            +ascent
-            +adios2
-            +cinema
-            +darshan
-            +faodel
-            +hdf5
-            +pnetcdf
-            +sensei
-            +sz
-            +unifyfs
-            +veloc
-            +vtkm
-            +zfp
+      - - ecp-data-vis-sdk +ascent +adios2 +cinema +darshan +faodel +hdf5 +pnetcdf
+          +sensei +sz +unifyfs +veloc +vtkm +zfp
       - - ~cuda ~rocm
         # Current testing of GPU supported configurations
         # is provided in the E4S stack
@@ -54,22 +48,20 @@ spack:
         # - ~cuda +rocm
 
   specs:
-    # Test ParaView builds with different GL backends
-    - matrix:
-      - [$sdk_base_spec]
-      - [$^paraview_specs]
-      - - ^hdf5@1.14 # Non-VisIt can build HDF5 1.14
-    # Test ParaView builds with differnt GL backends
-    # - matrix:
-    #   - [$sdk_base_spec]
-    #   - [$^visit_specs]
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/data-vis-sdk" }
+    # Test ParaView and VisIt builds with different GL backends
+  - matrix:
+    - [$sdk_base_spec]
+    - ["+paraview ~visit"]
+    - [$^paraview_specs]
+  - matrix:
+    - [$sdk_base_spec]
+    - ["~paraview +visit"]
+    - [$^visit_specs]
 
   ci:
     pipeline-gen:
     - build-job:
-        image: { "name": "ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01", "entrypoint": [""] }
+        image: {name: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01, entrypoint: [''] }
 
   cdash:
-    build-group:: Data and Vis SDK
+    'build-group:': Data and Vis SDK
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
new file mode 100644
index 00000000000000..413fdf34eb28f9
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
@@ -0,0 +1,176 @@
+spack:
+  include:
+  - $spack/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
+
+  view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  packages:
+    all:
+      require: '%cce'
+      compiler: [cce]
+      providers:
+        blas: [cray-libsci]
+        lapack: [cray-libsci]
+        mpi: [cray-mpich]
+        tbb: [intel-tbb]
+        scalapack: [netlib-scalapack]
+      target: [zen4]
+      variants: +mpi
+
+    tbb:
+      require: "intel-tbb"
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+    boost:
+      variants: +python +filesystem +iostreams +system
+    cuda:
+      version: [11.7.0]
+    elfutils:
+      variants: +bzip2 ~nls +xz
+      require: "%gcc"
+    hdf5:
+      variants: +fortran +hl +shared
+    libfabric:
+      variants: fabrics=sockets,tcp,udp,rxm
+    libunwind:
+      variants: +pic +xz
+    mpich:
+      variants: ~wrapperrpath
+    ncurses:
+      variants: +termlib
+    paraview:
+      # Don't build GUI support or GLX rendering for HPC/container deployments
+      require: "@5.11 ~qt+osmesa"
+    python:
+      version: [3.8.13]
+    trilinos:
+      require:
+      - one_of: [+amesos +amesos2 +anasazi +aztec +boost +epetra +epetraext +ifpack
+            +intrepid +intrepid2 +isorropia +kokkos +minitensor +nox +piro +phalanx
+            +rol +rythmos +sacado +stk +shards +stratimikos +tempus +tpetra
+            +trilinoscouplings +zoltan]
+      - one_of: [gotype=long_long, gotype=all]
+      - one_of: [~ml ~muelu ~zoltan2 ~teko, +ml +muelu +zoltan2 +teko]
+      - one_of: [+superlu-dist, ~superlu-dist]
+      - one_of: [+shylu, ~shylu]
+    xz:
+      variants: +pic
+    mesa:
+      version: [21.3.8]
+    unzip:
+      require: "%gcc"
+
+  specs:
+  # CPU
+  - adios
+  - aml
+  - arborx
+  - argobots
+  - bolt
+  - butterflypack
+  - boost +python +filesystem +iostreams +system
+  - cabana
+  - caliper
+  - chai ~benchmarks ~tests
+  - charliecloud
+  - conduit
+  - datatransferkit
+  - flecsi
+  - flit
+  - flux-core
+  - fortrilinos
+  - ginkgo
+  - globalarrays
+  - gmp
+  - gotcha
+  - h5bench
+  - hdf5-vol-async
+  - hdf5-vol-cache
+  - hdf5-vol-log
+  - heffte +fftw
+  - hpx max_cpu_count=512 networking=mpi
+  - hypre
+  - kokkos +openmp
+  - kokkos-kernels +openmp
+  - lammps
+  - legion
+  - libnrm
+  - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed
+    +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +mgard
+  - libquo
+  - libunwind
+  - mercury
+  - metall
+  - mfem
+  - mgard +serial +openmp +timing +unstructured ~cuda
+  - mpark-variant
+  - mpifileutils ~xattr
+  - nccmp
+  - nco
+  - netlib-scalapack
+  - omega-h
+  - openmpi
+  - openpmd-api
+  - papi
+  - papyrus
+  - pdt
+  - petsc
+  - plumed
+  - precice
+  - pumi
+  - py-h5py +mpi
+  - py-h5py ~mpi
+  - py-libensemble +mpi +nlopt
+  - py-petsc4py
+  - qthreads scheduler=distrib
+  - raja
+  - slate ~cuda
+  - slepc
+  - stc
+  - sundials
+  - superlu
+  - superlu-dist
+  - swig
+  - swig@4.0.2-fortran
+  - sz3
+  - tasmanian
+  - tau +mpi +python
+  - trilinos@13.0.1 +belos +ifpack2 +stokhos
+  - turbine
+  - umap
+  - umpire
+  - veloc
+  - wannier90
+
+  # - alquimia            # pflotran: petsc-3.19.4-c6pmpdtpzarytxo434zf76jqdkhdyn37/lib/petsc/conf/rules:169: material_aux.o] Error 1: fortran errors
+  # - amrex               # disabled temporarily pending resolution of unreproducible CI failure
+  # - archer              # subsumed by llvm +omp_tsan
+  # - axom                # axom: CMake Error at axom/sidre/cmake_install.cmake:154 (file): file INSTALL cannot find "/tmp/gitlab-runner-2/spack-stage/spack-stage-axom-0.8.1-jvol6riu34vuyqvrd5ft2gyhrxdqvf63/spack-build-jvol6ri/lib/fortran/axom_spio.mod": No such file or directory.
+  # - bricks              # bricks: clang-15: error: clang frontend command failed with exit code 134 (use -v to see invocation)
+  # - dealii              # llvm@14.0.6: ?; intel-tbb@2020.3: clang-15: error: unknown argument: '-flifetime-dse=1'; assimp@5.2.5: clang-15: error: clang frontend command failed with exit code 134 (use -v to see invocation)
+  # - dyninst             # requires %gcc
+  # - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc ~visit +vtkm +zfp ^hdf5@1.14 # llvm@14.0.6: ?;
+  # - exaworks            # rust: ld.lld: error: relocation R_X86_64_32 cannot be used against local symbol; recompile with -fPIC'; defined in /opt/cray/pe/cce/15.0.1/cce/x86_64/lib/no_mmap.o, referenced by /opt/cray/pe/cce/15.0.1/cce/x86_64/lib/no_mmap.o:(__no_mmap_for_malloc)
+  # - gasnet              # configure error: User requested --enable-ofi but I don't know how to build ofi programs for your system
+  # - gptune              # py-scipy: meson.build:82:0: ERROR: Unknown compiler(s): [['/home/gitlab-runner-3/builds/dWfnZWPh/0/spack/spack/lib/spack/env/cce/ftn']]
+  # - hpctoolkit          # dyninst requires %gcc
+  # - nrm                 # py-scipy: meson.build:82:0: ERROR: Unknown compiler(s): [['/home/gitlab-runner-3/builds/dWfnZWPh/0/spack/spack/lib/spack/env/cce/ftn']]
+  # - nvhpc               # requires %gcc
+  # - parsec ~cuda        # parsec: parsec/fortran/CMakeFiles/parsec_fortran.dir/parsecf.F90.o: ftn-2103 ftn: WARNING in command line. The -W extra option is not supported or invalid and will be ignored.
+  # - phist               # fortran_bindings/CMakeFiles/phist_fort.dir/phist_testing.F90.o: ftn-78 ftn: ERROR in command line. The -f option has an invalid argument, "no-math-errno".
+  # - plasma              # %cce conflict
+  # - py-jupyterhub       # rust: ld.lld: error: relocation R_X86_64_32 cannot be used against local symbol; recompile with -fPIC'; defined in /opt/cray/pe/cce/15.0.1/cce/x86_64/lib/no_mmap.o, referenced by /opt/cray/pe/cce/15.0.1/cce/x86_64/lib/no_mmap.o:(__no_mmap_for_malloc)
+  # - py-warpx            # py-scipy: meson.build:82:0: ERROR: Unknown compiler(s): [['/home/gitlab-runner-3/builds/dWfnZWPh/0/spack/spack/lib/spack/env/cce/ftn']]
+  # - quantum-espresso    # quantum-espresso: CMake Error at cmake/FindSCALAPACK.cmake:503 (message): A required library with SCALAPACK API not found.  Please specify library
+  # - scr                 # scr: make[2]: *** [examples/CMakeFiles/test_ckpt_F.dir/build.make:112: examples/test_ckpt_F] Error 1: /opt/cray/pe/cce/15.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: /opt/cray/pe/mpich/8.1.25/ofi/cray/10.0/lib/libmpi_cray.so: undefined reference to `PMI_Barrier'
+  # - strumpack ~slate    # strumpack: [test/CMakeFiles/test_HSS_seq.dir/build.make:117: test/test_HSS_seq] Error 1: ld.lld: error: undefined reference due to --no-allow-shlib-undefined: mpi_abort_
+  # - upcxx               # upcxx: configure error: User requested --enable-ofi but I don't know how to build ofi programs for your system
+  # - variorum            # variorum: /opt/cray/pe/cce/15.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: /opt/cray/pe/lib64/libpals.so.0: undefined reference to `json_array_append_new@@libjansson.so.4'
+  # - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu # openblas: ftn-2307 ftn: ERROR in command line: The "-m" option must be followed by 0, 1, 2, 3 or 4.; make[2]: *** [: spotrf2.o] Error 1; make[1]: *** [Makefile:27: lapacklib] Error 2; make: *** [Makefile:250: netlib] Error 2
+
+  cdash:
+    build-group: E4S Cray
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
new file mode 100644
index 00000000000000..c141cd9bf9233a
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
@@ -0,0 +1,175 @@
+spack:
+  include:
+  - $spack/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
+
+  view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  packages:
+    all:
+      require: '%gcc'
+      providers:
+        blas: [cray-libsci]
+        lapack: [cray-libsci]
+        mpi: [cray-mpich]
+        tbb: [intel-tbb]
+        scalapack: [netlib-scalapack]
+      target: [zen4]
+      variants: +mpi
+
+    tbb:
+      require: "intel-tbb"
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+    boost:
+      variants: +python +filesystem +iostreams +system
+    cuda:
+      version: [11.7.0]
+    elfutils:
+      variants: +bzip2 ~nls +xz
+      require: "%gcc"
+    hdf5:
+      variants: +fortran +hl +shared
+    libfabric:
+      variants: fabrics=sockets,tcp,udp,rxm
+    libunwind:
+      variants: +pic +xz
+    mpich:
+      variants: ~wrapperrpath
+    ncurses:
+      variants: +termlib
+    paraview:
+      # Don't build GUI support or GLX rendering for HPC/container deployments
+      require: "@5.11 ~qt+osmesa"
+    python:
+      version: [3.8.13]
+    trilinos:
+      require:
+      - one_of: [+amesos +amesos2 +anasazi +aztec +boost +epetra +epetraext +ifpack
+            +intrepid +intrepid2 +isorropia +kokkos +minitensor +nox +piro +phalanx
+            +rol +rythmos +sacado +stk +shards +stratimikos +tempus +tpetra
+            +trilinoscouplings +zoltan]
+      - one_of: [gotype=long_long, gotype=all]
+      - one_of: [~ml ~muelu ~zoltan2 ~teko, +ml +muelu +zoltan2 +teko]
+    xz:
+      variants: +pic
+    mesa:
+      version: [21.3.8]
+    unzip:
+      require: "%gcc"
+
+  specs:
+  # CPU
+  - adios
+  - aml
+  - arborx
+  - argobots
+  - bolt
+  - butterflypack
+  - boost +python +filesystem +iostreams +system
+  - cabana
+  - chai ~benchmarks ~tests
+  - conduit
+  - datatransferkit
+  - flecsi
+  - fortrilinos
+  - ginkgo
+  - globalarrays
+  - gmp
+  - gotcha
+  - h5bench
+  - hdf5-vol-async
+  - hdf5-vol-cache
+  - hdf5-vol-log
+  - heffte +fftw
+  - hypre
+  - kokkos +openmp
+  - kokkos-kernels +openmp
+  - lammps
+  - legion
+  - libnrm
+  - libquo
+  - libunwind
+  - mercury
+  - metall
+  - mfem
+  - mgard +serial +openmp +timing +unstructured ~cuda
+  - mpark-variant
+  - mpifileutils ~xattr
+  - nccmp
+  - nco
+  - netlib-scalapack
+  - omega-h
+  - openmpi
+  - openpmd-api
+  - papi
+  - papyrus
+  - pdt
+  - pumi
+  - qthreads scheduler=distrib
+  - raja
+  - slate ~cuda
+  - stc
+  - sundials
+  - superlu
+  - superlu-dist
+  - swig
+  - swig@4.0.2-fortran
+  - sz3
+  - tasmanian
+  - trilinos +belos +ifpack2 +stokhos
+  - turbine
+  - umap
+  - umpire
+  - veloc
+  - wannier90
+
+  # ERRORS
+  # - caliper                     # caliper: ModuleNotFoundError: No module named 'math'; src/mpi/services/mpiwrap/CMakeFiles/caliper-mpiwrap.dir/build.make:77: src/mpi/services/mpiwrap/Wrapper.cpp] Error 1
+  # - charliecloud                # python: Could not find platform dependent libraries 
+  # - flit                        # python: Could not find platform dependent libraries 
+  # - flux-core                   # python: Could not find platform dependent libraries 
+  # - hpx max_cpu_count=512 networking=mpi # python: Could not find platform dependent libraries 
+  # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +mgard # python: Could not find platform dependent libraries 
+  # - petsc                       # petsc: SyntaxError: (unicode error) \N escapes not supported (can't load unicodedata module)
+  # - plumed                      # python: Could not find platform dependent libraries 
+  # - precice                     # petsc: SyntaxError: (unicode error) \N escapes not supported (can't load unicodedata module)
+  # - py-h5py +mpi                # python: Could not find platform dependent libraries 
+  # - py-h5py ~mpi                # python: Could not find platform dependent libraries 
+  # - py-libensemble +mpi +nlopt  # python: Could not find platform dependent libraries 
+  # - py-petsc4py                 # python: Could not find platform dependent libraries 
+  # - slepc                       # petsc: SyntaxError: (unicode error) \N escapes not supported (can't load unicodedata module)
+  # - tau +mpi +python            # tau: ERROR: Cannot find python library (libpython*.[so|dylib]
+
+  # HOLDING THESE BACK UNTIL CRAY SLES CAPACITY IS EXPANDED AT UO
+  # - alquimia
+  # - amrex
+  # - archer
+  # - axom
+  # - bricks
+  # - dealii
+  # - dyninst
+  # - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc ~visit +vtkm +zfp ^hdf5@1.14 # llvm@14.0.6: ?;
+  # - exaworks
+  # - gasnet
+  # - gptune
+  # - hpctoolkit
+  # - nrm
+  # - nvhpc
+  # - parsec ~cuda
+  # - phist
+  # - plasma
+  # - py-jupyterhub
+  # - py-warpx
+  # - quantum-espresso
+  # - scr
+  # - strumpack ~slate
+  # - upcxx
+  # - variorum
+  # - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu
+
+  cdash:
+    build-group: E4S Cray SLES
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml
deleted file mode 100644
index bd74bc09255b3b..00000000000000
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml
+++ /dev/null
@@ -1,100 +0,0 @@
-spack:
-  view: false
-
-  concretizer:
-    reuse: false
-    unify: false
-
-  compilers:
-  - compiler:
-      spec: cce@15.0.1
-      paths:
-        cc: cc
-        cxx: CC
-        f77: ftn
-        fc: ftn
-      flags: {}
-      operating_system: rhel8
-      target: any
-      modules:
-      - PrgEnv-cray/8.3.3
-      - cce/15.0.1
-      environment:
-        set:
-          MACHTYPE: x86_64
-  - compiler:
-      spec: gcc@11.2.0
-      paths:
-        cc: gcc
-        cxx: g++
-        f77: gfortran
-        fc: gfortran
-      flags: {}
-      operating_system: rhel8
-      target: any
-      modules:
-      - PrgEnv-gnu
-      - gcc/11.2.0
-      environment: {}
-
-  packages:
-    all:
-      require: '%cce@15.0.1'
-      compiler: [cce@15.0.1]
-      providers:
-        blas: [cray-libsci]
-        lapack: [cray-libsci]
-        mpi: [cray-mpich]
-        tbb: [intel-tbb]
-        scalapack: [netlib-scalapack]
-      target: [zen4]
-      variants: +mpi
-
-    binutils:
-      variants: +ld +gold +headers +libiberty ~nls
-    hdf5:
-      variants: +fortran +hl +shared
-    libunwind:
-      variants: +pic +xz
-    ncurses:
-      require: '@6.3 +termlib'
-    openblas:
-      require: '@0.3.20'
-      variants: threads=openmp
-    xz:
-      variants: +pic
-    elfutils:
-      variants: +bzip2 ~nls +xz
-      require: '%gcc'
-
-    # EXTERNALS
-    cray-mpich:
-      buildable: false
-      externals:
-      - spec: cray-mpich@8.1.25 %cce@15.0.1
-        prefix: /opt/cray/pe/mpich/8.1.25/ofi/cray/10.0
-        modules:
-        - cray-mpich/8.1.25
-    cray-libsci:
-      buildable: false
-      externals:
-      - spec: cray-libsci@23.02.1.1 %cce@15.0.1
-        prefix: /opt/cray/pe/libsci/23.02.1.1/CRAY/9.0/x86_64/
-        modules:
-        - cray-libsci/23.02.1.1
-
-  specs:
-  - butterflypack
-  - hypre
-  - kokkos
-  - kokkos-kernels
-  - petsc
-  - raja
-  - slepc
-  - superlu-dist
-  - tau
-
-  mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray" }
-
-  cdash:
-    build-group: E4S Cray
\ No newline at end of file
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
new file mode 100644
index 00000000000000..82a1f07c8d4186
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse_v1/spack.yaml
@@ -0,0 +1,335 @@
+spack:
+  view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  packages:
+    all:
+      require: '%gcc@11.4.0 target=neoverse_v1'
+      providers:
+        blas: [openblas]
+        mpi: [mpich]
+      variants: +mpi
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+    elfutils:
+      variants: +bzip2 ~nls +xz
+    hdf5:
+      variants: +fortran +hl +shared
+    libfabric:
+      variants: fabrics=sockets,tcp,udp,rxm
+    libunwind:
+      variants: +pic +xz
+    openblas:
+      variants: threads=openmp
+    trilinos:
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    xz:
+      variants: +pic
+    mesa:
+      version: [21.3.8]
+    mpi:
+      require: mpich
+    mpich:
+      require: '~wrapperrpath ~hwloc'
+    ncurses:
+      require: '@6.3 +termlib'
+    tbb:
+      require: intel-tbb
+    boost:
+      version: [1.79.0]
+      variants: +atomic +chrono +container +date_time +exception +filesystem +graph
+        +iostreams +locale +log +math +mpi +multithreaded +program_options +random
+        +regex +serialization +shared +signals +stacktrace +system +test +thread +timer
+        cxxstd=17 visibility=global
+    libffi:
+      require: "@3.4.4"
+    vtk-m:
+      require: "+examples"
+    cuda:
+      version: [11.8.0]
+
+  specs:
+  # CPU
+  - adios
+  - alquimia
+  - aml
+  - amrex
+  - arborx
+  - argobots
+  - ascent # ecp dav
+  - axom
+  - bolt
+  - boost
+  - butterflypack
+  - cabana
+  - caliper
+  - chai ~benchmarks ~tests
+  - charliecloud
+  - conduit
+  - datatransferkit
+  - dyninst
+  - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc ~visit +vtkm +zfp  # +visit: ?
+  - exaworks
+  - flecsi
+  - flit
+  - flux-core
+  - fortrilinos
+  - gasnet
+  - ginkgo
+  - globalarrays
+  - gmp
+  - gotcha
+  - gptune ~mpispawn
+  - gromacs +cp2k ^cp2k build_system=cmake
+  - h5bench
+  - hdf5-vol-async
+  - hdf5-vol-cache
+  - hdf5-vol-log
+  - heffte +fftw
+  - hpctoolkit
+  - hpx networking=mpi
+  - hypre
+  - kokkos +openmp
+  - kokkos-kernels +openmp
+  - lammps
+  - lbann
+  - legion
+  - libnrm
+  - libquo
+  - libunwind
+  - loki
+  - mercury
+  - metall
+  - mfem
+  - mgard +serial +openmp +timing +unstructured ~cuda
+  - mpark-variant
+  - mpifileutils ~xattr
+  - nccmp
+  - nco
+  - netlib-scalapack
+  - nrm
+  - nvhpc
+  - omega-h
+  - openfoam
+  - openmpi
+  - openpmd-api
+  - papi
+  - papyrus
+  - parsec ~cuda
+  - pdt
+  - petsc
+  - phist
+  - plasma
+  - plumed
+  - precice
+  - pruners-ninja
+  - pumi
+  - py-h5py
+  - py-jupyterhub
+  - py-libensemble
+  - py-petsc4py
+  - py-warpx
+  - qthreads scheduler=distrib
+  - quantum-espresso
+  - raja
+  - rempi
+  - scr
+  - slate ~cuda
+  - slepc
+  - stc
+  - strumpack ~slate
+  - sundials
+  - superlu
+  - superlu-dist
+  - swig@4.0.2-fortran
+  - sz3
+  - tasmanian
+  - tau +mpi +python +syscall
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+  - turbine
+  - umap
+  - umpire
+  - upcxx
+  - wannier90
+  - xyce +mpi +shared +pymi +pymi_static_tpls
+  # INCLUDED IN ECP DAV CPU
+  - adios2
+  - darshan-runtime
+  - darshan-util
+  - faodel
+  - hdf5
+  - libcatalyst
+  - parallel-netcdf
+  - paraview
+  - py-cinemasci
+  - sz
+  - unifyfs
+  - veloc
+  # - visit                         # silo: https://github.com/spack/spack/issues/39538
+  - vtk-m
+  - zfp
+  # --
+  # - archer                        # part of llvm +omp_tsan
+  # - bricks ~cuda                  # not respecting target=aarch64?
+  # - dealii                        # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - geopm                         # geopm: https://github.com/spack/spack/issues/38795
+  # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp # py-numcodecs@0.7.3: gcc: error: unrecognized command-line option '-mno-sse2'
+  # - variorum                      # variorum: https://github.com/spack/spack/issues/38786
+
+  # CUDA NOARCH
+  - flux-core +cuda
+  - hpctoolkit +cuda
+  - papi +cuda
+  - tau +mpi +cuda +syscall
+  # --
+  # - bricks +cuda                  # not respecting target=aarch64?
+  # - legion +cuda                  # legion: needs NVIDIA driver
+
+  # CUDA 75
+  - amrex +cuda cuda_arch=75
+  - arborx +cuda cuda_arch=75 ^kokkos +wrapper
+  - cabana +cuda cuda_arch=75 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=75
+  - caliper +cuda cuda_arch=75
+  - chai ~benchmarks ~tests +cuda cuda_arch=75 ^umpire ~shared
+  - flecsi +cuda cuda_arch=75
+  - ginkgo +cuda cuda_arch=75
+  - heffte +cuda cuda_arch=75
+  - hpx +cuda cuda_arch=75
+  - hypre +cuda cuda_arch=75
+  - kokkos +wrapper +cuda cuda_arch=75
+  - kokkos-kernels +cuda cuda_arch=75 ^kokkos +wrapper +cuda cuda_arch=75
+  - magma +cuda cuda_arch=75
+  - mfem +cuda cuda_arch=75
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=75
+  - omega-h +cuda cuda_arch=75
+  - parsec +cuda cuda_arch=75
+  - petsc +cuda cuda_arch=75
+  - raja +cuda cuda_arch=75
+  - slate +cuda cuda_arch=75
+  - strumpack ~slate +cuda cuda_arch=75
+  - sundials +cuda cuda_arch=75
+  - superlu-dist +cuda cuda_arch=75
+  - tasmanian +cuda cuda_arch=75
+  - trilinos +cuda cuda_arch=75
+  - umpire ~shared +cuda cuda_arch=75
+  # INCLUDED IN ECP DAV CUDA
+  - adios2 +cuda cuda_arch=75
+  - paraview +cuda cuda_arch=75
+  - vtk-m +cuda cuda_arch=75
+  - zfp +cuda cuda_arch=75
+  # --
+  # - ascent +cuda cuda_arch=75     # ascent: https://github.com/spack/spack/issues/38045
+  # - axom +cuda cuda_arch=75       # axom: https://github.com/spack/spack/issues/29520
+  # - cusz +cuda cuda_arch=75       # cusz: https://github.com/spack/spack/issues/38787
+  # - dealii +cuda cuda_arch=75     # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=75  # embree: https://github.com/spack/spack/issues/39534
+  # - lammps +cuda cuda_arch=75     # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=75      # lbann: https://github.com/spack/spack/issues/38788
+  # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf ~cusz +mgard +cuda cuda_arch=75 # libpressio: CMake Error at CMakeLists.txt:498 (find_library): Could not find CUFile_LIBRARY using the following names: cufile ; +cusz: https://github.com/spack/spack/issues/38787
+  # - py-torch +cuda cuda_arch=75   # skipped, installed by other means
+  # - slepc +cuda cuda_arch=75      # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - upcxx +cuda cuda_arch=75      # upcxx: needs NVIDIA driver
+
+  # CUDA 80
+  - amrex +cuda cuda_arch=80
+  - arborx +cuda cuda_arch=80 ^kokkos +wrapper
+  - cabana +cuda cuda_arch=80 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=80
+  - caliper +cuda cuda_arch=80
+  - chai ~benchmarks ~tests +cuda cuda_arch=80 ^umpire ~shared
+  - flecsi +cuda cuda_arch=80
+  - ginkgo +cuda cuda_arch=80
+  - heffte +cuda cuda_arch=80
+  - hpx +cuda cuda_arch=80
+  - hypre +cuda cuda_arch=80
+  - kokkos +wrapper +cuda cuda_arch=80
+  - kokkos-kernels +cuda cuda_arch=80 ^kokkos +wrapper +cuda cuda_arch=80
+  - magma +cuda cuda_arch=80
+  - mfem +cuda cuda_arch=80
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=80
+  - omega-h +cuda cuda_arch=80
+  - parsec +cuda cuda_arch=80
+  - petsc +cuda cuda_arch=80
+  - raja +cuda cuda_arch=80
+  - slate +cuda cuda_arch=80
+  - strumpack ~slate +cuda cuda_arch=80
+  - sundials +cuda cuda_arch=80
+  - superlu-dist +cuda cuda_arch=80
+  - tasmanian +cuda cuda_arch=80
+  - trilinos +cuda cuda_arch=80
+  - umpire ~shared +cuda cuda_arch=80
+  # INCLUDED IN ECP DAV CUDA
+  - adios2 +cuda cuda_arch=80
+  - paraview +cuda cuda_arch=80
+  - vtk-m +cuda cuda_arch=80
+  - zfp +cuda cuda_arch=80
+  # --
+  # - ascent +cuda cuda_arch=80     # ascent: https://github.com/spack/spack/issues/38045
+  # - axom +cuda cuda_arch=80       # axom: https://github.com/spack/spack/issues/29520
+  # - cusz +cuda cuda_arch=80       # cusz: https://github.com/spack/spack/issues/38787
+  # - dealii +cuda cuda_arch=80     # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=80 # embree: https://github.com/spack/spack/issues/39534
+  # - lammps +cuda cuda_arch=80     # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=80      # lbann: https://github.com/spack/spack/issues/38788
+  # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf ~cusz +mgard +cuda cuda_arch=80 # libpressio: CMake Error at CMakeLists.txt:498 (find_library): Could not find CUFile_LIBRARY using the following names: cufile ; +cusz: https://github.com/spack/spack/issues/38787
+  # - py-torch +cuda cuda_arch=80   # skipped, installed by other means
+  # - slepc +cuda cuda_arch=80      # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - upcxx +cuda cuda_arch=80      # upcxx: needs NVIDIA driver
+
+  # CUDA 90
+  - amrex +cuda cuda_arch=90
+  - arborx +cuda cuda_arch=90 ^kokkos +wrapper
+  - cabana +cuda cuda_arch=90  ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=90
+  - caliper +cuda cuda_arch=90
+  - chai ~benchmarks ~tests +cuda cuda_arch=90 ^umpire ~shared
+  - flecsi +cuda cuda_arch=90
+  - ginkgo +cuda cuda_arch=90
+  - heffte +cuda cuda_arch=90
+  - hpx +cuda cuda_arch=90
+  - kokkos +wrapper +cuda cuda_arch=90
+  - kokkos-kernels +cuda cuda_arch=90 ^kokkos +wrapper +cuda cuda_arch=90
+  - magma +cuda cuda_arch=90
+  - mfem +cuda cuda_arch=90
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=90
+  - parsec +cuda cuda_arch=90
+  - petsc +cuda cuda_arch=90
+  - raja +cuda cuda_arch=90
+  - slate +cuda cuda_arch=90
+  - strumpack ~slate +cuda cuda_arch=90
+  - sundials +cuda cuda_arch=90
+  - superlu-dist +cuda cuda_arch=90
+  - trilinos +cuda cuda_arch=90
+  - umpire ~shared +cuda cuda_arch=90
+  # INCLUDED IN ECP DAV CUDA
+  - adios2 +cuda cuda_arch=90
+  # - paraview +cuda cuda_arch=90   # paraview: InstallError: Incompatible cuda_arch=90
+  - vtk-m +cuda cuda_arch=90
+  - zfp +cuda cuda_arch=90
+  # --
+  # - ascent +cuda cuda_arch=90     # ascent: https://github.com/spack/spack/issues/38045
+  # - axom +cuda cuda_arch=90       # axom: https://github.com/spack/spack/issues/29520
+  # - cusz +cuda cuda_arch=90       # cusz: https://github.com/spack/spack/issues/38787
+  # - dealii +cuda cuda_arch=90     # dealii: https://github.com/spack/spack/issues/39532
+  # - ecp-data-vis-sdk +adios2 +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=90 # embree: https://github.com/spack/spack/issues/39534
+  # - hypre +cuda cuda_arch=90      # concretizer: hypre +cuda requires cuda@:11, but cuda_arch=90 requires cuda@12:
+  # - lammps +cuda cuda_arch=90     # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=90      # concretizer: Cannot select a single "version" for package "lbann"
+  # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf ~cusz +mgard +cuda cuda_arch=90 # libpressio: CMake Error at CMakeLists.txt:498 (find_library): Could not find CUFile_LIBRARY using the following names: cufile ; +cusz: https://github.com/spack/spack/issues/38787
+  # - omega-h +cuda cuda_arch=90    # omega-h: https://github.com/spack/spack/issues/39535
+  # - py-torch +cuda cuda_arch=90   # skipped, installed by other means
+  # - slepc +cuda cuda_arch=90      # slepc: make[1]: *** internal error: invalid --jobserver-auth string 'fifo:/tmp/GMfifo1313'.
+  # - tasmanian +cuda cuda_arch=90  # tasmanian: conflicts with cuda@12
+  # - upcxx +cuda cuda_arch=90      # upcxx: needs NVIDIA driver
+
+  ci:
+    pipeline-gen:
+    - build-job:
+        image: "ghcr.io/spack/ubuntu20.04-runner-arm64-gcc-11.4:2023.08.01"
+
+  cdash:
+    build-group: E4S ARM Neoverse V1
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
index 461560d592255f..8c872240f9e608 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
@@ -1,5 +1,4 @@
 spack:
-
   view: false
 
   concretizer:
@@ -8,15 +7,17 @@ spack:
 
   packages:
     all:
-      require: '%oneapi'
+      require: '%oneapi target=x86_64_v3'
       providers:
         blas: [openblas]
         mpi: [mpich]
         tbb: [intel-tbb]
-      target: [x86_64]
       variants: +mpi
     elfutils:
       variants: +bzip2 ~nls +xz
+    hdf5:
+      require: "%gcc"
+      variants: +fortran +hl +shared
     libfabric:
       variants: fabrics=sockets,tcp,udp,rxm
     libunwind:
@@ -34,15 +35,12 @@ spack:
       variants: +pic
     mesa:
       version: [21.3.8]
-    hdf5:
-      require: "%gcc"
-      variants: +fortran +hl +shared
     mpi:
-      require: mpich
+      require: 'mpich@4:'
     mpich:
-      require: '@4.1.1 ~wrapperrpath ~hwloc'
+      require: '~wrapperrpath ~hwloc'
     py-cryptography:
-      require: '@38.0'
+      require: '@38.0.1'
     unzip:
       require: '%gcc'
     binutils:
@@ -60,40 +58,12 @@ spack:
       require: '%gcc'
     openssh:
       require: '%gcc'
-    bison:
-      require: '%gcc'
     libffi:
       require: "@3.4.4"
     dyninst:
       require: "%gcc"
-
-  compilers:
-  - compiler:
-      spec: oneapi@2023.1.0
-      paths:
-        cc: /opt/intel/oneapi/compiler/2023.1.0/linux/bin/icx
-        cxx: /opt/intel/oneapi/compiler/2023.1.0/linux/bin/icpx
-        f77: /opt/intel/oneapi/compiler/2023.1.0/linux/bin/ifx
-        fc: /opt/intel/oneapi/compiler/2023.1.0/linux/bin/ifx
-      flags: {}
-      operating_system: ubuntu20.04
-      target: x86_64
-      modules: []
-      environment: {}
-      extra_rpaths: []
-  - compiler:
-      spec: gcc@11.1.0
-      paths:
-        cc: /usr/bin/gcc
-        cxx: /usr/bin/g++
-        f77: /usr/bin/gfortran
-        fc: /usr/bin/gfortran
-      flags: {}
-      operating_system: ubuntu20.04
-      target: x86_64
-      modules: []
-      environment: {}
-      extra_rpaths: []
+    bison:
+      require: '%gcc'
 
   specs:
   # CPU
@@ -101,7 +71,6 @@ spack:
   - aml
   - amrex
   - arborx
-  - archer
   - argobots
   - axom
   - bolt
@@ -114,17 +83,21 @@ spack:
   - charliecloud
   - conduit
   - datatransferkit
+  - drishti
   - exaworks
   - flecsi
   - flit
   - flux-core
   - fortrilinos
   - gasnet
+  - ginkgo
   - globalarrays
   - gmp
   - gotcha
+  - gptune ~mpispawn
   - h5bench
   - hdf5-vol-async
+  - hdf5-vol-cache
   - hdf5-vol-log
   - heffte +fftw
   - hpx networking=mpi
@@ -134,22 +107,22 @@ spack:
   - lammps
   - lbann
   - legion
-  - libnrm 
+  - libnrm
+  - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp
   - libquo
   - libunwind
   - loki
   - mercury
   - metall
   - mfem
-  - mgard +serial +openmp +timing +unstructured ~cuda
   - mpark-variant
   - mpifileutils ~xattr
   - nccmp
   - nco
   - netlib-scalapack
+  - nrm
   - omega-h
   - openmpi
-  - openpmd-api
   - papi
   - papyrus
   - parsec ~cuda
@@ -159,14 +132,18 @@ spack:
   - plasma
   - plumed
   - precice
+  - pruners-ninja
   - pumi
   - py-h5py
+  - py-jupyterhub
   - py-libensemble
   - py-petsc4py
+  - py-warpx
   - qthreads scheduler=distrib
   - quantum-espresso
   - raja
   - rempi
+  - scr
   - slate ~cuda
   - slepc
   - stc
@@ -174,78 +151,67 @@ spack:
   - sundials
   - superlu
   - superlu-dist
-  - swig@4.0.2-fortran
   - sz3
   - tasmanian
-  - trilinos@13.0.1 +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+  - tau +mpi +python +syscall
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
   - turbine
   - umap
   - umpire
   - variorum
   - wannier90
+  - xyce +mpi +shared +pymi +pymi_static_tpls
   # INCLUDED IN ECP DAV CPU
-  # - adios2
-  # - ascent
-  # - darshan-runtime
-  # - darshan-util
-  # - faodel
-  # - hdf5
-  # - libcatalyst
-  # - parallel-netcdf
-  # - paraview
-  # - py-cinemasci
-  # - sz
-  # - unifyfs
-  # - veloc
-  # - visit
-  # - vtk-m ~openmp # https://github.com/spack/spack/issues/31830
-  # - zfp
+  - adios2                # mgard:  mgard.tpp:63:48: error: non-constant-expression cannot be narrowed from type 'int' to 'unsigned long' in initializer list [-Wc++11-narrowing]
+  - ascent
+  - darshan-runtime
+  - darshan-util
+  - faodel
+  - hdf5
+  - libcatalyst
+  - parallel-netcdf
+  # - paraview            # paraview: VTK/ThirdParty/cgns/vtkcgns/src/adfh/ADFH.c:2002:23: error: incompatible function pointer types passing 'herr_t (hid_t, const char *, const H5L_info1_t *, void *)' (aka 'int (long, const char *, const H5L_info1_t *, void *)') to parameter of type 'H5L_iterate2_t' (aka 'int (*)(long, const char *,const H5L_info2_t *, void *)') [-Wincompatible-function-pointer-types]
+  - py-cinemasci
+  - sz
+  - unifyfs
+  - veloc
+  # - visit               # silo: https://github.com/spack/spack/issues/39538
+  - vtk-m ~openmp         # https://github.com/spack/spack/issues/31830
+  - zfp
   # --
-  # - alquimia          # pflotran: pflotran/hdf5_aux.F90(5): error #7013: This module file was not generated by any release of this compiler.   [HDF5]
-  # - dealii            # intel-tbb: icpx: error: unknown argument: '-flifetime-dse=1'
-  # - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc +visit +vtkm +zfp # sz: hdf5-filter/H5Z-SZ/src/H5Z_SZ.c:24:9: error: call to undeclared function 'gettimeofday'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
-  # - geopm             # geopm: In file included from src/ProfileTable.cpp:34: ./src/ProfileTable.hpp:79:45: error: no type named 'string' in namespace 'std'
-  # - ginkgo            # ginkgo: icpx: error: clang frontend command failed with exit code 139 (use -v to see invocation)
-  # - gptune ~mpispawn  # py-scipy: for_main.c:(.text+0x19): undefined reference to `MAIN__'
-  # - hdf5-vol-cache    # /H5VLcache_ext.c:580:9: error: incompatible function pointer types initializing 'herr_t (*)(const void *, uint64_t *)' (aka 'int (*)(const void *, unsigned long *)') with an expression of type 'herr_t (const void *, unsigned int *)' (aka 'int (const void *, unsigned int *)') [-Wincompatible-function-pointer-types]
-  # - hpctoolkit        # intel-tbb: icpx: error: unknown argument: '-flifetime-dse=1'
-  # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp # py-numcodecs: c-blosc/internal-complibs/zlib-1.2.8/gzread.c:30:15: error: call to undeclared function 'read'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
-  # - nrm               # py-scipy: for_main.c:(.text+0x19): undefined reference to `MAIN__'
-  # - openfoam          # adios2: patch failed
-  # - pruners-ninja     # pruners-ninja: ninja_test_pingpong.c:79:5: error: call to undeclared library function 'memset' with type 'void *(void *, int, unsigned long)'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
-  # - py-jupyterhub     # py-ruamel-yaml-clib: setuptools/dist.py:287: SetuptoolsDeprecationWarning: The namespace_packages parameter is deprecated, consider using implicit namespaces instead (PEP 420). See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages
-  # - py-warpx ^warpx dims=2 # py-scipy: for_main.c:(.text+0x19): undefined reference to `MAIN__'
-  # - py-warpx ^warpx dims=3 # py-scipy: for_main.c:(.text+0x19): undefined reference to `MAIN__'
-  # - py-warpx ^warpx dims=rz # py-scipy: for_main.c:(.text+0x19): undefined reference to `MAIN__'  
-  # - scr               # libyogrt: configure: error: slurm is not in specified location!
-  # - tau +mpi +python  # tau: x86_64/lib/Makefile.tau-icpx-papi-mpi-pthread-python-pdt: No such file or directory
-  # - upcxx             # upcxx: /opt/intel/oneapi/mpi/2021.9.0//libfabric/bin/fi_info: error while loading shared libraries: libfabric.so.1: cannot open shared object file: No such file or directory
-  # - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu # cmake/tps.cmake:220 (message): Unable to compile against Trilinos.  It is possible Trilinos was not properly configured, or the environment has changed since Trilinos was installed.  See the CMake log files for more information.
+  # - alquimia            # pflotran: https://github.com/spack/spack/issues/39474
+  # - archer              # subsumed under llvm +libomp_tsan
+  # - dealii              # dealii: https://github.com/spack/spack/issues/39482
+  # - dxt-explorer        # r: https://github.com/spack/spack/issues/40257
+  # - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc +visit +vtkm +zfp # embree: CMake Error at CMakeLists.txt:215 (MESSAGE): Unsupported compiler: IntelLLVM; qt: qtbase/src/corelib/global/qendian.h:333:54: error: incomplete type 'std::numeric_limits' used in nested name specifier
+  # - geopm               # geopm issue: https://github.com/spack/spack/issues/38795
+  # - hpctoolkit          # dyninst@12.3.0%gcc: /usr/bin/ld: libiberty/./d-demangle.c:142: undefined reference to `_intel_fast_memcpy'; can't mix intel-tbb@%oneapi with dyninst%gcc
+  # - mgard +serial +openmp +timing +unstructured ~cuda # mgard: mgard.tpp:63:48: error: non-constant-expression cannot be narrowed from type 'int' to 'unsigned long' in initializer list [-Wc++11-narrowing]
+  # - openfoam            # cgal: https://github.com/spack/spack/issues/39481
+  # - openpmd-api         # mgard:  mgard.tpp:63:48: error: non-constant-expression cannot be narrowed from type 'int' to 'unsigned long' in initializer list [-Wc++11-narrowing]
+  # - swig@4.0.2-fortran  # ?
+  # - upcxx               # upcxx: /opt/intel/oneapi/mpi/2021.10.0//libfabric/bin/fi_info: error while loading shared libraries: libfabric.so.1: cannot open shared object file: No such file or directory
 
   # GPU
   - aml +ze
   - amrex +sycl
-  - arborx +sycl ^kokkos +sycl +openmp std=17 +tests +examples
-  - cabana +sycl ^kokkos +sycl +openmp std=17 +tests +examples
-  - kokkos +sycl +openmp std=17 +tests +examples
-  - kokkos-kernels build_type=Release %oneapi ^kokkos +sycl +openmp std=17 +tests +examples
+  - arborx +sycl ^kokkos +sycl +openmp cxxstd=17 +tests +examples
+  - cabana +sycl ^kokkos +sycl +openmp cxxstd=17 +tests +examples
+  - kokkos +sycl +openmp cxxstd=17 +tests +examples
+  - kokkos-kernels build_type=Release %oneapi ^kokkos +sycl +openmp cxxstd=17 +tests +examples
+  - slate +sycl
+  - sundials +sycl cxxstd=17 +examples-install
+  - tau +mpi +opencl +level_zero ~pdt +syscall   # tau: requires libdrm.so to be installed
   # --
   # - ginkgo +oneapi                    # InstallError: Ginkgo's oneAPI backend requires theDPC++ compiler as main CXX compiler.
-  # - hpctoolkit +level_zero            # intel-tbb: icpx: error: unknown argument: '-flifetime-dse=1'
-  # - sundials +sycl cxxstd=17          # sundials: include/sunmemory/sunmemory_sycl.h:20:10: fatal error: 'CL/sycl.hpp' file not found
-  # - tau +mpi +opencl +level_zero ~pdt # builds ok in container, but needs libdrm, will update container
-
-  # SKIPPED
-  # - nvhpc
-  # - dyninst  # only %gcc
-
+  # - hpctoolkit +level_zero            # dyninst@12.3.0%gcc: /usr/bin/ld: libiberty/./d-demangle.c:142: undefined reference to `_intel_fast_memcpy'; can't mix intel-tbb@%oneapi with dyninst%gcc
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-oneapi" }
+  - py-scipy
 
   ci:
     pipeline-gen:
     - build-job:
-        image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023.06.01
+        image: ghcr.io/spack/ubuntu20.04-runner-amd64-oneapi-2023.2.1:2023.08.01
 
   cdash:
-    build-group: E4S OneAPI
\ No newline at end of file
+    build-group: E4S OneAPI
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
index d138c63c35a9e1..511f48e7459408 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml
@@ -1,19 +1,20 @@
 spack:
   view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
   packages:
     all:
-      compiler: [gcc@11.1.0]
+      require: "%gcc@9.4.0 target=ppc64le"
+      compiler: [gcc@9.4.0]
       providers:
         blas: [openblas]
         mpi: [mpich]
-      target: [ppc64le]
       variants: +mpi cuda_arch=70
-    tbb:
-      require: intel-tbb
     binutils:
       variants: +ld +gold +headers +libiberty ~nls
-    cuda:
-      version: [11.7.0]
     elfutils:
       variants: +bzip2 ~nls +xz
     hdf5:
@@ -22,30 +23,34 @@ spack:
       variants: fabrics=sockets,tcp,udp,rxm
     libunwind:
       variants: +pic +xz
-    mpich:
-      variants: ~wrapperrpath
-    ncurses:
-      variants: +termlib
     openblas:
       variants: threads=openmp
-    paraview:
-      require: '@5.11 ~qt+osmesa'
     trilinos:
-      require:
-      - one_of: [+amesos +amesos2 +anasazi +aztec +boost +epetra +epetraext +ifpack
-            +intrepid +intrepid2 +isorropia +kokkos +minitensor +nox +piro +phalanx
-            +rol +rythmos +sacado +stk +shards +stratimikos +tempus +tpetra
-            +trilinoscouplings +zoltan]
-      - one_of: [gotype=long_long, gotype=all]
-      - one_of: [~ml ~muelu ~zoltan2 ~teko, +ml +muelu +zoltan2 +teko]
-      - one_of: [+superlu-dist, ~superlu-dist]
-      - one_of: [+shylu, ~shylu]
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
     xz:
       variants: +pic
     mesa:
       version: [21.3.8]
+    mpi:
+      require: mpich
+    mpich:
+      require: '~wrapperrpath ~hwloc'
+    ncurses:
+      require: '@6.3 +termlib'
     faodel:
-      require: ~tcmalloc # needed for ppc64le
+      require: "~tcmalloc"
+    tbb:
+      require: intel-tbb
+    libffi:
+      require: "@3.4.4"
+    vtk-m:
+      require: "+examples"
+    cuda:
+      require: "@11.4.4"
+
 
   specs:
   # CPU
@@ -57,6 +62,8 @@ spack:
   - argobots
   - axom
   - bolt
+  - boost
+  - bricks
   - butterflypack
   - cabana
   - caliper
@@ -64,8 +71,10 @@ spack:
   - charliecloud
   - conduit
   - datatransferkit
+  - drishti
+  - dxt-explorer
   - dyninst
-  - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 ~paraview +pnetcdf +sz +unifyfs +veloc ~visit +vtkm +zfp # +paraview fails: FAILED: VTK/Filters/Statistics/CMakeFiles/FiltersStatistics-objects.dir/vtkPCAStatistics.cxx.o: /tmp/ccgvkIk5.s: Assembler messages: /tmp/ccgvkIk5.s:260012: Error: invalid machine `power10'
+  # - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 ~paraview +pnetcdf +sz +unifyfs +veloc ~visit +vtkm +zfp # +visit: libext, libxkbfile, libxrender, libxt, silo (https://github.com/spack/spack/issues/39538), cairo
   - exaworks
   - flecsi
   - flit
@@ -77,21 +86,24 @@ spack:
   - gmp
   - gotcha
   - gptune
+  - gromacs +cp2k ^cp2k build_system=cmake
   - h5bench
   - hdf5-vol-async
   - hdf5-vol-cache
   - hdf5-vol-log
   - heffte +fftw
   - hpctoolkit
-  - hpx max_cpu_count=512 networking=mpi
+  - hpx networking=mpi
   - hypre
   - kokkos +openmp
   - kokkos-kernels +openmp
   - lammps
+  - lbann
   - legion
   - libnrm
   - libquo
   - libunwind
+  - loki
   - mercury
   - metall
   - mfem
@@ -104,20 +116,23 @@ spack:
   - nrm
   - nvhpc
   - omega-h
+  - openfoam
   - openmpi
   - openpmd-api
   - papi
   - papyrus
+  - paraview ~cuda ~rocm
   - parsec ~cuda
   - pdt
   - petsc
-  - phist
   - plasma
   - plumed
+  - precice
+  - pruners-ninja
   - pumi
   - py-h5py
   - py-jupyterhub
-  - py-libensemble +mpi +nlopt
+  - py-libensemble
   - py-petsc4py
   - py-warpx
   - qthreads scheduler=distrib
@@ -132,84 +147,101 @@ spack:
   - sundials
   - superlu
   - superlu-dist
-  - swig
   - swig@4.0.2-fortran
+  - sz3
   - tasmanian
-  - tau +mpi +python
-  - trilinos +belos +ifpack2 +stokhos
+  - tau +mpi +python                       # +syscall fails: https://github.com/spack/spack/pull/40830#issuecomment-1790799772; tau: has issue with `spack env depfile` build
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
   - turbine
   - umap
   - umpire
   - upcxx
   - wannier90
-  - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu
+  - xyce +mpi +shared +pymi +pymi_static_tpls
+  # INCLUDED IN ECP DAV CPU
+  - adios2
+  - ascent
+  - darshan-runtime
+  - darshan-util
+  - faodel
+  - hdf5
+  - libcatalyst
+  - parallel-netcdf
+  - paraview
+  - py-cinemasci
+  - sz
+  - unifyfs
+  - veloc
+  # - visit                                 # libext, libxkbfile, libxrender, libxt, silo (https://github.com/spack/spack/issues/39538), cairo
+  - vtk-m
+  - zfp
+  # --
+  # - archer                                # part of llvm +omp_tsan
+  # - dealii                                # fltk: https://github.com/spack/spack/issues/38791
+  # - geopm                                 # geopm: https://github.com/spack/spack/issues/38798
+  # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp # py-numcodecs: gcc: error: unrecognized command line option '-mno-sse2'; did you mean '-mno-isel'? gcc: error: unrecognized command line option '-mno-avx2'
+  # - phist +mpi                            # ghost@develop: gcc-9: error: unrecognized command line option '-march=native'; did you mean '-mcpu=native'?
+  # - variorum                              # variorum: https://github.com/spack/spack/issues/38786
 
-  # CUDA
-  - amrex +cuda
-  - arborx +cuda ^kokkos +wrapper
-  - cabana +cuda ^kokkos +wrapper +cuda_lambda +cuda
-  - caliper +cuda
-  - chai ~benchmarks ~tests +cuda ^umpire ~shared
-  - ecp-data-vis-sdk +cuda cuda_arch=70 +adios2 +hdf5 ~paraview +vtkm +zfp # +paraview fails:  FAILED: VTK/Filters/Statistics/CMakeFiles/FiltersStatistics-objects.dir/vtkPCAStatistics.cxx.o; /tmp/ccjmJhb6.s: Assembler messages: /tmp/ccjmJhb6.s:260012: Error: invalid machine `power10'
-  - flecsi +cuda
+  # CUDA NOARCH
+  - bricks +cuda
+  - cabana +cuda ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=70
   - flux-core +cuda
-  - ginkgo +cuda
-  - heffte +cuda
   - hpctoolkit +cuda
-  - hpx max_cpu_count=512 +cuda
-  - hypre +cuda
-  - kokkos +wrapper +cuda
-  - kokkos-kernels +cuda ^kokkos +wrapper +cuda +cuda_lambda
-  - magma +cuda
-  - mfem +cuda
-  - mgard +serial +openmp +timing +unstructured +cuda
-  - omega-h +cuda
   - papi +cuda
-  - petsc +cuda
-  - py-torch +cuda
-  - raja +cuda
-  - slate +cuda
-  - slepc +cuda
-  - strumpack ~slate +cuda
-  - sundials +cuda
-  - superlu-dist +cuda
-  - tasmanian +cuda
-  - tau +mpi +cuda
-  - "trilinos@13.4.0: +belos +ifpack2 +stokhos +cuda"
-  - umpire ~shared +cuda
-  - parsec +cuda
-
-  # CPU FAILURES
-  # - archer                  # llvm@8
-  # - bricks                  # bricks
-  # - geopm                   # geopm
-  # - hdf5-vol-daos           # hdf5-vol-daos: vhost/vhost_user.c:65:32: error: array size missing in 'vhost_message_handlers'
-  # - loki                    # loki
-  # - precice                 # precice
-  # - pruners-ninja           # pruners-ninja
-  # - variorum                # Intel/variorum_cpuid.c:11:5: error: impossible constraint in 'asm'
+  - tau +mpi +cuda                          # tau: has issue with `spack env depfile` build
   # --
-  # bricks: VSBrick-7pt.py-Scalar-8x8x8-1:30:3: error: 'vfloat512' was not declared in this scope
-  # fltk: /usr/bin/ld: ../lib/libfltk_png.a(pngrutil.o): in function `png_read_filter_row': pngrutil.c:(.text.png_read_filter_row+0x90): undefined reference to `png_init_filter_functions_vsx'
-  # geopm: libtool.m4: error: problem compiling CXX test program
-  # llvm@8: clang/lib/Lex/Lexer.cpp:2547:34: error: ISO C++ forbids declaration of 'type name' with no type [-fpermissive]
-  # loki: include/loki/SmallObj.h:462:57: error: ISO C++17 does not allow dynamic exception specifications
-  # precice: /tmp/ccYNMwgE.s: Assembler messages: /tmp/ccYNMwgE.s:278115: Error: invalid machine `power10'
-  # pruners-ninja: test/ninja_test_util.c:34: multiple definition of `a';
+  # - legion +cuda                          # legion: needs NVIDIA driver
 
-  # CUDA FAILURES
-  # - bricks +cuda              # bricks
-  # - dealii +cuda              # fltk
+  # CUDA 70
+  - amrex +cuda cuda_arch=70
+  - arborx +cuda cuda_arch=70 ^kokkos +wrapper
+  - caliper +cuda cuda_arch=70
+  - chai ~benchmarks ~tests +cuda cuda_arch=70 ^umpire ~shared
+  - ecp-data-vis-sdk ~rocm +adios2 ~ascent +hdf5 +vtkm +zfp ~paraview +cuda cuda_arch=70
+  - exago +mpi +python +raja +hiop ~rocm +cuda cuda_arch=70 ~ipopt ^hiop@1.0.0 ~sparse +mpi +raja ~rocm +cuda cuda_arch=70 #^raja@0.14.0
+  - flecsi +cuda cuda_arch=70
+  - ginkgo +cuda cuda_arch=70
+  - heffte +cuda cuda_arch=70
+  - hpx +cuda cuda_arch=70
+  - hypre +cuda cuda_arch=70
+  - kokkos +wrapper +cuda cuda_arch=70
+  - kokkos-kernels +cuda cuda_arch=70 ^kokkos +wrapper +cuda cuda_arch=70
+  - magma +cuda cuda_arch=70
+  - mfem +cuda cuda_arch=70
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=70
+  - omega-h +cuda cuda_arch=70
+  - parsec +cuda cuda_arch=70
+  - petsc +cuda cuda_arch=70
+  - raja +cuda cuda_arch=70
+  - slate +cuda cuda_arch=70
+  - slepc +cuda cuda_arch=70
+  - strumpack ~slate +cuda cuda_arch=70
+  - sundials +cuda cuda_arch=70
+  - superlu-dist +cuda cuda_arch=70
+  - tasmanian +cuda cuda_arch=70
+  - umpire ~shared +cuda cuda_arch=70
+  # INCLUDED IN ECP DAV CUDA
+  - adios2 +cuda cuda_arch=70
+  # - ascent +cuda cuda_arch=70             # ascent: https://github.com/spack/spack/issues/38045
+  - paraview +cuda cuda_arch=70
+  - vtk-m +cuda cuda_arch=70
+  - zfp +cuda cuda_arch=70
   # --
-  # bricks: VSBrick-7pt.py-Scalar-8x8x8-1:30:3: error: 'vfloat512' was not declared in this scope
-
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s-power" }
+  # - axom +cuda cuda_arch=70               # axom: https://github.com/spack/spack/issues/29520
+  # - cusz +cuda cuda_arch=70               # cusz: https://github.com/spack/spack/issues/38787
+  # - dealii +cuda cuda_arch=70             # fltk: https://github.com/spack/spack/issues/38791
+  # - lammps +cuda cuda_arch=70             # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=70              # lbann: https://github.com/spack/spack/issues/38788
+  # - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard +cuda cuda_arch=70 ^cusz +cuda cuda_arch=70 # depends_on("cuda@11.7.1:", when="+cuda")
+  # - py-torch +cuda cuda_arch=70           # skipped
+  # - trilinos +cuda cuda_arch=70           # trilinos: https://github.com/trilinos/Trilinos/issues/11630
+  # - upcxx +cuda cuda_arch=70              # upcxx: needs NVIDIA driver
 
   ci:
     pipeline-gen:
     - build-job:
-        image: ecpe4s/ubuntu20.04-runner-ppc64le:2023-01-01
+        image: ghcr.io/spack/ubuntu20.04-runner-ppc64-gcc-11.4:2023.08.01
 
   cdash:
     build-group: E4S Power
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
new file mode 100644
index 00000000000000..8f902aa6a8d4f4
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml
@@ -0,0 +1,333 @@
+spack:
+  view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
+  packages:
+    all:
+      require: '%gcc target=x86_64_v3'
+      providers:
+        blas: [openblas]
+        mpi: [mpich]
+      variants: +mpi
+    binutils:
+      variants: +ld +gold +headers +libiberty ~nls
+    elfutils:
+      variants: +bzip2 ~nls +xz
+    hdf5:
+      variants: +fortran +hl +shared
+    libfabric:
+      variants: fabrics=sockets,tcp,udp,rxm
+    libunwind:
+      variants: +pic +xz
+    openblas:
+      variants: threads=openmp
+    trilinos:
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    xz:
+      variants: +pic
+    mesa:
+      version: [21.3.8]
+    mpi:
+      require: mpich
+    mpich:
+      require: '~wrapperrpath ~hwloc'
+    ncurses:
+      require: '@6.3 +termlib'
+    tbb:
+      require: intel-tbb
+    boost:
+      version: [1.79.0]
+      variants: +atomic +chrono +container +date_time +exception +filesystem +graph
+        +iostreams +locale +log +math +mpi +multithreaded +program_options +random
+        +regex +serialization +shared +signals +stacktrace +system +test +thread +timer
+        cxxstd=17 visibility=global
+    libffi:
+      require: "@3.4.4"
+    vtk-m:
+      require: "+examples"
+    cuda:
+      version: [11.8.0]
+    paraview:
+      # Don't build GUI support or GLX rendering for HPC/container deployments
+      require: "@5.11 ~qt+osmesa"
+
+    # ROCm 5.4.3
+    comgr:
+      buildable: false
+      externals:
+      - spec: comgr@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hip-rocclr:
+      buildable: false
+      externals:
+      - spec: hip-rocclr@5.4.3
+        prefix: /opt/rocm-5.4.3/hip
+    hipblas:
+      buildable: false
+      externals:
+      - spec: hipblas@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hipcub:
+      buildable: false
+      externals:
+      - spec: hipcub@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hipfft:
+      buildable: false
+      externals:
+      - spec: hipfft@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hipsparse:
+      buildable: false
+      externals:
+      - spec: hipsparse@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    miopen-hip:
+      buildable: false
+      externals:
+      - spec: hip-rocclr@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    miopengemm:
+      buildable: false
+      externals:
+      - spec: miopengemm@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rccl:
+      buildable: false
+      externals:
+      - spec: rccl@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocblas:
+      buildable: false
+      externals:
+      - spec: rocblas@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocfft:
+      buildable: false
+      externals:
+      - spec: rocfft@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-clang-ocl:
+      buildable: false
+      externals:
+      - spec: rocm-clang-ocl@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-cmake:
+      buildable: false
+      externals:
+      - spec: rocm-cmake@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-dbgapi:
+      buildable: false
+      externals:
+      - spec: rocm-dbgapi@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-debug-agent:
+      buildable: false
+      externals:
+      - spec: rocm-debug-agent@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-device-libs:
+      buildable: false
+      externals:
+      - spec: rocm-device-libs@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-gdb:
+      buildable: false
+      externals:
+      - spec: rocm-gdb@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    rocm-opencl:
+      buildable: false
+      externals:
+      - spec: rocm-opencl@5.4.3
+        prefix: /opt/rocm-5.4.3/opencl
+    rocm-smi-lib:
+      buildable: false
+      externals:
+      - spec: rocm-smi-lib@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hip:
+      buildable: false
+      externals:
+      - spec: hip@5.4.3
+        prefix: /opt/rocm-5.4.3
+        extra_attributes:
+          compilers:
+            c: /opt/rocm-5.4.3/llvm/bin/clang++
+            c++: /opt/rocm-5.4.3/llvm/bin/clang++
+            hip: /opt/rocm-5.4.3/hip/bin/hipcc
+    hipify-clang:
+      buildable: false
+      externals:
+      - spec: hipify-clang@5.4.3
+        prefix: /opt/rocm-5.4.3
+    llvm-amdgpu:
+      buildable: false
+      externals:
+      - spec: llvm-amdgpu@5.4.3
+        prefix: /opt/rocm-5.4.3/llvm
+        extra_attributes:
+          compilers:
+            c: /opt/rocm-5.4.3/llvm/bin/clang++
+            cxx: /opt/rocm-5.4.3/llvm/bin/clang++
+    hsakmt-roct:
+      buildable: false
+      externals:
+      - spec: hsakmt-roct@5.4.3
+        prefix: /opt/rocm-5.4.3/
+    hsa-rocr-dev:
+      buildable: false
+      externals:
+      - spec: hsa-rocr-dev@5.4.3
+        prefix: /opt/rocm-5.4.3/
+        extra_atributes:
+          compilers:
+            c: /opt/rocm-5.4.3/llvm/bin/clang++
+            cxx: /opt/rocm-5.4.3/llvm/bin/clang++
+    roctracer-dev-api:
+      buildable: false
+      externals:
+      - spec: roctracer-dev-api@5.4.3
+        prefix: /opt/rocm-5.4.3
+    roctracer-dev:
+      buildable: false
+      externals:
+      - spec: roctracer-dev@4.5.3
+        prefix: /opt/rocm-5.4.3
+    rocprim:
+      buildable: false
+      externals:
+      - spec: rocprim@5.4.3
+        prefix: /opt/rocm-5.4.3
+    rocrand:
+      buildable: false
+      externals:
+      - spec: rocrand@5.4.3
+        prefix: /opt/rocm-5.4.3
+    hipsolver:
+      buildable: false
+      externals:
+      - spec: hipsolver@5.4.3
+        prefix: /opt/rocm-5.4.3
+    rocsolver:
+      buildable: false
+      externals:
+      - spec: rocsolver@5.4.3
+        prefix: /opt/rocm-5.4.3
+    rocsparse:
+      buildable: false
+      externals:
+      - spec: rocsparse@5.4.3
+        prefix: /opt/rocm-5.4.3
+    rocthrust:
+      buildable: false
+      externals:
+      - spec: rocthrust@5.4.3
+        prefix: /opt/rocm-5.4.3
+    rocprofiler-dev:
+      buildable: false
+      externals:
+      - spec: rocprofiler-dev@5.4.3
+        prefix: /opt/rocm-5.4.3
+
+  specs:
+  # ROCM NOARCH
+  - hpctoolkit +rocm
+  - tau +mpi +rocm +syscall  # tau: has issue with `spack env depfile` build
+
+  # ROCM 908
+  - adios2 +kokkos +rocm amdgpu_target=gfx908
+  - amrex +rocm amdgpu_target=gfx908
+  - arborx +rocm amdgpu_target=gfx908
+  - cabana +rocm amdgpu_target=gfx908
+  - caliper +rocm amdgpu_target=gfx908
+  - chai ~benchmarks +rocm amdgpu_target=gfx908
+  - ecp-data-vis-sdk +paraview +vtkm +rocm amdgpu_target=gfx908
+  - exago +mpi +python +raja +hiop +rocm amdgpu_target=gfx908 ~ipopt cxxflags="-Wno-error=non-pod-varargs" ^hiop@1.0.0 ~sparse +mpi +raja +rocm amdgpu_target=gfx908
+  - gasnet +rocm amdgpu_target=gfx908
+  - ginkgo +rocm amdgpu_target=gfx908
+  - heffte +rocm amdgpu_target=gfx908
+  - hpx +rocm amdgpu_target=gfx908
+  - hypre +rocm amdgpu_target=gfx908
+  - kokkos +rocm amdgpu_target=gfx908
+  - legion +rocm amdgpu_target=gfx908
+  - magma ~cuda +rocm amdgpu_target=gfx908
+  - mfem +rocm amdgpu_target=gfx908
+  - petsc +rocm amdgpu_target=gfx908
+  - raja ~openmp +rocm amdgpu_target=gfx908
+  - slate +rocm amdgpu_target=gfx908
+  - slepc +rocm amdgpu_target=gfx908 ^petsc +rocm amdgpu_target=gfx908
+  - strumpack ~slate +rocm amdgpu_target=gfx908
+  - sundials +rocm amdgpu_target=gfx908
+  - superlu-dist +rocm amdgpu_target=gfx908
+  - tasmanian ~openmp +rocm amdgpu_target=gfx908
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long +rocm amdgpu_target=gfx908
+  - umpire +rocm amdgpu_target=gfx908
+  - upcxx +rocm amdgpu_target=gfx908
+  # INCLUDED IN ECP DAV ROCM
+  # - hdf5
+  # - hdf5-vol-async
+  # - hdf5-vol-cache
+  # - hdf5-vol-log
+  # - libcatalyst
+  - paraview +rocm amdgpu_target=gfx908
+  # - vtk-m ~openmp +rocm amdgpu_target=gfx908  # vtk-m: https://github.com/spack/spack/issues/40268
+  # --
+  # - lbann ~cuda +rocm amdgpu_target=gfx908    # aluminum: https://github.com/spack/spack/issues/38807
+  # - papi +rocm amdgpu_target=gfx908           # papi: https://github.com/spack/spack/issues/27898
+
+  # ROCM 90a
+  - adios2 +kokkos +rocm amdgpu_target=gfx90a
+  - amrex +rocm amdgpu_target=gfx90a
+  - arborx +rocm amdgpu_target=gfx90a
+  - cabana +rocm amdgpu_target=gfx90a
+  - caliper +rocm amdgpu_target=gfx90a
+  - chai ~benchmarks +rocm amdgpu_target=gfx90a
+  - ecp-data-vis-sdk +paraview +vtkm +rocm amdgpu_target=gfx90a
+  - exago +mpi +python +raja +hiop +rocm amdgpu_target=gfx90a ~ipopt cxxflags="-Wno-error=non-pod-varargs" ^hiop@1.0.0 ~sparse +mpi +raja +rocm amdgpu_target=gfx90a  
+  - gasnet +rocm amdgpu_target=gfx90a
+  - ginkgo +rocm amdgpu_target=gfx90a
+  - heffte +rocm amdgpu_target=gfx90a
+  - hpx +rocm amdgpu_target=gfx90a
+  - hypre +rocm amdgpu_target=gfx90a
+  - kokkos +rocm amdgpu_target=gfx90a
+  - legion +rocm amdgpu_target=gfx90a
+  - magma ~cuda +rocm amdgpu_target=gfx90a
+  - mfem +rocm amdgpu_target=gfx90a
+  - petsc +rocm amdgpu_target=gfx90a
+  - raja ~openmp +rocm amdgpu_target=gfx90a
+  - slate +rocm amdgpu_target=gfx90a
+  - slepc +rocm amdgpu_target=gfx90a ^petsc +rocm amdgpu_target=gfx90a
+  - strumpack ~slate +rocm amdgpu_target=gfx90a
+  - sundials +rocm amdgpu_target=gfx90a
+  - superlu-dist +rocm amdgpu_target=gfx90a
+  - tasmanian ~openmp +rocm amdgpu_target=gfx90a
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long +rocm amdgpu_target=gfx90a
+  - umpire +rocm amdgpu_target=gfx90a
+  - upcxx +rocm amdgpu_target=gfx90a
+  # INCLUDED IN ECP DAV ROCM
+  # - hdf5
+  # - hdf5-vol-async
+  # - hdf5-vol-cache
+  # - hdf5-vol-log
+  # - libcatalyst
+  - paraview +rocm amdgpu_target=gfx90a
+  # - vtk-m ~openmp +rocm amdgpu_target=gfx90a  # vtk-m: https://github.com/spack/spack/issues/40268
+  # --
+  # - lbann ~cuda +rocm amdgpu_target=gfx90a    # aluminum: https://github.com/spack/spack/issues/38807
+  # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898
+
+  ci:
+    pipeline-gen:
+    - build-job:
+        image: "ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4-rocm5.4.3:2023.08.01"
+
+  cdash:
+    build-group: E4S ROCm External
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
index 1e20d8778f29be..11396a768f7cb1 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@@ -1,21 +1,19 @@
 spack:
   view: false
+
+  concretizer:
+    reuse: false
+    unify: false
+
   packages:
     all:
-      compiler: [gcc@11.1.0]
+      require: '%gcc target=x86_64_v3'
       providers:
         blas: [openblas]
         mpi: [mpich]
-      require: target=x86_64_v3
-      variants: +mpi amdgpu_target=gfx90a cuda_arch=80
-    tbb:
-      require: "intel-tbb"
+      variants: +mpi
     binutils:
       variants: +ld +gold +headers +libiberty ~nls
-    boost:
-      variants: +python +filesystem +iostreams +system
-    cuda:
-      version: [11.7.0]
     elfutils:
       variants: +bzip2 ~nls +xz
     hdf5:
@@ -24,31 +22,42 @@ spack:
       variants: fabrics=sockets,tcp,udp,rxm
     libunwind:
       variants: +pic +xz
-    mpich:
-      variants: ~wrapperrpath
-    ncurses:
-      variants: +termlib
     openblas:
       variants: threads=openmp
-    paraview:
-      # Don't build GUI support or GLX rendering for HPC/container deployments
-      require: "@5.11 ~qt+osmesa"
-    python:
-      version: [3.8.13]
     trilinos:
-      require:
-      - one_of: [+amesos +amesos2 +anasazi +aztec +boost +epetra +epetraext +ifpack
-            +intrepid +intrepid2 +isorropia +kokkos +minitensor +nox +piro +phalanx
-            +rol +rythmos +sacado +stk +shards +stratimikos +tempus +tpetra
-            +trilinoscouplings +zoltan]
-      - one_of: [gotype=long_long, gotype=all]
-      - one_of: [~ml ~muelu ~zoltan2 ~teko, +ml +muelu +zoltan2 +teko]
-      - one_of: [+superlu-dist, ~superlu-dist]
-      - one_of: [+shylu, ~shylu]
+      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
     xz:
       variants: +pic
     mesa:
       version: [21.3.8]
+    mpi:
+      require: mpich
+    mpich:
+      require: '~wrapperrpath ~hwloc'
+    ncurses:
+      require: '@6.3 +termlib'
+    tbb:
+      require: intel-tbb
+    boost:
+      version: [1.79.0]
+      variants: +atomic +chrono +container +date_time +exception +filesystem +graph
+        +iostreams +locale +log +math +mpi +multithreaded +program_options +random
+        +regex +serialization +shared +signals +stacktrace +system +test +thread +timer
+        cxxstd=17 visibility=global
+    libffi:
+      require: "@3.4.4"
+    vtk-m:
+      require: "+examples"
+    visit:
+      require: "~gui"
+    cuda:
+      version: [11.8.0]
+    paraview:
+      # Don't build GUI support or GLX rendering for HPC/container deployments
+      require: "@5.11 ~qt+osmesa"
 
   specs:
   # CPU
@@ -57,13 +66,12 @@ spack:
   - aml
   - amrex
   - arborx
-  - archer
   - argobots
   - axom
   - bolt
-  - bricks
+  - boost
+  - bricks ~cuda
   - butterflypack
-  - boost +python +filesystem +iostreams +system
   - cabana
   - caliper
   - chai ~benchmarks ~tests
@@ -71,8 +79,10 @@ spack:
   - conduit
   - datatransferkit
   - dealii
+  - drishti
+  - dxt-explorer
   - dyninst
-  - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc ~visit +vtkm +zfp ^hdf5@1.14
+  - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc +visit +vtkm +zfp                                                                                       # adios2~cuda, ascent~cuda, darshan-runtime, darshan-util, faodel, hdf5, libcatalyst, parallel-netcdf, paraview~cuda, py-cinemasci, sz, unifyfs, veloc, visit, vtk-m, zfp
   - exaworks
   - flecsi
   - flit
@@ -83,22 +93,26 @@ spack:
   - globalarrays
   - gmp
   - gotcha
-  - gptune
+  - gptune ~mpispawn
+  - gromacs +cp2k ^cp2k build_system=cmake
   - h5bench
   - hdf5-vol-async
   - hdf5-vol-cache
   - hdf5-vol-log
   - heffte +fftw
   - hpctoolkit
-  - hpx max_cpu_count=512 networking=mpi
+  - hpx networking=mpi
   - hypre
   - kokkos +openmp
   - kokkos-kernels +openmp
   - lammps
+  - lbann
   - legion
   - libnrm
+  - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp
   - libquo
   - libunwind
+  - loki
   - mercury
   - metall
   - mfem
@@ -111,6 +125,7 @@ spack:
   - nrm
   - nvhpc
   - omega-h
+  - openfoam
   - openmpi
   - openpmd-api
   - papi
@@ -122,16 +137,17 @@ spack:
   - plasma
   - plumed
   - precice
+  - pruners-ninja
   - pumi
-  - py-h5py +mpi
-  - py-h5py ~mpi
+  - py-h5py
   - py-jupyterhub
-  - py-libensemble +mpi +nlopt
+  - py-libensemble
   - py-petsc4py
   - py-warpx
   - qthreads scheduler=distrib
   - quantum-espresso
   - raja
+  - rempi
   - scr
   - slate ~cuda
   - slepc
@@ -140,104 +156,229 @@ spack:
   - sundials
   - superlu
   - superlu-dist
-  - swig
   - swig@4.0.2-fortran
   - sz3
   - tasmanian
-  - tau +mpi +python
-  - trilinos@13.0.1 +belos +ifpack2 +stokhos
+  - tau +mpi +python +syscall
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
   - turbine
   - umap
   - umpire
   - upcxx
   - variorum
-  - veloc
   - wannier90
-  - xyce +mpi +shared +pymi +pymi_static_tpls ^trilinos~shylu
+  - xyce +mpi +shared +pymi +pymi_static_tpls
+  # INCLUDED IN ECP DAV CPU
+  - adios2
+  - ascent
+  - darshan-runtime
+  - darshan-util
+  - faodel
+  - hdf5
+  - libcatalyst
+  - parallel-netcdf
+  - paraview
+  - py-cinemasci
+  - sz
+  - unifyfs
+  - veloc
+  # - visit                           # silo: https://github.com/spack/spack/issues/39538
+  - vtk-m
+  - zfp
+  # --
+  # - archer                          # submerged into llvm +libomp_tsan
+  # - geopm                           # geopm: https://github.com/spack/spack/issues/38795
 
-  # CUDA
-  - amrex +cuda
-  - arborx +cuda ^kokkos +wrapper
+  # CUDA NOARCH
   - bricks +cuda
-  - cabana +cuda ^kokkos +wrapper +cuda_lambda +cuda
-  - caliper +cuda
-  - chai ~benchmarks ~tests +cuda ^umpire ~shared
-  - cusz +cuda
-  - dealii +cuda
-  - ecp-data-vis-sdk +cuda ~ascent +adios2 +hdf5 +paraview +sz +vtkm +zfp  ^hdf5@1.14 # Removing ascent because RAJA build failure
-  - flecsi +cuda
   - flux-core +cuda
-  - ginkgo +cuda
-  - heffte +cuda
   - hpctoolkit +cuda
-  - hpx max_cpu_count=512 +cuda
-  - hypre +cuda
-  - kokkos +wrapper +cuda
-  - kokkos-kernels +cuda ^kokkos +wrapper +cuda +cuda_lambda
-  - magma +cuda
-  - mfem +cuda
-  - mgard +serial +openmp +timing +unstructured +cuda
-  - omega-h +cuda
   - papi +cuda
-  - petsc +cuda
-  - py-torch +cuda
-  - raja +cuda
-  - slate +cuda
-  - slepc +cuda
-  - strumpack ~slate +cuda
-  - sundials +cuda
-  - superlu-dist +cuda
-  - tasmanian +cuda
-  - tau +mpi +cuda
-  - "trilinos@13.4.0: +belos +ifpack2 +stokhos +cuda"
-  - umpire ~shared +cuda
+  - tau +mpi +cuda +syscall
+  # --
+  # - legion +cuda                    # legion: needs NVIDIA driver
 
-  # ROCm
-  - amrex +rocm
-  - arborx +rocm
-  - cabana +rocm
-  - caliper +rocm
-  - chai ~benchmarks +rocm
-  - ecp-data-vis-sdk +adios2 +hdf5 +paraview +pnetcdf +sz +vtkm +zfp +rocm ^hdf5@1.14 # Excludes ascent for now due to C++ standard issues
-  - gasnet +rocm
-  - ginkgo +rocm
-  - heffte +rocm
-  - hpctoolkit +rocm
-  - hpx max_cpu_count=512 +rocm
-  - hypre +rocm
-  - kokkos +rocm
-  - magma ~cuda +rocm
-  - mfem +rocm
-  - papi +rocm
-  - petsc +rocm
-  - raja ~openmp +rocm
-  - slate +rocm
-  - slepc +rocm ^petsc +rocm
-  - strumpack ~slate +rocm
-  - sundials +rocm
-  - superlu-dist +rocm
-  - tasmanian ~openmp +rocm
-  - tau +mpi +rocm
-  - "trilinos@13.4.0: +belos ~ifpack2 ~stokhos +rocm"
-  - umpire +rocm
-  - upcxx +rocm
+  # CUDA 80
+  - amrex +cuda cuda_arch=80
+  - arborx +cuda cuda_arch=80 ^kokkos +wrapper
+  - cabana +cuda cuda_arch=80 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=80
+  - caliper +cuda cuda_arch=80
+  - chai ~benchmarks ~tests +cuda cuda_arch=80 ^umpire ~shared
+  - cusz +cuda cuda_arch=80
+  - dealii +cuda cuda_arch=80
+  - ecp-data-vis-sdk ~rocm +adios2 ~ascent +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=80 # +ascent fails because fides fetch error
+  - exago +mpi +python +raja +hiop ~rocm +cuda cuda_arch=80 ~ipopt ^hiop@1.0.0 ~sparse +mpi +raja ~rocm +cuda cuda_arch=80 #^raja@0.14.0
+  - flecsi +cuda cuda_arch=80
+  - ginkgo +cuda cuda_arch=80
+  - heffte +cuda cuda_arch=80
+  - hpx +cuda cuda_arch=80
+  - hypre +cuda cuda_arch=80
+  - kokkos +wrapper +cuda cuda_arch=80
+  - kokkos-kernels +cuda cuda_arch=80 ^kokkos +wrapper +cuda cuda_arch=80
+  - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard +cuda cuda_arch=80 ^cusz +cuda cuda_arch=80
+  - magma +cuda cuda_arch=80
+  - mfem +cuda cuda_arch=80
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=80
+  - omega-h +cuda cuda_arch=80
+  - parsec +cuda cuda_arch=80
+  - petsc +cuda cuda_arch=80
+  - py-torch +cuda cuda_arch=80
+  - raja +cuda cuda_arch=80
+  - slate +cuda cuda_arch=80
+  - slepc +cuda cuda_arch=80
+  - strumpack ~slate +cuda cuda_arch=80
+  - sundials +cuda cuda_arch=80
+  - superlu-dist +cuda cuda_arch=80
+  - tasmanian +cuda cuda_arch=80
+  - trilinos +cuda cuda_arch=80
+  - umpire ~shared +cuda cuda_arch=80
+  # INCLUDED IN ECP DAV CUDA
+  # - adios2 +cuda cuda_arch=80
+  # - ascent +cuda cuda_arch=80       # ascent: https://github.com/spack/spack/issues/38045
+  # - paraview +cuda cuda_arch=80
+  # - vtk-m +cuda cuda_arch=80
+  # - zfp +cuda cuda_arch=80
+  # --
+  # - lammps +cuda cuda_arch=80       # lammps: needs NVIDIA driver
+  # - upcxx +cuda cuda_arch=80        # upcxx: needs NVIDIA driver
+  # - axom +cuda cuda_arch=80         # axom: https://github.com/spack/spack/issues/29520
+  # - lbann +cuda cuda_arch=80        # lbann: https://github.com/spack/spack/issues/38788
 
-  # CPU failures
-  # - geopm                 # /usr/include/x86_64-linux-gnu/bits/string_fortified.h:95:10: error:'__builtin_strncpy' specified bound 512 equals destination size [-Werror=stringop-truncation]
-  # - hdf5-vol-daos         # hdf5-vol-daos: vhost/vhost_user.c:65:32: error: array size missing in 'vhost_message_handlers'
-  # - loki                  # ../include/loki/Singleton.h:158:14: warning: 'template class std::auto_ptr' is deprecated: use 'std::unique_ptr' instead [-Wdeprecated-declarations]
-  # - pruners-ninja         # test/ninja_test_util.c:34: multiple definition of `a';
-  # - rempi                 # rempi_message_manager.h:53:3: error: 'string' does not name a type
+  # CUDA 90
+  - amrex +cuda cuda_arch=90
+  - arborx +cuda cuda_arch=90 ^kokkos +wrapper
+  - cabana +cuda cuda_arch=90 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=90
+  - caliper +cuda cuda_arch=90
+  - chai ~benchmarks ~tests +cuda cuda_arch=90 ^umpire ~shared
+  - cusz +cuda cuda_arch=90
+  - flecsi +cuda cuda_arch=90
+  - ginkgo +cuda cuda_arch=90
+  - heffte +cuda cuda_arch=90
+  - hpx +cuda cuda_arch=90
+  - kokkos +wrapper +cuda cuda_arch=90
+  - kokkos-kernels +cuda cuda_arch=90 ^kokkos +wrapper +cuda cuda_arch=90
+  - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard +cuda cuda_arch=90 ^cusz +cuda cuda_arch=90
+  - magma +cuda cuda_arch=90
+  - mfem +cuda cuda_arch=90
+  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=90
+  - parsec +cuda cuda_arch=90
+  - petsc +cuda cuda_arch=90
+  - py-torch +cuda cuda_arch=90
+  - raja +cuda cuda_arch=90
+  - slate +cuda cuda_arch=90
+  - slepc +cuda cuda_arch=90
+  - strumpack ~slate +cuda cuda_arch=90
+  - sundials +cuda cuda_arch=90
+  - superlu-dist +cuda cuda_arch=90
+  - trilinos +cuda cuda_arch=90
+  - umpire ~shared +cuda cuda_arch=90
+  # INCLUDED IN ECP DAV CUDA
+  - adios2 +cuda cuda_arch=90
+  # - ascent +cuda cuda_arch=90       # ascent: https://github.com/spack/spack/issues/38045
+  # - paraview +cuda cuda_arch=90     # paraview: InstallError: Incompatible cuda_arch=90
+  - vtk-m +cuda cuda_arch=90
+  - zfp +cuda cuda_arch=90
+  # --
+  # - axom +cuda cuda_arch=90         # axom: https://github.com/spack/spack/issues/29520
+  # - dealii +cuda cuda_arch=90       # dealii: https://github.com/spack/spack/issues/39532
+  # - ecp-data-vis-sdk ~rocm +adios2 +ascent +hdf5 +vtkm +zfp +paraview +cuda cuda_arch=90 # paraview: incompatible cuda_arch; vtk-m: CMake Error at CMake/VTKmWrappers.cmake:413 (message): vtkm_cont needs to be built STATIC as CUDA doesn't support virtual methods across dynamic library boundaries.  You need to set the CMake opt ion BUILD_SHARED_LIBS to `OFF` or (better) turn VTKm_NO_DEPRECATED_VIRTUAL to `ON`.
+  # - hypre +cuda cuda_arch=90        # concretizer: hypre +cuda requires cuda@:11, but cuda_arch=90 requires cuda@12:
+  # - lammps +cuda cuda_arch=90       # lammps: needs NVIDIA driver
+  # - lbann +cuda cuda_arch=90        # concretizer: Cannot select a single "version" for package "lbann"
+  # - omega-h +cuda cuda_arch=90      # omega-h: https://github.com/spack/spack/issues/39535
+  # - tasmanian +cuda cuda_arch=90    # tasmanian: conflicts with cuda@12
+  # - upcxx +cuda cuda_arch=90        # upcxx: needs NVIDIA driver
+
+  # ROCM NOARCH
+  - hpctoolkit +rocm
+  - tau +mpi +rocm +syscall  # tau: has issue with `spack env depfile` build
 
-  # CUDA failures
-  # - parsec +cuda          # parsec/mca/device/cuda/transfer.c:168: multiple definition of `parsec_CUDA_d2h_max_flows';
+  # ROCM 908
+  - adios2 +kokkos +rocm amdgpu_target=gfx908
+  - amrex +rocm amdgpu_target=gfx908
+  - arborx +rocm amdgpu_target=gfx908
+  - cabana +rocm amdgpu_target=gfx908
+  - caliper +rocm amdgpu_target=gfx908
+  - chai ~benchmarks +rocm amdgpu_target=gfx908
+  - ecp-data-vis-sdk +paraview +vtkm +rocm amdgpu_target=gfx908
+  - gasnet +rocm amdgpu_target=gfx908
+  - ginkgo +rocm amdgpu_target=gfx908
+  - heffte +rocm amdgpu_target=gfx908
+  - hpx +rocm amdgpu_target=gfx908
+  - hypre +rocm amdgpu_target=gfx908
+  - kokkos +rocm amdgpu_target=gfx908
+  - legion +rocm amdgpu_target=gfx908
+  - magma ~cuda +rocm amdgpu_target=gfx908
+  - mfem +rocm amdgpu_target=gfx908
+  - petsc +rocm amdgpu_target=gfx908
+  - raja ~openmp +rocm amdgpu_target=gfx908
+  - slate +rocm amdgpu_target=gfx908
+  - slepc +rocm amdgpu_target=gfx908 ^petsc +rocm amdgpu_target=gfx908
+  - strumpack ~slate +rocm amdgpu_target=gfx908
+  - sundials +rocm amdgpu_target=gfx908
+  - superlu-dist +rocm amdgpu_target=gfx908
+  - tasmanian ~openmp +rocm amdgpu_target=gfx908
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long +rocm amdgpu_target=gfx908
+  - umpire +rocm amdgpu_target=gfx908
+  - upcxx +rocm amdgpu_target=gfx908
+  # INCLUDED IN ECP DAV ROCM
+  # - hdf5
+  # - hdf5-vol-async
+  # - hdf5-vol-cache
+  # - hdf5-vol-log
+  # - libcatalyst
+  - paraview +rocm amdgpu_target=gfx908
+  # - vtk-m ~openmp +rocm amdgpu_target=gfx908  # vtk-m: https://github.com/spack/spack/issues/40268
+  # --
+  # - exago +mpi +python +raja +hiop +rocm amdgpu_target=gfx908 ~ipopt cxxflags="-Wno-error=non-pod-varargs" ^hiop@1.0.0 ~sparse +mpi +raja +rocm amdgpu_target=gfx908 # hiop: CMake Error at cmake/FindHiopHipLibraries.cmake:23 (find_package)
+  # - lbann ~cuda +rocm amdgpu_target=gfx908    # aluminum: https://github.com/spack/spack/issues/38807
+  # - papi +rocm amdgpu_target=gfx908           # papi: https://github.com/spack/spack/issues/27898
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/e4s" }
+  # ROCM 90a
+  - adios2 +kokkos +rocm amdgpu_target=gfx90a
+  - amrex +rocm amdgpu_target=gfx90a
+  - arborx +rocm amdgpu_target=gfx90a
+  - cabana +rocm amdgpu_target=gfx90a
+  - caliper +rocm amdgpu_target=gfx90a
+  - chai ~benchmarks +rocm amdgpu_target=gfx90a
+  - ecp-data-vis-sdk +paraview +vtkm +rocm amdgpu_target=gfx90a
+  - gasnet +rocm amdgpu_target=gfx90a
+  - ginkgo +rocm amdgpu_target=gfx90a
+  - heffte +rocm amdgpu_target=gfx90a
+  - hpx +rocm amdgpu_target=gfx90a
+  - hypre +rocm amdgpu_target=gfx90a
+  - kokkos +rocm amdgpu_target=gfx90a
+  - legion +rocm amdgpu_target=gfx90a
+  - magma ~cuda +rocm amdgpu_target=gfx90a
+  - mfem +rocm amdgpu_target=gfx90a
+  - petsc +rocm amdgpu_target=gfx90a
+  - raja ~openmp +rocm amdgpu_target=gfx90a
+  - slate +rocm amdgpu_target=gfx90a
+  - slepc +rocm amdgpu_target=gfx90a ^petsc +rocm amdgpu_target=gfx90a
+  - strumpack ~slate +rocm amdgpu_target=gfx90a
+  - sundials +rocm amdgpu_target=gfx90a
+  - superlu-dist +rocm amdgpu_target=gfx90a
+  - tasmanian ~openmp +rocm amdgpu_target=gfx90a
+  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long +rocm amdgpu_target=gfx90a
+  - umpire +rocm amdgpu_target=gfx90a
+  - upcxx +rocm amdgpu_target=gfx90a
+  # INCLUDED IN ECP DAV ROCM
+  # - hdf5
+  # - hdf5-vol-async
+  # - hdf5-vol-cache
+  # - hdf5-vol-log
+  # - libcatalyst
+  - paraview +rocm amdgpu_target=gfx90a
+  # - vtk-m ~openmp +rocm amdgpu_target=gfx90a  # vtk-m: https://github.com/spack/spack/issues/40268
+  # --
+  # - exago +mpi +python +raja +hiop +rocm amdgpu_target=gfx90a ~ipopt cxxflags="-Wno-error=non-pod-varargs" ^hiop@1.0.0 ~sparse +mpi +raja +rocm amdgpu_target=gfx90a # hiop: CMake Error at cmake/FindHiopHipLibraries.cmake:23 (find_package)
+  # - lbann ~cuda +rocm amdgpu_target=gfx90a    # aluminum: https://github.com/spack/spack/issues/38807
+  # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898
 
   ci:
     pipeline-gen:
     - build-job:
-        image: "ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01"
+        image: "ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01"
 
   cdash:
     build-group: E4S
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
index bb7c35063c7eba..263d8e29b30578 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
@@ -32,8 +32,6 @@ spack:
     paraview:
       # Don't build GUI support or GLX rendering for HPC/container deployments
       require: "@5.11 ~qt+osmesa"
-    python:
-      version: [3.8.13]
     trilinos:
       require: +amesos +amesos2 +anasazi +aztec +boost +epetra +epetraext
         +ifpack +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
@@ -51,12 +49,10 @@ spack:
   # FAILURES
   # - kokkos +wrapper +cuda cuda_arch=80 ^cuda@12.0.0     # https://github.com/spack/spack/issues/35378
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/gpu-tests" }
-
   ci:
     pipeline-gen:
     - build-job:
-        image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
+        image: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01
     - match_behavior: first
       submapping:
       - match:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
index 3b204f3f94aa6e..0905305113f083 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
@@ -1,19 +1,6 @@
 spack:
   view: false
 
-  concretizer:
-    unify: false
-    reuse: false
-
-  config:
-    concretizer: clingo
-    db_lock_timeout: 120
-    install_tree:
-      root: $spack/opt/spack
-      padded_length: 256
-      projections:
-        all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
-
   packages:
     all:
       require: target=aarch64
@@ -24,36 +11,52 @@ spack:
       require: ~fortran
 
   specs:
+  # Horovod
+  - py-horovod
+
   # Hugging Face
   - py-transformers
 
   # JAX
-  # - py-jax        # bazel codesign
-  # - py-jaxlib     # bazel codesign
+  # Bazel codesign issues
+  # - py-jax
+  # - py-jaxlib
 
   # Keras
+  # Bazel codesign issues
+  # - py-keras
   - py-keras-applications
   - py-keras-preprocessing
   - py-keras2onnx
-  # - py-keras      # bazel codesign
 
-  # MXNet
-  - mxnet
+  # MXNet not supported on darwin aarch64 yet
+  # - mxnet
 
   # PyTorch
   - py-botorch
+  - py-efficientnet-pytorch
   - py-gpytorch
+  - py-kornia
+  - py-lightning
   - py-pytorch-gradual-warmup-lr
+  - py-pytorch-lightning
   - py-segmentation-models-pytorch
   - py-timm
   - py-torch
   - py-torch-cluster
   - py-torch-geometric
+  - py-torch-nvidia-apex
+  - py-torch-scatter
   - py-torch-sparse
+  - py-torch-spline-conv
+  - py-torchaudio
   - py-torchdata
   - py-torchfile
   - py-torchgeo
+  - py-torchmetrics
+  - py-torchtext
   - py-torchvision
+  - py-vector-quantize-pytorch
 
   # scikit-learn
   - py-scikit-learn
@@ -66,73 +69,27 @@ spack:
   - py-tensorboardx
 
   # TensorFlow
-  # - py-tensorflow               # bazel codesign
-  # - py-tensorflow-datasets      # bazel codesign
-  # - py-tensorflow-hub           # bazel codesign
-  # - py-tensorflow-metadata      # bazel codesign
-  # - py-tensorflow-estimator     # bazel codesign
-  # - py-tensorflow-probability   # py-dm-tree due to bazel codesign
+  # Bazel codesign issues
+  # - py-tensorflow
+  # - py-tensorflow-datasets
+  # - py-tensorflow-estimator
+  # - py-tensorflow-hub
+  # - py-tensorflow-metadata
+  # - py-tensorflow-probability
 
   # XGBoost
   - py-xgboost
+  # - r-xgboost
   - xgboost
 
-  # ERRORS
-  # - py-efficientnet-pytorch     # py-torch
-  # - py-horovod                  # py-torch
-  # - py-kornia                   # py-torch
-  # - py-lightning                # py-torch
-  # - py-pytorch-lightning        # py-torch
-  # - py-torch-nvidia-apex        # py-torch
-  # - py-torch-scatter            # py-torch
-  # - py-torch-spline-conv        # py-torch
-  # - py-torchaudio               # py-torchaudio
-  # - py-torchmetrics             # py-torch
-  # - py-torchtext                # py-torchtext
-  # - py-vector-quantize-pytorch  # py-torch
-  # - r-xgboost                   # r
-
-  mirrors: { "mirror": "s3://spack-binaries/develop/ml-darwin-aarch64-cpu" }
-
   ci:
     pipeline-gen:
     - build-job-remove:
-        image: no-image
-        tags: [spack, public]
+        tags: [ spack, public ]
     - build-job:
-        tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ]
-        script::
-        - - spack compiler find
-          - cd ${SPACK_CONCRETE_ENV_DIR}
-          - spack env activate --without-view .
-          - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi
-          - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
-          # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
-          - if [[ -r /etc/protected-runner/e4s.gpg ]]; then spack gpg trust /etc/protected-runner/e4s.gpg; fi
-          # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
-          - if [[ -r /etc/protected-runner/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /etc/protected-runner/intermediate_ci_signing_key.gpg; fi
-          - if [[ -r /etc/protected-runner/spack_public_key.gpg ]]; then spack gpg trust /etc/protected-runner/spack_public_key.gpg; fi
-          - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
-        after_script:
-        - - cat /proc/loadavg || true
-    - signing-job:
-        image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
-        tags: ["aws"]
-        script:
-        - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
-          - /sign.sh
-          - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
-          - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
-    - any-job:
-        image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
-        tags: ["spack"]
-        before_script:
-        - - uname -a || true
-          - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
-          - nproc || true
-        - - . "./share/spack/setup-env.sh"
-          - spack --version
-          - spack arch
+        variables:
+          CI_GPG_KEY_ROOT: /etc/protected-runner
+        tags: [ "macos-ventura", "apple-clang-15", "aarch64-macos" ]
 
   cdash:
     build-group: Machine Learning MPS
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
index d244499fec0eac..71670d5a91568d 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
@@ -76,14 +76,11 @@ spack:
     # - r-xgboost
     - xgboost
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-cpu
-
   ci:
     pipeline-gen:
     - build-job:
         image:
-          name: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
+          name: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2023-07-01
           entrypoint: ['']
 
   cdash:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
index 4cb711402d47de..88291690382784 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
@@ -79,14 +79,11 @@ spack:
     # - r-xgboost
     - xgboost
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-cuda
-
   ci:
     pipeline-gen:
     - build-job:
         image:
-          name: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
+          name: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2023-07-01
           entrypoint: ['']
 
   cdash:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
index 38b141c9b357ad..620a95715b41e5 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
@@ -82,14 +82,11 @@ spack:
     # - r-xgboost
     - xgboost
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/ml-linux-x86_64-rocm
-
   ci:
     pipeline-gen:
     - build-job:
         image:
-          name: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
+          name: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2023-07-01
           entrypoint: ['']
 
   cdash:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
index b05b45f76378cf..6453d2a5fe6722 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws-aarch64/spack.yaml
@@ -38,8 +38,6 @@ spack:
     - - $compiler
     - - $target
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws-aarch64" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
index fd297ede91049a..ca7de563c44fe0 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
@@ -28,7 +28,7 @@ spack:
     - mfem +cuda ^hypre+cuda
     - raja
     - raja +cuda
-    - umpire 
+    - umpire
     - umpire +cuda
 
   - compiler:
@@ -44,8 +44,6 @@ spack:
     - - $compiler
     - - $target
 
-  mirrors: { "mirror": "s3://spack-binaries/develop/radiuss-aws" }
-
   ci:
     pipeline-gen:
     - build-job:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
index c80bcf10eed975..ca8e1a990519db 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
@@ -40,9 +40,6 @@ spack:
       - xbraid
       - zfp
 
-  mirrors:
-    mirror: "s3://spack-binaries/develop/radiuss"
-
   specs:
   - matrix:
     - [$radiuss]
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
index 4b39be884612f8..c320442cbe0f84 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
@@ -1,9 +1,4 @@
 spack:
-  config:
-    # allow deprecated versions in concretizations
-    # required for zlib
-    deprecated: true
-
   view: false
   packages:
     all:
@@ -13,49 +8,48 @@ spack:
   definitions:
   - gcc_system_packages:
     - matrix:
-      - - zlib
-        - zlib@1.2.8
-        - zlib@1.2.8 cflags=-O3
+      - - zlib-ng
+        - zlib-ng@2.0.7
+        - zlib-ng@2.0.7 cflags=-O3
         - tcl
-        - tcl ^zlib@1.2.8 cflags=-O3
+        - tcl ^zlib-ng@2.0.7 cflags=-O3
         - hdf5
         - hdf5~mpi
         - hdf5+hl+mpi ^mpich
         - trilinos
         - trilinos +hdf5 ^hdf5+hl+mpi ^mpich
-        - gcc@12.1.0
+        - gcc@12.3.0
         - mpileaks
-        - lmod
-        - macsio@1.1+scr^scr@2.0.0~fortran^silo~fortran^hdf5~fortran
-      - ['%gcc@11.3.0']
+        - lmod@8.7.18
+        - environment-modules
+        - macsio@1.1+scr ^scr@2.0.0~fortran ^silo~fortran ^hdf5~fortran
+      - ['%gcc@11']
   - gcc_old_packages:
-    - zlib%gcc@10.4.0
+    - zlib-ng%gcc@10
   - clang_packages:
     - matrix:
-      - [zlib, tcl ^zlib@1.2.8]
-      - ['%clang@14.0.0']
+      - [zlib-ng, tcl ^zlib-ng@2.0.7]
+      - ['%clang@14']
   - gcc_spack_built_packages:
     - matrix:
       - [netlib-scalapack]
       - [^mpich, ^openmpi]
       - [^openblas, ^netlib-lapack]
-      - ['%gcc@12.1.0']
+      - ['%gcc@12']
     - matrix:
-      - [py-scipy^openblas, armadillo^openblas, netlib-lapack, openmpi, mpich, elpa^mpich]
-      - ['%gcc@12.1.0']
+      - [py-scipy ^openblas, armadillo ^openblas, netlib-lapack, openmpi, mpich, elpa ^mpich]
+      - ['%gcc@12']
   specs:
   - $gcc_system_packages
   - $gcc_old_packages
   - $clang_packages
   - $gcc_spack_built_packages
 
-  mirrors:
-    mirror: s3://spack-binaries/develop/tutorial
   ci:
     pipeline-gen:
     - build-job:
         image:
-          name: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07
+          name: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-10-30
           entrypoint: ['']
   cdash:
     build-group: Spack Tutorial
diff --git a/share/spack/qa/completion-test.sh b/share/spack/qa/completion-test.sh
index 9559c56d0adadf..1071ed36dbeac2 100755
--- a/share/spack/qa/completion-test.sh
+++ b/share/spack/qa/completion-test.sh
@@ -61,6 +61,15 @@ contains 'python' _spack_completions spack extensions ''
 contains 'hdf5' _spack_completions spack -d install --jobs 8 ''
 contains 'hdf5' _spack_completions spack install -v ''
 
+title 'Testing alias handling'
+contains 'concretize' _spack_completions spack c
+contains 'concretise' _spack_completions spack c
+contains 'concretize' _spack_completions spack conc
+does_not_contain 'concretise' _spack_completions spack conc
+
+does_not_contain 'concretize' _spack_completions spack isnotacommand
+does_not_contain 'concretize' _spack_completions spack env isnotacommand
+
 # XFAIL: Fails for Python 2.6 because pkg_resources not found?
 #contains 'compilers.py' _spack_completions spack unit-test ''
 
diff --git a/share/spack/qa/setup-env-test.fish b/share/spack/qa/setup-env-test.fish
index 87158840b1f5a4..589f4cbfa8c353 100755
--- a/share/spack/qa/setup-env-test.fish
+++ b/share/spack/qa/setup-env-test.fish
@@ -285,7 +285,7 @@ spt_succeeds which spack
 # create a fake mock package install and store its location for later
 title "Setup"
 echo "Creating a mock package installation"
-spack -m install --fake a
+spack -m install --fake shell-a
 
 # create a test environment for testing environment commands
 echo "Creating a mock environment"
@@ -300,7 +300,7 @@ function spt_cleanup -p %self
 
     title "Cleanup"
     echo "Removing test packages before exiting."
-    spack -m uninstall -yf b a
+    spack -m uninstall -yf shell-b shell-a
 
     echo
     echo "$__spt_success tests succeeded."
@@ -322,7 +322,7 @@ spt_contains "usage: spack " spack help --all
 title 'Testing `spack cd`'
 spt_contains "usage: spack cd " spack cd -h
 spt_contains "usage: spack cd " spack cd --help
-spt_contains "cd $b_install" spack cd -i b
+spt_contains "cd $b_install" spack cd -i shell-b
 
 title 'Testing `spack module`'
 spt_contains "usage: spack module " spack -m module -h
@@ -330,34 +330,33 @@ spt_contains "usage: spack module " spack -m module --help
 spt_contains "usage: spack module " spack -m module
 
 title 'Testing `spack load`'
-set _b_loc (spack -m location -i b)
+set _b_loc (spack -m location -i shell-b)
 set _b_bin $_b_loc"/bin"
-set _a_loc (spack -m location -i a)
+set _a_loc (spack -m location -i shell-a)
 set _a_bin $_a_loc"/bin"
 
-spt_contains "set -gx PATH $_b_bin" spack -m load --only package --fish b
-spt_succeeds spack -m load b
-set LIST_CONTENT (spack -m load b; spack load --list)
-spt_contains "b@" echo $LIST_CONTENT
-spt_does_not_contain "a@" echo $LIST_CONTENT
+spt_contains "set -gx PATH $_b_bin" spack -m load --fish shell-b
+spt_succeeds spack -m load shell-b
+set LIST_CONTENT (spack -m load shell-b; spack load --list)
+spt_contains "shell-b@" echo $LIST_CONTENT
+spt_does_not_contain "shell-a@" echo $LIST_CONTENT
 # test a variable MacOS clears and one it doesn't for recursive loads
-spt_contains "set -gx PATH $_a_bin:$_b_bin" spack -m load --fish a
-spt_succeeds spack -m load --only dependencies a
-spt_succeeds spack -m load --only package a
+
+spt_succeeds spack -m load shell-a
 spt_fails spack -m load d
 spt_contains "usage: spack load " spack -m load -h
 spt_contains "usage: spack load " spack -m load -h d
 spt_contains "usage: spack load " spack -m load --help
 
 title 'Testing `spack unload`'
-spack -m load b a  # setup
-# spt_contains "module unload $b_module" spack -m unload b
-spt_succeeds spack -m unload b
+spack -m load shell-b shell-a  # setup
+# spt_contains "module unload $b_module" spack -m unload shell-b
+spt_succeeds spack -m unload shell-b
 spt_succeeds spack -m unload --all
 spack -m unload --all # cleanup
 spt_fails spack -m unload -l
-# spt_contains "module unload -l --arg $b_module" spack -m unload -l --arg b
-spt_fails spack -m unload d
+# spt_contains "module unload -l --arg $b_module" spack -m unload -l --arg shell-b
+spt_fails spack -m unload shell-d
 spt_contains "usage: spack unload " spack -m unload -h
 spt_contains "usage: spack unload " spack -m unload -h d
 spt_contains "usage: spack unload " spack -m unload --help
@@ -372,7 +371,6 @@ spt_contains " spack env list " spack env list --help
 
 title 'Testing `spack env activate`'
 spt_contains "No such environment:" spack env activate no_such_environment
-spt_contains "env activate requires an environment " spack env activate
 spt_contains "usage: spack env activate " spack env activate -h
 spt_contains "usage: spack env activate " spack env activate --help
 
@@ -416,6 +414,11 @@ spt_contains 'spack_test_2_env' 'fish' '-c' 'echo $PATH'
 spt_does_not_contain 'spack_test_env' 'fish' '-c' 'echo $PATH'
 despacktivate
 
+echo "Testing default environment"
+spack env activate
+contains "In environment default" spack env status
+despacktivate
+
 echo "Correct error exit codes for activate and deactivate"
 spt_fails spack env activate nonexisiting_environment
 spt_fails spack env deactivate
diff --git a/share/spack/qa/setup-env-test.sh b/share/spack/qa/setup-env-test.sh
index 94589e5bcb7818..b26619b9cde0a0 100755
--- a/share/spack/qa/setup-env-test.sh
+++ b/share/spack/qa/setup-env-test.sh
@@ -60,12 +60,12 @@ cd() {
 # Create a fake mock package install and store its location for later
 title "Setup"
 echo "Creating a mock package installation"
-spack -m install --fake a
-a_install=$(spack location -i a)
-a_module=$(spack -m module tcl find a)
+spack -m install --fake shell-a
+a_install=$(spack location -i shell-a)
+a_module=$(spack -m module tcl find shell-a)
 
-b_install=$(spack location -i b)
-b_module=$(spack -m module tcl find b)
+b_install=$(spack location -i shell-b)
+b_module=$(spack -m module tcl find shell-b)
 
 # Create a test environment for testing environment commands
 echo "Creating a mock environment"
@@ -80,7 +80,7 @@ cleanup() {
 
     title "Cleanup"
     echo "Removing test packages before exiting."
-    spack -m uninstall -yf b a
+    spack -m uninstall -yf shell-b shell-a
 }
 
 # -----------------------------------------------------------------------
@@ -96,7 +96,7 @@ contains "usage: spack " spack help --all
 title 'Testing `spack cd`'
 contains "usage: spack cd " spack cd -h
 contains "usage: spack cd " spack cd --help
-contains "cd $b_install" spack cd -i b
+contains "cd $b_install" spack cd -i shell-b
 
 title 'Testing `spack module`'
 contains "usage: spack module " spack -m module -h
@@ -104,25 +104,24 @@ contains "usage: spack module " spack -m module --help
 contains "usage: spack module " spack -m module
 
 title 'Testing `spack load`'
-contains "export PATH=$(spack -m location -i b)/bin" spack -m load --only package --sh b
-succeeds spack -m load b
-LIST_CONTENT=`spack -m load b; spack load --list`
-contains "b@" echo $LIST_CONTENT
-does_not_contain "a@" echo $LIST_CONTENT
+contains "export PATH=$(spack -m location -i shell-b)/bin" spack -m load --sh shell-b
+succeeds spack -m load shell-b
+LIST_CONTENT=`spack -m load shell-b; spack load --list`
+contains "shell-b@" echo $LIST_CONTENT
+does_not_contain "shell-a@" echo $LIST_CONTENT
 fails spack -m load -l
 # test a variable MacOS clears and one it doesn't for recursive loads
-contains "export PATH=$(spack -m location -i a)/bin" spack -m load --sh a
-contains "export PATH=$(spack -m location -i b)/bin" spack -m load --sh b
-succeeds spack -m load --only dependencies a
-succeeds spack -m load --only package a
+contains "export PATH=$(spack -m location -i shell-a)/bin" spack -m load --sh shell-a
+contains "export PATH=$(spack -m location -i shell-b)/bin" spack -m load --sh shell-b
+succeeds spack -m load shell-a
 fails spack -m load d
 contains "usage: spack load " spack -m load -h
 contains "usage: spack load " spack -m load -h d
 contains "usage: spack load " spack -m load --help
 
 title 'Testing `spack unload`'
-spack -m load b a  # setup
-succeeds spack -m unload b
+spack -m load shell-b shell-a  # setup
+succeeds spack -m unload shell-b
 succeeds spack -m unload --all
 spack -m unload --all # cleanup
 fails spack -m unload -l
@@ -141,7 +140,6 @@ contains " spack env list " spack env list --help
 
 title 'Testing `spack env activate`'
 contains "No such environment:" spack env activate no_such_environment
-contains "env activate requires an environment " spack env activate
 contains "usage: spack env activate " spack env activate -h
 contains "usage: spack env activate " spack env activate --help
 
@@ -198,6 +196,11 @@ contains "spack_test_2_env" sh -c 'echo $PATH'
 does_not_contain "spack_test_env" sh -c 'echo $PATH'
 despacktivate
 
+echo "Testing default environment"
+spack env activate
+contains "In environment default" spack env status
+despacktivate
+
 echo "Correct error exit codes for activate and deactivate"
 fails spack env activate nonexisiting_environment
 fails spack env deactivate
diff --git a/share/spack/setup-env.fish b/share/spack/setup-env.fish
index 901ffe129f7190..482c3eaa684c06 100755
--- a/share/spack/setup-env.fish
+++ b/share/spack/setup-env.fish
@@ -785,7 +785,15 @@ if test -z "$SPACK_SKIP_MODULES"
     sp_multi_pathadd MODULEPATH $_sp_tcl_roots
 end
 
+# Add programmable tab completion for fish
+#
+set -l fish_version (string split '.' $FISH_VERSION)
+if test $fish_version[1] -gt 3
+    or test $fish_version[1] -eq 3
+    and test $fish_version[2] -ge 2
 
+    source $sp_share_dir/spack-completion.fish
+end
 
 #
 # NOTES
diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh
index 7d4554359fa6ab..a42882266cf9fa 100755
--- a/share/spack/setup-env.sh
+++ b/share/spack/setup-env.sh
@@ -41,7 +41,7 @@
 
 # prevent infinite recursion when spack shells out (e.g., on cray for modules)
 if [ -n "${_sp_initializing:-}" ]; then
-    exit 0
+    return 0
 fi
 export _sp_initializing=true
 
@@ -98,7 +98,7 @@ _spack_shell_wrapper() {
             if [ "$_sp_arg" = "-h" ] || [ "$_sp_arg" = "--help" ]; then
                 command spack cd -h
             else
-                LOC="$(spack location $_sp_arg "$@")"
+                LOC="$(SPACK_COLOR="${SPACK_COLOR:-always}" spack location $_sp_arg "$@")"
                 if [ -d "$LOC" ] ; then
                     cd "$LOC"
                 else
@@ -126,8 +126,7 @@ _spack_shell_wrapper() {
                         # Space needed here to differentiate between `-h`
                         # argument and environments with "-h" in the name.
                         # Also see: https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html#Shell-Parameter-Expansion
-                        if [ -z ${1+x} ] || \
-                           [ "${_a#* --sh}" != "$_a" ] || \
+                        if [ "${_a#* --sh}" != "$_a" ] || \
                            [ "${_a#* --csh}" != "$_a" ] || \
                            [ "${_a#* -h}" != "$_a" ] || \
                            [ "${_a#* --help}" != "$_a" ];
@@ -136,7 +135,7 @@ _spack_shell_wrapper() {
                             command spack env activate "$@"
                         else
                             # Actual call to activate: source the output.
-                            stdout="$(command spack $_sp_flags env activate --sh "$@")" || return
+                            stdout="$(SPACK_COLOR="${SPACK_COLOR:-always}" command spack $_sp_flags env activate --sh "$@")" || return
                             eval "$stdout"
                         fi
                         ;;
@@ -158,7 +157,7 @@ _spack_shell_wrapper() {
                             command spack env deactivate -h
                         else
                             # No args: source the output of the command.
-                            stdout="$(command spack $_sp_flags env deactivate --sh)" || return
+                            stdout="$(SPACK_COLOR="${SPACK_COLOR:-always}" command spack $_sp_flags env deactivate --sh)" || return
                             eval "$stdout"
                         fi
                         ;;
@@ -186,7 +185,7 @@ _spack_shell_wrapper() {
                 # Args contain --sh, --csh, or -h/--help: just execute.
                 command spack $_sp_flags $_sp_subcommand "$@"
             else
-                stdout="$(command spack $_sp_flags $_sp_subcommand --sh "$@")" || return
+                stdout="$(SPACK_COLOR="${SPACK_COLOR:-always}" command spack $_sp_flags $_sp_subcommand --sh "$@")" || return
                 eval "$stdout"
             fi
             ;;
diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash
index 959e4924f22bc9..a54f7db414e409 100755
--- a/share/spack/spack-completion.bash
+++ b/share/spack/spack-completion.bash
@@ -7,7 +7,7 @@
 # NOTE: spack-completion.bash is auto-generated by:
 #
 #   $ spack commands --aliases --format=bash
-#       --header=bash/spack-completion.in --update=spack-completion.bash
+#       --header=bash/spack-completion.bash --update=spack-completion.bash
 #
 # Please do not manually modify this file.
 
@@ -52,6 +52,20 @@ if test -n "${ZSH_VERSION:-}" ; then
   fi
 fi
 
+# compgen -W doesn't work in some versions of zsh, so use this instead.
+# see https://www.zsh.org/mla/workers/2011/msg00582.html
+_compgen_w() {
+    if test -n "${ZSH_VERSION:-}" ; then
+        typeset -a words
+        words=( ${~=1} )
+        local find="$2"
+        results=(${(M)words[@]:#$find*})
+        echo "${results[@]}"
+    else
+        compgen -W "$1" -- "$2"
+    fi
+}
+
 # Bash programmable completion for Spack
 _bash_completion_spack() {
     # In all following examples, let the cursor be denoted by brackets, i.e. []
@@ -137,8 +151,11 @@ _bash_completion_spack() {
     if [[ "$(LC_ALL=C type $subfunction 2>&1)" =~ $rgx ]]
     then
         $subfunction
-        COMPREPLY=($(compgen -W "$SPACK_COMPREPLY" -- "$cur"))
+        COMPREPLY=($(_compgen_w "$SPACK_COMPREPLY" "$cur"))
     fi
+
+    # if every completion is an alias for the same thing, just return that thing.
+    _spack_compress_aliases
 }
 
 # Helper functions for subcommands
@@ -328,16 +345,63 @@ _spacktivate() {
   _spack_env_activate
 }
 
+# Simple function to get the spack alias for a command
+_spack_get_alias() {
+    local possible_alias="${1-}"
+    local IFS=";"
+
+    # spack aliases are a ;-separated list of :-separated pairs
+    for item in $SPACK_ALIASES; do
+        # maps a possible alias to its command
+        eval "local real_command=\"\${item#*${possible_alias}:}\""
+        if [ "$real_command" != "$item" ]; then
+            SPACK_ALIAS="$real_command"
+            return
+        fi
+    done
+
+    # no alias found -- just return $1
+    SPACK_ALIAS="$possible_alias"
+}
+
+# If all commands in COMPREPLY alias to the same thing, set COMPREPLY to
+# just the real command, not the aliases.
+_spack_compress_aliases() {
+    # If there are zero or one completions, don't do anything
+    # If this isn't the first argument, bail because aliases currently only apply
+    # to top-level commands.
+    if [ "${#COMPREPLY[@]}" -le "1" ] || [ "$COMP_CWORD_NO_FLAGS" != "1" ]; then
+        return
+    fi
+
+    # get the alias of the first thing in the list of completions
+    _spack_get_alias "${COMPREPLY[@]:0:1}"
+    local first_alias="$SPACK_ALIAS"
+
+    # if anything in the list would alias to something different, stop
+    for comp in "${COMPREPLY[@]:1}"; do
+        _spack_get_alias "$comp"
+        if [ "$SPACK_ALIAS" != "$first_alias" ]; then
+            return
+        fi
+    done
+
+    # all commands alias to first alias; just return that
+    COMPREPLY=("$first_alias")
+}
+
 # Spack commands
 #
 # Everything below here is auto-generated.
+SPACK_ALIASES="concretise:concretize;containerise:containerize;rm:remove"
+
 
 _spack() {
     if $list_options
     then
         SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace --backtrace -V --version --print-shell-vars"
     else
-        SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize config containerize create debug dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view"
+        SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view"
     fi
 }
 
@@ -359,7 +423,7 @@ _spack_audit() {
     then
         SPACK_COMPREPLY="-h --help"
     else
-        SPACK_COMPREPLY="configs packages-https packages list"
+        SPACK_COMPREPLY="configs externals packages-https packages list"
     fi
 }
 
@@ -367,6 +431,15 @@ _spack_audit_configs() {
     SPACK_COMPREPLY="-h --help"
 }
 
+_spack_audit_externals() {
+    if $list_options
+    then
+        SPACK_COMPREPLY="-h --help --list"
+    else
+        SPACK_COMPREPLY=""
+    fi
+}
+
 _spack_audit_packages_https() {
     if $list_options
     then
@@ -498,7 +571,7 @@ _spack_buildcache() {
 _spack_buildcache_push() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -f --force -u --unsigned -a --allow-root -k --key --update-index --rebuild-index --spec-file --only"
+        SPACK_COMPREPLY="-h --help -f --force --allow-root -a --unsigned -u --key -k --update-index --rebuild-index --spec-file --only --fail-fast --base-image -j --jobs"
     else
         _mirrors
     fi
@@ -507,7 +580,7 @@ _spack_buildcache_push() {
 _spack_buildcache_create() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -f --force -u --unsigned -a --allow-root -k --key --update-index --rebuild-index --spec-file --only"
+        SPACK_COMPREPLY="-h --help -f --force --allow-root -a --unsigned -u --key -k --update-index --rebuild-index --spec-file --only --fail-fast --base-image -j --jobs"
     else
         _mirrors
     fi
@@ -525,7 +598,7 @@ _spack_buildcache_install() {
 _spack_buildcache_list() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -l --long -L --very-long -v --variants -a --allarch"
+        SPACK_COMPREPLY="-h --help -l --long -L --very-long -N --namespaces -v --variants -a --allarch"
     else
         _all_packages
     fi
@@ -608,7 +681,7 @@ _spack_change() {
 _spack_checksum() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --keep-stage -b --batch -l --latest -p --preferred -a --add-to-package"
+        SPACK_COMPREPLY="-h --help --keep-stage --batch -b --latest -l --preferred -p --add-to-package -a --verify -j --jobs"
     else
         _all_packages
     fi
@@ -638,7 +711,7 @@ _spack_ci_rebuild() {
 _spack_ci_reproduce_build() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --working-dir"
+        SPACK_COMPREPLY="-h --help --runtime --working-dir -s --autostart --gpg-file --gpg-url"
     else
         SPACK_COMPREPLY=""
     fi
@@ -683,7 +756,7 @@ _spack_compiler() {
 _spack_compiler_find() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --scope"
+        SPACK_COMPREPLY="-h --help --mixed-toolchain --no-mixed-toolchain --scope"
     else
         SPACK_COMPREPLY=""
     fi
@@ -692,7 +765,7 @@ _spack_compiler_find() {
 _spack_compiler_add() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --scope"
+        SPACK_COMPREPLY="-h --help --mixed-toolchain --no-mixed-toolchain --scope"
     else
         SPACK_COMPREPLY=""
     fi
@@ -737,6 +810,10 @@ _spack_concretize() {
     SPACK_COMPREPLY="-h --help -f --force --test -q --quiet -U --fresh --reuse --reuse-deps -j --jobs"
 }
 
+_spack_concretise() {
+    SPACK_COMPREPLY="-h --help -f --force --test -q --quiet -U --fresh --reuse --reuse-deps -j --jobs"
+}
+
 _spack_config() {
     if $list_options
     then
@@ -830,6 +907,10 @@ _spack_containerize() {
     SPACK_COMPREPLY="-h --help --list-os --last-stage"
 }
 
+_spack_containerise() {
+    SPACK_COMPREPLY="-h --help --list-os --last-stage"
+}
+
 _spack_create() {
     if $list_options
     then
@@ -856,6 +937,15 @@ _spack_debug_report() {
     SPACK_COMPREPLY="-h --help"
 }
 
+_spack_deconcretize() {
+    if $list_options
+    then
+        SPACK_COMPREPLY="-h --help --root -y --yes-to-all -a --all"
+    else
+        _all_packages
+    fi
+}
+
 _spack_dependencies() {
     if $list_options
     then
@@ -935,14 +1025,14 @@ _spack_env() {
 _spack_env_activate() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --sh --csh --fish --bat --pwsh -v --with-view -V --without-view -p --prompt --temp -d --dir"
+        SPACK_COMPREPLY="-h --help --sh --csh --fish --bat --pwsh --with-view -v --without-view -V -p --prompt --temp -d --dir"
     else
         _environments
     fi
 }
 
 _spack_env_deactivate() {
-    SPACK_COMPREPLY="-h --help --sh --csh --fish --bat"
+    SPACK_COMPREPLY="-h --help --sh --csh --fish --bat --pwsh"
 }
 
 _spack_env_create() {
@@ -1049,7 +1139,7 @@ _spack_external() {
 _spack_external_find() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --not-buildable --exclude -p --path --scope --all -t --tag"
+        SPACK_COMPREPLY="-h --help --not-buildable --exclude -p --path --scope --all -t --tag -j --jobs"
     else
         _all_packages
     fi
@@ -1075,7 +1165,7 @@ _spack_fetch() {
 _spack_find() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --format -H --hashes --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tag -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated -N --namespace --start-date --end-date"
+        SPACK_COMPREPLY="-h --help --format -H --hashes --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tag -N --namespaces -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated --start-date --end-date"
     else
         _installed_packages
     fi
@@ -1186,7 +1276,7 @@ _spack_help() {
 _spack_info() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -a --all --detectable --maintainers --no-dependencies --no-variants --no-versions --phases --tags --tests --virtuals"
+        SPACK_COMPREPLY="-h --help -a --all --detectable --maintainers --no-dependencies --no-variants --no-versions --phases --tags --tests --virtuals --variants-by-name"
     else
         _all_packages
     fi
@@ -1234,7 +1324,7 @@ _spack_list() {
 _spack_load() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --sh --csh --fish --bat --first --only --list"
+        SPACK_COMPREPLY="-h --help --sh --csh --fish --bat --pwsh --first --only --list"
     else
         _installed_packages
     fi
@@ -1290,7 +1380,7 @@ _spack_mirror() {
     then
         SPACK_COMPREPLY="-h --help -n --no-checksum --deprecated"
     else
-        SPACK_COMPREPLY="create destroy add remove rm set-url list"
+        SPACK_COMPREPLY="create destroy add remove rm set-url set list"
     fi
 }
 
@@ -1310,7 +1400,7 @@ _spack_mirror_destroy() {
 _spack_mirror_add() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url"
+        SPACK_COMPREPLY="-h --help --scope --type --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url --oci-username --oci-password"
     else
         _mirrors
     fi
@@ -1337,7 +1427,16 @@ _spack_mirror_rm() {
 _spack_mirror_set_url() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --push --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url"
+        SPACK_COMPREPLY="-h --help --push --fetch --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url --oci-username --oci-password"
+    else
+        _mirrors
+    fi
+}
+
+_spack_mirror_set() {
+    if $list_options
+    then
+        SPACK_COMPREPLY="-h --help --push --fetch --type --url --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url --oci-username --oci-password"
     else
         _mirrors
     fi
@@ -1695,7 +1794,7 @@ _spack_restage() {
 _spack_solve() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --show -l --long -L --very-long -I --install-status --no-install-status -y --yaml -j --json -c --cover -N --namespaces -t --types --timers --stats -U --fresh --reuse --reuse-deps"
+        SPACK_COMPREPLY="-h --help --show -l --long -L --very-long -N --namespaces -I --install-status --no-install-status -y --yaml -j --json -c --cover -t --types --timers --stats -U --fresh --reuse --reuse-deps"
     else
         _all_packages
     fi
@@ -1704,7 +1803,7 @@ _spack_solve() {
 _spack_spec() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -l --long -L --very-long -I --install-status --no-install-status -y --yaml -j --json --format -c --cover -N --namespaces -t --types -U --fresh --reuse --reuse-deps"
+        SPACK_COMPREPLY="-h --help -l --long -L --very-long -N --namespaces -I --install-status --no-install-status -y --yaml -j --json --format -c --cover -t --types -U --fresh --reuse --reuse-deps"
     else
         _all_packages
     fi
@@ -1843,7 +1942,7 @@ _spack_unit_test() {
 _spack_unload() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --sh --csh --fish --bat -a --all"
+        SPACK_COMPREPLY="-h --help --sh --csh --fish --bat --pwsh -a --all"
     else
         _installed_packages
     fi
@@ -1891,7 +1990,7 @@ _spack_verify() {
 _spack_versions() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -s --safe --safe-only -r --remote -n --new -c --concurrency"
+        SPACK_COMPREPLY="-h --help -s --safe --safe-only -r --remote -n --new -j --jobs"
     else
         _all_packages
     fi
diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish
new file mode 100755
index 00000000000000..1029fa6b45e06c
--- /dev/null
+++ b/share/spack/spack-completion.fish
@@ -0,0 +1,3137 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+# NOTE: spack-completion.fish is auto-generated by:
+#
+#   $ spack commands --aliases --format=fish
+#       --header=fish/spack-completion.fish --update=spack-completion.fish
+#
+# Please do not manually modify this file.
+
+# Check fish version before proceeding
+set -l fish_version (string split '.' $FISH_VERSION)
+if test $fish_version[1] -lt 3
+    if test $fish_version[1] -eq 3
+        and test $fish_version[2] -lt 2
+        echo 'Fish version is older than 3.2.0. Some completion features may not work'
+        set -g __fish_spack_force_files
+    else
+        echo 'This script requires fish version 3.0 or later'
+        exit 1
+    end
+else
+    set -g __fish_spack_force_files -F
+end
+
+# The following global variables are used as a cache of `__fish_spack_argparse`
+
+# Cached command line
+set -g __fish_spack_argparse_cache_line
+# Parsed command
+set -g __fish_spack_argparse_command
+# Remaining arguments
+set -g __fish_spack_argparse_argv
+# Return value
+set -g __fish_spack_argparse_return
+
+# Spack command generates an optspec variable $__fish_spack_optspecs_.
+# We check if this command exists, and echo the optspec variable name.
+function __fish_spack_get_optspecs -d 'Get optspecs of spack command'
+    # Convert arguments to replace ' ' and '-' by '_'
+    set -l cmd_var (string replace -ra -- '[ -]' '_' $argv | string join '_')
+    # Set optspec variable name
+    set -l optspecs_var __fish_spack_optspecs_$cmd_var
+    # Query if variable $$optspecs_var exists
+    set -q $optspecs_var; or return 1
+    # If it exists, echo all optspecs line by line.
+    # String join returns 1 if no join was performed, so we return 0 in such case.
+    string join \n $$optspecs_var; or return 0
+end
+
+# Parse command-line arguments, save results to global variables,
+# and add found flags to __fish_spack_flag_.
+# Returns 1 if help flag is found.
+function __fish_spack_argparse
+    # Figure out if the current invocation already has a command.
+    set -l args $argv
+    set -l commands
+
+    # Return cached result if arguments haven't changed
+    if test "$__fish_spack_argparse_cache_line" = "$args"
+        return $__fish_spack_argparse_return
+    end
+
+    # Clear all flags found in last run
+    set -g | string replace -rf -- '^(__fish_spack_flag_\w+)(.*?)$' 'set -ge $1' | source
+
+    # Set default return value to 0, indicating success
+    set -g __fish_spack_argparse_return 0
+    # Set command line to current arguments
+    set -g __fish_spack_argparse_cache_line $argv
+
+    # Recursively check arguments for commands
+    while set -q args[1]
+        # Get optspecs of current command
+        set -l optspecs (__fish_spack_get_optspecs $commands $args[1])
+        or break
+
+        # If command exists, shift arguments
+        set -a commands $args[1]
+        set -e args[1]
+
+        # If command has no arguments, continue
+        set -q optspecs[1]; or continue
+
+        # Parse arguments. Set variable _flag_ if flag is found.
+        # We find all these variables and set them to the global variable __fish_spack_flag_.
+        argparse -i -s $optspecs -- $args 2>/dev/null; or break
+        set -l | string replace -rf -- '^(_flag_.*)$' 'set -g __fish_spack$1' | source
+
+        # Set args to not parsed arguments
+        set args $argv
+
+        # If command has help flag, we don't need to parse more so short circuit
+        if set -q _flag_help
+            set -g __fish_spack_argparse_return 1
+            break
+        end
+    end
+
+    # Set cached variables
+    set -g __fish_spack_argparse_command $commands
+    set -g __fish_spack_argparse_argv $args
+
+    return $__fish_spack_argparse_return
+end
+
+# Check if current commandline's command is "spack $argv"
+function __fish_spack_using_command
+    set -l line (commandline -opc)
+    __fish_spack_argparse $line; or return 1
+
+    set -p argv spack
+    test "$__fish_spack_argparse_command" = "$argv"
+end
+
+# Check if current commandline's command is "spack $argv[2..-1]",
+# and cursor is at $argv[1]-th positional argument
+function __fish_spack_using_command_pos
+    __fish_spack_using_command $argv[2..-1]
+    or return
+
+    test (count $__fish_spack_argparse_argv) -eq $argv[1]
+end
+
+function __fish_spack_using_command_pos_remainder
+    __fish_spack_using_command $argv[2..-1]
+    or return
+
+    test (count $__fish_spack_argparse_argv) -ge $argv[1]
+end
+
+# Helper functions for subcommands
+
+function __fish_spack_bootstrap_names
+    if set -q __fish_spack_flag_scope
+        spack bootstrap list --scope $__fish_spack_flag_scope | string replace -rf -- '^Name: (\w+).*?$' '$1'
+    else
+        spack bootstrap list | string replace -rf -- '^Name: (\w+).*?$' '$1'
+    end
+end
+
+# Reference: sudo's fish completion
+function __fish_spack_build_env_spec
+    set token (commandline -opt)
+
+    set -l index (contains -- -- $__fish_spack_argparse_argv)
+    if set -q index[1]
+        __fish_complete_subcommand --commandline $__fish_spack_argparse_argv[(math $index + 1)..-1]
+    else if set -q __fish_spack_argparse_argv[1]
+        __fish_complete_subcommand --commandline "$__fish_spack_argparse_argv[2..-1] $token"
+    else
+        __fish_spack_specs
+    end
+end
+
+function __fish_spack_commands
+    spack commands
+end
+
+function __fish_spack_colon_path
+    set token (string split -rm1 ':' (commandline -opt))
+
+    if test (count $token) -lt 2
+        __fish_complete_path $token[1]
+    else
+        __fish_complete_path $token[2] | string replace -r -- '^' "$token[1]:"
+    end
+end
+
+function __fish_spack_config_sections
+    if set -q __fish_spack_flag_scope
+        spack config --scope $__fish_spack_flag_scope list | string split ' '
+    else
+        spack config list | string split ' '
+    end
+end
+
+function __fish_spack_environments
+    string trim (spack env list)
+end
+
+function __fish_spack_extensions
+    # Skip optional flags, or it will be really slow
+    string match -q -- '-*' (commandline -opt)
+    and return
+
+    comm -1 -2 (spack extensions | string trim | psub) (__fish_spack_installed_packages | sort | psub)
+end
+
+function __fish_spack_gpg_keys
+    spack gpg list
+end
+
+function __fish_spack_installed_compilers
+    spack compilers | grep -v '^[=-]\|^$'
+end
+
+function __fish_spack_installed_packages
+    spack find --no-groups --format '{name}' | uniq
+end
+
+function __fish_spack_installed_specs
+    # Try match local hash first
+    __fish_spack_installed_specs_id
+    and return
+
+    spack find --no-groups --format '{name}@{version}'
+end
+
+function __fish_spack_installed_specs_id
+    set -l token (commandline -opt)
+    string match -q -- '/*' $token
+    or return 1
+
+    spack find --format '/{hash:7}'\t'{name}{@version}'
+end
+
+function __fish_spack_git_rev
+    type -q __fish_git_ranges
+    and __fish_git_ranges
+end
+
+function __fish_spack_mirrors
+    spack mirror list | awk {'printf ("%s\t%s", $1, $2)'}
+end
+
+function __fish_spack_package_versions
+    string trim (spack versions $argv)
+end
+
+function __fish_spack_packages
+    spack list
+end
+
+function __fish_spack_pkg_packages
+    spack pkg list
+end
+
+function __fish_spack_providers
+    string trim (spack providers | grep -v '^$')
+end
+
+function __fish_spack_repos
+    spack repo list | awk {'printf ("%s\t%s", $1, $2)'}
+end
+
+function __fish_spack_scopes
+    # TODO: how to list all scopes?
+    set -l scope system site user defaults
+    set -l platform cray darwin linux test
+
+    string join \n $scope
+end
+
+function __fish_spack_specs
+    set -l token (commandline -opt)
+
+    # Complete compilers
+    if string match -rq -- '^(?
.*%)[\w-]*(@[\w\.+~-]*)?$' $token
+        __fish_spack_installed_compilers | string replace -r -- '^' "$pre"
+        return
+    end
+
+    # Try to complete spec version
+    # Currently we can only match '@' after a package name
+    set -l package
+
+    # Match ^ following package name
+    if string match -rq -- '^(?
.*?\^)[\w\.+~-]*$' $token
+        # Package name is the nearest, assuming first character is always a letter or digit
+        set packages (string match -ar -- '^[\w-]+' $__fish_spack_argparse_argv $token)
+        set package $packages[-1]
+
+        if test -n "$package"
+            spack dependencies $package | string replace -r -- '^' "$pre"
+            return
+        end
+    end
+
+    # Match @ following package name
+    if string match -rq -- '^(?
.*?\^?(?[\w\.+~-]*)@)[\w\.]*$' $token
+        set package $packages[-1]
+
+        # Matched @ starting at next token
+        if test -z "$package"
+            string match -arq -- '(^|\^)(?[\w\.+~-]*)$' $__fish_spack_argparse_argv[-1]
+            if test -n "$inners[1]"
+                set package $inners[-1]
+            end
+        end
+    end
+
+    # Complete version if package found
+    if test -n "$package"
+        # Only list safe versions for speed
+        string trim (spack versions --safe $package) | string replace -r -- '^' "$pre"
+        return
+    end
+
+    # Else complete package name
+    __fish_spack_installed_packages | string replace -r -- '$' \t"installed"
+    spack list
+end
+
+function __fish_spack_specs_or_id
+    # Try to match local hash first
+    __fish_spack_installed_specs_id
+    and return
+
+    __fish_spack_specs
+end
+
+function __fish_spack_tags
+    string trim (spack tags)
+end
+
+function __fish_spack_tests
+    spack test list | grep -v '^[=-]'
+end
+
+function __fish_spack_unit_tests
+    # Skip optional flags, or it will be really slow
+    string match -q -- '-*' (commandline -opt)
+    and return
+
+    spack unit-test -l
+end
+
+function __fish_spack_yamls
+    # Trim flag from current token
+    string match -rq -- '(?
-.)?(?.*)' (commandline -opt)
+
+    if test -n "$token"
+        find $token* -type f '(' -iname '*.yaml' -or -iname '*.yml' ')'
+    else
+        find -maxdepth 2 -type f '(' -iname '*.yaml' -or -iname '*.yml' ')' | cut -c 3-
+    end
+end
+
+# Reset existing completions
+complete -c spack --erase
+
+# Spack commands
+#
+# Everything below here is auto-generated.
+
+# spack
+set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace backtrace V/version print-shell-vars=
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a add -d 'add a spec to an environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a arch -d 'print architecture information about this machine'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a audit -d 'audit configuration files, packages, etc.'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a blame -d 'show contributors to packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a bootstrap -d 'manage bootstrap configuration'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a build-env -d 'run a command in a spec\'s install environment, or dump its environment to screen or file'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a buildcache -d 'create, download and install binary packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a cd -d 'cd to spack directories in the shell'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a change -d 'change an existing spec in an environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a checksum -d 'checksum available versions of a package'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a ci -d 'manage continuous integration pipelines'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a clean -d 'remove temporary build files and/or downloaded archives'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a clone -d 'create a new installation of spack in another prefix'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a commands -d 'list available spack commands'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a compiler -d 'manage compilers'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a compilers -d 'list available compilers'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a concretize -d 'concretize an environment and write a lockfile'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a concretise -d 'concretize an environment and write a lockfile'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a config -d 'get and set configuration options'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a containerize -d 'creates recipes to build images for different container runtimes'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a containerise -d 'creates recipes to build images for different container runtimes'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a create -d 'create a new package file'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a debug -d 'debugging commands for troubleshooting Spack'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a deconcretize -d 'remove specs from the concretized lockfile of an environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a dependencies -d 'show dependencies of a package'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a dependents -d 'show packages that depend on another'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a deprecate -d 'replace one package with another via symlinks'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a dev-build -d 'developer build: build from code in current working directory'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a develop -d 'add a spec to an environment\'s dev-build information'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a diff -d 'compare two specs'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a docs -d 'open spack documentation in a web browser'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a edit -d 'open package files in $EDITOR'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a env -d 'manage virtual environments'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a extensions -d 'list extensions for package'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a external -d 'manage external packages in Spack configuration'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a fetch -d 'fetch archives for packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a find -d 'list and search installed packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a gc -d 'remove specs that are now no longer needed'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a gpg -d 'handle GPG actions for spack'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a graph -d 'generate graphs of package dependency relationships'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a help -d 'get help on spack and its commands'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a info -d 'get detailed information on a particular package'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a install -d 'build and install packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a license -d 'list and check license headers on files in spack'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a list -d 'list and search available packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a load -d 'add package to the user environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a location -d 'print out locations of packages and spack directories'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a log-parse -d 'filter errors and warnings from build logs'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a maintainers -d 'get information about package maintainers'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a make-installer -d 'generate Windows installer'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a mark -d 'mark packages as explicitly or implicitly installed'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a mirror -d 'manage mirrors (source and binary)'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a module -d 'generate/manage module files'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a patch -d 'patch expanded archive sources in preparation for install'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a pkg -d 'query packages associated with particular git revisions'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a providers -d 'list packages that provide a particular virtual package'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a pydoc -d 'run pydoc from within spack'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a python -d 'launch an interpreter as spack would launch a command'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a reindex -d 'rebuild Spack\'s package database'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a remove -d 'remove specs from an environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a rm -d 'remove specs from an environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a repo -d 'manage package source repositories'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a resource -d 'list downloadable resources (tarballs, repos, patches, etc.)'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a restage -d 'revert checked out package source code'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a solve -d 'concretize a specs using an ASP solver'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a spec -d 'show what would be installed, given a spec'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a stage -d 'expand downloaded archive in preparation for install'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a style -d 'runs source code style checks on spack'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a tags -d 'show package tags and associated packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a test -d 'run spack\'s tests for an install'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a test-env -d 'run a command in a spec\'s test environment, or dump its environment to screen or file'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a tutorial -d 'set up spack for our tutorial (WARNING: modifies config!)'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a undevelop -d 'remove specs from an environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a uninstall -d 'remove installed packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a unit-test -d 'run spack\'s unit tests (wrapper around pytest)'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a unload -d 'remove package from the user environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a url -d 'debugging tool for url parsing'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a verify -d 'check that all spack packages are on disk as installed'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a versions -d 'list available versions of a package'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a view -d 'project packages to a compact naming scheme on the filesystem'
+complete -c spack -n '__fish_spack_using_command ' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command ' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command ' -s H -l all-help -f -a help
+complete -c spack -n '__fish_spack_using_command ' -s H -l all-help -d 'show help for all commands (same as spack help --all)'
+complete -c spack -n '__fish_spack_using_command ' -l color -r -f -a 'always never auto'
+complete -c spack -n '__fish_spack_using_command ' -l color -r -d 'when to colorize output (default: auto)'
+complete -c spack -n '__fish_spack_using_command ' -s c -l config -r -f -a config_vars
+complete -c spack -n '__fish_spack_using_command ' -s c -l config -r -d 'add one or more custom, one off config settings'
+complete -c spack -n '__fish_spack_using_command ' -s C -l config-scope -r -f -a config_scopes
+complete -c spack -n '__fish_spack_using_command ' -s C -l config-scope -r -d 'add a custom configuration scope'
+complete -c spack -n '__fish_spack_using_command ' -s d -l debug -f -a debug
+complete -c spack -n '__fish_spack_using_command ' -s d -l debug -d 'write out debug messages'
+complete -c spack -n '__fish_spack_using_command ' -l timestamp -f -a timestamp
+complete -c spack -n '__fish_spack_using_command ' -l timestamp -d 'add a timestamp to tty output'
+complete -c spack -n '__fish_spack_using_command ' -l pdb -f -a pdb
+complete -c spack -n '__fish_spack_using_command ' -l pdb -d 'run spack under the pdb debugger'
+complete -c spack -n '__fish_spack_using_command ' -s e -l env -r -f -a env
+complete -c spack -n '__fish_spack_using_command ' -s e -l env -r -d 'run with a specific environment (see spack env)'
+complete -c spack -n '__fish_spack_using_command ' -s D -l env-dir -r -f -a env_dir
+complete -c spack -n '__fish_spack_using_command ' -s D -l env-dir -r -d 'run with an environment directory (ignore managed environments)'
+complete -c spack -n '__fish_spack_using_command ' -s E -l no-env -f -a no_env
+complete -c spack -n '__fish_spack_using_command ' -s E -l no-env -d 'run without any environments activated (see spack env)'
+complete -c spack -n '__fish_spack_using_command ' -l use-env-repo -f -a use_env_repo
+complete -c spack -n '__fish_spack_using_command ' -l use-env-repo -d 'when running in an environment, use its package repository'
+complete -c spack -n '__fish_spack_using_command ' -s k -l insecure -f -a insecure
+complete -c spack -n '__fish_spack_using_command ' -s k -l insecure -d 'do not check ssl certificates when downloading'
+complete -c spack -n '__fish_spack_using_command ' -s l -l enable-locks -f -a locks
+complete -c spack -n '__fish_spack_using_command ' -s l -l enable-locks -d 'use filesystem locking (default)'
+complete -c spack -n '__fish_spack_using_command ' -s L -l disable-locks -f -a locks
+complete -c spack -n '__fish_spack_using_command ' -s L -l disable-locks -d 'do not use filesystem locking (unsafe)'
+complete -c spack -n '__fish_spack_using_command ' -s m -l mock -f -a mock
+complete -c spack -n '__fish_spack_using_command ' -s m -l mock -d 'use mock packages instead of real ones'
+complete -c spack -n '__fish_spack_using_command ' -s b -l bootstrap -f -a bootstrap
+complete -c spack -n '__fish_spack_using_command ' -s b -l bootstrap -d 'use bootstrap configuration (bootstrap store, config, externals)'
+complete -c spack -n '__fish_spack_using_command ' -s p -l profile -f -a spack_profile
+complete -c spack -n '__fish_spack_using_command ' -s p -l profile -d 'profile execution using cProfile'
+complete -c spack -n '__fish_spack_using_command ' -l sorted-profile -r -f -a sorted_profile
+complete -c spack -n '__fish_spack_using_command ' -l sorted-profile -r -d 'profile and sort'
+complete -c spack -n '__fish_spack_using_command ' -l lines -r -f -a lines
+complete -c spack -n '__fish_spack_using_command ' -l lines -r -d 'lines of profile output or \'all\' (default: 20)'
+complete -c spack -n '__fish_spack_using_command ' -s v -l verbose -f -a verbose
+complete -c spack -n '__fish_spack_using_command ' -s v -l verbose -d 'print additional output during builds'
+complete -c spack -n '__fish_spack_using_command ' -l stacktrace -f -a stacktrace
+complete -c spack -n '__fish_spack_using_command ' -l stacktrace -d 'add stacktraces to all printed statements'
+complete -c spack -n '__fish_spack_using_command ' -l backtrace -f -a backtrace
+complete -c spack -n '__fish_spack_using_command ' -l backtrace -d 'always show backtraces for exceptions'
+complete -c spack -n '__fish_spack_using_command ' -s V -l version -f -a version
+complete -c spack -n '__fish_spack_using_command ' -s V -l version -d 'show version number and exit'
+complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -f -a print_shell_vars
+complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -d 'print info needed by setup-env.*sh'
+
+# spack add
+set -g __fish_spack_optspecs_spack_add h/help l/list-name=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 add' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command add' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command add' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command add' -s l -l list-name -r -f -a list_name
+complete -c spack -n '__fish_spack_using_command add' -s l -l list-name -r -d 'name of the list to add specs to'
+
+# spack arch
+set -g __fish_spack_optspecs_spack_arch h/help g/generic-target known-targets p/platform o/operating-system t/target f/frontend b/backend
+complete -c spack -n '__fish_spack_using_command arch' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command arch' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command arch' -s g -l generic-target -f -a generic_target
+complete -c spack -n '__fish_spack_using_command arch' -s g -l generic-target -d 'show the best generic target'
+complete -c spack -n '__fish_spack_using_command arch' -l known-targets -f -a known_targets
+complete -c spack -n '__fish_spack_using_command arch' -l known-targets -d 'show a list of all known targets and exit'
+complete -c spack -n '__fish_spack_using_command arch' -s p -l platform -f -a platform
+complete -c spack -n '__fish_spack_using_command arch' -s p -l platform -d 'print only the platform'
+complete -c spack -n '__fish_spack_using_command arch' -s o -l operating-system -f -a operating_system
+complete -c spack -n '__fish_spack_using_command arch' -s o -l operating-system -d 'print only the operating system'
+complete -c spack -n '__fish_spack_using_command arch' -s t -l target -f -a target
+complete -c spack -n '__fish_spack_using_command arch' -s t -l target -d 'print only the target'
+complete -c spack -n '__fish_spack_using_command arch' -s f -l frontend -f -a frontend
+complete -c spack -n '__fish_spack_using_command arch' -s f -l frontend -d 'print frontend'
+complete -c spack -n '__fish_spack_using_command arch' -s b -l backend -f -a backend
+complete -c spack -n '__fish_spack_using_command arch' -s b -l backend -d 'print backend'
+
+# spack audit
+set -g __fish_spack_optspecs_spack_audit h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 audit' -f -a configs -d 'audit configuration files'
+complete -c spack -n '__fish_spack_using_command_pos 0 audit' -f -a externals -d 'check external detection in packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 audit' -f -a packages-https -d 'check https in packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 audit' -f -a packages -d 'audit package recipes'
+complete -c spack -n '__fish_spack_using_command_pos 0 audit' -f -a list -d 'list available checks and exits'
+complete -c spack -n '__fish_spack_using_command audit' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command audit' -s h -l help -d 'show this help message and exit'
+
+# spack audit configs
+set -g __fish_spack_optspecs_spack_audit_configs h/help
+complete -c spack -n '__fish_spack_using_command audit configs' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command audit configs' -s h -l help -d 'show this help message and exit'
+
+# spack audit externals
+set -g __fish_spack_optspecs_spack_audit_externals h/help list
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 audit externals' -f -a '(__fish_spack_packages)'
+complete -c spack -n '__fish_spack_using_command audit externals' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command audit externals' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command audit externals' -l list -f -a list_externals
+complete -c spack -n '__fish_spack_using_command audit externals' -l list -d 'if passed, list which packages have detection tests'
+
+# spack audit packages-https
+set -g __fish_spack_optspecs_spack_audit_packages_https h/help all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 audit packages-https' -f -a '(__fish_spack_packages)'
+complete -c spack -n '__fish_spack_using_command audit packages-https' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command audit packages-https' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command audit packages-https' -l all -f -a check_all
+complete -c spack -n '__fish_spack_using_command audit packages-https' -l all -d 'audit all packages'
+
+# spack audit packages
+set -g __fish_spack_optspecs_spack_audit_packages h/help
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 audit packages' -f -a '(__fish_spack_packages)'
+complete -c spack -n '__fish_spack_using_command audit packages' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command audit packages' -s h -l help -d 'show this help message and exit'
+
+# spack audit list
+set -g __fish_spack_optspecs_spack_audit_list h/help
+complete -c spack -n '__fish_spack_using_command audit list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command audit list' -s h -l help -d 'show this help message and exit'
+
+# spack blame
+set -g __fish_spack_optspecs_spack_blame h/help t/time p/percent g/git json
+complete -c spack -n '__fish_spack_using_command_pos 0 blame' $__fish_spack_force_files -a '(__fish_spack_packages)'
+complete -c spack -n '__fish_spack_using_command blame' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command blame' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command blame' -s t -l time -f -a view
+complete -c spack -n '__fish_spack_using_command blame' -s t -l time -d 'sort by last modification date (default)'
+complete -c spack -n '__fish_spack_using_command blame' -s p -l percent -f -a view
+complete -c spack -n '__fish_spack_using_command blame' -s p -l percent -d 'sort by percent of code'
+complete -c spack -n '__fish_spack_using_command blame' -s g -l git -f -a view
+complete -c spack -n '__fish_spack_using_command blame' -s g -l git -d 'show git blame output instead of summary'
+complete -c spack -n '__fish_spack_using_command blame' -l json -f -a json
+complete -c spack -n '__fish_spack_using_command blame' -l json -d 'output blame as machine-readable json records'
+
+# spack bootstrap
+set -g __fish_spack_optspecs_spack_bootstrap h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap' -f -a now -d 'Spack ready, right now!'
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap' -f -a status -d 'get the status of Spack'
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap' -f -a enable -d 'enable bootstrapping'
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap' -f -a disable -d 'disable bootstrapping'
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap' -f -a reset -d 'reset bootstrapping configuration to Spack defaults'
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap' -f -a root -d 'get/set the root bootstrap directory'
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap' -f -a list -d 'list all the sources of software to bootstrap Spack'
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap' -f -a add -d 'add a new source for bootstrapping'
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap' -f -a remove -d 'remove a bootstrapping source'
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap' -f -a mirror -d 'create a local mirror to bootstrap Spack'
+complete -c spack -n '__fish_spack_using_command bootstrap' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command bootstrap' -s h -l help -d 'show this help message and exit'
+
+# spack bootstrap now
+set -g __fish_spack_optspecs_spack_bootstrap_now h/help dev
+complete -c spack -n '__fish_spack_using_command bootstrap now' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command bootstrap now' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command bootstrap now' -l dev -f -a dev
+complete -c spack -n '__fish_spack_using_command bootstrap now' -l dev -d 'bootstrap dev dependencies too'
+
+# spack bootstrap status
+set -g __fish_spack_optspecs_spack_bootstrap_status h/help optional dev
+complete -c spack -n '__fish_spack_using_command bootstrap status' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command bootstrap status' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command bootstrap status' -l optional -f -a optional
+complete -c spack -n '__fish_spack_using_command bootstrap status' -l optional -d 'show the status of rarely used optional dependencies'
+complete -c spack -n '__fish_spack_using_command bootstrap status' -l dev -f -a dev
+complete -c spack -n '__fish_spack_using_command bootstrap status' -l dev -d 'show the status of dependencies needed to develop Spack'
+
+# spack bootstrap enable
+set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)'
+complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify'
+
+# spack bootstrap disable
+set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)'
+complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify'
+
+# spack bootstrap reset
+set -g __fish_spack_optspecs_spack_bootstrap_reset h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command bootstrap reset' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command bootstrap reset' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command bootstrap reset' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command bootstrap reset' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack bootstrap root
+set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify'
+
+# spack bootstrap list
+set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope=
+complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify'
+
+# spack bootstrap add
+set -g __fish_spack_optspecs_spack_bootstrap_add h/help scope= trust
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__fish_spack_bootstrap_names)'
+complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)'
+complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify'
+complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust
+complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition'
+
+# spack bootstrap remove
+set -g __fish_spack_optspecs_spack_bootstrap_remove h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap remove' -f -a '(__fish_spack_bootstrap_names)'
+complete -c spack -n '__fish_spack_using_command bootstrap remove' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command bootstrap remove' -s h -l help -d 'show this help message and exit'
+
+# spack bootstrap mirror
+set -g __fish_spack_optspecs_spack_bootstrap_mirror h/help binary-packages dev
+
+complete -c spack -n '__fish_spack_using_command bootstrap mirror' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command bootstrap mirror' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command bootstrap mirror' -l binary-packages -f -a binary_packages
+complete -c spack -n '__fish_spack_using_command bootstrap mirror' -l binary-packages -d 'download public binaries in the mirror'
+complete -c spack -n '__fish_spack_using_command bootstrap mirror' -l dev -f -a dev
+complete -c spack -n '__fish_spack_using_command bootstrap mirror' -l dev -d 'download dev dependencies too'
+
+# spack build-env
+set -g __fish_spack_optspecs_spack_build_env h/help clean dirty U/fresh reuse reuse-deps dump= pickle=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 build-env' -f -a '(__fish_spack_build_env_spec)'
+complete -c spack -n '__fish_spack_using_command build-env' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command build-env' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command build-env' -l clean -f -a dirty
+complete -c spack -n '__fish_spack_using_command build-env' -l clean -d 'unset harmful variables in the build environment (default)'
+complete -c spack -n '__fish_spack_using_command build-env' -l dirty -f -a dirty
+complete -c spack -n '__fish_spack_using_command build-env' -l dirty -d 'preserve user environment in spack\'s build environment (danger!)'
+complete -c spack -n '__fish_spack_using_command build-env' -s U -l fresh -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command build-env' -s U -l fresh -d 'do not reuse installed deps; build newest configuration'
+complete -c spack -n '__fish_spack_using_command build-env' -l reuse -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command build-env' -l reuse -d 'reuse installed packages/buildcaches when possible'
+complete -c spack -n '__fish_spack_using_command build-env' -l reuse-deps -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command build-env' -l reuse-deps -d 'reuse installed dependencies only'
+complete -c spack -n '__fish_spack_using_command build-env' -l dump -r -f -a dump
+complete -c spack -n '__fish_spack_using_command build-env' -l dump -r -d 'dump a source-able environment to FILE'
+complete -c spack -n '__fish_spack_using_command build-env' -l pickle -r -f -a pickle
+complete -c spack -n '__fish_spack_using_command build-env' -l pickle -r -d 'dump a pickled source-able environment to FILE'
+
+# spack buildcache
+set -g __fish_spack_optspecs_spack_buildcache h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a push -d 'create a binary package and push it to a mirror'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a create -d 'create a binary package and push it to a mirror'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a install -d 'install from a binary package'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a list -d 'list binary packages available from mirrors'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a keys -d 'get public keys available on mirrors'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a preview -d 'analyze an installed spec and reports whether executables and libraries are relocatable'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a check -d 'check specs against remote binary mirror(s) to see if any need to be rebuilt'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a download -d 'download buildcache entry from a remote mirror to local folder'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a get-buildcache-name -d 'get name (prefix) of buildcache entries for this spec'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a save-specfile -d 'get full spec for dependencies and write them to files in the specified output directory'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a sync -d 'sync binaries (and associated metadata) from one mirror to another'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a update-index -d 'update a buildcache index'
+complete -c spack -n '__fish_spack_using_command_pos 0 buildcache' -f -a rebuild-index -d 'update a buildcache index'
+complete -c spack -n '__fish_spack_using_command buildcache' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache' -s h -l help -d 'show this help message and exit'
+
+# spack buildcache push
+set -g __fish_spack_optspecs_spack_buildcache_push h/help f/force a/allow-root u/unsigned k/key= update-index spec-file= only= fail-fast base-image= j/jobs=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 buildcache push' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command buildcache push' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache push' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache push' -s f -l force -f -a force
+complete -c spack -n '__fish_spack_using_command buildcache push' -s f -l force -d 'overwrite tarball if it exists'
+complete -c spack -n '__fish_spack_using_command buildcache push' -l allow-root -s a -f -a allow_root
+complete -c spack -n '__fish_spack_using_command buildcache push' -l allow-root -s a -d 'allow install root string in binary files after RPATH substitution'
+complete -c spack -n '__fish_spack_using_command buildcache push' -l unsigned -s u -f -a unsigned
+complete -c spack -n '__fish_spack_using_command buildcache push' -l unsigned -s u -d 'push unsigned buildcache tarballs'
+complete -c spack -n '__fish_spack_using_command buildcache push' -l key -s k -r -f -a key
+complete -c spack -n '__fish_spack_using_command buildcache push' -l key -s k -r -d 'key for signing'
+complete -c spack -n '__fish_spack_using_command buildcache push' -l update-index -l rebuild-index -f -a update_index
+complete -c spack -n '__fish_spack_using_command buildcache push' -l update-index -l rebuild-index -d 'regenerate buildcache index after building package(s)'
+complete -c spack -n '__fish_spack_using_command buildcache push' -l spec-file -r -f -a spec_file
+complete -c spack -n '__fish_spack_using_command buildcache push' -l spec-file -r -d 'create buildcache entry for spec from json or yaml file'
+complete -c spack -n '__fish_spack_using_command buildcache push' -l only -r -f -a 'package dependencies'
+complete -c spack -n '__fish_spack_using_command buildcache push' -l only -r -d 'select the buildcache mode. The default is to build a cache for the package along with all its dependencies. Alternatively, one can decide to build a cache for only the package or only the dependencies'
+complete -c spack -n '__fish_spack_using_command buildcache push' -l fail-fast -f -a fail_fast
+complete -c spack -n '__fish_spack_using_command buildcache push' -l fail-fast -d 'stop pushing on first failure (default is best effort)'
+complete -c spack -n '__fish_spack_using_command buildcache push' -l base-image -r -f -a base_image
+complete -c spack -n '__fish_spack_using_command buildcache push' -l base-image -r -d 'specify the base image for the buildcache. '
+complete -c spack -n '__fish_spack_using_command buildcache push' -s j -l jobs -r -f -a jobs
+complete -c spack -n '__fish_spack_using_command buildcache push' -s j -l jobs -r -d 'explicitly set number of parallel jobs'
+
+# spack buildcache create
+set -g __fish_spack_optspecs_spack_buildcache_create h/help f/force a/allow-root u/unsigned k/key= update-index spec-file= only= fail-fast base-image= j/jobs=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 buildcache create' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command buildcache create' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache create' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache create' -s f -l force -f -a force
+complete -c spack -n '__fish_spack_using_command buildcache create' -s f -l force -d 'overwrite tarball if it exists'
+complete -c spack -n '__fish_spack_using_command buildcache create' -l allow-root -s a -f -a allow_root
+complete -c spack -n '__fish_spack_using_command buildcache create' -l allow-root -s a -d 'allow install root string in binary files after RPATH substitution'
+complete -c spack -n '__fish_spack_using_command buildcache create' -l unsigned -s u -f -a unsigned
+complete -c spack -n '__fish_spack_using_command buildcache create' -l unsigned -s u -d 'push unsigned buildcache tarballs'
+complete -c spack -n '__fish_spack_using_command buildcache create' -l key -s k -r -f -a key
+complete -c spack -n '__fish_spack_using_command buildcache create' -l key -s k -r -d 'key for signing'
+complete -c spack -n '__fish_spack_using_command buildcache create' -l update-index -l rebuild-index -f -a update_index
+complete -c spack -n '__fish_spack_using_command buildcache create' -l update-index -l rebuild-index -d 'regenerate buildcache index after building package(s)'
+complete -c spack -n '__fish_spack_using_command buildcache create' -l spec-file -r -f -a spec_file
+complete -c spack -n '__fish_spack_using_command buildcache create' -l spec-file -r -d 'create buildcache entry for spec from json or yaml file'
+complete -c spack -n '__fish_spack_using_command buildcache create' -l only -r -f -a 'package dependencies'
+complete -c spack -n '__fish_spack_using_command buildcache create' -l only -r -d 'select the buildcache mode. The default is to build a cache for the package along with all its dependencies. Alternatively, one can decide to build a cache for only the package or only the dependencies'
+complete -c spack -n '__fish_spack_using_command buildcache create' -l fail-fast -f -a fail_fast
+complete -c spack -n '__fish_spack_using_command buildcache create' -l fail-fast -d 'stop pushing on first failure (default is best effort)'
+complete -c spack -n '__fish_spack_using_command buildcache create' -l base-image -r -f -a base_image
+complete -c spack -n '__fish_spack_using_command buildcache create' -l base-image -r -d 'specify the base image for the buildcache. '
+complete -c spack -n '__fish_spack_using_command buildcache create' -s j -l jobs -r -f -a jobs
+complete -c spack -n '__fish_spack_using_command buildcache create' -s j -l jobs -r -d 'explicitly set number of parallel jobs'
+
+# spack buildcache install
+set -g __fish_spack_optspecs_spack_buildcache_install h/help f/force m/multiple u/unsigned o/otherarch
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 buildcache install' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command buildcache install' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache install' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache install' -s f -l force -f -a force
+complete -c spack -n '__fish_spack_using_command buildcache install' -s f -l force -d 'overwrite install directory if it exists'
+complete -c spack -n '__fish_spack_using_command buildcache install' -s m -l multiple -f -a multiple
+complete -c spack -n '__fish_spack_using_command buildcache install' -s m -l multiple -d 'allow all matching packages'
+complete -c spack -n '__fish_spack_using_command buildcache install' -s u -l unsigned -f -a unsigned
+complete -c spack -n '__fish_spack_using_command buildcache install' -s u -l unsigned -d 'install unsigned buildcache tarballs for testing'
+complete -c spack -n '__fish_spack_using_command buildcache install' -s o -l otherarch -f -a otherarch
+complete -c spack -n '__fish_spack_using_command buildcache install' -s o -l otherarch -d 'install specs from other architectures instead of default platform and OS'
+
+# spack buildcache list
+set -g __fish_spack_optspecs_spack_buildcache_list h/help l/long L/very-long N/namespaces v/variants a/allarch
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 buildcache list' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command buildcache list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache list' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache list' -s l -l long -f -a long
+complete -c spack -n '__fish_spack_using_command buildcache list' -s l -l long -d 'show dependency hashes as well as versions'
+complete -c spack -n '__fish_spack_using_command buildcache list' -s L -l very-long -f -a very_long
+complete -c spack -n '__fish_spack_using_command buildcache list' -s L -l very-long -d 'show full dependency hashes as well as versions'
+complete -c spack -n '__fish_spack_using_command buildcache list' -s N -l namespaces -f -a namespaces
+complete -c spack -n '__fish_spack_using_command buildcache list' -s N -l namespaces -d 'show fully qualified package names'
+complete -c spack -n '__fish_spack_using_command buildcache list' -s v -l variants -f -a variants
+complete -c spack -n '__fish_spack_using_command buildcache list' -s v -l variants -d 'show variants in output (can be long)'
+complete -c spack -n '__fish_spack_using_command buildcache list' -s a -l allarch -f -a allarch
+complete -c spack -n '__fish_spack_using_command buildcache list' -s a -l allarch -d 'list specs for all available architectures instead of default platform and OS'
+
+# spack buildcache keys
+set -g __fish_spack_optspecs_spack_buildcache_keys h/help i/install t/trust f/force
+complete -c spack -n '__fish_spack_using_command buildcache keys' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache keys' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache keys' -s i -l install -f -a install
+complete -c spack -n '__fish_spack_using_command buildcache keys' -s i -l install -d 'install Keys pulled from mirror'
+complete -c spack -n '__fish_spack_using_command buildcache keys' -s t -l trust -f -a trust
+complete -c spack -n '__fish_spack_using_command buildcache keys' -s t -l trust -d 'trust all downloaded keys'
+complete -c spack -n '__fish_spack_using_command buildcache keys' -s f -l force -f -a force
+complete -c spack -n '__fish_spack_using_command buildcache keys' -s f -l force -d 'force new download of keys'
+
+# spack buildcache preview
+set -g __fish_spack_optspecs_spack_buildcache_preview h/help
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 buildcache preview' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command buildcache preview' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache preview' -s h -l help -d 'show this help message and exit'
+
+# spack buildcache check
+set -g __fish_spack_optspecs_spack_buildcache_check h/help m/mirror-url= o/output-file= scope= s/spec= spec-file=
+complete -c spack -n '__fish_spack_using_command buildcache check' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache check' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -f -a mirror_url
+complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL'
+complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file
+complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written'
+complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check'
+complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -f -a spec
+complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -d 'check single spec instead of release specs file'
+complete -c spack -n '__fish_spack_using_command buildcache check' -l spec-file -r -f -a spec_file
+complete -c spack -n '__fish_spack_using_command buildcache check' -l spec-file -r -d 'check single spec from json or yaml file instead of release specs file'
+
+# spack buildcache download
+set -g __fish_spack_optspecs_spack_buildcache_download h/help s/spec= spec-file= p/path=
+complete -c spack -n '__fish_spack_using_command buildcache download' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache download' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache download' -s s -l spec -r -f -a spec
+complete -c spack -n '__fish_spack_using_command buildcache download' -s s -l spec -r -d 'download built tarball for spec from mirror'
+complete -c spack -n '__fish_spack_using_command buildcache download' -l spec-file -r -f -a spec_file
+complete -c spack -n '__fish_spack_using_command buildcache download' -l spec-file -r -d 'download built tarball for spec (from json or yaml file) from mirror'
+complete -c spack -n '__fish_spack_using_command buildcache download' -s p -l path -r -f -a path
+complete -c spack -n '__fish_spack_using_command buildcache download' -s p -l path -r -d 'path to directory where tarball should be downloaded'
+
+# spack buildcache get-buildcache-name
+set -g __fish_spack_optspecs_spack_buildcache_get_buildcache_name h/help s/spec= spec-file=
+complete -c spack -n '__fish_spack_using_command buildcache get-buildcache-name' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache get-buildcache-name' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache get-buildcache-name' -s s -l spec -r -f -a spec
+complete -c spack -n '__fish_spack_using_command buildcache get-buildcache-name' -s s -l spec -r -d 'spec string for which buildcache name is desired'
+complete -c spack -n '__fish_spack_using_command buildcache get-buildcache-name' -l spec-file -r -f -a spec_file
+complete -c spack -n '__fish_spack_using_command buildcache get-buildcache-name' -l spec-file -r -d 'path to spec json or yaml file for which buildcache name is desired'
+
+# spack buildcache save-specfile
+set -g __fish_spack_optspecs_spack_buildcache_save_specfile h/help root-spec= root-specfile= s/specs= specfile-dir=
+complete -c spack -n '__fish_spack_using_command buildcache save-specfile' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache save-specfile' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache save-specfile' -l root-spec -r -f -a root_spec
+complete -c spack -n '__fish_spack_using_command buildcache save-specfile' -l root-spec -r -d 'root spec of dependent spec'
+complete -c spack -n '__fish_spack_using_command buildcache save-specfile' -l root-specfile -r -f -a root_specfile
+complete -c spack -n '__fish_spack_using_command buildcache save-specfile' -l root-specfile -r -d 'path to json or yaml file containing root spec of dependent spec'
+complete -c spack -n '__fish_spack_using_command buildcache save-specfile' -s s -l specs -r -f -a specs
+complete -c spack -n '__fish_spack_using_command buildcache save-specfile' -s s -l specs -r -d 'list of dependent specs for which saved yaml is desired'
+complete -c spack -n '__fish_spack_using_command buildcache save-specfile' -l specfile-dir -r -f -a specfile_dir
+complete -c spack -n '__fish_spack_using_command buildcache save-specfile' -l specfile-dir -r -d 'path to directory where spec yamls should be saved'
+
+# spack buildcache sync
+set -g __fish_spack_optspecs_spack_buildcache_sync h/help manifest-glob=
+
+complete -c spack -n '__fish_spack_using_command buildcache sync' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache sync' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache sync' -l manifest-glob -r -f -a manifest_glob
+complete -c spack -n '__fish_spack_using_command buildcache sync' -l manifest-glob -r -d 'a quoted glob pattern identifying copy manifest files'
+
+# spack buildcache update-index
+set -g __fish_spack_optspecs_spack_buildcache_update_index h/help k/keys
+
+complete -c spack -n '__fish_spack_using_command buildcache update-index' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache update-index' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache update-index' -s k -l keys -f -a keys
+complete -c spack -n '__fish_spack_using_command buildcache update-index' -s k -l keys -d 'if provided, key index will be updated as well as package index'
+
+# spack buildcache rebuild-index
+set -g __fish_spack_optspecs_spack_buildcache_rebuild_index h/help k/keys
+
+complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -s k -l keys -f -a keys
+complete -c spack -n '__fish_spack_using_command buildcache rebuild-index' -s k -l keys -d 'if provided, key index will be updated as well as package index'
+
+# spack cd
+set -g __fish_spack_optspecs_spack_cd h/help m/module-dir r/spack-root i/install-dir p/package-dir P/packages s/stage-dir S/stages source-dir b/build-dir e/env= first
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 cd' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command cd' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command cd' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command cd' -s m -l module-dir -f -a module_dir
+complete -c spack -n '__fish_spack_using_command cd' -s m -l module-dir -d 'spack python module directory'
+complete -c spack -n '__fish_spack_using_command cd' -s r -l spack-root -f -a spack_root
+complete -c spack -n '__fish_spack_using_command cd' -s r -l spack-root -d 'spack installation root'
+complete -c spack -n '__fish_spack_using_command cd' -s i -l install-dir -f -a install_dir
+complete -c spack -n '__fish_spack_using_command cd' -s i -l install-dir -d 'install prefix for spec (spec need not be installed)'
+complete -c spack -n '__fish_spack_using_command cd' -s p -l package-dir -f -a package_dir
+complete -c spack -n '__fish_spack_using_command cd' -s p -l package-dir -d 'directory enclosing a spec\'s package.py file'
+complete -c spack -n '__fish_spack_using_command cd' -s P -l packages -f -a packages
+complete -c spack -n '__fish_spack_using_command cd' -s P -l packages -d 'top-level packages directory for Spack'
+complete -c spack -n '__fish_spack_using_command cd' -s s -l stage-dir -f -a stage_dir
+complete -c spack -n '__fish_spack_using_command cd' -s s -l stage-dir -d 'stage directory for a spec'
+complete -c spack -n '__fish_spack_using_command cd' -s S -l stages -f -a stages
+complete -c spack -n '__fish_spack_using_command cd' -s S -l stages -d 'top level stage directory'
+complete -c spack -n '__fish_spack_using_command cd' -l source-dir -f -a source_dir
+complete -c spack -n '__fish_spack_using_command cd' -l source-dir -d 'source directory for a spec (requires it to be staged first)'
+complete -c spack -n '__fish_spack_using_command cd' -s b -l build-dir -f -a build_dir
+complete -c spack -n '__fish_spack_using_command cd' -s b -l build-dir -d 'build directory for a spec (requires it to be staged first)'
+complete -c spack -n '__fish_spack_using_command cd' -s e -l env -r -f -a location_env
+complete -c spack -n '__fish_spack_using_command cd' -s e -l env -r -d 'location of the named or current environment'
+complete -c spack -n '__fish_spack_using_command cd' -l first -f -a find_first
+complete -c spack -n '__fish_spack_using_command cd' -l first -d 'use the first match if multiple packages match the spec'
+
+# spack change
+set -g __fish_spack_optspecs_spack_change h/help l/list-name= match-spec= a/all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 change' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command change' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command change' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command change' -s l -l list-name -r -f -a list_name
+complete -c spack -n '__fish_spack_using_command change' -s l -l list-name -r -d 'name of the list to remove specs from'
+complete -c spack -n '__fish_spack_using_command change' -l match-spec -r -f -a match_spec
+complete -c spack -n '__fish_spack_using_command change' -l match-spec -r -d 'if name is ambiguous, supply a spec to match'
+complete -c spack -n '__fish_spack_using_command change' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command change' -s a -l all -d 'change all matching specs (allow changing more than one spec)'
+
+# spack checksum
+set -g __fish_spack_optspecs_spack_checksum h/help keep-stage b/batch l/latest p/preferred a/add-to-package verify j/jobs=
+complete -c spack -n '__fish_spack_using_command_pos 0 checksum' -f -a '(__fish_spack_packages)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 checksum' -f -a '(__fish_spack_package_versions $__fish_spack_argparse_argv[1])'
+complete -c spack -n '__fish_spack_using_command checksum' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command checksum' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command checksum' -l keep-stage -f -a keep_stage
+complete -c spack -n '__fish_spack_using_command checksum' -l keep-stage -d 'don\'t clean up staging area when command completes'
+complete -c spack -n '__fish_spack_using_command checksum' -l batch -s b -f -a batch
+complete -c spack -n '__fish_spack_using_command checksum' -l batch -s b -d 'don\'t ask which versions to checksum'
+complete -c spack -n '__fish_spack_using_command checksum' -l latest -s l -f -a latest
+complete -c spack -n '__fish_spack_using_command checksum' -l latest -s l -d 'checksum the latest available version'
+complete -c spack -n '__fish_spack_using_command checksum' -l preferred -s p -f -a preferred
+complete -c spack -n '__fish_spack_using_command checksum' -l preferred -s p -d 'checksum the known Spack preferred version'
+complete -c spack -n '__fish_spack_using_command checksum' -l add-to-package -s a -f -a add_to_package
+complete -c spack -n '__fish_spack_using_command checksum' -l add-to-package -s a -d 'add new versions to package'
+complete -c spack -n '__fish_spack_using_command checksum' -l verify -f -a verify
+complete -c spack -n '__fish_spack_using_command checksum' -l verify -d 'verify known package checksums'
+complete -c spack -n '__fish_spack_using_command checksum' -s j -l jobs -r -f -a jobs
+complete -c spack -n '__fish_spack_using_command checksum' -s j -l jobs -r -d 'explicitly set number of parallel jobs'
+
+# spack ci
+set -g __fish_spack_optspecs_spack_ci h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a generate -d 'generate jobs file from a CI-aware spack file'
+complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a rebuild-index -d 'rebuild the buildcache index for the remote mirror'
+complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a rebuild -d 'rebuild a spec if it is not on the remote mirror'
+complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a reproduce-build -d 'generate instructions for reproducing the spec rebuild job'
+complete -c spack -n '__fish_spack_using_command ci' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command ci' -s h -l help -d 'show this help message and exit'
+
+# spack ci generate
+set -g __fish_spack_optspecs_spack_ci_generate h/help output-file= copy-to= optimize dependencies buildcache-destination= prune-dag no-prune-dag check-index-only artifacts-root=
+complete -c spack -n '__fish_spack_using_command ci generate' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command ci generate' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command ci generate' -l output-file -r -f -a output_file
+complete -c spack -n '__fish_spack_using_command ci generate' -l output-file -r -d 'pathname for the generated gitlab ci yaml file'
+complete -c spack -n '__fish_spack_using_command ci generate' -l copy-to -r -f -a copy_to
+complete -c spack -n '__fish_spack_using_command ci generate' -l copy-to -r -d 'path to additional directory for job files'
+complete -c spack -n '__fish_spack_using_command ci generate' -l optimize -f -a optimize
+complete -c spack -n '__fish_spack_using_command ci generate' -l optimize -d '(experimental) optimize the gitlab yaml file for size'
+complete -c spack -n '__fish_spack_using_command ci generate' -l dependencies -f -a dependencies
+complete -c spack -n '__fish_spack_using_command ci generate' -l dependencies -d '(experimental) disable DAG scheduling (use \'plain\' dependencies)'
+complete -c spack -n '__fish_spack_using_command ci generate' -l buildcache-destination -r -f -a buildcache_destination
+complete -c spack -n '__fish_spack_using_command ci generate' -l buildcache-destination -r -d 'override the mirror configured in the environment'
+complete -c spack -n '__fish_spack_using_command ci generate' -l prune-dag -f -a prune_dag
+complete -c spack -n '__fish_spack_using_command ci generate' -l prune-dag -d 'skip up-to-date specs'
+complete -c spack -n '__fish_spack_using_command ci generate' -l no-prune-dag -f -a prune_dag
+complete -c spack -n '__fish_spack_using_command ci generate' -l no-prune-dag -d 'process up-to-date specs'
+complete -c spack -n '__fish_spack_using_command ci generate' -l check-index-only -f -a index_only
+complete -c spack -n '__fish_spack_using_command ci generate' -l check-index-only -d 'only check spec state from buildcache indices'
+complete -c spack -n '__fish_spack_using_command ci generate' -l artifacts-root -r -f -a artifacts_root
+complete -c spack -n '__fish_spack_using_command ci generate' -l artifacts-root -r -d 'path to the root of the artifacts directory'
+
+# spack ci rebuild-index
+set -g __fish_spack_optspecs_spack_ci_rebuild_index h/help
+complete -c spack -n '__fish_spack_using_command ci rebuild-index' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command ci rebuild-index' -s h -l help -d 'show this help message and exit'
+
+# spack ci rebuild
+set -g __fish_spack_optspecs_spack_ci_rebuild h/help t/tests fail-fast
+complete -c spack -n '__fish_spack_using_command ci rebuild' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command ci rebuild' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command ci rebuild' -s t -l tests -f -a tests
+complete -c spack -n '__fish_spack_using_command ci rebuild' -s t -l tests -d 'run stand-alone tests after the build'
+complete -c spack -n '__fish_spack_using_command ci rebuild' -l fail-fast -f -a fail_fast
+complete -c spack -n '__fish_spack_using_command ci rebuild' -l fail-fast -d 'stop stand-alone tests after the first failure'
+
+# spack ci reproduce-build
+set -g __fish_spack_optspecs_spack_ci_reproduce_build h/help runtime= working-dir= s/autostart gpg-file= gpg-url=
+complete -c spack -n '__fish_spack_using_command_pos 0 ci reproduce-build' -f
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l runtime -r -f -a 'docker podman'
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l runtime -r -d 'Container runtime to use.'
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l working-dir -r -f -a working_dir
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l working-dir -r -d 'where to unpack artifacts'
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -s s -l autostart -f -a autostart
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -s s -l autostart -d 'Run docker reproducer automatically'
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l gpg-file -r -f -a gpg_file
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l gpg-file -r -d 'Path to public GPG key for validating binary cache installs'
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l gpg-url -r -f -a gpg_url
+complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l gpg-url -r -d 'URL to public GPG key for validating binary cache installs'
+
+# spack clean
+set -g __fish_spack_optspecs_spack_clean h/help s/stage d/downloads f/failures m/misc-cache p/python-cache b/bootstrap a/all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 clean' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command clean' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command clean' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command clean' -s s -l stage -f -a stage
+complete -c spack -n '__fish_spack_using_command clean' -s s -l stage -d 'remove all temporary build stages (default)'
+complete -c spack -n '__fish_spack_using_command clean' -s d -l downloads -f -a downloads
+complete -c spack -n '__fish_spack_using_command clean' -s d -l downloads -d 'remove cached downloads'
+complete -c spack -n '__fish_spack_using_command clean' -s f -l failures -f -a failures
+complete -c spack -n '__fish_spack_using_command clean' -s f -l failures -d 'force removal of all install failure tracking markers'
+complete -c spack -n '__fish_spack_using_command clean' -s m -l misc-cache -f -a misc_cache
+complete -c spack -n '__fish_spack_using_command clean' -s m -l misc-cache -d 'remove long-lived caches, like the virtual package index'
+complete -c spack -n '__fish_spack_using_command clean' -s p -l python-cache -f -a python_cache
+complete -c spack -n '__fish_spack_using_command clean' -s p -l python-cache -d 'remove .pyc, .pyo files and __pycache__ folders'
+complete -c spack -n '__fish_spack_using_command clean' -s b -l bootstrap -f -a bootstrap
+complete -c spack -n '__fish_spack_using_command clean' -s b -l bootstrap -d 'remove software and configuration needed to bootstrap Spack'
+complete -c spack -n '__fish_spack_using_command clean' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command clean' -s a -l all -d 'equivalent to -sdfmp (does not include --bootstrap)'
+
+# spack clone
+set -g __fish_spack_optspecs_spack_clone h/help r/remote=
+complete -c spack -n '__fish_spack_using_command_pos 0 clone' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command clone' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command clone' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command clone' -s r -l remote -r -f -a remote
+complete -c spack -n '__fish_spack_using_command clone' -s r -l remote -r -d 'name of the remote to clone from'
+
+# spack commands
+set -g __fish_spack_optspecs_spack_commands h/help update-completion a/aliases format= header= update=
+
+complete -c spack -n '__fish_spack_using_command commands' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command commands' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command commands' -l update-completion -f -a update_completion
+complete -c spack -n '__fish_spack_using_command commands' -l update-completion -d 'regenerate spack\'s tab completion scripts'
+complete -c spack -n '__fish_spack_using_command commands' -s a -l aliases -f -a aliases
+complete -c spack -n '__fish_spack_using_command commands' -s a -l aliases -d 'include command aliases'
+complete -c spack -n '__fish_spack_using_command commands' -l format -r -f -a 'subcommands rst names bash fish'
+complete -c spack -n '__fish_spack_using_command commands' -l format -r -d 'format to be used to print the output (default: names)'
+complete -c spack -n '__fish_spack_using_command commands' -l header -r -f -a header
+complete -c spack -n '__fish_spack_using_command commands' -l header -r -d 'prepend contents of FILE to the output (useful for rst format)'
+complete -c spack -n '__fish_spack_using_command commands' -l update -r -f -a update
+complete -c spack -n '__fish_spack_using_command commands' -l update -r -d 'write output to the specified file, if any command is newer'
+
+# spack compiler
+set -g __fish_spack_optspecs_spack_compiler h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 compiler' -f -a find -d 'search the system for compilers to add to Spack configuration'
+complete -c spack -n '__fish_spack_using_command_pos 0 compiler' -f -a add -d 'search the system for compilers to add to Spack configuration'
+complete -c spack -n '__fish_spack_using_command_pos 0 compiler' -f -a remove -d 'remove compiler by spec'
+complete -c spack -n '__fish_spack_using_command_pos 0 compiler' -f -a rm -d 'remove compiler by spec'
+complete -c spack -n '__fish_spack_using_command_pos 0 compiler' -f -a list -d 'list available compilers'
+complete -c spack -n '__fish_spack_using_command_pos 0 compiler' -f -a info -d 'show compiler paths'
+complete -c spack -n '__fish_spack_using_command compiler' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command compiler' -s h -l help -d 'show this help message and exit'
+
+# spack compiler find
+set -g __fish_spack_optspecs_spack_compiler_find h/help mixed-toolchain no-mixed-toolchain scope=
+
+complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -f -a mixed_toolchain
+complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)'
+complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain
+complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)'
+complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify'
+
+# spack compiler add
+set -g __fish_spack_optspecs_spack_compiler_add h/help mixed-toolchain no-mixed-toolchain scope=
+
+complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -f -a mixed_toolchain
+complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)'
+complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain
+complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)'
+complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify'
+
+# spack compiler remove
+set -g __fish_spack_optspecs_spack_compiler_remove h/help a/all scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 compiler remove' -f -a '(__fish_spack_installed_compilers)'
+complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec'
+complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify'
+
+# spack compiler rm
+set -g __fish_spack_optspecs_spack_compiler_rm h/help a/all scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 compiler rm' -f -a '(__fish_spack_installed_compilers)'
+complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec'
+complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify'
+
+# spack compiler list
+set -g __fish_spack_optspecs_spack_compiler_list h/help scope=
+complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from'
+
+# spack compiler info
+set -g __fish_spack_optspecs_spack_compiler_info h/help scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)'
+complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from'
+
+# spack compilers
+set -g __fish_spack_optspecs_spack_compilers h/help scope=
+complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify'
+
+# spack concretize
+set -g __fish_spack_optspecs_spack_concretize h/help f/force test= q/quiet U/fresh reuse reuse-deps j/jobs=
+complete -c spack -n '__fish_spack_using_command concretize' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command concretize' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command concretize' -s f -l force -f -a force
+complete -c spack -n '__fish_spack_using_command concretize' -s f -l force -d 're-concretize even if already concretized'
+complete -c spack -n '__fish_spack_using_command concretize' -l test -r -f -a 'root all'
+complete -c spack -n '__fish_spack_using_command concretize' -l test -r -d 'concretize with test dependencies of only root packages or all packages'
+complete -c spack -n '__fish_spack_using_command concretize' -s q -l quiet -f -a quiet
+complete -c spack -n '__fish_spack_using_command concretize' -s q -l quiet -d 'don\'t print concretized specs'
+complete -c spack -n '__fish_spack_using_command concretize' -s U -l fresh -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command concretize' -s U -l fresh -d 'do not reuse installed deps; build newest configuration'
+complete -c spack -n '__fish_spack_using_command concretize' -l reuse -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command concretize' -l reuse -d 'reuse installed packages/buildcaches when possible'
+complete -c spack -n '__fish_spack_using_command concretize' -l reuse-deps -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command concretize' -l reuse-deps -d 'reuse installed dependencies only'
+complete -c spack -n '__fish_spack_using_command concretize' -s j -l jobs -r -f -a jobs
+complete -c spack -n '__fish_spack_using_command concretize' -s j -l jobs -r -d 'explicitly set number of parallel jobs'
+
+# spack concretise
+set -g __fish_spack_optspecs_spack_concretise h/help f/force test= q/quiet U/fresh reuse reuse-deps j/jobs=
+complete -c spack -n '__fish_spack_using_command concretise' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command concretise' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command concretise' -s f -l force -f -a force
+complete -c spack -n '__fish_spack_using_command concretise' -s f -l force -d 're-concretize even if already concretized'
+complete -c spack -n '__fish_spack_using_command concretise' -l test -r -f -a 'root all'
+complete -c spack -n '__fish_spack_using_command concretise' -l test -r -d 'concretize with test dependencies of only root packages or all packages'
+complete -c spack -n '__fish_spack_using_command concretise' -s q -l quiet -f -a quiet
+complete -c spack -n '__fish_spack_using_command concretise' -s q -l quiet -d 'don\'t print concretized specs'
+complete -c spack -n '__fish_spack_using_command concretise' -s U -l fresh -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command concretise' -s U -l fresh -d 'do not reuse installed deps; build newest configuration'
+complete -c spack -n '__fish_spack_using_command concretise' -l reuse -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command concretise' -l reuse -d 'reuse installed packages/buildcaches when possible'
+complete -c spack -n '__fish_spack_using_command concretise' -l reuse-deps -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command concretise' -l reuse-deps -d 'reuse installed dependencies only'
+complete -c spack -n '__fish_spack_using_command concretise' -s j -l jobs -r -f -a jobs
+complete -c spack -n '__fish_spack_using_command concretise' -s j -l jobs -r -d 'explicitly set number of parallel jobs'
+
+# spack config
+set -g __fish_spack_optspecs_spack_config h/help scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a get -d 'print configuration values'
+complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a blame -d 'print configuration annotated with source file:line'
+complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a edit -d 'edit configuration file'
+complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a list -d 'list configuration sections'
+complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a add -d 'add configuration parameters'
+complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a prefer-upstream -d 'set package preferences from upstream'
+complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a remove -d 'remove configuration parameters'
+complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a rm -d 'remove configuration parameters'
+complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d 'update configuration files to the latest format'
+complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update'
+complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify'
+
+# spack config get
+set -g __fish_spack_optspecs_spack_config_get h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 config get' -f -a 'bootstrap cdash ci compilers concretizer config definitions mirrors modules packages repos upstreams'
+complete -c spack -n '__fish_spack_using_command config get' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command config get' -s h -l help -d 'show this help message and exit'
+
+# spack config blame
+set -g __fish_spack_optspecs_spack_config_blame h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 config blame' -f -a 'bootstrap cdash ci compilers concretizer config definitions mirrors modules packages repos upstreams'
+complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -d 'show this help message and exit'
+
+# spack config edit
+set -g __fish_spack_optspecs_spack_config_edit h/help print-file
+complete -c spack -n '__fish_spack_using_command_pos 0 config edit' -f -a 'bootstrap cdash ci compilers concretizer config definitions mirrors modules packages repos upstreams'
+complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command config edit' -l print-file -f -a print_file
+complete -c spack -n '__fish_spack_using_command config edit' -l print-file -d 'print the file name that would be edited'
+
+# spack config list
+set -g __fish_spack_optspecs_spack_config_list h/help
+complete -c spack -n '__fish_spack_using_command config list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command config list' -s h -l help -d 'show this help message and exit'
+
+# spack config add
+set -g __fish_spack_optspecs_spack_config_add h/help f/file=
+complete -c spack -n '__fish_spack_using_command_pos 0 config add' -f -a '(__fish_spack_colon_path)'
+complete -c spack -n '__fish_spack_using_command config add' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command config add' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command config add' -s f -l file -r -f -a file
+complete -c spack -n '__fish_spack_using_command config add' -s f -l file -r -d 'file from which to set all config values'
+
+# spack config prefer-upstream
+set -g __fish_spack_optspecs_spack_config_prefer_upstream h/help local
+complete -c spack -n '__fish_spack_using_command config prefer-upstream' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command config prefer-upstream' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command config prefer-upstream' -l local -f -a local
+complete -c spack -n '__fish_spack_using_command config prefer-upstream' -l local -d 'set packages preferences based on local installs, rather than upstream'
+
+# spack config remove
+set -g __fish_spack_optspecs_spack_config_remove h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 config remove' -f -a '(__fish_spack_colon_path)'
+complete -c spack -n '__fish_spack_using_command config remove' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command config remove' -s h -l help -d 'show this help message and exit'
+
+# spack config rm
+set -g __fish_spack_optspecs_spack_config_rm h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 config rm' -f -a '(__fish_spack_colon_path)'
+complete -c spack -n '__fish_spack_using_command config rm' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command config rm' -s h -l help -d 'show this help message and exit'
+
+# spack config update
+set -g __fish_spack_optspecs_spack_config_update h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command_pos 0 config update' -f -a '(__fish_spack_config_sections)'
+complete -c spack -n '__fish_spack_using_command config update' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command config update' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command config update' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command config update' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack config revert
+set -g __fish_spack_optspecs_spack_config_revert h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command_pos 0 config revert' -f -a '(__fish_spack_config_sections)'
+complete -c spack -n '__fish_spack_using_command config revert' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command config revert' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command config revert' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command config revert' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack containerize
+set -g __fish_spack_optspecs_spack_containerize h/help list-os last-stage=
+complete -c spack -n '__fish_spack_using_command containerize' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command containerize' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command containerize' -l list-os -f -a list_os
+complete -c spack -n '__fish_spack_using_command containerize' -l list-os -d 'list all the OS that can be used in the bootstrap phase and exit'
+complete -c spack -n '__fish_spack_using_command containerize' -l last-stage -r -f -a 'bootstrap build final'
+complete -c spack -n '__fish_spack_using_command containerize' -l last-stage -r -d 'last stage in the container recipe'
+
+# spack containerise
+set -g __fish_spack_optspecs_spack_containerise h/help list-os last-stage=
+complete -c spack -n '__fish_spack_using_command containerise' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command containerise' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command containerise' -l list-os -f -a list_os
+complete -c spack -n '__fish_spack_using_command containerise' -l list-os -d 'list all the OS that can be used in the bootstrap phase and exit'
+complete -c spack -n '__fish_spack_using_command containerise' -l last-stage -r -f -a 'bootstrap build final'
+complete -c spack -n '__fish_spack_using_command containerise' -l last-stage -r -d 'last stage in the container recipe'
+
+# spack create
+set -g __fish_spack_optspecs_spack_create h/help keep-stage n/name= t/template= r/repo= N/namespace= f/force skip-editor b/batch
+
+complete -c spack -n '__fish_spack_using_command create' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command create' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command create' -l keep-stage -f -a keep_stage
+complete -c spack -n '__fish_spack_using_command create' -l keep-stage -d 'don\'t clean up staging area when command completes'
+complete -c spack -n '__fish_spack_using_command create' -s n -l name -r -f -a name
+complete -c spack -n '__fish_spack_using_command create' -s n -l name -r -d 'name of the package to create'
+complete -c spack -n '__fish_spack_using_command create' -s t -l template -r -f -a 'autoreconf autotools bazel bundle cmake generic intel lua makefile maven meson octave perlbuild perlmake python qmake r racket ruby scons sip waf'
+complete -c spack -n '__fish_spack_using_command create' -s t -l template -r -d 'build system template to use'
+complete -c spack -n '__fish_spack_using_command create' -s r -l repo -r -f -a repo
+complete -c spack -n '__fish_spack_using_command create' -s r -l repo -r -d 'path to a repository where the package should be created'
+complete -c spack -n '__fish_spack_using_command create' -s N -l namespace -r -f -a namespace
+complete -c spack -n '__fish_spack_using_command create' -s N -l namespace -r -d 'specify a namespace for the package'
+complete -c spack -n '__fish_spack_using_command create' -s f -l force -f -a force
+complete -c spack -n '__fish_spack_using_command create' -s f -l force -d 'overwrite any existing package file with the same name'
+complete -c spack -n '__fish_spack_using_command create' -l skip-editor -f -a skip_editor
+complete -c spack -n '__fish_spack_using_command create' -l skip-editor -d 'skip the edit session for the package (e.g., automation)'
+complete -c spack -n '__fish_spack_using_command create' -s b -l batch -f -a batch
+complete -c spack -n '__fish_spack_using_command create' -s b -l batch -d 'don\'t ask which versions to checksum'
+
+# spack debug
+set -g __fish_spack_optspecs_spack_debug h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 debug' -f -a create-db-tarball -d 'create a tarball of Spack\'s installation metadata'
+complete -c spack -n '__fish_spack_using_command_pos 0 debug' -f -a report -d 'print information useful for bug reports'
+complete -c spack -n '__fish_spack_using_command debug' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command debug' -s h -l help -d 'show this help message and exit'
+
+# spack debug create-db-tarball
+set -g __fish_spack_optspecs_spack_debug_create_db_tarball h/help
+complete -c spack -n '__fish_spack_using_command debug create-db-tarball' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command debug create-db-tarball' -s h -l help -d 'show this help message and exit'
+
+# spack debug report
+set -g __fish_spack_optspecs_spack_debug_report h/help
+complete -c spack -n '__fish_spack_using_command debug report' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command debug report' -s h -l help -d 'show this help message and exit'
+
+# spack deconcretize
+set -g __fish_spack_optspecs_spack_deconcretize h/help root y/yes-to-all a/all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 deconcretize' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command deconcretize' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command deconcretize' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command deconcretize' -l root -f -a root
+complete -c spack -n '__fish_spack_using_command deconcretize' -l root -d 'deconcretize only specific environment roots'
+complete -c spack -n '__fish_spack_using_command deconcretize' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command deconcretize' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+complete -c spack -n '__fish_spack_using_command deconcretize' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command deconcretize' -s a -l all -d 'deconcretize ALL specs that match each supplied spec'
+
+# spack dependencies
+set -g __fish_spack_optspecs_spack_dependencies h/help i/installed t/transitive deptype= V/no-expand-virtuals
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 dependencies' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command dependencies' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command dependencies' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command dependencies' -s i -l installed -f -a installed
+complete -c spack -n '__fish_spack_using_command dependencies' -s i -l installed -d 'list installed dependencies of an installed spec instead of possible dependencies of a package'
+complete -c spack -n '__fish_spack_using_command dependencies' -s t -l transitive -f -a transitive
+complete -c spack -n '__fish_spack_using_command dependencies' -s t -l transitive -d 'show all transitive dependencies'
+complete -c spack -n '__fish_spack_using_command dependencies' -l deptype -r -f -a deptype
+complete -c spack -n '__fish_spack_using_command dependencies' -l deptype -r -d 'comma-separated list of deptypes to traverse (default=build,link,run,test)'
+complete -c spack -n '__fish_spack_using_command dependencies' -s V -l no-expand-virtuals -f -a expand_virtuals
+complete -c spack -n '__fish_spack_using_command dependencies' -s V -l no-expand-virtuals -d 'do not expand virtual dependencies'
+
+# spack dependents
+set -g __fish_spack_optspecs_spack_dependents h/help i/installed t/transitive
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 dependents' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command dependents' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command dependents' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command dependents' -s i -l installed -f -a installed
+complete -c spack -n '__fish_spack_using_command dependents' -s i -l installed -d 'list installed dependents of an installed spec instead of possible dependents of a package'
+complete -c spack -n '__fish_spack_using_command dependents' -s t -l transitive -f -a transitive
+complete -c spack -n '__fish_spack_using_command dependents' -s t -l transitive -d 'show all transitive dependents'
+
+# spack deprecate
+set -g __fish_spack_optspecs_spack_deprecate h/help y/yes-to-all d/dependencies D/no-dependencies i/install-deprecator I/no-install-deprecator l/link-type=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 deprecate' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command deprecate' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command deprecate' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command deprecate' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command deprecate' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+complete -c spack -n '__fish_spack_using_command deprecate' -s d -l dependencies -f -a dependencies
+complete -c spack -n '__fish_spack_using_command deprecate' -s d -l dependencies -d 'deprecate dependencies (default)'
+complete -c spack -n '__fish_spack_using_command deprecate' -s D -l no-dependencies -f -a dependencies
+complete -c spack -n '__fish_spack_using_command deprecate' -s D -l no-dependencies -d 'do not deprecate dependencies'
+complete -c spack -n '__fish_spack_using_command deprecate' -s i -l install-deprecator -f -a install
+complete -c spack -n '__fish_spack_using_command deprecate' -s i -l install-deprecator -d 'concretize and install deprecator spec'
+complete -c spack -n '__fish_spack_using_command deprecate' -s I -l no-install-deprecator -f -a install
+complete -c spack -n '__fish_spack_using_command deprecate' -s I -l no-install-deprecator -d 'deprecator spec must already be installed (default)'
+complete -c spack -n '__fish_spack_using_command deprecate' -s l -l link-type -r -f -a 'soft hard'
+complete -c spack -n '__fish_spack_using_command deprecate' -s l -l link-type -r -d 'type of filesystem link to use for deprecation (default soft)'
+
+# spack dev-build
+set -g __fish_spack_optspecs_spack_dev_build h/help j/jobs= d/source-path= i/ignore-dependencies n/no-checksum deprecated keep-prefix skip-patch q/quiet drop-in= test= b/before= u/until= clean dirty U/fresh reuse reuse-deps
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 dev-build' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command dev-build' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command dev-build' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command dev-build' -s j -l jobs -r -f -a jobs
+complete -c spack -n '__fish_spack_using_command dev-build' -s j -l jobs -r -d 'explicitly set number of parallel jobs'
+complete -c spack -n '__fish_spack_using_command dev-build' -s d -l source-path -r -f -a source_path
+complete -c spack -n '__fish_spack_using_command dev-build' -s d -l source-path -r -d 'path to source directory (defaults to the current directory)'
+complete -c spack -n '__fish_spack_using_command dev-build' -s i -l ignore-dependencies -f -a ignore_deps
+complete -c spack -n '__fish_spack_using_command dev-build' -s i -l ignore-dependencies -d 'do not try to install dependencies of requested packages'
+complete -c spack -n '__fish_spack_using_command dev-build' -s n -l no-checksum -f -a no_checksum
+complete -c spack -n '__fish_spack_using_command dev-build' -s n -l no-checksum -d 'do not use checksums to verify downloaded files (unsafe)'
+complete -c spack -n '__fish_spack_using_command dev-build' -l deprecated -f -a deprecated
+complete -c spack -n '__fish_spack_using_command dev-build' -l deprecated -d 'fetch deprecated versions without warning'
+complete -c spack -n '__fish_spack_using_command dev-build' -l keep-prefix -f -a keep_prefix
+complete -c spack -n '__fish_spack_using_command dev-build' -l keep-prefix -d 'do not remove the install prefix if installation fails'
+complete -c spack -n '__fish_spack_using_command dev-build' -l skip-patch -f -a skip_patch
+complete -c spack -n '__fish_spack_using_command dev-build' -l skip-patch -d 'skip patching for the developer build'
+complete -c spack -n '__fish_spack_using_command dev-build' -s q -l quiet -f -a quiet
+complete -c spack -n '__fish_spack_using_command dev-build' -s q -l quiet -d 'do not display verbose build output while installing'
+complete -c spack -n '__fish_spack_using_command dev-build' -l drop-in -r -f -a shell
+complete -c spack -n '__fish_spack_using_command dev-build' -l drop-in -r -d 'drop into a build environment in a new shell, e.g., bash'
+complete -c spack -n '__fish_spack_using_command dev-build' -l test -r -f -a 'root all'
+complete -c spack -n '__fish_spack_using_command dev-build' -l test -r -d 'run tests on only root packages or all packages'
+complete -c spack -n '__fish_spack_using_command dev-build' -s b -l before -r -f -a before
+complete -c spack -n '__fish_spack_using_command dev-build' -s b -l before -r -d 'phase to stop before when installing (default None)'
+complete -c spack -n '__fish_spack_using_command dev-build' -s u -l until -r -f -a until
+complete -c spack -n '__fish_spack_using_command dev-build' -s u -l until -r -d 'phase to stop after when installing (default None)'
+complete -c spack -n '__fish_spack_using_command dev-build' -l clean -f -a dirty
+complete -c spack -n '__fish_spack_using_command dev-build' -l clean -d 'unset harmful variables in the build environment (default)'
+complete -c spack -n '__fish_spack_using_command dev-build' -l dirty -f -a dirty
+complete -c spack -n '__fish_spack_using_command dev-build' -l dirty -d 'preserve user environment in spack\'s build environment (danger!)'
+complete -c spack -n '__fish_spack_using_command dev-build' -s U -l fresh -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command dev-build' -s U -l fresh -d 'do not reuse installed deps; build newest configuration'
+complete -c spack -n '__fish_spack_using_command dev-build' -l reuse -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command dev-build' -l reuse -d 'reuse installed packages/buildcaches when possible'
+complete -c spack -n '__fish_spack_using_command dev-build' -l reuse-deps -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command dev-build' -l reuse-deps -d 'reuse installed dependencies only'
+
+# spack develop
+set -g __fish_spack_optspecs_spack_develop h/help p/path= no-clone clone f/force=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 develop' -f -k -a '(__fish_spack_specs_or_id)'
+complete -c spack -n '__fish_spack_using_command develop' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command develop' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command develop' -s p -l path -r -f -a path
+complete -c spack -n '__fish_spack_using_command develop' -s p -l path -r -d 'source location of package'
+complete -c spack -n '__fish_spack_using_command develop' -l no-clone -f -a clone
+complete -c spack -n '__fish_spack_using_command develop' -l no-clone -d 'do not clone, the package already exists at the source path'
+complete -c spack -n '__fish_spack_using_command develop' -l clone -f -a clone
+complete -c spack -n '__fish_spack_using_command develop' -l clone -d 'clone the package even if the path already exists'
+complete -c spack -n '__fish_spack_using_command develop' -s f -l force -r -f -a force
+complete -c spack -n '__fish_spack_using_command develop' -s f -l force -r -d 'remove any files or directories that block cloning source code'
+
+# spack diff
+set -g __fish_spack_optspecs_spack_diff h/help json first a/attribute=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 diff' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command diff' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command diff' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command diff' -l json -f -a dump_json
+complete -c spack -n '__fish_spack_using_command diff' -l json -d 'dump json output instead of pretty printing'
+complete -c spack -n '__fish_spack_using_command diff' -l first -f -a load_first
+complete -c spack -n '__fish_spack_using_command diff' -l first -d 'load the first match if multiple packages match the spec'
+complete -c spack -n '__fish_spack_using_command diff' -s a -l attribute -r -f -a attribute
+complete -c spack -n '__fish_spack_using_command diff' -s a -l attribute -r -d 'select the attributes to show (defaults to all)'
+
+# spack docs
+set -g __fish_spack_optspecs_spack_docs h/help
+complete -c spack -n '__fish_spack_using_command docs' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command docs' -s h -l help -d 'show this help message and exit'
+
+# spack edit
+set -g __fish_spack_optspecs_spack_edit h/help b/build-system c/command d/docs t/test m/module r/repo= N/namespace=
+complete -c spack -n '__fish_spack_using_command_pos 0 edit' -f -a '(__fish_spack_packages)'
+complete -c spack -n '__fish_spack_using_command edit' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command edit' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command edit' -s b -l build-system -f -a path
+complete -c spack -n '__fish_spack_using_command edit' -s b -l build-system -d 'edit the build system with the supplied name'
+complete -c spack -n '__fish_spack_using_command edit' -s c -l command -f -a path
+complete -c spack -n '__fish_spack_using_command edit' -s c -l command -d 'edit the command with the supplied name'
+complete -c spack -n '__fish_spack_using_command edit' -s d -l docs -f -a path
+complete -c spack -n '__fish_spack_using_command edit' -s d -l docs -d 'edit the docs with the supplied name'
+complete -c spack -n '__fish_spack_using_command edit' -s t -l test -f -a path
+complete -c spack -n '__fish_spack_using_command edit' -s t -l test -d 'edit the test with the supplied name'
+complete -c spack -n '__fish_spack_using_command edit' -s m -l module -f -a path
+complete -c spack -n '__fish_spack_using_command edit' -s m -l module -d 'edit the main spack module with the supplied name'
+complete -c spack -n '__fish_spack_using_command edit' -s r -l repo -r -f -a repo
+complete -c spack -n '__fish_spack_using_command edit' -s r -l repo -r -d 'path to repo to edit package in'
+complete -c spack -n '__fish_spack_using_command edit' -s N -l namespace -r -f -a namespace
+complete -c spack -n '__fish_spack_using_command edit' -s N -l namespace -r -d 'namespace of package to edit'
+
+# spack env
+set -g __fish_spack_optspecs_spack_env h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a activate -d 'set the current environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a deactivate -d 'deactivate any active environment in the shell'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a create -d 'create a new environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a remove -d 'remove an existing environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a rm -d 'remove an existing environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a list -d 'list available environments'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a ls -d 'list available environments'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a status -d 'print whether there is an active environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a st -d 'print whether there is an active environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a loads -d 'list modules for an installed environment \'(see spack module loads)\''
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a view -d 'manage a view associated with the environment'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a update -d 'update environments to the latest format'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a revert -d 'restore environments to their state before update'
+complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a depfile -d 'generate a depfile from the concrete environment specs'
+complete -c spack -n '__fish_spack_using_command env' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env' -s h -l help -d 'show this help message and exit'
+
+# spack env activate
+set -g __fish_spack_optspecs_spack_env_activate h/help sh csh fish bat pwsh v/with-view= V/without-view p/prompt temp d/dir=
+complete -c spack -n '__fish_spack_using_command_pos 0 env activate' -f -a '(__fish_spack_environments)'
+complete -c spack -n '__fish_spack_using_command env activate' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env activate' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command env activate' -l sh -f -a shell
+complete -c spack -n '__fish_spack_using_command env activate' -l sh -d 'print sh commands to activate the environment'
+complete -c spack -n '__fish_spack_using_command env activate' -l csh -f -a shell
+complete -c spack -n '__fish_spack_using_command env activate' -l csh -d 'print csh commands to activate the environment'
+complete -c spack -n '__fish_spack_using_command env activate' -l fish -f -a shell
+complete -c spack -n '__fish_spack_using_command env activate' -l fish -d 'print fish commands to activate the environment'
+complete -c spack -n '__fish_spack_using_command env activate' -l bat -f -a shell
+complete -c spack -n '__fish_spack_using_command env activate' -l bat -d 'print bat commands to activate the environment'
+complete -c spack -n '__fish_spack_using_command env activate' -l pwsh -f -a shell
+complete -c spack -n '__fish_spack_using_command env activate' -l pwsh -d 'print powershell commands to activate environment'
+complete -c spack -n '__fish_spack_using_command env activate' -l with-view -s v -r -f -a with_view
+complete -c spack -n '__fish_spack_using_command env activate' -l with-view -s v -r -d 'set runtime environment variables for specific view'
+complete -c spack -n '__fish_spack_using_command env activate' -l without-view -s V -f -a without_view
+complete -c spack -n '__fish_spack_using_command env activate' -l without-view -s V -d 'do not set runtime environment variables for any view'
+complete -c spack -n '__fish_spack_using_command env activate' -s p -l prompt -f -a prompt
+complete -c spack -n '__fish_spack_using_command env activate' -s p -l prompt -d 'decorate the command line prompt when activating'
+complete -c spack -n '__fish_spack_using_command env activate' -l temp -f -a temp
+complete -c spack -n '__fish_spack_using_command env activate' -l temp -d 'create and activate an environment in a temporary directory'
+complete -c spack -n '__fish_spack_using_command env activate' -s d -l dir -r -f -a dir
+complete -c spack -n '__fish_spack_using_command env activate' -s d -l dir -r -d 'activate the environment in this directory'
+
+# spack env deactivate
+set -g __fish_spack_optspecs_spack_env_deactivate h/help sh csh fish bat pwsh
+complete -c spack -n '__fish_spack_using_command env deactivate' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env deactivate' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command env deactivate' -l sh -f -a shell
+complete -c spack -n '__fish_spack_using_command env deactivate' -l sh -d 'print sh commands to deactivate the environment'
+complete -c spack -n '__fish_spack_using_command env deactivate' -l csh -f -a shell
+complete -c spack -n '__fish_spack_using_command env deactivate' -l csh -d 'print csh commands to deactivate the environment'
+complete -c spack -n '__fish_spack_using_command env deactivate' -l fish -f -a shell
+complete -c spack -n '__fish_spack_using_command env deactivate' -l fish -d 'print fish commands to activate the environment'
+complete -c spack -n '__fish_spack_using_command env deactivate' -l bat -f -a shell
+complete -c spack -n '__fish_spack_using_command env deactivate' -l bat -d 'print bat commands to activate the environment'
+complete -c spack -n '__fish_spack_using_command env deactivate' -l pwsh -f -a shell
+complete -c spack -n '__fish_spack_using_command env deactivate' -l pwsh -d 'print pwsh commands to activate the environment'
+
+# spack env create
+set -g __fish_spack_optspecs_spack_env_create h/help d/dir keep-relative without-view with-view=
+complete -c spack -n '__fish_spack_using_command_pos 0 env create' -f -a '(__fish_spack_environments)'
+complete -c spack -n '__fish_spack_using_command env create' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env create' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command env create' -s d -l dir -f -a dir
+complete -c spack -n '__fish_spack_using_command env create' -s d -l dir -d 'create an environment in a specific directory'
+complete -c spack -n '__fish_spack_using_command env create' -l keep-relative -f -a keep_relative
+complete -c spack -n '__fish_spack_using_command env create' -l keep-relative -d 'copy relative develop paths verbatim into the new environment when initializing from envfile'
+complete -c spack -n '__fish_spack_using_command env create' -l without-view -f -a without_view
+complete -c spack -n '__fish_spack_using_command env create' -l without-view -d 'do not maintain a view for this environment'
+complete -c spack -n '__fish_spack_using_command env create' -l with-view -r -f -a with_view
+complete -c spack -n '__fish_spack_using_command env create' -l with-view -r -d 'specify that this environment should maintain a view at the specified path (by default the view is maintained in the environment directory)'
+
+# spack env remove
+set -g __fish_spack_optspecs_spack_env_remove h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 env remove' -f -a '(__fish_spack_environments)'
+complete -c spack -n '__fish_spack_using_command env remove' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env remove' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command env remove' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command env remove' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack env rm
+set -g __fish_spack_optspecs_spack_env_rm h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 env rm' -f -a '(__fish_spack_environments)'
+complete -c spack -n '__fish_spack_using_command env rm' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env rm' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command env rm' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command env rm' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack env list
+set -g __fish_spack_optspecs_spack_env_list h/help
+complete -c spack -n '__fish_spack_using_command env list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env list' -s h -l help -d 'show this help message and exit'
+
+# spack env ls
+set -g __fish_spack_optspecs_spack_env_ls h/help
+complete -c spack -n '__fish_spack_using_command env ls' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env ls' -s h -l help -d 'show this help message and exit'
+
+# spack env status
+set -g __fish_spack_optspecs_spack_env_status h/help
+complete -c spack -n '__fish_spack_using_command env status' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env status' -s h -l help -d 'show this help message and exit'
+
+# spack env st
+set -g __fish_spack_optspecs_spack_env_st h/help
+complete -c spack -n '__fish_spack_using_command env st' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env st' -s h -l help -d 'show this help message and exit'
+
+# spack env loads
+set -g __fish_spack_optspecs_spack_env_loads h/help n/module-set-name= m/module-type= input-only p/prefix= x/exclude= r/dependencies
+complete -c spack -n '__fish_spack_using_command env loads' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env loads' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command env loads' -s n -l module-set-name -r -f -a module_set_name
+complete -c spack -n '__fish_spack_using_command env loads' -s n -l module-set-name -r -d 'module set for which to generate load operations'
+complete -c spack -n '__fish_spack_using_command env loads' -s m -l module-type -r -f -a 'tcl lmod'
+complete -c spack -n '__fish_spack_using_command env loads' -s m -l module-type -r -d 'type of module system to generate loads for'
+complete -c spack -n '__fish_spack_using_command env loads' -l input-only -f -a shell
+complete -c spack -n '__fish_spack_using_command env loads' -l input-only -d 'generate input for module command (instead of a shell script)'
+complete -c spack -n '__fish_spack_using_command env loads' -s p -l prefix -r -f -a prefix
+complete -c spack -n '__fish_spack_using_command env loads' -s p -l prefix -r -d 'prepend to module names when issuing module load commands'
+complete -c spack -n '__fish_spack_using_command env loads' -s x -l exclude -r -f -a exclude
+complete -c spack -n '__fish_spack_using_command env loads' -s x -l exclude -r -d 'exclude package from output; may be specified multiple times'
+complete -c spack -n '__fish_spack_using_command env loads' -s r -l dependencies -f -a recurse_dependencies
+complete -c spack -n '__fish_spack_using_command env loads' -s r -l dependencies -d 'recursively traverse spec dependencies'
+
+# spack env view
+set -g __fish_spack_optspecs_spack_env_view h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 env view' -f -a 'regenerate enable disable'
+complete -c spack -n '__fish_spack_using_command env view' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env view' -s h -l help -d 'show this help message and exit'
+
+# spack env update
+set -g __fish_spack_optspecs_spack_env_update h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command_pos 0 env update' -f -a '(__fish_spack_environments)'
+complete -c spack -n '__fish_spack_using_command env update' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env update' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command env update' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command env update' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack env revert
+set -g __fish_spack_optspecs_spack_env_revert h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command_pos 0 env revert' -f -a '(__fish_spack_environments)'
+complete -c spack -n '__fish_spack_using_command env revert' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env revert' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command env revert' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command env revert' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack env depfile
+set -g __fish_spack_optspecs_spack_env_depfile h/help make-prefix= make-disable-jobserver use-buildcache= o/output= G/generator=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 env depfile' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command env depfile' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command env depfile' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command env depfile' -l make-prefix -l make-target-prefix -r -f -a make_prefix
+complete -c spack -n '__fish_spack_using_command env depfile' -l make-prefix -l make-target-prefix -r -d 'prefix Makefile targets (and variables) with /'
+complete -c spack -n '__fish_spack_using_command env depfile' -l make-disable-jobserver -f -a jobserver
+complete -c spack -n '__fish_spack_using_command env depfile' -l make-disable-jobserver -d 'disable POSIX jobserver support'
+complete -c spack -n '__fish_spack_using_command env depfile' -l use-buildcache -r -f -a use_buildcache
+complete -c spack -n '__fish_spack_using_command env depfile' -l use-buildcache -r -d 'when using `only`, redundant build dependencies are pruned from the DAG'
+complete -c spack -n '__fish_spack_using_command env depfile' -s o -l output -r -f -a output
+complete -c spack -n '__fish_spack_using_command env depfile' -s o -l output -r -d 'write the depfile to FILE rather than to stdout'
+complete -c spack -n '__fish_spack_using_command env depfile' -s G -l generator -r -f -a make
+complete -c spack -n '__fish_spack_using_command env depfile' -s G -l generator -r -d 'specify the depfile type'
+
+# spack extensions
+set -g __fish_spack_optspecs_spack_extensions h/help l/long L/very-long d/deps p/paths s/show=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 extensions' -f -a '(__fish_spack_extensions)'
+complete -c spack -n '__fish_spack_using_command extensions' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command extensions' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command extensions' -s l -l long -f -a long
+complete -c spack -n '__fish_spack_using_command extensions' -s l -l long -d 'show dependency hashes as well as versions'
+complete -c spack -n '__fish_spack_using_command extensions' -s L -l very-long -f -a very_long
+complete -c spack -n '__fish_spack_using_command extensions' -s L -l very-long -d 'show full dependency hashes as well as versions'
+complete -c spack -n '__fish_spack_using_command extensions' -s d -l deps -f -a deps
+complete -c spack -n '__fish_spack_using_command extensions' -s d -l deps -d 'output dependencies along with found specs'
+complete -c spack -n '__fish_spack_using_command extensions' -s p -l paths -f -a paths
+complete -c spack -n '__fish_spack_using_command extensions' -s p -l paths -d 'show paths to package install directories'
+complete -c spack -n '__fish_spack_using_command extensions' -s s -l show -r -f -a 'packages installed all'
+complete -c spack -n '__fish_spack_using_command extensions' -s s -l show -r -d 'show only part of output'
+
+# spack external
+set -g __fish_spack_optspecs_spack_external h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 external' -f -a find -d 'add external packages to packages.yaml'
+complete -c spack -n '__fish_spack_using_command_pos 0 external' -f -a list -d 'list detectable packages, by repository and name'
+complete -c spack -n '__fish_spack_using_command_pos 0 external' -f -a read-cray-manifest -d 'consume a Spack-compatible description of externally-installed packages, including dependency relationships'
+complete -c spack -n '__fish_spack_using_command external' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command external' -s h -l help -d 'show this help message and exit'
+
+# spack external find
+set -g __fish_spack_optspecs_spack_external_find h/help not-buildable exclude= p/path= scope= all t/tag= j/jobs=
+
+complete -c spack -n '__fish_spack_using_command external find' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command external find' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command external find' -l not-buildable -f -a not_buildable
+complete -c spack -n '__fish_spack_using_command external find' -l not-buildable -d 'packages with detected externals won\'t be built with Spack'
+complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f -a exclude
+complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search'
+complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path
+complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals'
+complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify'
+complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all
+complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about'
+complete -c spack -n '__fish_spack_using_command external find' -s t -l tag -r -f -a tags
+complete -c spack -n '__fish_spack_using_command external find' -s t -l tag -r -d 'filter a package query by tag (multiple use allowed)'
+complete -c spack -n '__fish_spack_using_command external find' -s j -l jobs -r -f -a jobs
+complete -c spack -n '__fish_spack_using_command external find' -s j -l jobs -r -d 'explicitly set number of parallel jobs'
+
+# spack external list
+set -g __fish_spack_optspecs_spack_external_list h/help
+complete -c spack -n '__fish_spack_using_command external list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command external list' -s h -l help -d 'show this help message and exit'
+
+# spack external read-cray-manifest
+set -g __fish_spack_optspecs_spack_external_read_cray_manifest h/help file= directory= ignore-default-dir dry-run fail-on-error
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -l file -r -f -a file
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -l file -r -d 'specify a location other than the default'
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -l directory -r -f -a directory
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -l directory -r -d 'specify a directory storing a group of manifest files'
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -l ignore-default-dir -f -a ignore_default_dir
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -l ignore-default-dir -d 'ignore the default directory of manifest files'
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -l dry-run -f -a dry_run
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -l dry-run -d 'don\'t modify DB with files that are read'
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -l fail-on-error -f -a fail_on_error
+complete -c spack -n '__fish_spack_using_command external read-cray-manifest' -l fail-on-error -d 'if a manifest file cannot be parsed, fail and report the full stack trace'
+
+# spack fetch
+set -g __fish_spack_optspecs_spack_fetch h/help n/no-checksum deprecated m/missing D/dependencies
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 fetch' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command fetch' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command fetch' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command fetch' -s n -l no-checksum -f -a no_checksum
+complete -c spack -n '__fish_spack_using_command fetch' -s n -l no-checksum -d 'do not use checksums to verify downloaded files (unsafe)'
+complete -c spack -n '__fish_spack_using_command fetch' -l deprecated -f -a deprecated
+complete -c spack -n '__fish_spack_using_command fetch' -l deprecated -d 'fetch deprecated versions without warning'
+complete -c spack -n '__fish_spack_using_command fetch' -s m -l missing -f -a missing
+complete -c spack -n '__fish_spack_using_command fetch' -s m -l missing -d 'fetch only missing (not yet installed) dependencies'
+complete -c spack -n '__fish_spack_using_command fetch' -s D -l dependencies -f -a dependencies
+complete -c spack -n '__fish_spack_using_command fetch' -s D -l dependencies -d 'also fetch all dependencies'
+
+# spack find
+set -g __fish_spack_optspecs_spack_find h/help format= H/hashes json d/deps p/paths groups no-groups l/long L/very-long t/tag= N/namespaces c/show-concretized f/show-flags show-full-compiler x/explicit X/implicit u/unknown m/missing v/variants loaded M/only-missing deprecated only-deprecated start-date= end-date=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 find' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command find' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command find' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command find' -l format -r -f -a format
+complete -c spack -n '__fish_spack_using_command find' -l format -r -d 'output specs with the specified format string'
+complete -c spack -n '__fish_spack_using_command find' -s H -l hashes -f -a format
+complete -c spack -n '__fish_spack_using_command find' -s H -l hashes -d 'same as \'--format {/hash}\'; use with xargs or $()'
+complete -c spack -n '__fish_spack_using_command find' -l json -f -a json
+complete -c spack -n '__fish_spack_using_command find' -l json -d 'output specs as machine-readable json records'
+complete -c spack -n '__fish_spack_using_command find' -s d -l deps -f -a deps
+complete -c spack -n '__fish_spack_using_command find' -s d -l deps -d 'output dependencies along with found specs'
+complete -c spack -n '__fish_spack_using_command find' -s p -l paths -f -a paths
+complete -c spack -n '__fish_spack_using_command find' -s p -l paths -d 'show paths to package install directories'
+complete -c spack -n '__fish_spack_using_command find' -l groups -f -a groups
+complete -c spack -n '__fish_spack_using_command find' -l groups -d 'display specs in arch/compiler groups (default on)'
+complete -c spack -n '__fish_spack_using_command find' -l no-groups -f -a groups
+complete -c spack -n '__fish_spack_using_command find' -l no-groups -d 'do not group specs by arch/compiler'
+complete -c spack -n '__fish_spack_using_command find' -s l -l long -f -a long
+complete -c spack -n '__fish_spack_using_command find' -s l -l long -d 'show dependency hashes as well as versions'
+complete -c spack -n '__fish_spack_using_command find' -s L -l very-long -f -a very_long
+complete -c spack -n '__fish_spack_using_command find' -s L -l very-long -d 'show full dependency hashes as well as versions'
+complete -c spack -n '__fish_spack_using_command find' -s t -l tag -r -f -a tags
+complete -c spack -n '__fish_spack_using_command find' -s t -l tag -r -d 'filter a package query by tag (multiple use allowed)'
+complete -c spack -n '__fish_spack_using_command find' -s N -l namespaces -f -a namespaces
+complete -c spack -n '__fish_spack_using_command find' -s N -l namespaces -d 'show fully qualified package names'
+complete -c spack -n '__fish_spack_using_command find' -s c -l show-concretized -f -a show_concretized
+complete -c spack -n '__fish_spack_using_command find' -s c -l show-concretized -d 'show concretized specs in an environment'
+complete -c spack -n '__fish_spack_using_command find' -s f -l show-flags -f -a show_flags
+complete -c spack -n '__fish_spack_using_command find' -s f -l show-flags -d 'show spec compiler flags'
+complete -c spack -n '__fish_spack_using_command find' -l show-full-compiler -f -a show_full_compiler
+complete -c spack -n '__fish_spack_using_command find' -l show-full-compiler -d 'show full compiler specs'
+complete -c spack -n '__fish_spack_using_command find' -s x -l explicit -f -a explicit
+complete -c spack -n '__fish_spack_using_command find' -s x -l explicit -d 'show only specs that were installed explicitly'
+complete -c spack -n '__fish_spack_using_command find' -s X -l implicit -f -a implicit
+complete -c spack -n '__fish_spack_using_command find' -s X -l implicit -d 'show only specs that were installed as dependencies'
+complete -c spack -n '__fish_spack_using_command find' -s u -l unknown -f -a unknown
+complete -c spack -n '__fish_spack_using_command find' -s u -l unknown -d 'show only specs Spack does not have a package for'
+complete -c spack -n '__fish_spack_using_command find' -s m -l missing -f -a missing
+complete -c spack -n '__fish_spack_using_command find' -s m -l missing -d 'show missing dependencies as well as installed specs'
+complete -c spack -n '__fish_spack_using_command find' -s v -l variants -f -a variants
+complete -c spack -n '__fish_spack_using_command find' -s v -l variants -d 'show variants in output (can be long)'
+complete -c spack -n '__fish_spack_using_command find' -l loaded -f -a loaded
+complete -c spack -n '__fish_spack_using_command find' -l loaded -d 'show only packages loaded in the user environment'
+complete -c spack -n '__fish_spack_using_command find' -s M -l only-missing -f -a only_missing
+complete -c spack -n '__fish_spack_using_command find' -s M -l only-missing -d 'show only missing dependencies'
+complete -c spack -n '__fish_spack_using_command find' -l deprecated -f -a deprecated
+complete -c spack -n '__fish_spack_using_command find' -l deprecated -d 'show deprecated packages as well as installed specs'
+complete -c spack -n '__fish_spack_using_command find' -l only-deprecated -f -a only_deprecated
+complete -c spack -n '__fish_spack_using_command find' -l only-deprecated -d 'show only deprecated packages'
+complete -c spack -n '__fish_spack_using_command find' -l start-date -r -f -a start_date
+complete -c spack -n '__fish_spack_using_command find' -l start-date -r -d 'earliest date of installation [YYYY-MM-DD]'
+complete -c spack -n '__fish_spack_using_command find' -l end-date -r -f -a end_date
+complete -c spack -n '__fish_spack_using_command find' -l end-date -r -d 'latest date of installation [YYYY-MM-DD]'
+
+# spack gc
+set -g __fish_spack_optspecs_spack_gc h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command gc' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command gc' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command gc' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command gc' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack gpg
+set -g __fish_spack_optspecs_spack_gpg h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 gpg' -f -a verify -d 'verify a signed package'
+complete -c spack -n '__fish_spack_using_command_pos 0 gpg' -f -a trust -d 'add a key to the keyring'
+complete -c spack -n '__fish_spack_using_command_pos 0 gpg' -f -a untrust -d 'remove a key from the keyring'
+complete -c spack -n '__fish_spack_using_command_pos 0 gpg' -f -a sign -d 'sign a package'
+complete -c spack -n '__fish_spack_using_command_pos 0 gpg' -f -a create -d 'create a new key'
+complete -c spack -n '__fish_spack_using_command_pos 0 gpg' -f -a list -d 'list keys available in the keyring'
+complete -c spack -n '__fish_spack_using_command_pos 0 gpg' -f -a init -d 'add the default keys to the keyring'
+complete -c spack -n '__fish_spack_using_command_pos 0 gpg' -f -a export -d 'export a gpg key, optionally including secret key'
+complete -c spack -n '__fish_spack_using_command_pos 0 gpg' -f -a publish -d 'publish public keys to a build cache'
+complete -c spack -n '__fish_spack_using_command gpg' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command gpg' -s h -l help -d 'show this help message and exit'
+
+# spack gpg verify
+set -g __fish_spack_optspecs_spack_gpg_verify h/help
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 gpg verify' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command gpg verify' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command gpg verify' -s h -l help -d 'show this help message and exit'
+
+# spack gpg trust
+set -g __fish_spack_optspecs_spack_gpg_trust h/help
+
+complete -c spack -n '__fish_spack_using_command gpg trust' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command gpg trust' -s h -l help -d 'show this help message and exit'
+
+# spack gpg untrust
+set -g __fish_spack_optspecs_spack_gpg_untrust h/help signing
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 gpg untrust' -f -a '(__fish_spack_gpg_keys)'
+complete -c spack -n '__fish_spack_using_command gpg untrust' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command gpg untrust' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command gpg untrust' -l signing -f -a signing
+complete -c spack -n '__fish_spack_using_command gpg untrust' -l signing -d 'allow untrusting signing keys'
+
+# spack gpg sign
+set -g __fish_spack_optspecs_spack_gpg_sign h/help output= key= clearsign
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 gpg sign' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command gpg sign' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command gpg sign' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command gpg sign' -l output -r -f -a output
+complete -c spack -n '__fish_spack_using_command gpg sign' -l output -r -d 'the directory to place signatures'
+complete -c spack -n '__fish_spack_using_command gpg sign' -l key -r -f -a key
+complete -c spack -n '__fish_spack_using_command gpg sign' -l key -r -d 'the key to use for signing'
+complete -c spack -n '__fish_spack_using_command gpg sign' -l clearsign -f -a clearsign
+complete -c spack -n '__fish_spack_using_command gpg sign' -l clearsign -d 'if specified, create a clearsign signature'
+
+# spack gpg create
+set -g __fish_spack_optspecs_spack_gpg_create h/help comment= expires= export= export-secret=
+
+complete -c spack -n '__fish_spack_using_command gpg create' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command gpg create' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command gpg create' -l comment -r -f -a comment
+complete -c spack -n '__fish_spack_using_command gpg create' -l comment -r -d 'a description for the intended use of the key'
+complete -c spack -n '__fish_spack_using_command gpg create' -l expires -r -f -a expires
+complete -c spack -n '__fish_spack_using_command gpg create' -l expires -r -d 'when the key should expire'
+complete -c spack -n '__fish_spack_using_command gpg create' -l export -r -f -a export
+complete -c spack -n '__fish_spack_using_command gpg create' -l export -r -d 'export the public key to a file'
+complete -c spack -n '__fish_spack_using_command gpg create' -l export-secret -r -f -a secret
+complete -c spack -n '__fish_spack_using_command gpg create' -l export-secret -r -d 'export the private key to a file'
+
+# spack gpg list
+set -g __fish_spack_optspecs_spack_gpg_list h/help trusted signing
+complete -c spack -n '__fish_spack_using_command gpg list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command gpg list' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command gpg list' -l trusted -f -a trusted
+complete -c spack -n '__fish_spack_using_command gpg list' -l trusted -d 'list trusted keys'
+complete -c spack -n '__fish_spack_using_command gpg list' -l signing -f -a signing
+complete -c spack -n '__fish_spack_using_command gpg list' -l signing -d 'list keys which may be used for signing'
+
+# spack gpg init
+set -g __fish_spack_optspecs_spack_gpg_init h/help from=
+complete -c spack -n '__fish_spack_using_command gpg init' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command gpg init' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command gpg init' -l from -r -f -a import_dir
+
+# spack gpg export
+set -g __fish_spack_optspecs_spack_gpg_export h/help secret
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 gpg export' -f -a '(__fish_spack_gpg_keys)'
+complete -c spack -n '__fish_spack_using_command gpg export' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command gpg export' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command gpg export' -l secret -f -a secret
+complete -c spack -n '__fish_spack_using_command gpg export' -l secret -d 'export secret keys'
+
+# spack gpg publish
+set -g __fish_spack_optspecs_spack_gpg_publish h/help d/directory= m/mirror-name= mirror-url= rebuild-index
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 gpg publish' -f -a '(__fish_spack_gpg_keys)'
+complete -c spack -n '__fish_spack_using_command gpg publish' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command gpg publish' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command gpg publish' -s d -l directory -r -f -a directory
+complete -c spack -n '__fish_spack_using_command gpg publish' -s d -l directory -r -d 'local directory where keys will be published'
+complete -c spack -n '__fish_spack_using_command gpg publish' -s m -l mirror-name -r -f -a mirror_name
+complete -c spack -n '__fish_spack_using_command gpg publish' -s m -l mirror-name -r -d 'name of the mirror where keys will be published'
+complete -c spack -n '__fish_spack_using_command gpg publish' -l mirror-url -r -f -a mirror_url
+complete -c spack -n '__fish_spack_using_command gpg publish' -l mirror-url -r -d 'URL of the mirror where keys will be published'
+complete -c spack -n '__fish_spack_using_command gpg publish' -l rebuild-index -f -a rebuild_index
+complete -c spack -n '__fish_spack_using_command gpg publish' -l rebuild-index -d 'regenerate buildcache key index after publishing key(s)'
+
+# spack graph
+set -g __fish_spack_optspecs_spack_graph h/help a/ascii d/dot s/static c/color i/installed deptype=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 graph' -f -k -a '(__fish_spack_specs_or_id)'
+complete -c spack -n '__fish_spack_using_command graph' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command graph' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command graph' -s a -l ascii -f -a ascii
+complete -c spack -n '__fish_spack_using_command graph' -s a -l ascii -d 'draw graph as ascii to stdout (default)'
+complete -c spack -n '__fish_spack_using_command graph' -s d -l dot -f -a dot
+complete -c spack -n '__fish_spack_using_command graph' -s d -l dot -d 'generate graph in dot format and print to stdout'
+complete -c spack -n '__fish_spack_using_command graph' -s s -l static -f -a static
+complete -c spack -n '__fish_spack_using_command graph' -s s -l static -d 'graph static (possible) deps, don\'t concretize (implies --dot)'
+complete -c spack -n '__fish_spack_using_command graph' -s c -l color -f -a color
+complete -c spack -n '__fish_spack_using_command graph' -s c -l color -d 'use different colors for different dependency types'
+complete -c spack -n '__fish_spack_using_command graph' -s i -l installed -f -a installed
+complete -c spack -n '__fish_spack_using_command graph' -s i -l installed -d 'graph installed specs, or specs in the active env (implies --dot)'
+complete -c spack -n '__fish_spack_using_command graph' -l deptype -r -f -a deptype
+complete -c spack -n '__fish_spack_using_command graph' -l deptype -r -d 'comma-separated list of deptypes to traverse (default=build,link,run,test)'
+
+# spack help
+set -g __fish_spack_optspecs_spack_help h/help a/all spec
+complete -c spack -n '__fish_spack_using_command_pos 0 help' -f -a '(__fish_spack_commands)'
+complete -c spack -n '__fish_spack_using_command help' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command help' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command help' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command help' -s a -l all -d 'list all available commands and options'
+complete -c spack -n '__fish_spack_using_command help' -l spec -f -a guide
+complete -c spack -n '__fish_spack_using_command help' -l spec -d 'help on the package specification syntax'
+
+# spack info
+set -g __fish_spack_optspecs_spack_info h/help a/all detectable maintainers no-dependencies no-variants no-versions phases tags tests virtuals variants-by-name
+complete -c spack -n '__fish_spack_using_command_pos 0 info' -f -a '(__fish_spack_packages)'
+complete -c spack -n '__fish_spack_using_command info' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command info' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command info' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command info' -s a -l all -d 'output all package information'
+complete -c spack -n '__fish_spack_using_command info' -l detectable -f -a detectable
+complete -c spack -n '__fish_spack_using_command info' -l detectable -d 'output information on external detection'
+complete -c spack -n '__fish_spack_using_command info' -l maintainers -f -a maintainers
+complete -c spack -n '__fish_spack_using_command info' -l maintainers -d 'output package maintainers'
+complete -c spack -n '__fish_spack_using_command info' -l no-dependencies -f -a no_dependencies
+complete -c spack -n '__fish_spack_using_command info' -l no-dependencies -d 'do not output build, link, and run package dependencies'
+complete -c spack -n '__fish_spack_using_command info' -l no-variants -f -a no_variants
+complete -c spack -n '__fish_spack_using_command info' -l no-variants -d 'do not output variants'
+complete -c spack -n '__fish_spack_using_command info' -l no-versions -f -a no_versions
+complete -c spack -n '__fish_spack_using_command info' -l no-versions -d 'do not output versions'
+complete -c spack -n '__fish_spack_using_command info' -l phases -f -a phases
+complete -c spack -n '__fish_spack_using_command info' -l phases -d 'output installation phases'
+complete -c spack -n '__fish_spack_using_command info' -l tags -f -a tags
+complete -c spack -n '__fish_spack_using_command info' -l tags -d 'output package tags'
+complete -c spack -n '__fish_spack_using_command info' -l tests -f -a tests
+complete -c spack -n '__fish_spack_using_command info' -l tests -d 'output relevant build-time and stand-alone tests'
+complete -c spack -n '__fish_spack_using_command info' -l virtuals -f -a virtuals
+complete -c spack -n '__fish_spack_using_command info' -l virtuals -d 'output virtual packages'
+complete -c spack -n '__fish_spack_using_command info' -l variants-by-name -f -a variants_by_name
+complete -c spack -n '__fish_spack_using_command info' -l variants-by-name -d 'list variants in strict name order; don\'t group by condition'
+
+# spack install
+set -g __fish_spack_optspecs_spack_install h/help only= u/until= j/jobs= overwrite fail-fast keep-prefix keep-stage dont-restage use-cache no-cache cache-only use-buildcache= include-build-deps no-check-signature show-log-on-error source n/no-checksum deprecated v/verbose fake only-concrete add no-add f/file= clean dirty test= log-format= log-file= help-cdash cdash-upload-url= cdash-build= cdash-site= cdash-track= cdash-buildstamp= y/yes-to-all U/fresh reuse reuse-deps
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 install' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command install' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command install' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command install' -l only -r -f -a 'package dependencies'
+complete -c spack -n '__fish_spack_using_command install' -l only -r -d 'select the mode of installation'
+complete -c spack -n '__fish_spack_using_command install' -s u -l until -r -f -a until
+complete -c spack -n '__fish_spack_using_command install' -s u -l until -r -d 'phase to stop after when installing (default None)'
+complete -c spack -n '__fish_spack_using_command install' -s j -l jobs -r -f -a jobs
+complete -c spack -n '__fish_spack_using_command install' -s j -l jobs -r -d 'explicitly set number of parallel jobs'
+complete -c spack -n '__fish_spack_using_command install' -l overwrite -f -a overwrite
+complete -c spack -n '__fish_spack_using_command install' -l overwrite -d 'reinstall an existing spec, even if it has dependents'
+complete -c spack -n '__fish_spack_using_command install' -l fail-fast -f -a fail_fast
+complete -c spack -n '__fish_spack_using_command install' -l fail-fast -d 'stop all builds if any build fails (default is best effort)'
+complete -c spack -n '__fish_spack_using_command install' -l keep-prefix -f -a keep_prefix
+complete -c spack -n '__fish_spack_using_command install' -l keep-prefix -d 'don\'t remove the install prefix if installation fails'
+complete -c spack -n '__fish_spack_using_command install' -l keep-stage -f -a keep_stage
+complete -c spack -n '__fish_spack_using_command install' -l keep-stage -d 'don\'t remove the build stage if installation succeeds'
+complete -c spack -n '__fish_spack_using_command install' -l dont-restage -f -a dont_restage
+complete -c spack -n '__fish_spack_using_command install' -l dont-restage -d 'if a partial install is detected, don\'t delete prior state'
+complete -c spack -n '__fish_spack_using_command install' -l use-cache -f -a use_cache
+complete -c spack -n '__fish_spack_using_command install' -l use-cache -d 'check for pre-built Spack packages in mirrors (default)'
+complete -c spack -n '__fish_spack_using_command install' -l no-cache -f -a use_cache
+complete -c spack -n '__fish_spack_using_command install' -l no-cache -d 'do not check for pre-built Spack packages in mirrors'
+complete -c spack -n '__fish_spack_using_command install' -l cache-only -f -a cache_only
+complete -c spack -n '__fish_spack_using_command install' -l cache-only -d 'only install package from binary mirrors'
+complete -c spack -n '__fish_spack_using_command install' -l use-buildcache -r -f -a use_buildcache
+complete -c spack -n '__fish_spack_using_command install' -l use-buildcache -r -d 'select the mode of buildcache for the \'package\' and \'dependencies\''
+complete -c spack -n '__fish_spack_using_command install' -l include-build-deps -f -a include_build_deps
+complete -c spack -n '__fish_spack_using_command install' -l include-build-deps -d 'include build deps when installing from cache, useful for CI pipeline troubleshooting'
+complete -c spack -n '__fish_spack_using_command install' -l no-check-signature -f -a unsigned
+complete -c spack -n '__fish_spack_using_command install' -l no-check-signature -d 'do not check signatures of binary packages'
+complete -c spack -n '__fish_spack_using_command install' -l show-log-on-error -f -a show_log_on_error
+complete -c spack -n '__fish_spack_using_command install' -l show-log-on-error -d 'print full build log to stderr if build fails'
+complete -c spack -n '__fish_spack_using_command install' -l source -f -a install_source
+complete -c spack -n '__fish_spack_using_command install' -l source -d 'install source files in prefix'
+complete -c spack -n '__fish_spack_using_command install' -s n -l no-checksum -f -a no_checksum
+complete -c spack -n '__fish_spack_using_command install' -s n -l no-checksum -d 'do not use checksums to verify downloaded files (unsafe)'
+complete -c spack -n '__fish_spack_using_command install' -l deprecated -f -a deprecated
+complete -c spack -n '__fish_spack_using_command install' -l deprecated -d 'fetch deprecated versions without warning'
+complete -c spack -n '__fish_spack_using_command install' -s v -l verbose -f -a install_verbose
+complete -c spack -n '__fish_spack_using_command install' -s v -l verbose -d 'display verbose build output while installing'
+complete -c spack -n '__fish_spack_using_command install' -l fake -f -a fake
+complete -c spack -n '__fish_spack_using_command install' -l fake -d 'fake install for debug purposes'
+complete -c spack -n '__fish_spack_using_command install' -l only-concrete -f -a only_concrete
+complete -c spack -n '__fish_spack_using_command install' -l only-concrete -d '(with environment) only install already concretized specs'
+complete -c spack -n '__fish_spack_using_command install' -l add -f -a add
+complete -c spack -n '__fish_spack_using_command install' -l add -d '(with environment) add spec to the environment as a root'
+complete -c spack -n '__fish_spack_using_command install' -l no-add -f -a add
+complete -c spack -n '__fish_spack_using_command install' -l no-add -d '(with environment) do not add spec to the environment as a root'
+complete -c spack -n '__fish_spack_using_command install' -s f -l file -r -f -a specfiles
+complete -c spack -n '__fish_spack_using_command install' -s f -l file -r -d 'read specs to install from .yaml files'
+complete -c spack -n '__fish_spack_using_command install' -l clean -f -a dirty
+complete -c spack -n '__fish_spack_using_command install' -l clean -d 'unset harmful variables in the build environment (default)'
+complete -c spack -n '__fish_spack_using_command install' -l dirty -f -a dirty
+complete -c spack -n '__fish_spack_using_command install' -l dirty -d 'preserve user environment in spack\'s build environment (danger!)'
+complete -c spack -n '__fish_spack_using_command install' -l test -r -f -a 'root all'
+complete -c spack -n '__fish_spack_using_command install' -l test -r -d 'run tests on only root packages or all packages'
+complete -c spack -n '__fish_spack_using_command install' -l log-format -r -f -a 'junit cdash'
+complete -c spack -n '__fish_spack_using_command install' -l log-format -r -d 'format to be used for log files'
+complete -c spack -n '__fish_spack_using_command install' -l log-file -r -f -a log_file
+complete -c spack -n '__fish_spack_using_command install' -l log-file -r -d 'filename for the log file'
+complete -c spack -n '__fish_spack_using_command install' -l help-cdash -f -a help_cdash
+complete -c spack -n '__fish_spack_using_command install' -l help-cdash -d 'show usage instructions for CDash reporting'
+complete -c spack -n '__fish_spack_using_command install' -l cdash-upload-url -r -f -a cdash_upload_url
+complete -c spack -n '__fish_spack_using_command install' -l cdash-build -r -f -a cdash_build
+complete -c spack -n '__fish_spack_using_command install' -l cdash-site -r -f -a cdash_site
+complete -c spack -n '__fish_spack_using_command install' -l cdash-track -r -f -a cdash_track
+complete -c spack -n '__fish_spack_using_command install' -l cdash-buildstamp -r -f -a cdash_buildstamp
+complete -c spack -n '__fish_spack_using_command install' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command install' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+complete -c spack -n '__fish_spack_using_command install' -s U -l fresh -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command install' -s U -l fresh -d 'do not reuse installed deps; build newest configuration'
+complete -c spack -n '__fish_spack_using_command install' -l reuse -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command install' -l reuse -d 'reuse installed packages/buildcaches when possible'
+complete -c spack -n '__fish_spack_using_command install' -l reuse-deps -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command install' -l reuse-deps -d 'reuse installed dependencies only'
+
+# spack license
+set -g __fish_spack_optspecs_spack_license h/help root=
+complete -c spack -n '__fish_spack_using_command_pos 0 license' -f -a list-files -d 'list files in spack that should have license headers'
+complete -c spack -n '__fish_spack_using_command_pos 0 license' -f -a verify -d 'verify that files in spack have the right license header'
+complete -c spack -n '__fish_spack_using_command_pos 0 license' -f -a update-copyright-year -d 'update copyright for the current year in all licensed files'
+complete -c spack -n '__fish_spack_using_command license' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command license' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command license' -l root -r -f -a root
+complete -c spack -n '__fish_spack_using_command license' -l root -r -d 'scan a different prefix for license issues'
+
+# spack license list-files
+set -g __fish_spack_optspecs_spack_license_list_files h/help
+complete -c spack -n '__fish_spack_using_command license list-files' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command license list-files' -s h -l help -d 'show this help message and exit'
+
+# spack license verify
+set -g __fish_spack_optspecs_spack_license_verify h/help
+complete -c spack -n '__fish_spack_using_command license verify' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command license verify' -s h -l help -d 'show this help message and exit'
+
+# spack license update-copyright-year
+set -g __fish_spack_optspecs_spack_license_update_copyright_year h/help
+complete -c spack -n '__fish_spack_using_command license update-copyright-year' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command license update-copyright-year' -s h -l help -d 'show this help message and exit'
+
+# spack list
+set -g __fish_spack_optspecs_spack_list h/help d/search-description format= v/virtuals t/tag= count update=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 list' -f -a '(__fish_spack_packages)'
+complete -c spack -n '__fish_spack_using_command list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command list' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command list' -s d -l search-description -f -a search_description
+complete -c spack -n '__fish_spack_using_command list' -s d -l search-description -d 'filtering will also search the description for a match'
+complete -c spack -n '__fish_spack_using_command list' -l format -r -f -a 'name_only version_json html'
+complete -c spack -n '__fish_spack_using_command list' -l format -r -d 'format to be used to print the output [default: name_only]'
+complete -c spack -n '__fish_spack_using_command list' -s v -l virtuals -f -a virtuals
+complete -c spack -n '__fish_spack_using_command list' -s v -l virtuals -d 'include virtual packages in list'
+complete -c spack -n '__fish_spack_using_command list' -s t -l tag -r -f -a tags
+complete -c spack -n '__fish_spack_using_command list' -s t -l tag -r -d 'filter a package query by tag (multiple use allowed)'
+complete -c spack -n '__fish_spack_using_command list' -l count -f -a count
+complete -c spack -n '__fish_spack_using_command list' -l count -d 'display the number of packages that would be listed'
+complete -c spack -n '__fish_spack_using_command list' -l update -r -f -a update
+complete -c spack -n '__fish_spack_using_command list' -l update -r -d 'write output to the specified file, if any package is newer'
+
+# spack load
+set -g __fish_spack_optspecs_spack_load h/help sh csh fish bat pwsh first only= list
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 load' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command load' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command load' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command load' -l sh -f -a shell
+complete -c spack -n '__fish_spack_using_command load' -l sh -d 'print sh commands to load the package'
+complete -c spack -n '__fish_spack_using_command load' -l csh -f -a shell
+complete -c spack -n '__fish_spack_using_command load' -l csh -d 'print csh commands to load the package'
+complete -c spack -n '__fish_spack_using_command load' -l fish -f -a shell
+complete -c spack -n '__fish_spack_using_command load' -l fish -d 'print fish commands to load the package'
+complete -c spack -n '__fish_spack_using_command load' -l bat -f -a shell
+complete -c spack -n '__fish_spack_using_command load' -l bat -d 'print bat commands to load the package'
+complete -c spack -n '__fish_spack_using_command load' -l pwsh -f -a shell
+complete -c spack -n '__fish_spack_using_command load' -l pwsh -d 'print pwsh commands to load the package'
+complete -c spack -n '__fish_spack_using_command load' -l first -f -a load_first
+complete -c spack -n '__fish_spack_using_command load' -l first -d 'load the first match if multiple packages match the spec'
+complete -c spack -n '__fish_spack_using_command load' -l only -r -f -a 'package dependencies'
+complete -c spack -n '__fish_spack_using_command load' -l only -r -d 'select whether to load the package and its dependencies'
+complete -c spack -n '__fish_spack_using_command load' -l list -f -a list
+complete -c spack -n '__fish_spack_using_command load' -l list -d 'show loaded packages: same as `spack find --loaded`'
+
+# spack location
+set -g __fish_spack_optspecs_spack_location h/help m/module-dir r/spack-root i/install-dir p/package-dir P/packages s/stage-dir S/stages source-dir b/build-dir e/env= first
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 location' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command location' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command location' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command location' -s m -l module-dir -f -a module_dir
+complete -c spack -n '__fish_spack_using_command location' -s m -l module-dir -d 'spack python module directory'
+complete -c spack -n '__fish_spack_using_command location' -s r -l spack-root -f -a spack_root
+complete -c spack -n '__fish_spack_using_command location' -s r -l spack-root -d 'spack installation root'
+complete -c spack -n '__fish_spack_using_command location' -s i -l install-dir -f -a install_dir
+complete -c spack -n '__fish_spack_using_command location' -s i -l install-dir -d 'install prefix for spec (spec need not be installed)'
+complete -c spack -n '__fish_spack_using_command location' -s p -l package-dir -f -a package_dir
+complete -c spack -n '__fish_spack_using_command location' -s p -l package-dir -d 'directory enclosing a spec\'s package.py file'
+complete -c spack -n '__fish_spack_using_command location' -s P -l packages -f -a packages
+complete -c spack -n '__fish_spack_using_command location' -s P -l packages -d 'top-level packages directory for Spack'
+complete -c spack -n '__fish_spack_using_command location' -s s -l stage-dir -f -a stage_dir
+complete -c spack -n '__fish_spack_using_command location' -s s -l stage-dir -d 'stage directory for a spec'
+complete -c spack -n '__fish_spack_using_command location' -s S -l stages -f -a stages
+complete -c spack -n '__fish_spack_using_command location' -s S -l stages -d 'top level stage directory'
+complete -c spack -n '__fish_spack_using_command location' -l source-dir -f -a source_dir
+complete -c spack -n '__fish_spack_using_command location' -l source-dir -d 'source directory for a spec (requires it to be staged first)'
+complete -c spack -n '__fish_spack_using_command location' -s b -l build-dir -f -a build_dir
+complete -c spack -n '__fish_spack_using_command location' -s b -l build-dir -d 'build directory for a spec (requires it to be staged first)'
+complete -c spack -n '__fish_spack_using_command location' -s e -l env -r -f -a location_env
+complete -c spack -n '__fish_spack_using_command location' -s e -l env -r -d 'location of the named or current environment'
+complete -c spack -n '__fish_spack_using_command location' -l first -f -a find_first
+complete -c spack -n '__fish_spack_using_command location' -l first -d 'use the first match if multiple packages match the spec'
+
+# spack log-parse
+set -g __fish_spack_optspecs_spack_log_parse h/help show= c/context= p/profile w/width= j/jobs=
+
+complete -c spack -n '__fish_spack_using_command log-parse' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command log-parse' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command log-parse' -l show -r -f -a show
+complete -c spack -n '__fish_spack_using_command log-parse' -l show -r -d 'comma-separated list of what to show; options: errors, warnings'
+complete -c spack -n '__fish_spack_using_command log-parse' -s c -l context -r -f -a context
+complete -c spack -n '__fish_spack_using_command log-parse' -s c -l context -r -d 'lines of context to show around lines of interest'
+complete -c spack -n '__fish_spack_using_command log-parse' -s p -l profile -f -a profile
+complete -c spack -n '__fish_spack_using_command log-parse' -s p -l profile -d 'print out a profile of time spent in regexes during parse'
+complete -c spack -n '__fish_spack_using_command log-parse' -s w -l width -r -f -a width
+complete -c spack -n '__fish_spack_using_command log-parse' -s w -l width -r -d 'wrap width: auto-size to terminal by default; 0 for no wrap'
+complete -c spack -n '__fish_spack_using_command log-parse' -s j -l jobs -r -f -a jobs
+complete -c spack -n '__fish_spack_using_command log-parse' -s j -l jobs -r -d 'number of jobs to parse log file (default: 1 for short logs, ncpus for long logs)'
+
+# spack maintainers
+set -g __fish_spack_optspecs_spack_maintainers h/help maintained unmaintained a/all by-user
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 maintainers' -f -a '(__fish_spack_packages)'
+complete -c spack -n '__fish_spack_using_command maintainers' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command maintainers' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command maintainers' -l maintained -f -a maintained
+complete -c spack -n '__fish_spack_using_command maintainers' -l maintained -d 'show names of maintained packages'
+complete -c spack -n '__fish_spack_using_command maintainers' -l unmaintained -f -a unmaintained
+complete -c spack -n '__fish_spack_using_command maintainers' -l unmaintained -d 'show names of unmaintained packages'
+complete -c spack -n '__fish_spack_using_command maintainers' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command maintainers' -s a -l all -d 'show maintainers for all packages'
+complete -c spack -n '__fish_spack_using_command maintainers' -l by-user -f -a by_user
+complete -c spack -n '__fish_spack_using_command maintainers' -l by-user -d 'show packages for users instead of users for packages'
+
+# spack make-installer
+set -g __fish_spack_optspecs_spack_make_installer h/help v/spack-version= s/spack-source= g/git-installer-verbosity=
+complete -c spack -n '__fish_spack_using_command_pos 0 make-installer' -f -a '(__fish_spack_environments)'
+complete -c spack -n '__fish_spack_using_command make-installer' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command make-installer' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command make-installer' -s v -l spack-version -r -f -a spack_version
+complete -c spack -n '__fish_spack_using_command make-installer' -s v -l spack-version -r -d 'download given spack version'
+complete -c spack -n '__fish_spack_using_command make-installer' -s s -l spack-source -r -f -a spack_source
+complete -c spack -n '__fish_spack_using_command make-installer' -s s -l spack-source -r -d 'full path to spack source'
+complete -c spack -n '__fish_spack_using_command make-installer' -s g -l git-installer-verbosity -r -f -a 'SILENT VERYSILENT'
+complete -c spack -n '__fish_spack_using_command make-installer' -s g -l git-installer-verbosity -r -d 'level of verbosity provided by bundled git installer (default is fully verbose)'
+
+# spack mark
+set -g __fish_spack_optspecs_spack_mark h/help a/all e/explicit i/implicit
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 mark' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command mark' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command mark' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command mark' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command mark' -s a -l all -d 'mark ALL installed packages that match each supplied spec'
+complete -c spack -n '__fish_spack_using_command mark' -s e -l explicit -f -a explicit
+complete -c spack -n '__fish_spack_using_command mark' -s e -l explicit -d 'mark packages as explicitly installed'
+complete -c spack -n '__fish_spack_using_command mark' -s i -l implicit -f -a implicit
+complete -c spack -n '__fish_spack_using_command mark' -s i -l implicit -d 'mark packages as implicitly installed'
+
+# spack mirror
+set -g __fish_spack_optspecs_spack_mirror h/help n/no-checksum deprecated
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror' -f -a create -d 'create a directory to be used as a spack mirror, and fill it with package archives'
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror' -f -a destroy -d 'given a url, recursively delete everything under it'
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror' -f -a add -d 'add a mirror to Spack'
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror' -f -a remove -d 'remove a mirror by name'
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror' -f -a rm -d 'remove a mirror by name'
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror' -f -a set-url -d 'change the URL of a mirror'
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror' -f -a set -d 'configure the connection details of a mirror'
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror' -f -a list -d 'print out available mirrors to the console'
+complete -c spack -n '__fish_spack_using_command mirror' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command mirror' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command mirror' -s n -l no-checksum -f -a no_checksum
+complete -c spack -n '__fish_spack_using_command mirror' -s n -l no-checksum -d 'do not use checksums to verify downloaded files (unsafe)'
+complete -c spack -n '__fish_spack_using_command mirror' -l deprecated -f -a deprecated
+complete -c spack -n '__fish_spack_using_command mirror' -l deprecated -d 'fetch deprecated versions without warning'
+
+# spack mirror create
+set -g __fish_spack_optspecs_spack_mirror_create h/help d/directory= a/all f/file= exclude-file= exclude-specs= skip-unstable-versions D/dependencies n/versions-per-spec=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 mirror create' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command mirror create' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command mirror create' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command mirror create' -s d -l directory -r -f -a directory
+complete -c spack -n '__fish_spack_using_command mirror create' -s d -l directory -r -d 'directory in which to create mirror'
+complete -c spack -n '__fish_spack_using_command mirror create' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command mirror create' -s a -l all -d 'mirror all versions of all packages in Spack, or all packages in the current environment if there is an active environment (this requires significant time and space)'
+complete -c spack -n '__fish_spack_using_command mirror create' -s f -l file -r -f -a file
+complete -c spack -n '__fish_spack_using_command mirror create' -s f -l file -r -d 'file with specs of packages to put in mirror'
+complete -c spack -n '__fish_spack_using_command mirror create' -l exclude-file -r -f -a exclude_file
+complete -c spack -n '__fish_spack_using_command mirror create' -l exclude-file -r -d 'specs which Spack should not try to add to a mirror (listed in a file, one per line)'
+complete -c spack -n '__fish_spack_using_command mirror create' -l exclude-specs -r -f -a exclude_specs
+complete -c spack -n '__fish_spack_using_command mirror create' -l exclude-specs -r -d 'specs which Spack should not try to add to a mirror (specified on command line)'
+complete -c spack -n '__fish_spack_using_command mirror create' -l skip-unstable-versions -f -a skip_unstable_versions
+complete -c spack -n '__fish_spack_using_command mirror create' -l skip-unstable-versions -d 'don\'t cache versions unless they identify a stable (unchanging) source code'
+complete -c spack -n '__fish_spack_using_command mirror create' -s D -l dependencies -f -a dependencies
+complete -c spack -n '__fish_spack_using_command mirror create' -s D -l dependencies -d 'also fetch all dependencies'
+complete -c spack -n '__fish_spack_using_command mirror create' -s n -l versions-per-spec -r -f -a versions_per_spec
+complete -c spack -n '__fish_spack_using_command mirror create' -s n -l versions-per-spec -r -d 'the number of versions to fetch for each spec, choose \'all\' to retrieve all versions of each package'
+
+# spack mirror destroy
+set -g __fish_spack_optspecs_spack_mirror_destroy h/help m/mirror-name= mirror-url=
+complete -c spack -n '__fish_spack_using_command mirror destroy' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command mirror destroy' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command mirror destroy' -s m -l mirror-name -r -f -a mirror_name
+complete -c spack -n '__fish_spack_using_command mirror destroy' -s m -l mirror-name -r -d 'find mirror to destroy by name'
+complete -c spack -n '__fish_spack_using_command mirror destroy' -l mirror-url -r -f -a mirror_url
+complete -c spack -n '__fish_spack_using_command mirror destroy' -l mirror-url -r -d 'find mirror to destroy by url'
+
+# spack mirror add
+set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= s3-access-key-id= s3-access-key-secret= s3-access-token= s3-profile= s3-endpoint-url= oci-username= oci-password=
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f
+complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify'
+complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source'
+complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)'
+complete -c spack -n '__fish_spack_using_command mirror add' -l s3-access-key-id -r -f -a s3_access_key_id
+complete -c spack -n '__fish_spack_using_command mirror add' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror add' -l s3-access-key-secret -r -f -a s3_access_key_secret
+complete -c spack -n '__fish_spack_using_command mirror add' -l s3-access-key-secret -r -d 'secret string to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror add' -l s3-access-token -r -f -a s3_access_token
+complete -c spack -n '__fish_spack_using_command mirror add' -l s3-access-token -r -d 'access token to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror add' -l s3-profile -r -f -a s3_profile
+complete -c spack -n '__fish_spack_using_command mirror add' -l s3-profile -r -d 'S3 profile name to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror add' -l s3-endpoint-url -r -f -a s3_endpoint_url
+complete -c spack -n '__fish_spack_using_command mirror add' -l s3-endpoint-url -r -d 'endpoint URL to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror add' -l oci-username -r -f -a oci_username
+complete -c spack -n '__fish_spack_using_command mirror add' -l oci-username -r -d 'username to use to connect to this OCI mirror'
+complete -c spack -n '__fish_spack_using_command mirror add' -l oci-password -r -f -a oci_password
+complete -c spack -n '__fish_spack_using_command mirror add' -l oci-password -r -d 'password to use to connect to this OCI mirror'
+
+# spack mirror remove
+set -g __fish_spack_optspecs_spack_mirror_remove h/help scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)'
+complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify'
+
+# spack mirror rm
+set -g __fish_spack_optspecs_spack_mirror_rm h/help scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)'
+complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify'
+
+# spack mirror set-url
+set -g __fish_spack_optspecs_spack_mirror_set_url h/help push fetch scope= s3-access-key-id= s3-access-key-secret= s3-access-token= s3-profile= s3-endpoint-url= oci-username= oci-password=
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror set-url' -f -a '(__fish_spack_mirrors)'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command mirror set-url' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a push
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-secret -r -f -a s3_access_key_secret
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-secret -r -d 'secret string to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-token -r -f -a s3_access_token
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-token -r -d 'access token to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-profile -r -f -a s3_profile
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-profile -r -d 'S3 profile name to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-endpoint-url -r -f -a s3_endpoint_url
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-endpoint-url -r -d 'endpoint URL to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l oci-username -r -f -a oci_username
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l oci-username -r -d 'username to use to connect to this OCI mirror'
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l oci-password -r -f -a oci_password
+complete -c spack -n '__fish_spack_using_command mirror set-url' -l oci-password -r -d 'password to use to connect to this OCI mirror'
+
+# spack mirror set
+set -g __fish_spack_optspecs_spack_mirror_set h/help push fetch type= url= scope= s3-access-key-id= s3-access-key-secret= s3-access-token= s3-profile= s3-endpoint-url= oci-username= oci-password=
+complete -c spack -n '__fish_spack_using_command_pos 0 mirror set' -f -a '(__fish_spack_mirrors)'
+complete -c spack -n '__fish_spack_using_command mirror set' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command mirror set' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command mirror set' -l push -f -a push
+complete -c spack -n '__fish_spack_using_command mirror set' -l push -d 'modify just the push connection details'
+complete -c spack -n '__fish_spack_using_command mirror set' -l fetch -f -a fetch
+complete -c spack -n '__fish_spack_using_command mirror set' -l fetch -d 'modify just the fetch connection details'
+complete -c spack -n '__fish_spack_using_command mirror set' -l type -r -f -a 'binary source'
+complete -c spack -n '__fish_spack_using_command mirror set' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source`'
+complete -c spack -n '__fish_spack_using_command mirror set' -l url -r -f -a url
+complete -c spack -n '__fish_spack_using_command mirror set' -l url -r -d 'url of mirror directory from \'spack mirror create\''
+complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify'
+complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id
+complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-secret -r -f -a s3_access_key_secret
+complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-secret -r -d 'secret string to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-token -r -f -a s3_access_token
+complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-token -r -d 'access token to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror set' -l s3-profile -r -f -a s3_profile
+complete -c spack -n '__fish_spack_using_command mirror set' -l s3-profile -r -d 'S3 profile name to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror set' -l s3-endpoint-url -r -f -a s3_endpoint_url
+complete -c spack -n '__fish_spack_using_command mirror set' -l s3-endpoint-url -r -d 'endpoint URL to use to connect to this S3 mirror'
+complete -c spack -n '__fish_spack_using_command mirror set' -l oci-username -r -f -a oci_username
+complete -c spack -n '__fish_spack_using_command mirror set' -l oci-username -r -d 'username to use to connect to this OCI mirror'
+complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password -r -f -a oci_password
+complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password -r -d 'password to use to connect to this OCI mirror'
+
+# spack mirror list
+set -g __fish_spack_optspecs_spack_mirror_list h/help scope=
+complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from'
+
+# spack module
+set -g __fish_spack_optspecs_spack_module h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 module' -f -a lmod -d 'manipulate hierarchical module files'
+complete -c spack -n '__fish_spack_using_command_pos 0 module' -f -a tcl -d 'manipulate non-hierarchical module files'
+complete -c spack -n '__fish_spack_using_command module' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module' -s h -l help -d 'show this help message and exit'
+
+# spack module lmod
+set -g __fish_spack_optspecs_spack_module_lmod h/help n/name=
+complete -c spack -n '__fish_spack_using_command_pos 0 module lmod' -f -a refresh -d 'regenerate module files'
+complete -c spack -n '__fish_spack_using_command_pos 0 module lmod' -f -a find -d 'find module files for packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 module lmod' -f -a rm -d 'remove module files'
+complete -c spack -n '__fish_spack_using_command_pos 0 module lmod' -f -a loads -d 'prompt the list of modules associated with a constraint'
+complete -c spack -n '__fish_spack_using_command_pos 0 module lmod' -f -a setdefault -d 'set the default module file for a package'
+complete -c spack -n '__fish_spack_using_command module lmod' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module lmod' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command module lmod' -s n -l name -r -f -a module_set_name
+complete -c spack -n '__fish_spack_using_command module lmod' -s n -l name -r -d 'named module set to use from modules configuration'
+
+# spack module lmod refresh
+set -g __fish_spack_optspecs_spack_module_lmod_refresh h/help delete-tree upstream-modules y/yes-to-all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 module lmod refresh' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command module lmod refresh' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module lmod refresh' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command module lmod refresh' -l delete-tree -f -a delete_tree
+complete -c spack -n '__fish_spack_using_command module lmod refresh' -l delete-tree -d 'delete the module file tree before refresh'
+complete -c spack -n '__fish_spack_using_command module lmod refresh' -l upstream-modules -f -a upstream_modules
+complete -c spack -n '__fish_spack_using_command module lmod refresh' -l upstream-modules -d 'generate modules for packages installed upstream'
+complete -c spack -n '__fish_spack_using_command module lmod refresh' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command module lmod refresh' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack module lmod find
+set -g __fish_spack_optspecs_spack_module_lmod_find h/help full-path r/dependencies
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 module lmod find' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command module lmod find' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module lmod find' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command module lmod find' -l full-path -f -a full_path
+complete -c spack -n '__fish_spack_using_command module lmod find' -l full-path -d 'display full path to module file'
+complete -c spack -n '__fish_spack_using_command module lmod find' -s r -l dependencies -f -a recurse_dependencies
+complete -c spack -n '__fish_spack_using_command module lmod find' -s r -l dependencies -d 'recursively traverse spec dependencies'
+
+# spack module lmod rm
+set -g __fish_spack_optspecs_spack_module_lmod_rm h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 module lmod rm' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command module lmod rm' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module lmod rm' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command module lmod rm' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command module lmod rm' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack module lmod loads
+set -g __fish_spack_optspecs_spack_module_lmod_loads h/help input-only p/prefix= x/exclude= r/dependencies
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 module lmod loads' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command module lmod loads' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module lmod loads' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command module lmod loads' -l input-only -f -a shell
+complete -c spack -n '__fish_spack_using_command module lmod loads' -l input-only -d 'generate input for module command (instead of a shell script)'
+complete -c spack -n '__fish_spack_using_command module lmod loads' -s p -l prefix -r -f -a prefix
+complete -c spack -n '__fish_spack_using_command module lmod loads' -s p -l prefix -r -d 'prepend to module names when issuing module load commands'
+complete -c spack -n '__fish_spack_using_command module lmod loads' -s x -l exclude -r -f -a exclude
+complete -c spack -n '__fish_spack_using_command module lmod loads' -s x -l exclude -r -d 'exclude package from output; may be specified multiple times'
+complete -c spack -n '__fish_spack_using_command module lmod loads' -s r -l dependencies -f -a recurse_dependencies
+complete -c spack -n '__fish_spack_using_command module lmod loads' -s r -l dependencies -d 'recursively traverse spec dependencies'
+
+# spack module lmod setdefault
+set -g __fish_spack_optspecs_spack_module_lmod_setdefault h/help
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 module lmod setdefault' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command module lmod setdefault' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module lmod setdefault' -s h -l help -d 'show this help message and exit'
+
+# spack module tcl
+set -g __fish_spack_optspecs_spack_module_tcl h/help n/name=
+complete -c spack -n '__fish_spack_using_command_pos 0 module tcl' -f -a refresh -d 'regenerate module files'
+complete -c spack -n '__fish_spack_using_command_pos 0 module tcl' -f -a find -d 'find module files for packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 module tcl' -f -a rm -d 'remove module files'
+complete -c spack -n '__fish_spack_using_command_pos 0 module tcl' -f -a loads -d 'prompt the list of modules associated with a constraint'
+complete -c spack -n '__fish_spack_using_command_pos 0 module tcl' -f -a setdefault -d 'set the default module file for a package'
+complete -c spack -n '__fish_spack_using_command module tcl' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module tcl' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command module tcl' -s n -l name -r -f -a module_set_name
+complete -c spack -n '__fish_spack_using_command module tcl' -s n -l name -r -d 'named module set to use from modules configuration'
+
+# spack module tcl refresh
+set -g __fish_spack_optspecs_spack_module_tcl_refresh h/help delete-tree upstream-modules y/yes-to-all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 module tcl refresh' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command module tcl refresh' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module tcl refresh' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command module tcl refresh' -l delete-tree -f -a delete_tree
+complete -c spack -n '__fish_spack_using_command module tcl refresh' -l delete-tree -d 'delete the module file tree before refresh'
+complete -c spack -n '__fish_spack_using_command module tcl refresh' -l upstream-modules -f -a upstream_modules
+complete -c spack -n '__fish_spack_using_command module tcl refresh' -l upstream-modules -d 'generate modules for packages installed upstream'
+complete -c spack -n '__fish_spack_using_command module tcl refresh' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command module tcl refresh' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack module tcl find
+set -g __fish_spack_optspecs_spack_module_tcl_find h/help full-path r/dependencies
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 module tcl find' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command module tcl find' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module tcl find' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command module tcl find' -l full-path -f -a full_path
+complete -c spack -n '__fish_spack_using_command module tcl find' -l full-path -d 'display full path to module file'
+complete -c spack -n '__fish_spack_using_command module tcl find' -s r -l dependencies -f -a recurse_dependencies
+complete -c spack -n '__fish_spack_using_command module tcl find' -s r -l dependencies -d 'recursively traverse spec dependencies'
+
+# spack module tcl rm
+set -g __fish_spack_optspecs_spack_module_tcl_rm h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 module tcl rm' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command module tcl rm' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module tcl rm' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command module tcl rm' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command module tcl rm' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack module tcl loads
+set -g __fish_spack_optspecs_spack_module_tcl_loads h/help input-only p/prefix= x/exclude= r/dependencies
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 module tcl loads' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command module tcl loads' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module tcl loads' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command module tcl loads' -l input-only -f -a shell
+complete -c spack -n '__fish_spack_using_command module tcl loads' -l input-only -d 'generate input for module command (instead of a shell script)'
+complete -c spack -n '__fish_spack_using_command module tcl loads' -s p -l prefix -r -f -a prefix
+complete -c spack -n '__fish_spack_using_command module tcl loads' -s p -l prefix -r -d 'prepend to module names when issuing module load commands'
+complete -c spack -n '__fish_spack_using_command module tcl loads' -s x -l exclude -r -f -a exclude
+complete -c spack -n '__fish_spack_using_command module tcl loads' -s x -l exclude -r -d 'exclude package from output; may be specified multiple times'
+complete -c spack -n '__fish_spack_using_command module tcl loads' -s r -l dependencies -f -a recurse_dependencies
+complete -c spack -n '__fish_spack_using_command module tcl loads' -s r -l dependencies -d 'recursively traverse spec dependencies'
+
+# spack module tcl setdefault
+set -g __fish_spack_optspecs_spack_module_tcl_setdefault h/help
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 module tcl setdefault' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command module tcl setdefault' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command module tcl setdefault' -s h -l help -d 'show this help message and exit'
+
+# spack patch
+set -g __fish_spack_optspecs_spack_patch h/help n/no-checksum deprecated U/fresh reuse reuse-deps
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 patch' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command patch' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command patch' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command patch' -s n -l no-checksum -f -a no_checksum
+complete -c spack -n '__fish_spack_using_command patch' -s n -l no-checksum -d 'do not use checksums to verify downloaded files (unsafe)'
+complete -c spack -n '__fish_spack_using_command patch' -l deprecated -f -a deprecated
+complete -c spack -n '__fish_spack_using_command patch' -l deprecated -d 'fetch deprecated versions without warning'
+complete -c spack -n '__fish_spack_using_command patch' -s U -l fresh -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command patch' -s U -l fresh -d 'do not reuse installed deps; build newest configuration'
+complete -c spack -n '__fish_spack_using_command patch' -l reuse -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command patch' -l reuse -d 'reuse installed packages/buildcaches when possible'
+complete -c spack -n '__fish_spack_using_command patch' -l reuse-deps -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command patch' -l reuse-deps -d 'reuse installed dependencies only'
+
+# spack pkg
+set -g __fish_spack_optspecs_spack_pkg h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg' -f -a add -d 'add a package to the git stage with `git add`'
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg' -f -a list -d 'list packages associated with a particular spack git revision'
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg' -f -a diff -d 'compare packages available in two different git revisions'
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg' -f -a added -d 'show packages added since a commit'
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg' -f -a changed -d 'show packages changed since a commit'
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg' -f -a removed -d 'show packages removed since a commit'
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg' -f -a grep -d 'grep for strings in package.py files from all repositories'
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg' -f -a source -d 'dump source code for a package'
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg' -f -a hash -d 'dump canonical source code hash for a package spec'
+complete -c spack -n '__fish_spack_using_command pkg' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command pkg' -s h -l help -d 'show this help message and exit'
+
+# spack pkg add
+set -g __fish_spack_optspecs_spack_pkg_add h/help
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 pkg add' -f -a '(__fish_spack_pkg_packages)'
+complete -c spack -n '__fish_spack_using_command pkg add' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command pkg add' -s h -l help -d 'show this help message and exit'
+
+# spack pkg list
+set -g __fish_spack_optspecs_spack_pkg_list h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg list' -f -a '(__fish_spack_git_rev)'
+complete -c spack -n '__fish_spack_using_command pkg list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command pkg list' -s h -l help -d 'show this help message and exit'
+
+# spack pkg diff
+set -g __fish_spack_optspecs_spack_pkg_diff h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg diff' -f -a '(__fish_spack_git_rev)'
+complete -c spack -n '__fish_spack_using_command_pos 1 pkg diff' -f -a '(__fish_spack_git_rev)'
+complete -c spack -n '__fish_spack_using_command pkg diff' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command pkg diff' -s h -l help -d 'show this help message and exit'
+
+# spack pkg added
+set -g __fish_spack_optspecs_spack_pkg_added h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg added' -f -a '(__fish_spack_git_rev)'
+complete -c spack -n '__fish_spack_using_command_pos 1 pkg added' -f -a '(__fish_spack_git_rev)'
+complete -c spack -n '__fish_spack_using_command pkg added' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command pkg added' -s h -l help -d 'show this help message and exit'
+
+# spack pkg changed
+set -g __fish_spack_optspecs_spack_pkg_changed h/help t/type=
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg changed' -f -a '(__fish_spack_git_rev)'
+complete -c spack -n '__fish_spack_using_command_pos 1 pkg changed' -f -a '(__fish_spack_git_rev)'
+complete -c spack -n '__fish_spack_using_command pkg changed' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command pkg changed' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command pkg changed' -s t -l type -r -f -a type
+complete -c spack -n '__fish_spack_using_command pkg changed' -s t -l type -r -d 'types of changes to show (A: added, R: removed, C: changed); default is \'C\''
+
+# spack pkg removed
+set -g __fish_spack_optspecs_spack_pkg_removed h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 pkg removed' -f -a '(__fish_spack_git_rev)'
+complete -c spack -n '__fish_spack_using_command_pos 1 pkg removed' -f -a '(__fish_spack_git_rev)'
+complete -c spack -n '__fish_spack_using_command pkg removed' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command pkg removed' -s h -l help -d 'show this help message and exit'
+
+# spack pkg grep
+set -g __fish_spack_optspecs_spack_pkg_grep help
+
+complete -c spack -n '__fish_spack_using_command pkg grep' -l help -f -a help
+complete -c spack -n '__fish_spack_using_command pkg grep' -l help -d 'show this help message and exit'
+
+# spack pkg source
+set -g __fish_spack_optspecs_spack_pkg_source h/help c/canonical
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 pkg source' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command pkg source' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command pkg source' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command pkg source' -s c -l canonical -f -a canonical
+complete -c spack -n '__fish_spack_using_command pkg source' -s c -l canonical -d 'dump canonical source as used by package hash'
+
+# spack pkg hash
+set -g __fish_spack_optspecs_spack_pkg_hash h/help
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 pkg hash' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command pkg hash' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command pkg hash' -s h -l help -d 'show this help message and exit'
+
+# spack providers
+set -g __fish_spack_optspecs_spack_providers h/help
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 providers' -f -a '(__fish_spack_providers)'
+complete -c spack -n '__fish_spack_using_command providers' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command providers' -s h -l help -d 'show this help message and exit'
+
+# spack pydoc
+set -g __fish_spack_optspecs_spack_pydoc h/help
+
+complete -c spack -n '__fish_spack_using_command pydoc' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command pydoc' -s h -l help -d 'show this help message and exit'
+
+# spack python
+set -g __fish_spack_optspecs_spack_python h/help V/version c/= u/ i/= m/= path
+
+complete -c spack -n '__fish_spack_using_command python' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command python' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command python' -s V -l version -f -a python_version
+complete -c spack -n '__fish_spack_using_command python' -s V -l version -d 'print the Python version number and exit'
+complete -c spack -n '__fish_spack_using_command python' -s c -r -f -a python_command
+complete -c spack -n '__fish_spack_using_command python' -s c -r -d 'command to execute'
+complete -c spack -n '__fish_spack_using_command python' -s u -f -a unbuffered
+complete -c spack -n '__fish_spack_using_command python' -s u -d 'for compatibility with xdist, do not use without adding -u to the interpreter'
+complete -c spack -n '__fish_spack_using_command python' -s i -r -f -a 'python ipython'
+complete -c spack -n '__fish_spack_using_command python' -s i -r -d 'python interpreter'
+complete -c spack -n '__fish_spack_using_command python' -s m -r -f -a module
+complete -c spack -n '__fish_spack_using_command python' -s m -r -d 'run library module as a script'
+complete -c spack -n '__fish_spack_using_command python' -l path -f -a show_path
+complete -c spack -n '__fish_spack_using_command python' -l path -d 'show path to python interpreter that spack uses'
+
+# spack reindex
+set -g __fish_spack_optspecs_spack_reindex h/help
+complete -c spack -n '__fish_spack_using_command reindex' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command reindex' -s h -l help -d 'show this help message and exit'
+
+# spack remove
+set -g __fish_spack_optspecs_spack_remove h/help a/all l/list-name= f/force
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 remove' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command remove' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command remove' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command remove' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command remove' -s a -l all -d 'remove all specs from (clear) the environment'
+complete -c spack -n '__fish_spack_using_command remove' -s l -l list-name -r -f -a list_name
+complete -c spack -n '__fish_spack_using_command remove' -s l -l list-name -r -d 'name of the list to remove specs from'
+complete -c spack -n '__fish_spack_using_command remove' -s f -l force -f -a force
+complete -c spack -n '__fish_spack_using_command remove' -s f -l force -d 'remove concretized spec (if any) immediately'
+
+# spack rm
+set -g __fish_spack_optspecs_spack_rm h/help a/all l/list-name= f/force
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 rm' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command rm' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command rm' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command rm' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command rm' -s a -l all -d 'remove all specs from (clear) the environment'
+complete -c spack -n '__fish_spack_using_command rm' -s l -l list-name -r -f -a list_name
+complete -c spack -n '__fish_spack_using_command rm' -s l -l list-name -r -d 'name of the list to remove specs from'
+complete -c spack -n '__fish_spack_using_command rm' -s f -l force -f -a force
+complete -c spack -n '__fish_spack_using_command rm' -s f -l force -d 'remove concretized spec (if any) immediately'
+
+# spack repo
+set -g __fish_spack_optspecs_spack_repo h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a create -d 'create a new package repository'
+complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a list -d 'show registered repositories and their namespaces'
+complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a add -d 'add a package source to Spack\'s configuration'
+complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a remove -d 'remove a repository from Spack\'s configuration'
+complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a rm -d 'remove a repository from Spack\'s configuration'
+complete -c spack -n '__fish_spack_using_command repo' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command repo' -s h -l help -d 'show this help message and exit'
+
+# spack repo create
+set -g __fish_spack_optspecs_spack_repo_create h/help d/subdirectory=
+complete -c spack -n '__fish_spack_using_command_pos 0 repo create' -f -a '(__fish_spack_environments)'
+complete -c spack -n '__fish_spack_using_command repo create' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command repo create' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirectory -r -f -a subdir
+complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirectory -r -d 'subdirectory to store packages in the repository'
+
+# spack repo list
+set -g __fish_spack_optspecs_spack_repo_list h/help scope=
+complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from'
+
+# spack repo add
+set -g __fish_spack_optspecs_spack_repo_add h/help scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 repo add' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify'
+
+# spack repo remove
+set -g __fish_spack_optspecs_spack_repo_remove h/help scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)'
+complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify'
+
+# spack repo rm
+set -g __fish_spack_optspecs_spack_repo_rm h/help scope=
+complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)'
+complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system site user command_line'
+complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify'
+
+# spack resource
+set -g __fish_spack_optspecs_spack_resource h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 resource' -f -a list -d 'list all resources known to spack (currently just patches)'
+complete -c spack -n '__fish_spack_using_command_pos 0 resource' -f -a show -d 'show a resource, identified by its checksum'
+complete -c spack -n '__fish_spack_using_command resource' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command resource' -s h -l help -d 'show this help message and exit'
+
+# spack resource list
+set -g __fish_spack_optspecs_spack_resource_list h/help only-hashes
+complete -c spack -n '__fish_spack_using_command resource list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command resource list' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command resource list' -l only-hashes -f -a only_hashes
+complete -c spack -n '__fish_spack_using_command resource list' -l only-hashes -d 'only print sha256 hashes of resources'
+
+# spack resource show
+set -g __fish_spack_optspecs_spack_resource_show h/help
+
+complete -c spack -n '__fish_spack_using_command resource show' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command resource show' -s h -l help -d 'show this help message and exit'
+
+# spack restage
+set -g __fish_spack_optspecs_spack_restage h/help
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 restage' -f -k -a '(__fish_spack_specs_or_id)'
+complete -c spack -n '__fish_spack_using_command restage' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command restage' -s h -l help -d 'show this help message and exit'
+
+# spack solve
+set -g __fish_spack_optspecs_spack_solve h/help show= l/long L/very-long N/namespaces I/install-status no-install-status y/yaml j/json c/cover= t/types timers stats U/fresh reuse reuse-deps
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 solve' -f -k -a '(__fish_spack_specs_or_id)'
+complete -c spack -n '__fish_spack_using_command solve' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command solve' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command solve' -l show -r -f -a show
+complete -c spack -n '__fish_spack_using_command solve' -l show -r -d 'select outputs'
+complete -c spack -n '__fish_spack_using_command solve' -s l -l long -f -a long
+complete -c spack -n '__fish_spack_using_command solve' -s l -l long -d 'show dependency hashes as well as versions'
+complete -c spack -n '__fish_spack_using_command solve' -s L -l very-long -f -a very_long
+complete -c spack -n '__fish_spack_using_command solve' -s L -l very-long -d 'show full dependency hashes as well as versions'
+complete -c spack -n '__fish_spack_using_command solve' -s N -l namespaces -f -a namespaces
+complete -c spack -n '__fish_spack_using_command solve' -s N -l namespaces -d 'show fully qualified package names'
+complete -c spack -n '__fish_spack_using_command solve' -s I -l install-status -f -a install_status
+complete -c spack -n '__fish_spack_using_command solve' -s I -l install-status -d 'show install status of packages'
+complete -c spack -n '__fish_spack_using_command solve' -l no-install-status -f -a install_status
+complete -c spack -n '__fish_spack_using_command solve' -l no-install-status -d 'do not show install status annotations'
+complete -c spack -n '__fish_spack_using_command solve' -s y -l yaml -f -a format
+complete -c spack -n '__fish_spack_using_command solve' -s y -l yaml -d 'print concrete spec as yaml'
+complete -c spack -n '__fish_spack_using_command solve' -s j -l json -f -a format
+complete -c spack -n '__fish_spack_using_command solve' -s j -l json -d 'print concrete spec as json'
+complete -c spack -n '__fish_spack_using_command solve' -s c -l cover -r -f -a 'nodes edges paths'
+complete -c spack -n '__fish_spack_using_command solve' -s c -l cover -r -d 'how extensively to traverse the DAG (default: nodes)'
+complete -c spack -n '__fish_spack_using_command solve' -s t -l types -f -a types
+complete -c spack -n '__fish_spack_using_command solve' -s t -l types -d 'show dependency types'
+complete -c spack -n '__fish_spack_using_command solve' -l timers -f -a timers
+complete -c spack -n '__fish_spack_using_command solve' -l timers -d 'print out timers for different solve phases'
+complete -c spack -n '__fish_spack_using_command solve' -l stats -f -a stats
+complete -c spack -n '__fish_spack_using_command solve' -l stats -d 'print out statistics from clingo'
+complete -c spack -n '__fish_spack_using_command solve' -s U -l fresh -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command solve' -s U -l fresh -d 'do not reuse installed deps; build newest configuration'
+complete -c spack -n '__fish_spack_using_command solve' -l reuse -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command solve' -l reuse -d 'reuse installed packages/buildcaches when possible'
+complete -c spack -n '__fish_spack_using_command solve' -l reuse-deps -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command solve' -l reuse-deps -d 'reuse installed dependencies only'
+
+# spack spec
+set -g __fish_spack_optspecs_spack_spec h/help l/long L/very-long N/namespaces I/install-status no-install-status y/yaml j/json format= c/cover= t/types U/fresh reuse reuse-deps
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 spec' -f -k -a '(__fish_spack_specs_or_id)'
+complete -c spack -n '__fish_spack_using_command spec' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command spec' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command spec' -s l -l long -f -a long
+complete -c spack -n '__fish_spack_using_command spec' -s l -l long -d 'show dependency hashes as well as versions'
+complete -c spack -n '__fish_spack_using_command spec' -s L -l very-long -f -a very_long
+complete -c spack -n '__fish_spack_using_command spec' -s L -l very-long -d 'show full dependency hashes as well as versions'
+complete -c spack -n '__fish_spack_using_command spec' -s N -l namespaces -f -a namespaces
+complete -c spack -n '__fish_spack_using_command spec' -s N -l namespaces -d 'show fully qualified package names'
+complete -c spack -n '__fish_spack_using_command spec' -s I -l install-status -f -a install_status
+complete -c spack -n '__fish_spack_using_command spec' -s I -l install-status -d 'show install status of packages'
+complete -c spack -n '__fish_spack_using_command spec' -l no-install-status -f -a install_status
+complete -c spack -n '__fish_spack_using_command spec' -l no-install-status -d 'do not show install status annotations'
+complete -c spack -n '__fish_spack_using_command spec' -s y -l yaml -f -a format
+complete -c spack -n '__fish_spack_using_command spec' -s y -l yaml -d 'print concrete spec as YAML'
+complete -c spack -n '__fish_spack_using_command spec' -s j -l json -f -a format
+complete -c spack -n '__fish_spack_using_command spec' -s j -l json -d 'print concrete spec as JSON'
+complete -c spack -n '__fish_spack_using_command spec' -l format -r -f -a format
+complete -c spack -n '__fish_spack_using_command spec' -l format -r -d 'print concrete spec with the specified format string'
+complete -c spack -n '__fish_spack_using_command spec' -s c -l cover -r -f -a 'nodes edges paths'
+complete -c spack -n '__fish_spack_using_command spec' -s c -l cover -r -d 'how extensively to traverse the DAG (default: nodes)'
+complete -c spack -n '__fish_spack_using_command spec' -s t -l types -f -a types
+complete -c spack -n '__fish_spack_using_command spec' -s t -l types -d 'show dependency types'
+complete -c spack -n '__fish_spack_using_command spec' -s U -l fresh -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command spec' -s U -l fresh -d 'do not reuse installed deps; build newest configuration'
+complete -c spack -n '__fish_spack_using_command spec' -l reuse -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command spec' -l reuse -d 'reuse installed packages/buildcaches when possible'
+complete -c spack -n '__fish_spack_using_command spec' -l reuse-deps -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command spec' -l reuse-deps -d 'reuse installed dependencies only'
+
+# spack stage
+set -g __fish_spack_optspecs_spack_stage h/help n/no-checksum deprecated p/path= U/fresh reuse reuse-deps
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 stage' -f -k -a '(__fish_spack_specs_or_id)'
+complete -c spack -n '__fish_spack_using_command stage' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command stage' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command stage' -s n -l no-checksum -f -a no_checksum
+complete -c spack -n '__fish_spack_using_command stage' -s n -l no-checksum -d 'do not use checksums to verify downloaded files (unsafe)'
+complete -c spack -n '__fish_spack_using_command stage' -l deprecated -f -a deprecated
+complete -c spack -n '__fish_spack_using_command stage' -l deprecated -d 'fetch deprecated versions without warning'
+complete -c spack -n '__fish_spack_using_command stage' -s p -l path -r -f -a path
+complete -c spack -n '__fish_spack_using_command stage' -s p -l path -r -d 'path to stage package, does not add to spack tree'
+complete -c spack -n '__fish_spack_using_command stage' -s U -l fresh -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command stage' -s U -l fresh -d 'do not reuse installed deps; build newest configuration'
+complete -c spack -n '__fish_spack_using_command stage' -l reuse -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command stage' -l reuse -d 'reuse installed packages/buildcaches when possible'
+complete -c spack -n '__fish_spack_using_command stage' -l reuse-deps -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command stage' -l reuse-deps -d 'reuse installed dependencies only'
+
+# spack style
+set -g __fish_spack_optspecs_spack_style h/help b/base= a/all r/root-relative U/no-untracked f/fix root= t/tool= s/skip=
+
+complete -c spack -n '__fish_spack_using_command style' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command style' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command style' -s b -l base -r -f -a base
+complete -c spack -n '__fish_spack_using_command style' -s b -l base -r -d 'branch to compare against to determine changed files (default: develop)'
+complete -c spack -n '__fish_spack_using_command style' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command style' -s a -l all -d 'check all files, not just changed files'
+complete -c spack -n '__fish_spack_using_command style' -s r -l root-relative -f -a root_relative
+complete -c spack -n '__fish_spack_using_command style' -s r -l root-relative -d 'print root-relative paths (default: cwd-relative)'
+complete -c spack -n '__fish_spack_using_command style' -s U -l no-untracked -f -a untracked
+complete -c spack -n '__fish_spack_using_command style' -s U -l no-untracked -d 'exclude untracked files from checks'
+complete -c spack -n '__fish_spack_using_command style' -s f -l fix -f -a fix
+complete -c spack -n '__fish_spack_using_command style' -s f -l fix -d 'format automatically if possible (e.g., with isort, black)'
+complete -c spack -n '__fish_spack_using_command style' -l root -r -f -a root
+complete -c spack -n '__fish_spack_using_command style' -l root -r -d 'style check a different spack instance'
+complete -c spack -n '__fish_spack_using_command style' -s t -l tool -r -f -a tool
+complete -c spack -n '__fish_spack_using_command style' -s t -l tool -r -d 'specify which tools to run (default: isort,black,flake8,mypy)'
+complete -c spack -n '__fish_spack_using_command style' -s s -l skip -r -f -a skip
+complete -c spack -n '__fish_spack_using_command style' -s s -l skip -r -d 'specify tools to skip (choose from isort,black,flake8,mypy)'
+
+# spack tags
+set -g __fish_spack_optspecs_spack_tags h/help i/installed a/all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 tags' -f -a '(__fish_spack_tags)'
+complete -c spack -n '__fish_spack_using_command tags' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command tags' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command tags' -s i -l installed -f -a installed
+complete -c spack -n '__fish_spack_using_command tags' -s i -l installed -d 'show information for installed packages only'
+complete -c spack -n '__fish_spack_using_command tags' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command tags' -s a -l all -d 'show packages for all available tags'
+
+# spack test
+set -g __fish_spack_optspecs_spack_test h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 test' -f -a run -d 'run tests for the specified installed packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 test' -f -a list -d 'list installed packages with available tests'
+complete -c spack -n '__fish_spack_using_command_pos 0 test' -f -a find -d 'find tests that are running or have available results'
+complete -c spack -n '__fish_spack_using_command_pos 0 test' -f -a status -d 'get the current status for the specified Spack test suite(s)'
+complete -c spack -n '__fish_spack_using_command_pos 0 test' -f -a results -d 'get the results from Spack test suite(s) (default all)'
+complete -c spack -n '__fish_spack_using_command_pos 0 test' -f -a remove -d 'remove results from Spack test suite(s) (default all)'
+complete -c spack -n '__fish_spack_using_command test' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command test' -s h -l help -d 'show this help message and exit'
+
+# spack test run
+set -g __fish_spack_optspecs_spack_test_run h/help alias= fail-fast fail-first externals x/explicit keep-stage log-format= log-file= cdash-upload-url= cdash-build= cdash-site= cdash-track= cdash-buildstamp= help-cdash clean dirty
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 test run' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command test run' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command test run' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command test run' -l alias -r -f -a alias
+complete -c spack -n '__fish_spack_using_command test run' -l alias -r -d 'provide an alias for this test-suite for subsequent access'
+complete -c spack -n '__fish_spack_using_command test run' -l fail-fast -f -a fail_fast
+complete -c spack -n '__fish_spack_using_command test run' -l fail-fast -d 'stop tests for each package after the first failure'
+complete -c spack -n '__fish_spack_using_command test run' -l fail-first -f -a fail_first
+complete -c spack -n '__fish_spack_using_command test run' -l fail-first -d 'stop after the first failed package'
+complete -c spack -n '__fish_spack_using_command test run' -l externals -f -a externals
+complete -c spack -n '__fish_spack_using_command test run' -l externals -d 'test packages that are externally installed'
+complete -c spack -n '__fish_spack_using_command test run' -s x -l explicit -f -a explicit
+complete -c spack -n '__fish_spack_using_command test run' -s x -l explicit -d 'only test packages that are explicitly installed'
+complete -c spack -n '__fish_spack_using_command test run' -l keep-stage -f -a keep_stage
+complete -c spack -n '__fish_spack_using_command test run' -l keep-stage -d 'keep testing directory for debugging'
+complete -c spack -n '__fish_spack_using_command test run' -l log-format -r -f -a 'junit cdash'
+complete -c spack -n '__fish_spack_using_command test run' -l log-format -r -d 'format to be used for log files'
+complete -c spack -n '__fish_spack_using_command test run' -l log-file -r -f -a log_file
+complete -c spack -n '__fish_spack_using_command test run' -l log-file -r -d 'filename for the log file'
+complete -c spack -n '__fish_spack_using_command test run' -l cdash-upload-url -r -f -a cdash_upload_url
+complete -c spack -n '__fish_spack_using_command test run' -l cdash-build -r -f -a cdash_build
+complete -c spack -n '__fish_spack_using_command test run' -l cdash-site -r -f -a cdash_site
+complete -c spack -n '__fish_spack_using_command test run' -l cdash-track -r -f -a cdash_track
+complete -c spack -n '__fish_spack_using_command test run' -l cdash-buildstamp -r -f -a cdash_buildstamp
+complete -c spack -n '__fish_spack_using_command test run' -l help-cdash -f -a help_cdash
+complete -c spack -n '__fish_spack_using_command test run' -l help-cdash -d 'show usage instructions for CDash reporting'
+complete -c spack -n '__fish_spack_using_command test run' -l clean -f -a dirty
+complete -c spack -n '__fish_spack_using_command test run' -l clean -d 'unset harmful variables in the build environment (default)'
+complete -c spack -n '__fish_spack_using_command test run' -l dirty -f -a dirty
+complete -c spack -n '__fish_spack_using_command test run' -l dirty -d 'preserve user environment in spack\'s build environment (danger!)'
+
+# spack test list
+set -g __fish_spack_optspecs_spack_test_list h/help a/all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 test list' -f -a '(__fish_spack_tags)'
+complete -c spack -n '__fish_spack_using_command test list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command test list' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command test list' -s a -l all -f -a list_all
+complete -c spack -n '__fish_spack_using_command test list' -s a -l all -d 'list all packages with tests (not just installed)'
+
+# spack test find
+set -g __fish_spack_optspecs_spack_test_find h/help
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 test find' -f -a '(__fish_spack_tests)'
+complete -c spack -n '__fish_spack_using_command test find' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command test find' -s h -l help -d 'show this help message and exit'
+
+# spack test status
+set -g __fish_spack_optspecs_spack_test_status h/help
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 test status' -f -a '(__fish_spack_tests)'
+complete -c spack -n '__fish_spack_using_command test status' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command test status' -s h -l help -d 'show this help message and exit'
+
+# spack test results
+set -g __fish_spack_optspecs_spack_test_results h/help l/logs f/failed
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 test results' -f -a '(__fish_spack_tests)'
+complete -c spack -n '__fish_spack_using_command test results' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command test results' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command test results' -s l -l logs -f -a logs
+complete -c spack -n '__fish_spack_using_command test results' -s l -l logs -d 'print the test log for each matching package'
+complete -c spack -n '__fish_spack_using_command test results' -s f -l failed -f -a failed
+complete -c spack -n '__fish_spack_using_command test results' -s f -l failed -d 'only show results for failed tests of matching packages'
+
+# spack test remove
+set -g __fish_spack_optspecs_spack_test_remove h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 test remove' -f -a '(__fish_spack_tests)'
+complete -c spack -n '__fish_spack_using_command test remove' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command test remove' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command test remove' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command test remove' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack test-env
+set -g __fish_spack_optspecs_spack_test_env h/help clean dirty U/fresh reuse reuse-deps dump= pickle=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 test-env' -f -a '(__fish_spack_build_env_spec)'
+complete -c spack -n '__fish_spack_using_command test-env' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command test-env' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command test-env' -l clean -f -a dirty
+complete -c spack -n '__fish_spack_using_command test-env' -l clean -d 'unset harmful variables in the build environment (default)'
+complete -c spack -n '__fish_spack_using_command test-env' -l dirty -f -a dirty
+complete -c spack -n '__fish_spack_using_command test-env' -l dirty -d 'preserve user environment in spack\'s build environment (danger!)'
+complete -c spack -n '__fish_spack_using_command test-env' -s U -l fresh -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command test-env' -s U -l fresh -d 'do not reuse installed deps; build newest configuration'
+complete -c spack -n '__fish_spack_using_command test-env' -l reuse -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command test-env' -l reuse -d 'reuse installed packages/buildcaches when possible'
+complete -c spack -n '__fish_spack_using_command test-env' -l reuse-deps -f -a concretizer_reuse
+complete -c spack -n '__fish_spack_using_command test-env' -l reuse-deps -d 'reuse installed dependencies only'
+complete -c spack -n '__fish_spack_using_command test-env' -l dump -r -f -a dump
+complete -c spack -n '__fish_spack_using_command test-env' -l dump -r -d 'dump a source-able environment to FILE'
+complete -c spack -n '__fish_spack_using_command test-env' -l pickle -r -f -a pickle
+complete -c spack -n '__fish_spack_using_command test-env' -l pickle -r -d 'dump a pickled source-able environment to FILE'
+
+# spack tutorial
+set -g __fish_spack_optspecs_spack_tutorial h/help y/yes-to-all
+complete -c spack -n '__fish_spack_using_command tutorial' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command tutorial' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command tutorial' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command tutorial' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+
+# spack undevelop
+set -g __fish_spack_optspecs_spack_undevelop h/help a/all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 undevelop' -f -k -a '(__fish_spack_specs_or_id)'
+complete -c spack -n '__fish_spack_using_command undevelop' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command undevelop' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command undevelop' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command undevelop' -s a -l all -d 'remove all specs from (clear) the environment'
+
+# spack uninstall
+set -g __fish_spack_optspecs_spack_uninstall h/help f/force remove R/dependents y/yes-to-all a/all origin=
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 uninstall' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command uninstall' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command uninstall' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command uninstall' -s f -l force -f -a force
+complete -c spack -n '__fish_spack_using_command uninstall' -s f -l force -d 'remove regardless of whether other packages or environments depend on this one'
+complete -c spack -n '__fish_spack_using_command uninstall' -l remove -f -a remove
+complete -c spack -n '__fish_spack_using_command uninstall' -l remove -d 'if in an environment, then the spec should also be removed from the environment description'
+complete -c spack -n '__fish_spack_using_command uninstall' -s R -l dependents -f -a dependents
+complete -c spack -n '__fish_spack_using_command uninstall' -s R -l dependents -d 'also uninstall any packages that depend on the ones given via command line'
+complete -c spack -n '__fish_spack_using_command uninstall' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command uninstall' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+complete -c spack -n '__fish_spack_using_command uninstall' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command uninstall' -s a -l all -d 'remove ALL installed packages that match each supplied spec'
+complete -c spack -n '__fish_spack_using_command uninstall' -l origin -r -f -a origin
+complete -c spack -n '__fish_spack_using_command uninstall' -l origin -r -d 'only remove DB records with the specified origin'
+
+# spack unit-test
+set -g __fish_spack_optspecs_spack_unit_test h/help H/pytest-help l/list L/list-long N/list-names extension= s/ k/= showlocals
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 unit-test' -f -a '(__fish_spack_unit_tests)'
+complete -c spack -n '__fish_spack_using_command unit-test' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command unit-test' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command unit-test' -s H -l pytest-help -f -a pytest_help
+complete -c spack -n '__fish_spack_using_command unit-test' -s H -l pytest-help -d 'show full pytest help, with advanced options'
+complete -c spack -n '__fish_spack_using_command unit-test' -s l -l list -f -a list
+complete -c spack -n '__fish_spack_using_command unit-test' -s l -l list -d 'list test filenames'
+complete -c spack -n '__fish_spack_using_command unit-test' -s L -l list-long -f -a list
+complete -c spack -n '__fish_spack_using_command unit-test' -s L -l list-long -d 'list all test functions'
+complete -c spack -n '__fish_spack_using_command unit-test' -s N -l list-names -f -a list
+complete -c spack -n '__fish_spack_using_command unit-test' -s N -l list-names -d 'list full names of all tests'
+complete -c spack -n '__fish_spack_using_command unit-test' -l extension -r -f -a extension
+complete -c spack -n '__fish_spack_using_command unit-test' -l extension -r -d 'run test for a given spack extension'
+complete -c spack -n '__fish_spack_using_command unit-test' -s s -f -a parsed_args
+complete -c spack -n '__fish_spack_using_command unit-test' -s s -d 'print output while tests run (disable capture)'
+complete -c spack -n '__fish_spack_using_command unit-test' -s k -r -f -a expression
+complete -c spack -n '__fish_spack_using_command unit-test' -s k -r -d 'filter tests by keyword (can also use w/list options)'
+complete -c spack -n '__fish_spack_using_command unit-test' -l showlocals -f -a parsed_args
+complete -c spack -n '__fish_spack_using_command unit-test' -l showlocals -d 'show local variable values in tracebacks'
+
+# spack unload
+set -g __fish_spack_optspecs_spack_unload h/help sh csh fish bat pwsh a/all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 unload' -f -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command unload' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command unload' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command unload' -l sh -f -a shell
+complete -c spack -n '__fish_spack_using_command unload' -l sh -d 'print sh commands to activate the environment'
+complete -c spack -n '__fish_spack_using_command unload' -l csh -f -a shell
+complete -c spack -n '__fish_spack_using_command unload' -l csh -d 'print csh commands to activate the environment'
+complete -c spack -n '__fish_spack_using_command unload' -l fish -f -a shell
+complete -c spack -n '__fish_spack_using_command unload' -l fish -d 'print fish commands to load the package'
+complete -c spack -n '__fish_spack_using_command unload' -l bat -f -a shell
+complete -c spack -n '__fish_spack_using_command unload' -l bat -d 'print bat commands to load the package'
+complete -c spack -n '__fish_spack_using_command unload' -l pwsh -f -a shell
+complete -c spack -n '__fish_spack_using_command unload' -l pwsh -d 'print pwsh commands to load the package'
+complete -c spack -n '__fish_spack_using_command unload' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command unload' -s a -l all -d 'unload all loaded Spack packages'
+
+# spack url
+set -g __fish_spack_optspecs_spack_url h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 url' -f -a parse -d 'attempt to parse a url'
+complete -c spack -n '__fish_spack_using_command_pos 0 url' -f -a list -d 'list urls in all packages'
+complete -c spack -n '__fish_spack_using_command_pos 0 url' -f -a summary -d 'print a summary of how well we are parsing package urls'
+complete -c spack -n '__fish_spack_using_command_pos 0 url' -f -a stats -d 'print statistics on versions and checksums for all packages'
+complete -c spack -n '__fish_spack_using_command url' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command url' -s h -l help -d 'show this help message and exit'
+
+# spack url parse
+set -g __fish_spack_optspecs_spack_url_parse h/help s/spider
+
+complete -c spack -n '__fish_spack_using_command url parse' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command url parse' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command url parse' -s s -l spider -f -a spider
+complete -c spack -n '__fish_spack_using_command url parse' -s s -l spider -d 'spider the source page for versions'
+
+# spack url list
+set -g __fish_spack_optspecs_spack_url_list h/help c/color e/extrapolation n/incorrect-name N/correct-name v/incorrect-version V/correct-version
+complete -c spack -n '__fish_spack_using_command url list' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command url list' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command url list' -s c -l color -f -a color
+complete -c spack -n '__fish_spack_using_command url list' -s c -l color -d 'color the parsed version and name in the urls shown (versions will be cyan, name red)'
+complete -c spack -n '__fish_spack_using_command url list' -s e -l extrapolation -f -a extrapolation
+complete -c spack -n '__fish_spack_using_command url list' -s e -l extrapolation -d 'color the versions used for extrapolation as well (additional versions will be green, names magenta)'
+complete -c spack -n '__fish_spack_using_command url list' -s n -l incorrect-name -f -a incorrect_name
+complete -c spack -n '__fish_spack_using_command url list' -s n -l incorrect-name -d 'only list urls for which the name was incorrectly parsed'
+complete -c spack -n '__fish_spack_using_command url list' -s N -l correct-name -f -a correct_name
+complete -c spack -n '__fish_spack_using_command url list' -s N -l correct-name -d 'only list urls for which the name was correctly parsed'
+complete -c spack -n '__fish_spack_using_command url list' -s v -l incorrect-version -f -a incorrect_version
+complete -c spack -n '__fish_spack_using_command url list' -s v -l incorrect-version -d 'only list urls for which the version was incorrectly parsed'
+complete -c spack -n '__fish_spack_using_command url list' -s V -l correct-version -f -a correct_version
+complete -c spack -n '__fish_spack_using_command url list' -s V -l correct-version -d 'only list urls for which the version was correctly parsed'
+
+# spack url summary
+set -g __fish_spack_optspecs_spack_url_summary h/help
+complete -c spack -n '__fish_spack_using_command url summary' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command url summary' -s h -l help -d 'show this help message and exit'
+
+# spack url stats
+set -g __fish_spack_optspecs_spack_url_stats h/help show-issues
+complete -c spack -n '__fish_spack_using_command url stats' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command url stats' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command url stats' -l show-issues -f -a show_issues
+complete -c spack -n '__fish_spack_using_command url stats' -l show-issues -d 'show packages with issues (md5 hashes, http urls)'
+
+# spack verify
+set -g __fish_spack_optspecs_spack_verify h/help l/local j/json a/all s/specs f/files
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 verify' $__fish_spack_force_files -a '(__fish_spack_installed_specs)'
+complete -c spack -n '__fish_spack_using_command verify' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command verify' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command verify' -s l -l local -f -a local
+complete -c spack -n '__fish_spack_using_command verify' -s l -l local -d 'verify only locally installed packages'
+complete -c spack -n '__fish_spack_using_command verify' -s j -l json -f -a json
+complete -c spack -n '__fish_spack_using_command verify' -s j -l json -d 'ouptut json-formatted errors'
+complete -c spack -n '__fish_spack_using_command verify' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command verify' -s a -l all -d 'verify all packages'
+complete -c spack -n '__fish_spack_using_command verify' -s s -l specs -f -a type
+complete -c spack -n '__fish_spack_using_command verify' -s s -l specs -d 'treat entries as specs (default)'
+complete -c spack -n '__fish_spack_using_command verify' -s f -l files -f -a type
+complete -c spack -n '__fish_spack_using_command verify' -s f -l files -d 'treat entries as absolute filenames'
+
+# spack versions
+set -g __fish_spack_optspecs_spack_versions h/help s/safe safe-only r/remote n/new j/jobs=
+complete -c spack -n '__fish_spack_using_command_pos 0 versions' -f -a '(__fish_spack_packages)'
+complete -c spack -n '__fish_spack_using_command versions' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command versions' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command versions' -s s -l safe -f -a safe
+complete -c spack -n '__fish_spack_using_command versions' -s s -l safe -d 'only list safe versions of the package'
+complete -c spack -n '__fish_spack_using_command versions' -l safe-only -f -a safe_only
+complete -c spack -n '__fish_spack_using_command versions' -l safe-only -d '[deprecated] only list safe versions of the package'
+complete -c spack -n '__fish_spack_using_command versions' -s r -l remote -f -a remote
+complete -c spack -n '__fish_spack_using_command versions' -s r -l remote -d 'only list remote versions of the package'
+complete -c spack -n '__fish_spack_using_command versions' -s n -l new -f -a new
+complete -c spack -n '__fish_spack_using_command versions' -s n -l new -d 'only list remote versions newer than the latest checksummed version'
+complete -c spack -n '__fish_spack_using_command versions' -s j -l jobs -r -f -a jobs
+complete -c spack -n '__fish_spack_using_command versions' -s j -l jobs -r -d 'explicitly set number of parallel jobs'
+
+# spack view
+set -g __fish_spack_optspecs_spack_view h/help v/verbose e/exclude= d/dependencies=
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a symlink -d 'add package files to a filesystem view via symbolic links'
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a add -d 'add package files to a filesystem view via symbolic links'
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a soft -d 'add package files to a filesystem view via symbolic links'
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a hardlink -d 'add packages files to a filesystem view via hard links'
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a hard -d 'add packages files to a filesystem view via hard links'
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a copy -d 'add package files to a filesystem view via copy/relocate'
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a relocate -d 'add package files to a filesystem view via copy/relocate'
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a remove -d 'remove packages from a filesystem view'
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a rm -d 'remove packages from a filesystem view'
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a statlink -d 'check status of packages in a filesystem view'
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a status -d 'check status of packages in a filesystem view'
+complete -c spack -n '__fish_spack_using_command_pos 0 view' -f -a check -d 'check status of packages in a filesystem view'
+complete -c spack -n '__fish_spack_using_command view' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command view' -s v -l verbose -f -a verbose
+complete -c spack -n '__fish_spack_using_command view' -s v -l verbose -d 'if not verbose only warnings/errors will be printed'
+complete -c spack -n '__fish_spack_using_command view' -s e -l exclude -r -f -a exclude
+complete -c spack -n '__fish_spack_using_command view' -s e -l exclude -r -d 'exclude packages with names matching the given regex pattern'
+complete -c spack -n '__fish_spack_using_command view' -s d -l dependencies -r -f -a 'true false yes no'
+complete -c spack -n '__fish_spack_using_command view' -s d -l dependencies -r -d 'link/remove/list dependencies'
+
+# spack view symlink
+set -g __fish_spack_optspecs_spack_view_symlink h/help projection-file= i/ignore-conflicts
+complete -c spack -n '__fish_spack_using_command_pos 0 view symlink' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view symlink' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view symlink' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view symlink' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command view symlink' -l projection-file -r -f -a projection_file
+complete -c spack -n '__fish_spack_using_command view symlink' -l projection-file -r -d 'initialize view using projections from file'
+complete -c spack -n '__fish_spack_using_command view symlink' -s i -l ignore-conflicts -f -a ignore_conflicts
+
+# spack view add
+set -g __fish_spack_optspecs_spack_view_add h/help projection-file= i/ignore-conflicts
+complete -c spack -n '__fish_spack_using_command_pos 0 view add' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view add' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view add' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view add' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command view add' -l projection-file -r -f -a projection_file
+complete -c spack -n '__fish_spack_using_command view add' -l projection-file -r -d 'initialize view using projections from file'
+complete -c spack -n '__fish_spack_using_command view add' -s i -l ignore-conflicts -f -a ignore_conflicts
+
+# spack view soft
+set -g __fish_spack_optspecs_spack_view_soft h/help projection-file= i/ignore-conflicts
+complete -c spack -n '__fish_spack_using_command_pos 0 view soft' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view soft' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view soft' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view soft' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command view soft' -l projection-file -r -f -a projection_file
+complete -c spack -n '__fish_spack_using_command view soft' -l projection-file -r -d 'initialize view using projections from file'
+complete -c spack -n '__fish_spack_using_command view soft' -s i -l ignore-conflicts -f -a ignore_conflicts
+
+# spack view hardlink
+set -g __fish_spack_optspecs_spack_view_hardlink h/help projection-file= i/ignore-conflicts
+complete -c spack -n '__fish_spack_using_command_pos 0 view hardlink' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view hardlink' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view hardlink' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view hardlink' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command view hardlink' -l projection-file -r -f -a projection_file
+complete -c spack -n '__fish_spack_using_command view hardlink' -l projection-file -r -d 'initialize view using projections from file'
+complete -c spack -n '__fish_spack_using_command view hardlink' -s i -l ignore-conflicts -f -a ignore_conflicts
+
+# spack view hard
+set -g __fish_spack_optspecs_spack_view_hard h/help projection-file= i/ignore-conflicts
+complete -c spack -n '__fish_spack_using_command_pos 0 view hard' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view hard' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view hard' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view hard' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command view hard' -l projection-file -r -f -a projection_file
+complete -c spack -n '__fish_spack_using_command view hard' -l projection-file -r -d 'initialize view using projections from file'
+complete -c spack -n '__fish_spack_using_command view hard' -s i -l ignore-conflicts -f -a ignore_conflicts
+
+# spack view copy
+set -g __fish_spack_optspecs_spack_view_copy h/help projection-file= i/ignore-conflicts
+complete -c spack -n '__fish_spack_using_command_pos 0 view copy' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view copy' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view copy' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view copy' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command view copy' -l projection-file -r -f -a projection_file
+complete -c spack -n '__fish_spack_using_command view copy' -l projection-file -r -d 'initialize view using projections from file'
+complete -c spack -n '__fish_spack_using_command view copy' -s i -l ignore-conflicts -f -a ignore_conflicts
+
+# spack view relocate
+set -g __fish_spack_optspecs_spack_view_relocate h/help projection-file= i/ignore-conflicts
+complete -c spack -n '__fish_spack_using_command_pos 0 view relocate' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view relocate' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view relocate' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view relocate' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command view relocate' -l projection-file -r -f -a projection_file
+complete -c spack -n '__fish_spack_using_command view relocate' -l projection-file -r -d 'initialize view using projections from file'
+complete -c spack -n '__fish_spack_using_command view relocate' -s i -l ignore-conflicts -f -a ignore_conflicts
+
+# spack view remove
+set -g __fish_spack_optspecs_spack_view_remove h/help no-remove-dependents a/all
+complete -c spack -n '__fish_spack_using_command_pos 0 view remove' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view remove' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view remove' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view remove' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command view remove' -l no-remove-dependents -f -a no_remove_dependents
+complete -c spack -n '__fish_spack_using_command view remove' -l no-remove-dependents -d 'do not remove dependents of specified specs'
+complete -c spack -n '__fish_spack_using_command view remove' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command view remove' -s a -l all -d 'act on all specs in view'
+
+# spack view rm
+set -g __fish_spack_optspecs_spack_view_rm h/help no-remove-dependents a/all
+complete -c spack -n '__fish_spack_using_command_pos 0 view rm' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view rm' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view rm' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view rm' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command view rm' -l no-remove-dependents -f -a no_remove_dependents
+complete -c spack -n '__fish_spack_using_command view rm' -l no-remove-dependents -d 'do not remove dependents of specified specs'
+complete -c spack -n '__fish_spack_using_command view rm' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command view rm' -s a -l all -d 'act on all specs in view'
+
+# spack view statlink
+set -g __fish_spack_optspecs_spack_view_statlink h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 view statlink' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view statlink' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view statlink' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view statlink' -s h -l help -d 'show this help message and exit'
+
+# spack view status
+set -g __fish_spack_optspecs_spack_view_status h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 view status' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view status' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view status' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view status' -s h -l help -d 'show this help message and exit'
+
+# spack view check
+set -g __fish_spack_optspecs_spack_view_check h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 view check' -f -a '(__fish_complete_directories)'
+complete -c spack -n '__fish_spack_using_command_pos_remainder 1 view check' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command view check' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command view check' -s h -l help -d 'show this help message and exit'
diff --git a/share/spack/templates/bootstrap/spack.yaml b/share/spack/templates/bootstrap/spack.yaml
index 4573bb485eb998..251977d0c5b4e3 100644
--- a/share/spack/templates/bootstrap/spack.yaml
+++ b/share/spack/templates/bootstrap/spack.yaml
@@ -15,6 +15,7 @@ spack:
   config:
     install_tree:
       root: {{ store_path }}
+      padded_length: 0
 
   packages:
     python:
diff --git a/share/spack/templates/container/Dockerfile b/share/spack/templates/container/Dockerfile
index 7e9c2f4231d39d..213302876c15f3 100644
--- a/share/spack/templates/container/Dockerfile
+++ b/share/spack/templates/container/Dockerfile
@@ -43,9 +43,6 @@ RUN find -L {{ paths.view }}/* -type f -exec readlink -f '{}' \; | \
 RUN cd {{ paths.environment }} && \
     spack env activate --sh -d . > activate.sh
 
-{% if extra_instructions.build %}
-{{ extra_instructions.build }}
-{% endif %}
 {% endblock build_stage %}
 {% endif %}
 
@@ -55,15 +52,17 @@ FROM {{ run.image }}
 
 COPY --from=builder {{ paths.environment }} {{ paths.environment }}
 COPY --from=builder {{ paths.store }} {{ paths.store }}
-COPY --from=builder {{ paths.hidden_view }} {{ paths.hidden_view }}
-COPY --from=builder {{ paths.view }} {{ paths.view }}
+
+# paths.view is a symlink, so copy the parent to avoid dereferencing and duplicating it
+COPY --from=builder {{ paths.view_parent }} {{ paths.view_parent }}
 
 RUN { \
       echo '#!/bin/sh' \
       && echo '.' {{ paths.environment }}/activate.sh \
       && echo 'exec "$@"'; \
     } > /entrypoint.sh \
-&&  chmod a+x /entrypoint.sh
+&& chmod a+x /entrypoint.sh \
+&& ln -s {{ paths.view }} {{ paths.former_view }}
 
 {% block final_stage %}
 
@@ -76,10 +75,6 @@ RUN {% if os_package_update %}{{ os_packages_final.update }} \
  && {% endif %}{{ os_packages_final.install }} {{ os_packages_final.list | join | replace('\n', ' ') }} \
  && {{ os_packages_final.clean }}
 {% endif %}
-{% if extra_instructions.final %}
-
-{{ extra_instructions.final }}
-{% endif %}
 {% endblock final_stage %}
 {% for label, value in labels.items() %}
 LABEL "{{ label }}"="{{ value }}"
diff --git a/share/spack/templates/container/amazonlinux_2.dockerfile b/share/spack/templates/container/amazonlinux_2.dockerfile
index c5b69e317da2da..e11dd0c482ca14 100644
--- a/share/spack/templates/container/amazonlinux_2.dockerfile
+++ b/share/spack/templates/container/amazonlinux_2.dockerfile
@@ -10,6 +10,7 @@ RUN yum update -y \
         gcc-gfortran \
         git \
         gnupg2 \
+        hg \
         hostname \
         iproute \
         make \
@@ -18,6 +19,7 @@ RUN yum update -y \
         python3-pip \
         python3-setuptools \
         unzip \
+        zstd \
  && pip3 install boto3 \
  && rm -rf /var/cache/yum \
  && yum clean all
diff --git a/share/spack/templates/container/bootstrap-base.dockerfile b/share/spack/templates/container/bootstrap-base.dockerfile
index e002cce02e168e..38de6c88494ac2 100644
--- a/share/spack/templates/container/bootstrap-base.dockerfile
+++ b/share/spack/templates/container/bootstrap-base.dockerfile
@@ -23,7 +23,7 @@ RUN ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
 RUN mkdir -p /root/.spack \
  && cp $SPACK_ROOT/share/spack/docker/modules.yaml \
         /root/.spack/modules.yaml \
- && rm -rf /root/*.* /run/nologin $SPACK_ROOT/.git
+ && rm -rf /root/*.* /run/nologin
 
 # [WORKAROUND]
 # https://superuser.com/questions/1241548/
@@ -39,7 +39,9 @@ WORKDIR /root
 SHELL ["docker-shell"]
 
 # Creates the package cache
-RUN spack bootstrap now && spack spec hdf5+mpi
+RUN spack bootstrap now \
+    && spack bootstrap status --optional \
+    && spack spec hdf5+mpi
 
 ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
 CMD ["interactive-shell"]
diff --git a/share/spack/templates/container/centos_7.dockerfile b/share/spack/templates/container/centos_7.dockerfile
index d4401c5413c40f..2d99ecad91586a 100644
--- a/share/spack/templates/container/centos_7.dockerfile
+++ b/share/spack/templates/container/centos_7.dockerfile
@@ -13,6 +13,7 @@ RUN yum update -y \
         git \
         gnupg2 \
         hostname \
+        hg \
         iproute \
         make \
         patch \
@@ -20,6 +21,7 @@ RUN yum update -y \
         python3-pip \
         python3-setuptools \
         unzip \
+        zstd \
  && pip3 install boto3 \
  && rm -rf /var/cache/yum \
  && yum clean all
diff --git a/share/spack/templates/container/centos_stream.dockerfile b/share/spack/templates/container/centos_stream.dockerfile
index 292d76d73c6423..c11a658fc7d960 100644
--- a/share/spack/templates/container/centos_stream.dockerfile
+++ b/share/spack/templates/container/centos_stream.dockerfile
@@ -15,15 +15,18 @@ RUN dnf update -y \
         gcc-gfortran \
         git \
         gnupg2 \
+        hg \
         hostname \
         iproute \
         make \
+        svn \
         patch \
-        python38 \
-        python38-pip \
-        python38-setuptools \
+        python3.11 \
+        python3.11-setuptools \
         unzip \
- && pip3 install boto3 \
+        zstd \
+ && python3.11 -m ensurepip \
+ && pip3.11 install boto3 \
  && rm -rf /var/cache/dnf \
  && dnf clean all
 {% endblock %}
diff --git a/share/spack/templates/container/leap-15.dockerfile b/share/spack/templates/container/leap-15.dockerfile
index 2427c3775456da..386f0b5b2fedcb 100644
--- a/share/spack/templates/container/leap-15.dockerfile
+++ b/share/spack/templates/container/leap-15.dockerfile
@@ -9,12 +9,16 @@ RUN zypper ref && \
     gcc-c++\
     gcc-fortran\
     make\
+    mercurial\
     git\
     gzip\
     patch\
     python3-base \
     python3-boto3\
+    subversion\
     tar\
+    unzip\
     xz\
+    zstd\
 &&  zypper clean
 {% endblock %}
diff --git a/share/spack/templates/container/singularity.def b/share/spack/templates/container/singularity.def
index 7c15ce12212cd2..4d9fdd617c912d 100644
--- a/share/spack/templates/container/singularity.def
+++ b/share/spack/templates/container/singularity.def
@@ -44,15 +44,12 @@ EOF
     grep 'x-executable\|x-archive\|x-sharedlib' | \
     awk -F: '{print $1}' | xargs strip
 {% endif %}
-{% if extra_instructions.build %}
-{{ extra_instructions.build }}
-{% endif %}
 {% endblock build_stage %}
 {% if apps %}
 {% for application, help_text in apps.items() %}
 
 %apprun {{ application }}
-    exec /opt/view/bin/{{ application }} "$@"
+    exec {{ paths.view }}/bin/{{ application }} "$@"
 
 %apphelp {{ application }}
     {{help_text }}
@@ -66,11 +63,14 @@ Stage: final
 %files from build
   {{ paths.environment }} /opt
   {{ paths.store }} /opt
-  {{ paths.hidden_view }} /opt
-  {{ paths.view }} /opt
+  {{ paths.view_parent }} /opt
   {{ paths.environment }}/environment_modifications.sh {{ paths.environment }}/environment_modifications.sh
 
 %post
+
+  # Symlink the old view location
+  ln -s {{ paths.view }} {{ paths.former_view }}
+
 {% block final_stage %}
 {% if extra_instructions.pre_final %}
  {{ extra_instructions.pre_final }}
@@ -85,9 +85,6 @@ Stage: final
 {% endif %}
   # Modify the environment without relying on sourcing shell specific files at startup
   cat {{ paths.environment }}/environment_modifications.sh >> $SINGULARITY_ENVIRONMENT
-{% if extra_instructions.final %}
-{{ extra_instructions.final }}
-{% endif %}
 {% endblock final_stage %}
 {% if runscript %}
 
diff --git a/share/spack/templates/container/ubuntu_2004.dockerfile b/share/spack/templates/container/ubuntu_2004.dockerfile
index 26b656295248f6..f1c3ca456de1a4 100644
--- a/share/spack/templates/container/ubuntu_2004.dockerfile
+++ b/share/spack/templates/container/ubuntu_2004.dockerfile
@@ -22,10 +22,13 @@ RUN apt-get -yqq update \
         iproute2 \
         locales \
         make \
+        mercurial \
+        subversion \
         python3 \
         python3-pip \
         python3-setuptools \
         unzip \
+        zstd \
  && locale-gen en_US.UTF-8 \
  && pip3 install boto3 \
  && rm -rf /var/lib/apt/lists/*
diff --git a/share/spack/templates/depfile/Makefile b/share/spack/templates/depfile/Makefile
index a50304a8be66cf..4b764752678c0f 100644
--- a/share/spack/templates/depfile/Makefile
+++ b/share/spack/templates/depfile/Makefile
@@ -1,4 +1,4 @@
-SPACK ?= spack
+SPACK ?= spack -c config:install_status:false
 SPACK_INSTALL_FLAGS ?=
 
 # This variable can be used to add post install hooks
@@ -8,7 +8,7 @@ SPACK_INSTALL_FLAGS ?=
 
 {{ all_target }}: {{ env_target }}
 
-{{ env_target }}: {{ root_install_targets }}
+{{ env_target }}: {{ root_install_targets }} | {{ dirs_target }}
 	@touch $@
 
 {{ dirs_target }}:
diff --git a/share/spack/templates/modules/modulefile.lua b/share/spack/templates/modules/modulefile.lua
index f86e76cfe68954..65e97f2410eda6 100644
--- a/share/spack/templates/modules/modulefile.lua
+++ b/share/spack/templates/modules/modulefile.lua
@@ -69,6 +69,12 @@ setenv("LMOD_{{ name|upper() }}_VERSION", "{{ version_part }}")
 depends_on("{{ module }}")
 {% endfor %}
 {% endblock %}
+{#  #}
+{% block conflict %}
+{% for name in conflicts %}
+conflict("{{ name }}")
+{% endfor %}
+{% endblock %}
 
 {% block environment %}
 {% for command_name, cmd in environment_modifications %}
diff --git a/share/spack/templates/modules/modulefile.tcl b/share/spack/templates/modules/modulefile.tcl
index 52d987da61b136..d1593b88280e65 100644
--- a/share/spack/templates/modules/modulefile.tcl
+++ b/share/spack/templates/modules/modulefile.tcl
@@ -11,16 +11,16 @@
 
 {% block header %}
 {% if short_description %}
-module-whatis "{{ short_description }}"
+module-whatis {{ '{' }}{{ short_description }}{{ '}' }}
 {% endif %}
 
 proc ModulesHelp { } {
-    puts stderr "Name   : {{ spec.name }}"
-    puts stderr "Version: {{ spec.version }}"
-    puts stderr "Target : {{ spec.target }}"
+    puts stderr {{ '{' }}Name   : {{ spec.name }}{{ '}' }}
+    puts stderr {{ '{' }}Version: {{ spec.version }}{{ '}' }}
+    puts stderr {{ '{' }}Target : {{ spec.target }}{{ '}' }}
 {% if long_description %}
-    puts stderr ""
-{{ long_description| textwrap(72)| quote()| prepend_to_line('    puts stderr ')| join() }}
+    puts stderr {}
+{{ long_description| textwrap(72)| curly_quote()| prepend_to_line('    puts stderr ')| join() }}
 {% endif %}
 }
 {% endblock %}
@@ -54,13 +54,25 @@ conflict {{ name }}
 {% block environment %}
 {% for command_name, cmd in environment_modifications %}
 {% if command_name == 'PrependPath' %}
-prepend-path --delim "{{ cmd.separator }}" {{ cmd.name }} "{{ cmd.value }}"
+{% if cmd.separator == ':' %}
+prepend-path {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }}
+{% else %}
+prepend-path --delim {{ '{' }}{{ cmd.separator }}{{ '}' }} {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }}
+{% endif %}
 {% elif command_name in ('AppendPath', 'AppendFlagsEnv') %}
-append-path --delim "{{ cmd.separator }}" {{ cmd.name }} "{{ cmd.value }}"
+{% if cmd.separator == ':' %}
+append-path {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }}
+{% else %}
+append-path --delim {{ '{' }}{{ cmd.separator }}{{ '}' }} {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }}
+{% endif %}
 {% elif command_name in ('RemovePath', 'RemoveFlagsEnv') %}
-remove-path --delim "{{ cmd.separator }}" {{ cmd.name }} "{{ cmd.value }}"
+{% if cmd.separator == ':' %}
+remove-path {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }}
+{% else %}
+remove-path --delim {{ '{' }}{{ cmd.separator }}{{ '}' }} {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }}
+{% endif %}
 {% elif command_name == 'SetEnv' %}
-setenv {{ cmd.name }} "{{ cmd.value }}"
+setenv {{ cmd.name }} {{ '{' }}{{ cmd.value }}{{ '}' }}
 {% elif command_name == 'UnsetEnv' %}
 unsetenv {{ cmd.name }}
 {% endif %}
@@ -68,7 +80,7 @@ unsetenv {{ cmd.name }}
 {% endfor %}
 {# Make sure system man pages are enabled by appending trailing delimiter to MANPATH #}
 {% if has_manpath_modifications %}
-append-path --delim ":" MANPATH ""
+append-path MANPATH {{ '{' }}{{ '}' }}
 {% endif %}
 {% endblock %}
 
diff --git a/var/spack/repos/builder.test/packages/gmake/package.py b/var/spack/repos/builder.test/packages/gmake/package.py
new file mode 100644
index 00000000000000..b3d5c50086e291
--- /dev/null
+++ b/var/spack/repos/builder.test/packages/gmake/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Gmake(Package):
+    """Dummy GMake Package"""
+
+    homepage = "https://www.gnu.org/software/make"
+    url = "https://ftpmirror.gnu.org/make/make-4.4.tar.gz"
+
+    version("4.4", sha256="ce35865411f0490368a8fc383f29071de6690cbadc27704734978221f25e2bed")
+
+    def do_stage(self):
+        mkdirp(self.stage.source_path)
diff --git a/var/spack/repos/builtin.mock/packages/adios2/package.py b/var/spack/repos/builtin.mock/packages/adios2/package.py
new file mode 100644
index 00000000000000..fb2f43ea0e154a
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/adios2/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class Adios2(Package):
+    """This packagae has the variants shared and
+    bzip2, both defaulted to True"""
+
+    homepage = "https://example.com"
+    url = "https://example.com/adios2.tar.gz"
+
+    version("2.9.1", sha256="ddfa32c14494250ee8a48ef1c97a1bf6442c15484bbbd4669228a0f90242f4f9")
+
+    variant("shared", default=True, description="Build shared libraries")
+    variant("bzip2", default=True, description="Enable BZip2 compression")
+
+    depends_on("bzip2")
diff --git a/var/spack/repos/builtin.mock/packages/ascent/package.py b/var/spack/repos/builtin.mock/packages/ascent/package.py
new file mode 100644
index 00000000000000..9a8db472dc07ab
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/ascent/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Ascent(Package):
+    """This packagae has the variants shared, defaulted
+    to True and adios2 defaulted to False"""
+
+    homepage = "https://github.com/Alpine-DAV/ascent"
+    url = "http://www.example.com/ascent-1.0.tar.gz"
+
+    version("0.9.2", sha256="44cd954aa5db478ab40042cd54fd6fcedf25000c3bb510ca23fcff8090531b91")
+
+    variant("adios2", default=False, description="Build Adios2 filter support")
+    variant("shared", default=True, description="Build Ascent as shared libs")
+
+    depends_on("adios2", when="+adios2")
diff --git a/var/spack/repos/builtin.mock/packages/b/package.py b/var/spack/repos/builtin.mock/packages/b/package.py
index 06d82860850808..1685711825f9ea 100644
--- a/var/spack/repos/builtin.mock/packages/b/package.py
+++ b/var/spack/repos/builtin.mock/packages/b/package.py
@@ -15,4 +15,8 @@ class B(Package):
     version("1.0", md5="0123456789abcdef0123456789abcdef")
     version("0.9", md5="abcd456789abcdef0123456789abcdef")
 
+    variant(
+        "foo", description="", values=any_combination_of("bar", "baz", "fee").with_default("bar")
+    )
+
     depends_on("test-dependency", type="test")
diff --git a/var/spack/repos/builtin.mock/packages/build-env-compiler-var-a/package.py b/var/spack/repos/builtin.mock/packages/build-env-compiler-var-a/package.py
new file mode 100644
index 00000000000000..ea6f0f34e8ee6d
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/build-env-compiler-var-a/package.py
@@ -0,0 +1,14 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class BuildEnvCompilerVarA(Package):
+    """Package with runtime variable that should be dropped in the parent's build environment."""
+
+    url = "https://www.example.com"
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+    depends_on("build-env-compiler-var-b", type="build")
diff --git a/var/spack/repos/builtin.mock/packages/build-env-compiler-var-b/package.py b/var/spack/repos/builtin.mock/packages/build-env-compiler-var-b/package.py
new file mode 100644
index 00000000000000..7905869b344de6
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/build-env-compiler-var-b/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class BuildEnvCompilerVarB(Package):
+    """Package with runtime variable that should be dropped in the parent's build environment."""
+
+    url = "https://www.example.com"
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+
+    def setup_run_environment(self, env):
+        env.set("CC", "this-should-be-dropped")
+        env.set("CXX", "this-should-be-dropped")
+        env.set("FC", "this-should-be-dropped")
+        env.set("F77", "this-should-be-dropped")
+        env.set("ANOTHER_VAR", "this-should-be-present")
diff --git a/var/spack/repos/builtin.mock/packages/build-error/package.py b/var/spack/repos/builtin.mock/packages/build-error/package.py
index 29342bc5fcc7e8..2df3699822640f 100644
--- a/var/spack/repos/builtin.mock/packages/build-error/package.py
+++ b/var/spack/repos/builtin.mock/packages/build-error/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import sys
+
 from spack.package import *
 
 
@@ -15,18 +17,38 @@ class BuildError(Package):
     version("1.0", md5="0123456789abcdef0123456789abcdef")
 
     def install(self, spec, prefix):
-        with open("configure", "w") as f:
-            f.write(
-                """#!/bin/sh\n
-echo 'checking build system type... x86_64-apple-darwin16.6.0'
-echo 'checking host system type... x86_64-apple-darwin16.6.0'
-echo 'checking for gcc... /Users/gamblin2/src/spack/lib/spack/env/clang/clang'
-echo 'checking whether the C compiler works... yes'
-echo 'checking for C compiler default output file name... a.out'
-echo 'checking for suffix of executables...'
-echo 'configure: error: in /path/to/some/file:'
-echo 'configure: error: cannot run C compiled programs.'
-exit 1
-"""
-            )
-        configure()
+        if sys.platform == "win32":
+            with open("configure.bat", "w") as f:
+                f.write(
+                    """
+    @ECHO off
+    ECHO checking build system type... x86_64-apple-darwin16.6.0
+    ECHO checking host system type... x86_64-apple-darwin16.6.0
+    ECHO checking for gcc... /Users/gamblin2/src/spack/lib/spack/env/clang/clang
+    ECHO checking whether the C compiler works... yes
+    ECHO checking for C compiler default output file name... a.out
+    ECHO checking for suffix of executables...
+    ECHO configure: error: in /path/to/some/file:
+    ECHO configure: error: cannot run C compiled programs.
+    EXIT /B 1
+                  """
+                )
+
+            Executable("configure.bat")("--prefix=%s" % self.prefix)
+            configure()
+        else:
+            with open("configure", "w") as f:
+                f.write(
+                    """#!/bin/sh\n
+    echo 'checking build system type... x86_64-apple-darwin16.6.0'
+    echo 'checking host system type... x86_64-apple-darwin16.6.0'
+    echo 'checking for gcc... /Users/gamblin2/src/spack/lib/spack/env/clang/clang'
+    echo 'checking whether the C compiler works... yes'
+    echo 'checking for C compiler default output file name... a.out'
+    echo 'checking for suffix of executables...'
+    echo 'configure: error: in /path/to/some/file:'
+    echo 'configure: error: cannot run C compiled programs.'
+    exit 1
+    """
+                )
+            configure()
diff --git a/var/spack/repos/builtin.mock/packages/build-warnings/package.py b/var/spack/repos/builtin.mock/packages/build-warnings/package.py
index 06026896e2bfe5..ce4af309b93e0e 100644
--- a/var/spack/repos/builtin.mock/packages/build-warnings/package.py
+++ b/var/spack/repos/builtin.mock/packages/build-warnings/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import sys
+
 from spack.package import *
 
 
@@ -15,16 +17,33 @@ class BuildWarnings(Package):
     version("1.0", md5="0123456789abcdef0123456789abcdef")
 
     def install(self, spec, prefix):
-        with open("configure", "w") as f:
-            f.write(
-                """#!/bin/sh\n
-echo 'checking for gcc... /Users/gamblin2/src/spack/lib/spack/env/clang/clang'
-echo 'checking whether the C compiler works... yes'
-echo 'checking for C compiler default output file name... a.out'
-echo 'WARNING: ALL CAPITAL WARNING!'
-echo 'checking for suffix of executables...'
-echo 'foo.c:89: warning: some weird warning!'
-exit 1
-"""
-            )
-        configure()
+        if sys.platform == "win32":
+            with open("configure.bat", "w") as f:
+                f.write(
+                    """
+  @ECHO off
+  ECHO 'checking for gcc... /Users/gamblin2/src/spack/lib/spack/env/clang/clang'
+  ECHO 'checking whether the C compiler works... yes'
+  ECHO 'checking for C compiler default output file name... a.out'
+  ECHO 'WARNING: ALL CAPITAL WARNING!'
+  ECHO 'checking for suffix of executables...'
+  ECHO 'foo.c:89: warning: some weird warning!'
+  EXIT /B 1
+                  """
+                )
+
+            Executable("configure.bat")("--prefix=%s" % self.prefix)
+        else:
+            with open("configure", "w") as f:
+                f.write(
+                    """#!/bin/sh\n
+  echo 'checking for gcc... /Users/gamblin2/src/spack/lib/spack/env/clang/clang'
+  echo 'checking whether the C compiler works... yes'
+  echo 'checking for C compiler default output file name... a.out'
+  echo 'WARNING: ALL CAPITAL WARNING!'
+  echo 'checking for suffix of executables...'
+  echo 'foo.c:89: warning: some weird warning!'
+  exit 1
+  """
+                )
+            configure()
diff --git a/var/spack/repos/builtin.mock/packages/bzip2/package.py b/var/spack/repos/builtin.mock/packages/bzip2/package.py
new file mode 100644
index 00000000000000..326533ac5ea117
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/bzip2/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class Bzip2(Package):
+    """This packagae has the variants shared
+    defaulted to True"""
+
+    homepage = "https://example.com"
+    url = "https://example.com/bzip2-1.0.8tar.gz"
+
+    version("1.0.8", sha256="ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269")
+
+    variant("shared", default=True, description="Enables the build of shared libraries.")
diff --git a/var/spack/repos/builtin.mock/packages/client-not-foo/package.py b/var/spack/repos/builtin.mock/packages/client-not-foo/package.py
new file mode 100644
index 00000000000000..03c9374b3acce1
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/client-not-foo/package.py
@@ -0,0 +1,17 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class ClientNotFoo(Package):
+    """This package has a variant "foo", which is False by default."""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/c-1.0.tar.gz"
+
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+
+    variant("foo", default=False, description="")
diff --git a/var/spack/repos/builtin.mock/packages/cmake-client/package.py b/var/spack/repos/builtin.mock/packages/cmake-client/package.py
index 2ce5a98dd1462c..a0694f2d6d7221 100644
--- a/var/spack/repos/builtin.mock/packages/cmake-client/package.py
+++ b/var/spack/repos/builtin.mock/packages/cmake-client/package.py
@@ -109,7 +109,7 @@ def install(self, spec, prefix):
         print(cmake)
         print(cmake.exe)
         check(
-            cmake.exe[0].startswith(spec["cmake"].prefix.bin),
+            cmake.path.startswith(spec["cmake"].prefix.bin),
             "Wrong cmake was in environment: %s" % cmake,
         )
 
diff --git a/var/spack/repos/builtin.mock/packages/dependency-foo-bar/package.py b/var/spack/repos/builtin.mock/packages/dependency-foo-bar/package.py
new file mode 100644
index 00000000000000..21e67f8a61bc12
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/dependency-foo-bar/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class DependencyFooBar(Package):
+    """This package has a variant "bar", which is False by default, and
+    variant "foo" which is True by default.
+    """
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/dependency-foo-bar-1.0.tar.gz"
+
+    version("1.0", md5="1234567890abcdefg1234567890098765")
+
+    variant("foo", default=True, description="")
+    variant("bar", default=False, description="")
diff --git a/var/spack/repos/builtin.mock/packages/gmake/package.py b/var/spack/repos/builtin.mock/packages/gmake/package.py
index aa5dd8452bf27e..b3d5c50086e291 100644
--- a/var/spack/repos/builtin.mock/packages/gmake/package.py
+++ b/var/spack/repos/builtin.mock/packages/gmake/package.py
@@ -13,3 +13,6 @@ class Gmake(Package):
     url = "https://ftpmirror.gnu.org/make/make-4.4.tar.gz"
 
     version("4.4", sha256="ce35865411f0490368a8fc383f29071de6690cbadc27704734978221f25e2bed")
+
+    def do_stage(self):
+        mkdirp(self.stage.source_path)
diff --git a/var/spack/repos/builtin.mock/packages/intel-parallel-studio/package.py b/var/spack/repos/builtin.mock/packages/intel-parallel-studio/package.py
new file mode 100644
index 00000000000000..1ec5cf6932619e
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/intel-parallel-studio/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class IntelParallelStudio(Package):
+    """Intel Parallel Studio."""
+
+    homepage = "https://software.intel.com/en-us/intel-parallel-studio-xe"
+    url = "http://tec/16225/parallel_studio_xe_2020_cluster_edition.tgz"
+
+    version("cluster.2020.0", sha256="b1d3e3e425b2e44a06760ff173104bdf")
+
+    provides("mpi@:3")
+    provides("scalapack")
+    provides("blas", "lapack")
diff --git a/var/spack/repos/builtin.mock/packages/invalid-gitlab-patch-url/package.py b/var/spack/repos/builtin.mock/packages/invalid-gitlab-patch-url/package.py
new file mode 100644
index 00000000000000..527a1815e62863
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/invalid-gitlab-patch-url/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class InvalidGitlabPatchUrl(Package):
+    """Package that has GitLab patch URLs that fail auditing."""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/patch-1.0.tar.gz"
+
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+
+    patch(
+        "https://gitlab.com/QEF/q-e/-/commit/4ca3afd4c6f27afcf3f42415a85a353a7be1bd37.patch",
+        sha256="d7dec588efb5c04f99d949d8b9bb4a0fbc98b917ae79e12e4b87ad7c3dc9e268",
+    )
diff --git a/var/spack/repos/builtin.mock/packages/invalid-selfhosted-gitlab-patch-url/package.py b/var/spack/repos/builtin.mock/packages/invalid-selfhosted-gitlab-patch-url/package.py
new file mode 100644
index 00000000000000..818876405c26f6
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/invalid-selfhosted-gitlab-patch-url/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class InvalidSelfhostedGitlabPatchUrl(Package):
+    """Package that has GitLab patch URLs that fail auditing."""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/patch-1.0.tar.gz"
+
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+
+    patch(
+        "https://gitlab.gnome.org/GNOME/glib/-/commit/bda87264372c006c94e21ffb8ff9c50ecb3e14bd.patch",
+        sha256="2e811ec62cb09044c95a4d0213993f09af70cdcc1c709257b33bc9248ae950ed",
+    )
diff --git a/var/spack/repos/builtin.mock/packages/licenses-1/package.py b/var/spack/repos/builtin.mock/packages/licenses-1/package.py
new file mode 100644
index 00000000000000..d5c67830c98f2e
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/licenses-1/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Licenses1(Package):
+    """Package with a licenses field."""
+
+    homepage = "https://www.example.com"
+    url = "https://www.example.com/license"
+
+    license("MIT", when="+foo")
+    license("Apache-2.0", when="~foo")
+
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
diff --git a/var/spack/repos/builtin.mock/packages/low-priority-provider/package.py b/var/spack/repos/builtin.mock/packages/low-priority-provider/package.py
index 5b7bfc03c1aad2..940dea3dafc13c 100644
--- a/var/spack/repos/builtin.mock/packages/low-priority-provider/package.py
+++ b/var/spack/repos/builtin.mock/packages/low-priority-provider/package.py
@@ -14,5 +14,5 @@ class LowPriorityProvider(Package):
 
     version("1.0", md5="0123456789abcdef0123456789abcdef")
 
-    provides("lapack")
-    provides("mpi")
+    # A low priority provider that provides both these specs together
+    provides("mpi", "lapack")
diff --git a/var/spack/repos/builtin.mock/packages/many-virtual-consumer/package.py b/var/spack/repos/builtin.mock/packages/many-virtual-consumer/package.py
index 070adf60bc80e1..087cfb77cccac1 100644
--- a/var/spack/repos/builtin.mock/packages/many-virtual-consumer/package.py
+++ b/var/spack/repos/builtin.mock/packages/many-virtual-consumer/package.py
@@ -19,4 +19,4 @@ class ManyVirtualConsumer(Package):
     # This directive is an example of imposing a constraint on a
     # dependency is that dependency is in the DAG. This pattern
     # is mainly used with virtual providers.
-    depends_on("low-priority-provider@1.0", when="^low-priority-provider")
+    depends_on("low-priority-provider@1.0", when="^[virtuals=mpi,lapack] low-priority-provider")
diff --git a/var/spack/repos/builtin.mock/packages/module-long-help/package.py b/var/spack/repos/builtin.mock/packages/module-long-help/package.py
new file mode 100644
index 00000000000000..43a0da412a8d43
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/module-long-help/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class ModuleLongHelp(Package):
+    """Package to test long description message generated in modulefile.
+    Message too long is wrapped over multiple lines."""
+
+    homepage = "http://www.llnl.gov"
+    url = "http://www.llnl.gov/module-long-help-1.0.tar.gz"
+
+    version("1.0", "0123456789abcdef0123456789abcdef")
+
+    def setup_run_environment(self, env):
+        env.set("FOO", "bar")
diff --git a/var/spack/repos/builtin.mock/packages/multivalue-variant/package.py b/var/spack/repos/builtin.mock/packages/multivalue-variant/package.py
index 136646324191dd..b0f7ac9501cd18 100644
--- a/var/spack/repos/builtin.mock/packages/multivalue-variant/package.py
+++ b/var/spack/repos/builtin.mock/packages/multivalue-variant/package.py
@@ -19,7 +19,7 @@ class MultivalueVariant(Package):
     variant(
         "foo",
         description="Multi-valued variant",
-        values=any_combination_of("bar", "baz", "barbaz"),
+        values=any_combination_of("bar", "baz", "barbaz", "fee"),
     )
 
     variant(
diff --git a/var/spack/repos/builtin.mock/packages/netlib-scalapack/package.py b/var/spack/repos/builtin.mock/packages/netlib-scalapack/package.py
new file mode 100644
index 00000000000000..fe5d7f90a1f27d
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/netlib-scalapack/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class NetlibScalapack(Package):
+    homepage = "http://www.netlib.org/scalapack/"
+    url = "http://www.netlib.org/scalapack/scalapack-2.1.0.tgz"
+
+    version("2.1.0", "b1d3e3e425b2e44a06760ff173104bdf")
+
+    provides("scalapack")
+
+    depends_on("mpi")
+    depends_on("lapack")
+    depends_on("blas")
diff --git a/var/spack/repos/builtin.mock/packages/old-sbang/package.py b/var/spack/repos/builtin.mock/packages/old-sbang/package.py
index c82696460d35da..2f7e8505b0efb3 100644
--- a/var/spack/repos/builtin.mock/packages/old-sbang/package.py
+++ b/var/spack/repos/builtin.mock/packages/old-sbang/package.py
@@ -30,7 +30,7 @@ def install(self, spec, prefix):
 
 {1}
 """.format(
-            spack.store.unpadded_root, prefix.bin
+            spack.store.STORE.unpadded_root, prefix.bin
         )
         with open("%s/sbang-style-1.sh" % self.prefix.bin, "w") as f:
             f.write(sbang_style_1)
diff --git a/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py b/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py
index 015608587756c6..1273b70def2127 100644
--- a/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py
+++ b/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py
@@ -14,5 +14,4 @@ class OpenblasWithLapack(Package):
 
     version("0.2.15", md5="b1190f3d3471685f17cfd1ec1d252ac9")
 
-    provides("lapack")
-    provides("blas")
+    provides("lapack", "blas")
diff --git a/var/spack/repos/builtin.mock/packages/parent-foo-bar/package.py b/var/spack/repos/builtin.mock/packages/parent-foo-bar/package.py
new file mode 100644
index 00000000000000..14516566a9f7d4
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/parent-foo-bar/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class ParentFooBar(Package):
+    """This package has a variant "bar", which is True by default, and depends on another
+    package which has the same variant defaulting to False.
+    """
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/parent-foo-bar-1.0.tar.gz"
+
+    version("1.0", md5="abcdefg0123456789abcdefghfedcba0")
+
+    variant("foo", default=True, description="")
+    variant("bar", default=True, description="")
+
+    depends_on("dependency-foo-bar")
diff --git a/var/spack/repos/builtin.mock/packages/parent-foo/package.py b/var/spack/repos/builtin.mock/packages/parent-foo/package.py
new file mode 100644
index 00000000000000..61d15231f70822
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/parent-foo/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class ParentFoo(Package):
+    """This package has a variant "foo", which is True by default, and depends on another
+    package which has the same variant defaulting to False.
+    """
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/c-1.0.tar.gz"
+
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+
+    variant("foo", default=True, description="")
+
+    depends_on("client-not-foo")
diff --git a/var/spack/repos/builtin.mock/packages/printing-package/package.py b/var/spack/repos/builtin.mock/packages/printing-package/package.py
index 9d41e0a3a62859..999b78a09536ee 100644
--- a/var/spack/repos/builtin.mock/packages/printing-package/package.py
+++ b/var/spack/repos/builtin.mock/packages/printing-package/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import os
+
 from spack.package import *
 
 
@@ -20,9 +22,8 @@ class PrintingPackage(Package):
     def install(self, spec, prefix):
         print("BEFORE INSTALL")
 
-        configure("--prefix=%s" % prefix)
-        make()
-        make("install")
+        mkdirp(prefix)
+        touch(os.path.join(prefix, "dummyfile"))
 
         print("AFTER INSTALL")
 
diff --git a/var/spack/repos/builtin.mock/packages/python/package.py b/var/spack/repos/builtin.mock/packages/python/package.py
index 4ba957e050384d..9fbee0ec2ed45b 100644
--- a/var/spack/repos/builtin.mock/packages/python/package.py
+++ b/var/spack/repos/builtin.mock/packages/python/package.py
@@ -14,6 +14,7 @@ class Python(Package):
 
     extendable = True
 
+    version("3.8.0", md5="d41d8cd98f00b204e9800998ecf8427e")
     version("3.7.1", md5="aaabbbcccdddeeefffaaabbbcccddd12")
     version("3.5.1", md5="be78e48cdfc1a7ad90efff146dce6cfe")
     version("3.5.0", md5="a56c0c0b45d75a0ec9c6dee933c41c36")
diff --git a/var/spack/repos/builtin.mock/packages/shell-a/package.py b/var/spack/repos/builtin.mock/packages/shell-a/package.py
new file mode 100644
index 00000000000000..3ff34102bfd628
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/shell-a/package.py
@@ -0,0 +1,17 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class ShellA(Package):
+    """Simple package with one dependency for shell tests"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/shell-a-1.0.tar.gz"
+
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+    version("2.0", md5="abcdef0123456789abcdef0123456789")
+
+    depends_on("shell-b")
diff --git a/var/spack/repos/builtin.mock/packages/shell-b/package.py b/var/spack/repos/builtin.mock/packages/shell-b/package.py
new file mode 100644
index 00000000000000..3db70f12189bcc
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/shell-b/package.py
@@ -0,0 +1,16 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class ShellB(Package):
+    """Simple package with no dependencies for shell tests"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/shell-b-1.0.tar.gz"
+
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+    version("0.9", md5="abcd456789abcdef0123456789abcdef")
diff --git a/var/spack/repos/builtin.mock/packages/vendorsb/package.py b/var/spack/repos/builtin.mock/packages/vendorsb/package.py
new file mode 100644
index 00000000000000..fbf14236a10be2
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/vendorsb/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Vendorsb(Package):
+    """A package that vendors another, and thus conflicts with it"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/b-1.0.tar.gz"
+
+    version("1.1", md5="0123456789abcdef0123456789abcdef")
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+
+    # b is not a dependency
+    conflicts("b", when="@=1.1")
diff --git a/var/spack/repos/builtin.mock/packages/vtk-m/package.py b/var/spack/repos/builtin.mock/packages/vtk-m/package.py
new file mode 100644
index 00000000000000..9c1eadf4c666d8
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/vtk-m/package.py
@@ -0,0 +1,37 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class VtkM(CMakePackage):
+    """This is a fake vtk-m package used to demonstrate virtual package providers
+    with dependencies."""
+
+    homepage = "http://www.spack-fake-vtk-m.org"
+    url = "http://www.spack-fake-vtk-m.org/downloads/vtk-m-1.0.tar.gz"
+
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+
+    variant("cuda", default=False, description="Build with CUDA")
+    variant(
+        "cuda_arch",
+        description="CUDA architecture",
+        default="none",
+        values=("70", "none"),
+        multi=False,
+        when="+cuda",
+    )
+
+    variant("rocm", default=False, description="Enable ROCm support")
+    variant(
+        "amdgpu_target",
+        default="none",
+        description="AMD GPU architecture",
+        values=("gfx900", "none"),
+        multi=False,
+        when="+rocm",
+    )
+    depends_on("cmake@3.18:")
diff --git a/var/spack/repos/builtin/packages/3dtk/package.py b/var/spack/repos/builtin/packages/3dtk/package.py
index c01faaaba62d1e..a3bac5bb6771b3 100644
--- a/var/spack/repos/builtin/packages/3dtk/package.py
+++ b/var/spack/repos/builtin/packages/3dtk/package.py
@@ -50,7 +50,7 @@ class _3dtk(CMakePackage):
         "boost@:1.75+serialization+graph+regex+filesystem+system+thread+date_time+program_options"
     )
     depends_on("suite-sparse")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libpng")
     depends_on("eigen")
     depends_on("cgal", when="+cgal")
diff --git a/var/spack/repos/builtin/packages/3proxy/package.py b/var/spack/repos/builtin/packages/3proxy/package.py
index 8cae9f900528cc..78e52895145b65 100644
--- a/var/spack/repos/builtin/packages/3proxy/package.py
+++ b/var/spack/repos/builtin/packages/3proxy/package.py
@@ -24,9 +24,9 @@ class _3proxy(MakefilePackage):
     depends_on("m4", type="build")
 
     def build(self, spec, prefix):
-        make("-f", "Makefile.{0}".format(platform.system()))
+        make("-f", f"Makefile.{platform.system()}", f"CC={spack_cc}")
 
     def install(self, spec, prefix):
         make(
-            "-f", "Makefile.{0}".format(platform.system()), "prefix={0}".format(prefix), "install"
+            "-f", f"Makefile.{platform.system()}", f"prefix={prefix}", f"CC={spack_cc}", "install"
         )
diff --git a/var/spack/repos/builtin/packages/7zip/package.py b/var/spack/repos/builtin/packages/7zip/package.py
index 2e9d09bbb9c63b..123e233ab188d2 100644
--- a/var/spack/repos/builtin/packages/7zip/package.py
+++ b/var/spack/repos/builtin/packages/7zip/package.py
@@ -75,8 +75,8 @@ def is_64bit(self):
     def build(self, spec, prefix):
         link_type = "1" if "static" in spec.variants["link_type"].value else "0"
         nmake_args = [
-            "PLATFORM=%s" % self.plat_arch,
-            "MY_STATIC_LINK=%s" % link_type,
+            f"PLATFORM={self.plat_arch}",
+            f"MY_STATIC_LINK={link_type}",
             "NEW_COMPILER=1",
         ]
         # 7zips makefile is configured in such as way that if this value is set
diff --git a/var/spack/repos/builtin/packages/abacus/package.py b/var/spack/repos/builtin/packages/abacus/package.py
index 68ee2f4894cc26..ccf89f82b8db66 100644
--- a/var/spack/repos/builtin/packages/abacus/package.py
+++ b/var/spack/repos/builtin/packages/abacus/package.py
@@ -65,7 +65,7 @@ def edit(self, spec, prefix):
                 spec["fftw"].prefix,
                 spec["elpa"].prefix,
                 inc_var,
-                "{0}".format(spec["elpa"].version),
+                f"{spec['elpa'].version}",
                 spec["cereal"].prefix,
             )
         )
diff --git a/var/spack/repos/builtin/packages/abduco/package.py b/var/spack/repos/builtin/packages/abduco/package.py
index 8287ecc7d94a35..9c3ea0c4487961 100644
--- a/var/spack/repos/builtin/packages/abduco/package.py
+++ b/var/spack/repos/builtin/packages/abduco/package.py
@@ -21,4 +21,4 @@ class Abduco(MakefilePackage):
     version("0.4", sha256="bda3729df116ce41f9a087188d71d934da2693ffb1ebcf33b803055eb478bcbb")
 
     def install(self, spec, prefix):
-        make("PREFIX={0}".format(prefix), "install")
+        make(f"PREFIX={prefix}", "install")
diff --git a/var/spack/repos/builtin/packages/abi-compliance-checker/package.py b/var/spack/repos/builtin/packages/abi-compliance-checker/package.py
index 05d57471ba24f2..6f575badd208dc 100644
--- a/var/spack/repos/builtin/packages/abi-compliance-checker/package.py
+++ b/var/spack/repos/builtin/packages/abi-compliance-checker/package.py
@@ -22,4 +22,4 @@ class AbiComplianceChecker(MakefilePackage):
     depends_on("universal-ctags")
 
     def install(self, spec, prefix):
-        make("prefix={0}".format(prefix), "install")
+        make(f"prefix={prefix}", "install")
diff --git a/var/spack/repos/builtin/packages/abi-dumper/package.py b/var/spack/repos/builtin/packages/abi-dumper/package.py
index f649bf9db20874..584eed3664cdd0 100644
--- a/var/spack/repos/builtin/packages/abi-dumper/package.py
+++ b/var/spack/repos/builtin/packages/abi-dumper/package.py
@@ -25,4 +25,4 @@ class AbiDumper(Package):
     depends_on("vtable-dumper@1.1:")
 
     def install(self, spec, prefix):
-        make("prefix={0}".format(prefix), "install")
+        make(f"prefix={prefix}", "install")
diff --git a/var/spack/repos/builtin/packages/abinit/package.py b/var/spack/repos/builtin/packages/abinit/package.py
index 445fc60bbe0985..9f0938daf1433b 100644
--- a/var/spack/repos/builtin/packages/abinit/package.py
+++ b/var/spack/repos/builtin/packages/abinit/package.py
@@ -85,6 +85,11 @@ class Abinit(AutotoolsPackage):
     # libxml2
     depends_on("libxml2", when="@9:+libxml2")
 
+    # If the Intel suite is used for Lapack, it must be used for fftw and vice-versa
+    for _intel_pkg in INTEL_MATH_LIBRARIES:
+        requires(f"^[virtuals=fftw-api] {_intel_pkg}", when=f"^[virtuals=lapack]   {_intel_pkg}")
+        requires(f"^[virtuals=lapack]   {_intel_pkg}", when=f"^[virtuals=fftw-api] {_intel_pkg}")
+
     # Cannot ask for +scalapack if it does not depend on MPI
     conflicts("+scalapack", when="~mpi")
 
@@ -101,11 +106,7 @@ class Abinit(AutotoolsPackage):
     # TODO: The logic here can be reversed with the new concretizer. Instead of
     # using `conflicts`, `depends_on` could be used instead.
     for fftw in ["amdfftw", "cray-fftw", "fujitsu-fftw", "fftw"]:
-        conflicts(
-            "+openmp",
-            when="^{0}~openmp".format(fftw),
-            msg="Need to request {0} +openmp".format(fftw),
-        )
+        conflicts("+openmp", when=f"^{fftw}~openmp", msg=f"Need to request {fftw} +openmp")
 
     mkl_message = "Need to set dependent variant to threads=openmp"
     conflicts("+openmp", when="^intel-mkl threads=none", msg=mkl_message)
@@ -137,34 +138,28 @@ def configure_args(self):
 
         oapp = options.append
         if "@:8" in spec:
-            oapp("--enable-optim={0}".format(self.spec.variants["optimization-flavor"].value))
+            oapp(f"--enable-optim={self.spec.variants['optimization-flavor'].value}")
         else:
-            oapp("--with-optim-flavor={0}".format(self.spec.variants["optimization-flavor"].value))
+            oapp(f"--with-optim-flavor={self.spec.variants['optimization-flavor'].value}")
 
         if "+wannier90" in spec:
             if "@:8" in spec:
-                oapp(
-                    "--with-wannier90-libs=-L{0}".format(
-                        spec["wannier90"].prefix.lib + " -lwannier -lm"
-                    )
-                )
-                oapp("--with-wannier90-incs=-I{0}".format(spec["wannier90"].prefix.modules))
-                oapp("--with-wannier90-bins={0}".format(spec["wannier90"].prefix.bin))
+                oapp(f"--with-wannier90-libs=-L{spec['wannier90'].prefix.lib} -lwannier -lm")
+                oapp(f"--with-wannier90-incs=-I{spec['wannier90'].prefix.modules}")
+                oapp(f"--with-wannier90-bins={spec['wannier90'].prefix.bin}")
                 oapp("--enable-connectors")
                 oapp("--with-dft-flavor=atompaw+libxc+wannier90")
             else:
                 options.extend(
                     [
-                        "WANNIER90_CPPFLAGS=-I{0}".format(spec["wannier90"].prefix.modules),
-                        "WANNIER90_LIBS=-L{0} {1}".format(
-                            spec["wannier90"].prefix.lib, "-lwannier"
-                        ),
+                        f"WANNIER90_CPPFLAGS=-I{spec['wannier90'].prefix.modules}",
+                        f"WANNIER90_LIBS=-L{spec['wannier90'].prefix.lib} -lwannier",
                     ]
                 )
         else:
             if "@:9.8" in spec:
-                oapp("--with-fftw={0}".format(spec["fftw-api"].prefix))
-                oapp("--with-hdf5={0}".format(spec["hdf5"].prefix))
+                oapp(f"--with-fftw={spec['fftw-api'].prefix}")
+                oapp(f"--with-hdf5={spec['hdf5'].prefix}")
 
             if "@:8" in spec:
                 oapp("--with-dft-flavor=atompaw+libxc")
@@ -172,9 +167,9 @@ def configure_args(self):
                 "--without-wannier90",
 
         if "+mpi" in spec:
-            oapp("CC={0}".format(spec["mpi"].mpicc))
-            oapp("CXX={0}".format(spec["mpi"].mpicxx))
-            oapp("FC={0}".format(spec["mpi"].mpifc))
+            oapp(f"CC={spec['mpi'].mpicc}")
+            oapp(f"CXX={spec['mpi'].mpicxx}")
+            oapp(f"FC={spec['mpi'].mpifc}")
 
             # MPI version:
             # let the configure script auto-detect MPI support from mpi_prefix
@@ -196,7 +191,8 @@ def configure_args(self):
 
         # BLAS/LAPACK/SCALAPACK-ELPA
         linalg = spec["lapack"].libs + spec["blas"].libs
-        if "^mkl" in spec:
+        is_using_intel_libraries = spec["lapack"].name in INTEL_MATH_LIBRARIES
+        if is_using_intel_libraries:
             linalg_flavor = "mkl"
         elif "@9:" in spec and "^openblas" in spec:
             linalg_flavor = "openblas"
@@ -208,16 +204,16 @@ def configure_args(self):
         if "+scalapack" in spec:
             linalg = spec["scalapack"].libs + linalg
             if "@:8" in spec:
-                linalg_flavor = "scalapack+{0}".format(linalg_flavor)
+                linalg_flavor = f"scalapack+{linalg_flavor}"
 
         if "@:8" in spec:
-            oapp("--with-linalg-libs={0}".format(linalg.ld_flags))
+            oapp(f"--with-linalg-libs={linalg.ld_flags}")
         else:
-            oapp("LINALG_LIBS={0}".format(linalg.ld_flags))
+            oapp(f"LINALG_LIBS={linalg.ld_flags}")
 
-        oapp("--with-linalg-flavor={0}".format(linalg_flavor))
+        oapp(f"--with-linalg-flavor={linalg_flavor}")
 
-        if "^mkl" in spec:
+        if is_using_intel_libraries:
             fftflavor = "dfti"
         else:
             if "+openmp" in spec:
@@ -225,32 +221,32 @@ def configure_args(self):
             else:
                 fftflavor, fftlibs = "fftw3", "-lfftw3 -lfftw3f"
 
-        oapp("--with-fft-flavor={0}".format(fftflavor))
+        oapp(f"--with-fft-flavor={fftflavor}")
 
         if "@:8" in spec:
-            if "^mkl" in spec:
-                oapp("--with-fft-incs={0}".format(spec["fftw-api"].headers.cpp_flags))
-                oapp("--with-fft-libs={0}".format(spec["fftw-api"].libs.ld_flags))
+            if is_using_intel_libraries:
+                oapp(f"--with-fft-incs={spec['fftw-api'].headers.cpp_flags}")
+                oapp(f"--with-fft-libs={spec['fftw-api'].libs.ld_flags}")
             else:
                 options.extend(
                     [
-                        "--with-fft-incs={0}".format(spec["fftw-api"].headers.cpp_flags),
-                        "--with-fft-libs=-L{0} {1}".format(spec["fftw-api"].prefix.lib, fftlibs),
+                        f"--with-fft-incs={spec['fftw-api'].headers.cpp_flags}",
+                        f"--with-fft-libs=-L{spec['fftw-api'].prefix.lib} {fftlibs}",
                     ]
                 )
         else:
-            if "^mkl" in spec:
+            if is_using_intel_libraries:
                 options.extend(
                     [
-                        "FFT_CPPFLAGS={0}".format(spec["fftw-api"].headers.cpp_flags),
-                        "FFT_LIBs={0}".format(spec["fftw-api"].libs.ld_flags),
+                        f"FFT_CPPFLAGS={spec['fftw-api'].headers.cpp_flags}",
+                        f"FFT_LIBs={spec['fftw-api'].libs.ld_flags}",
                     ]
                 )
             else:
                 options.extend(
                     [
-                        "FFTW3_CPPFLAGS={0}".format(spec["fftw-api"].headers.cpp_flags),
-                        "FFTW3_LIBS=-L{0} {1}".format(spec["fftw-api"].prefix.lib, fftlibs),
+                        f"FFTW3_CPPFLAGS={spec['fftw-api'].headers.cpp_flags}",
+                        f"FFTW3_LIBS=-L{spec['fftw-api'].prefix.lib} {fftlibs}",
                     ]
                 )
 
@@ -259,12 +255,12 @@ def configure_args(self):
         if "@:8" in spec:
             options.extend(
                 [
-                    "--with-libxc-incs={0}".format(libxc.headers.cpp_flags),
-                    "--with-libxc-libs={0}".format(libxc.libs.ld_flags + " -lm"),
+                    f"--with-libxc-incs={libxc.headers.cpp_flags}",
+                    f"--with-libxc-libs={libxc.libs.ld_flags + ' -lm'}",
                 ]
             )
         else:
-            oapp("--with-libxc={0}".format(libxc.prefix))
+            oapp(f"--with-libxc={libxc.prefix}")
 
         # Netcdf4/HDF5
         hdf5 = spec["hdf5:hl"]
@@ -276,24 +272,21 @@ def configure_args(self):
             # to link with the high level HDF5 library
             options.extend(
                 [
-                    "--with-netcdf-incs={0}".format(
+                    "--with-netcdf-incs={}".format(
                         netcdfc.headers.cpp_flags + " " + netcdff.headers.cpp_flags
                     ),
-                    "--with-netcdf-libs={0}".format(
+                    "--with-netcdf-libs={}".format(
                         netcdff.libs.ld_flags + " " + hdf5.libs.ld_flags
                     ),
                 ]
             )
         else:
             options.extend(
-                [
-                    "--with-netcdf={0}".format(netcdfc.prefix),
-                    "--with-netcdf-fortran={0}".format(netcdff.prefix),
-                ]
+                [f"--with-netcdf={netcdfc.prefix}", f"--with-netcdf-fortran={netcdff.prefix}"]
             )
 
         if self.spec.satisfies("%fj"):
-            oapp("FCFLAGS_MODDIR=-M{0}".format(join_path(self.stage.source_path, "src/mods")))
+            oapp(f"FCFLAGS_MODDIR=-M{join_path(self.stage.source_path, 'src/mods')}")
 
         return options
 
diff --git a/var/spack/repos/builtin/packages/abseil-cpp/package.py b/var/spack/repos/builtin/packages/abseil-cpp/package.py
index 77eb9b35bfaafc..a6b69015f8d9f2 100644
--- a/var/spack/repos/builtin/packages/abseil-cpp/package.py
+++ b/var/spack/repos/builtin/packages/abseil-cpp/package.py
@@ -15,6 +15,9 @@ class AbseilCpp(CMakePackage):
     maintainers("jcftang")
     tags = ["windows"]
 
+    version(
+        "20230125.3", sha256="5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36"
+    )
     version(
         "20230125.2", sha256="9a2b5752d7bfade0bdeee2701de17c9480620f8b237e1964c1b9967c75374906"
     )
diff --git a/var/spack/repos/builtin/packages/abyss/package.py b/var/spack/repos/builtin/packages/abyss/package.py
index c345626761d1c3..1cb46a8957eed1 100644
--- a/var/spack/repos/builtin/packages/abyss/package.py
+++ b/var/spack/repos/builtin/packages/abyss/package.py
@@ -60,12 +60,12 @@ class Abyss(AutotoolsPackage):
     def configure_args(self):
         maxk = int(self.spec.variants["maxk"].value)
         args = [
-            "--with-boost=%s" % self.spec["boost"].prefix,
-            "--with-sqlite=%s" % self.spec["sqlite"].prefix,
-            "--with-mpi=%s" % self.spec["mpi"].prefix,
+            f"--with-boost={self.spec['boost'].prefix}",
+            f"--with-sqlite={self.spec['sqlite'].prefix}",
+            f"--with-mpi={self.spec['mpi'].prefix}",
         ]
         if maxk:
-            args.append("--enable-maxk=%s" % maxk)
+            args.append(f"--enable-maxk={maxk}")
         if self.spec["mpi"].name == "mpich":
             args.append("--enable-mpich")
         return args
diff --git a/var/spack/repos/builtin/packages/accfft/package.py b/var/spack/repos/builtin/packages/accfft/package.py
index aa32f1b0a53e81..eb99aec48492d9 100644
--- a/var/spack/repos/builtin/packages/accfft/package.py
+++ b/var/spack/repos/builtin/packages/accfft/package.py
@@ -32,15 +32,15 @@ class Accfft(CMakePackage, CudaPackage):
     def cmake_args(self):
         spec = self.spec
         args = [
-            "-DFFTW_ROOT={0}".format(spec["fftw"].prefix),
-            "-DFFTW_USE_STATIC_LIBS=false",
-            "-DBUILD_GPU={0}".format("true" if "+cuda" in spec else "false"),
-            "-DBUILD_SHARED={0}".format("true" if "+shared" in spec else "false"),
+            self.define("FFTW_ROOT", spec["fftw"].prefix),
+            self.define("FFTW_USE_STATIC_LIBS", "false"),
+            self.define("BUILD_GPU", str(spec.satisfies("+cuda")).lower()),
+            self.define("BUILD_SHARED", str(spec.satisfies("+shared")).lower()),
         ]
 
         if "+cuda" in spec:
             cuda_arch = [x for x in spec.variants["cuda_arch"].value if x]
             if cuda_arch:
-                args.append("-DCUDA_NVCC_FLAGS={0}".format(" ".join(self.cuda_flags(cuda_arch))))
+                args.append(f"-DCUDA_NVCC_FLAGS={' '.join(self.cuda_flags(cuda_arch))}")
 
         return args
diff --git a/var/spack/repos/builtin/packages/ace/package.py b/var/spack/repos/builtin/packages/ace/package.py
index c152bbdeb60fc8..afd164fc31527c 100644
--- a/var/spack/repos/builtin/packages/ace/package.py
+++ b/var/spack/repos/builtin/packages/ace/package.py
@@ -43,4 +43,4 @@ def edit(self, spec, prefix):
                     "include $(ACE_ROOT)/include/makeinclude/"
                     "platform_linux" + supported[self.compiler.name] + ".GNU\n"
                 )
-                f.write("INSTALL_PREFIX=%s" % prefix)
+                f.write(f"INSTALL_PREFIX={prefix}")
diff --git a/var/spack/repos/builtin/packages/acfl/package.py b/var/spack/repos/builtin/packages/acfl/package.py
index 9147fa6dc704f2..bbe476a0198770 100644
--- a/var/spack/repos/builtin/packages/acfl/package.py
+++ b/var/spack/repos/builtin/packages/acfl/package.py
@@ -31,11 +31,46 @@
     "rhel8": "RHEL-8",
     "rhel9": "RHEL-9",
     "rocky8": "RHEL-8",
+    "rocky9": "RHEL-9",
     "amzn2": "AmazonLinux-2",
     "amzn2023": "AmazonLinux-2023",
 }
 
 _versions = {
+    "23.10": {
+        "RHEL-7": (
+            "c3bd4df3e5f6c97369237b0067e0a421dceb9c167d73f22f3da87f5025258314",
+            "https://developer.arm.com/-/media/Files/downloads/hpc/arm-compiler-for-linux/23-10/arm-compiler-for-linux_23.10_RHEL-7_aarch64.tar",
+        ),
+        "RHEL-8": (
+            "2aea8890a0c0f60bbcc5ddb043d13bd7cd10501218b04cbeb19129449e7d7053",
+            "https://developer.arm.com/-/media/Files/downloads/hpc/arm-compiler-for-linux/23-10/arm-compiler-for-linux_23.10_RHEL-8_aarch64.tar",
+        ),
+        "RHEL-9": (
+            "6c5c63c701875da7e87c6362be189bcbfaad678c08b81ec91e1e0252a321fae7",
+            "https://developer.arm.com/-/media/Files/downloads/hpc/arm-compiler-for-linux/23-10/arm-compiler-for-linux_23.10_RHEL-9_aarch64.tar",
+        ),
+        "SLES-15": (
+            "e1e62544210bae495cd2503ef280a748fda637c373f1eb76f5ff30c9ec92c4c1",
+            "https://developer.arm.com/-/media/Files/downloads/hpc/arm-compiler-for-linux/23-10/arm-compiler-for-linux_23.10_SLES-15_aarch64.tar",
+        ),
+        "Ubuntu-20.04": (
+            "83dce8ea03de3b9b937ecfc611961a8e4d15eba4c267a4e47e22a876e403da96",
+            "https://developer.arm.com/-/media/Files/downloads/hpc/arm-compiler-for-linux/23-10/arm-compiler-for-linux_23.10_Ubuntu-20.04_aarch64.tar",
+        ),
+        "Ubuntu-22.04": (
+            "3354f0ab73856a8a5cd99364cbec7a6b22621701790cb36c3e5f756b363e6d43",
+            "https://developer.arm.com/-/media/Files/downloads/hpc/arm-compiler-for-linux/23-10/arm-compiler-for-linux_23.10_Ubuntu-22.04_aarch64.tar",
+        ),
+        "AmazonLinux-2": (
+            "ee4fa47246f16323d05d91135ef70a8c355ff60209307754b8532b5744d9cfe9",
+            "https://developer.arm.com/-/media/Files/downloads/hpc/arm-compiler-for-linux/23-10/arm-compiler-for-linux_23.10_AmazonLinux-2_aarch64.tar",
+        ),
+        "AmazonLinux-2023": (
+            "640487dfc7ab6eca48b448264013c9aa972b84af9f0c6fc8734fa5e8dc008e43",
+            "https://developer.arm.com/-/media/Files/downloads/hpc/arm-compiler-for-linux/23-10/arm-compiler-for-linux_23.10_AmazonLinux-2023_aarch64.tar",
+        ),
+    },
     "23.04.1": {
         "RHEL-7": (
             "5e84daaf0510f73c235723112f9241bbd744ed89eb4f70f089bac05cf2aad2c4",
@@ -184,8 +219,7 @@ def get_acfl_prefix(spec):
         )
     else:
         return join_path(
-            spec.prefix,
-            "arm-linux-compiler-{0}_{1}".format(spec.version, get_os(spec.version.string)),
+            spec.prefix, f"arm-linux-compiler-{spec.version}_{get_os(spec.version.string)}"
         )
 
 
@@ -200,7 +234,7 @@ class Acfl(Package):
     """
 
     homepage = "https://developer.arm.com/Tools%20and%20Software/Arm%20Compiler%20for%20Linux"
-    url = "https://developer.arm.com/-/media/Files/downloads/hpc/arm-compiler-for-linux/23-04-1/arm-compiler-for-linux_23.04.1_Ubuntu-22.04_aarch64.tar"
+    url = "https://developer.arm.com/-/media/Files/downloads/hpc/arm-compiler-for-linux/23-10/arm-compiler-for-linux_23.10_Ubuntu-22.04_aarch64.tar"
 
     maintainers("annop-w")
 
@@ -237,7 +271,7 @@ class Acfl(Package):
     # Run the installer with the desired install directory
     def install(self, spec, prefix):
         exe = Executable(
-            "./arm-compiler-for-linux_{0}_{1}.sh".format(spec.version, get_os(spec.version.string))
+            f"./arm-compiler-for-linux_{spec.version}_{get_os(spec.version.string)}.sh"
         )
         exe("--accept", "--force", "--install-to", prefix)
 
diff --git a/var/spack/repos/builtin/packages/ack/package.py b/var/spack/repos/builtin/packages/ack/package.py
index 684106ff5db260..320f9e818cc60a 100644
--- a/var/spack/repos/builtin/packages/ack/package.py
+++ b/var/spack/repos/builtin/packages/ack/package.py
@@ -41,7 +41,7 @@ class Ack(Package):
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
-        ack_source = "ack-{0}-single-file".format(self.version)
+        ack_source = f"ack-{self.version}-single-file"
         ack_installed = join_path(prefix.bin, "ack")
 
         # install source
diff --git a/var/spack/repos/builtin/packages/acpica-tools/package.py b/var/spack/repos/builtin/packages/acpica-tools/package.py
index 55fee583c7841b..c9d063a5a975f1 100644
--- a/var/spack/repos/builtin/packages/acpica-tools/package.py
+++ b/var/spack/repos/builtin/packages/acpica-tools/package.py
@@ -19,4 +19,4 @@ class AcpicaTools(MakefilePackage):
     depends_on("bison", type="build")
 
     def install(self, spec, prefix):
-        make("PREFIX={0}".format(prefix), "install")
+        make(f"PREFIX={prefix}", "install")
diff --git a/var/spack/repos/builtin/packages/activeharmony/package.py b/var/spack/repos/builtin/packages/activeharmony/package.py
index e3f2d92955a6c4..62af3515b9051c 100644
--- a/var/spack/repos/builtin/packages/activeharmony/package.py
+++ b/var/spack/repos/builtin/packages/activeharmony/package.py
@@ -29,7 +29,7 @@ def setup_build_environment(self, spack_env):
 
     @when("@:4.5")
     def install(self, spec, prefix):
-        make("install", "PREFIX=%s" % prefix)
+        make("install", f"PREFIX={prefix}")
 
     @when("@4.6.0:")
     def install(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/acts/package.py b/var/spack/repos/builtin/packages/acts/package.py
index 2c7036d0d70f50..f474b92cc98c99 100644
--- a/var/spack/repos/builtin/packages/acts/package.py
+++ b/var/spack/repos/builtin/packages/acts/package.py
@@ -40,6 +40,28 @@ class Acts(CMakePackage, CudaPackage):
     # Supported Acts versions
     version("main", branch="main")
     version("master", branch="main", deprecated=True)  # For compatibility
+    version("30.3.2", commit="76826f208f5929d8326798c87263f2563d0ae7e9", submodules=True)
+    version("30.3.1", commit="bbee459dd93855417d5717d53cbbb2bace7de2bb", submodules=True)
+    version("30.3.0", commit="311acb9ab41c2d79a4b90b193e5b25297182d670", submodules=True)
+    version("30.2.0", commit="264b0a3214cbf8ca013623fc196e2d90d647c58f", submodules=True)
+    version("30.1.1", commit="3d43492b2775e62051e9ad31f06b91d6e2357ab9", submodules=True)
+    version("30.1.0", commit="60d9eec916f6c81373858c8d99d821861d7efeb8", submodules=True)
+    version("30.0.0", commit="00fa3fabac86a1e65198d4b94dd263b1c731a84c", submodules=True)
+    version("29.2.0", commit="b2d65308399d8f653fa8bdd73a2a203c58608358", submodules=True)
+    version("29.1.0", commit="4681c3b142db469b00ca03e92e6b237f7c89d141", submodules=True)
+    version("29.0.0", commit="9c6e4597af39f826e17d46850fdb407a48817ba6", submodules=True)
+    version("28.2.0", commit="c612e7c625f961330e383fb7856cc7398dd82881", submodules=True)
+    version("28.1.0", commit="08e51b5f93c0d09f2d1e7e4f062e715072ec3e9b", submodules=True)
+    version("28.0.0", commit="0d8aa418c00e8f79bab2cf88234f3433670b447c", submodules=True)
+    version("27.1.0", commit="219480220738318fbedb943cac85415687d75b66", submodules=True)
+    version("27.0.0", commit="4d7029bd4e9285fcda2770aef6d78a7f833cb14f", submodules=True)
+    version("26.0.0", commit="d43af5c3bcf44f593721940e569ba267579e0334", submodules=True)
+    version("25.0.1", commit="1772d2a1a96acfd918a538c14a9e24fe7ce7f50b", submodules=True)
+    version("25.0.0", commit="c7b538d7b92fe6bffd619de7885b7ea97ddcd26a", submodules=True)
+    version("24.0.0", commit="4da149cd27ae3802a54f21a48e1757e475aa8189", submodules=True)
+    version("23.5.0", commit="8fad985ab78ceb9aaec25d1f658197833e4586fa", submodules=True)
+    version("23.4.0", commit="52723f7e7a2e6f9f59d6d7ca1cf183ca1cd43380", submodules=True)
+    version("23.3.0", commit="ec3e69da90b9dff52bdbe30cb7953417b6184d4b", submodules=True)
     version("23.2.1", commit="a9fe5167d4d3b6b53b28d3b17060a5f3e380cf3a", submodules=True)
     version("23.2.0", commit="bc3120d23a72cfdd0ea8f9a0997f59caf311672b", submodules=True)
     version("23.1.0", commit="4479f182a37650a538344f749b967d6f757bdf60", submodules=True)
@@ -189,6 +211,7 @@ class Acts(CMakePackage, CudaPackage):
         description="Build the geometric digitization plugin",
         when="@:16",
     )
+    variant("edm4hep", default=False, description="Build EDM4hep plugin", when="@25:")
     # FIXME: Can't build Exa.TrkX plugin+examples yet, missing cuGraph dep
     variant(
         "fatras",
@@ -200,8 +223,10 @@ class Acts(CMakePackage, CudaPackage):
     variant("identification", default=False, description="Build the Identification plugin")
     variant("json", default=False, description="Build the Json plugin")
     variant("legacy", default=False, description="Build the Legacy package")
+    variant("mlpack", default=False, description="Build MLpack plugin", when="@25:")
     variant("onnx", default=False, description="Build ONNX plugin")
     variant("odd", default=False, description="Build the Open Data Detector", when="@19.1:")
+    variant("podio", default=False, description="Build Podio plugin", when="@30.3:")
     variant(
         "profilecpu",
         default=False,
@@ -218,11 +243,14 @@ class Acts(CMakePackage, CudaPackage):
     variant("tgeo", default=False, description="Build the TGeo plugin", when="+identification")
 
     # Variants that only affect Acts examples for now
+    variant(
+        "binaries", default=False, description="Build the examples binaries", when="@23: +examples"
+    )
     variant(
         "edm4hep",
         default=False,
         description="Build the EDM4hep examples",
-        when="@19.4.0: +examples",
+        when="@19.4.0:24 +examples",
     )
     variant(
         "geant4",
@@ -261,6 +289,10 @@ class Acts(CMakePackage, CudaPackage):
     depends_on("acts-dd4hep", when="@19 +dd4hep")
     depends_on("actsvg@0.4.20:", when="@20.1: +svg")
     depends_on("actsvg@0.4.28:", when="@23.2: +svg")
+    depends_on("actsvg@0.4.29:", when="@23.4: +svg")
+    depends_on("actsvg@0.4.30:", when="@23.5: +svg")
+    depends_on("actsvg@0.4.33:", when="@25:27 +svg")
+    depends_on("actsvg@0.4.35:", when="@28: +svg")
     depends_on("autodiff @0.6:", when="@17: +autodiff")
     depends_on("autodiff @0.5.11:0.5.99", when="@1.2:16 +autodiff")
     depends_on("boost @1.62:1.69 +program_options +test", when="@:0.10.3")
@@ -270,6 +302,7 @@ class Acts(CMakePackage, CudaPackage):
     depends_on("dd4hep @1.21: +dddetectors +ddrec", when="@20: +dd4hep")
     depends_on("dd4hep +ddg4", when="+dd4hep +geant4 +examples")
     depends_on("edm4hep @0.4.1:", when="+edm4hep")
+    depends_on("edm4hep @0.7:", when="@25: +edm4hep")
     depends_on("eigen @3.3.7:", when="@15.1:")
     depends_on("eigen @3.3.7:3.3.99", when="@:15.0")
     depends_on("geant4", when="+fatras_geant4")
@@ -280,12 +313,20 @@ class Acts(CMakePackage, CudaPackage):
     depends_on("hepmc3 @3.2.1:", when="+hepmc3")
     depends_on("heppdt", when="+hepmc3 @:4.0")
     depends_on("intel-tbb @2020.1:", when="+examples +tbb")
+    depends_on("mlpack@3.1.1:", when="+mlpack")
     depends_on("nlohmann-json @3.9.1:", when="@0.14: +json")
+    depends_on("podio @0.6:", when="@25: +edm4hep")
+    depends_on("podio @0.16:", when="@30.3: +edm4hep")
+    depends_on("podio @0.16:", when="+podio")
     depends_on("pythia8", when="+pythia8")
     depends_on("python", when="+python")
     depends_on("python@3.8:", when="+python @19.11:19")
     depends_on("python@3.8:", when="+python @21:")
-    depends_on("py-onnxruntime", when="+onnx")
+    depends_on("py-onnxruntime@:1.12", when="+onnx @:23.2")
+    # FIXME py-onnxruntime@1.12: required but not yet available
+    # Ref: https://github.com/spack/spack/pull/37064
+    # depends_on("py-onnxruntime@1.12:", when="+onnx @23.3:")
+    conflicts("+onnx", when="@23.3:", msg="py-onnxruntime@1.12: required but not yet available")
     depends_on("py-pybind11 @2.6.2:", when="+python @18:")
     depends_on("py-pytest", when="+python +unit_tests")
     depends_on("root @6.10:", when="+tgeo @:0.8.0")
@@ -297,8 +338,12 @@ class Acts(CMakePackage, CudaPackage):
     for _cxxstd in _cxxstd_values:
         if isinstance(_cxxstd, _ConditionalVariantValues):
             for _v in _cxxstd:
+                depends_on(
+                    f"geant4 cxxstd={_v.value}", when=f"cxxstd={_v.value} {_v.when} ^geant4"
+                )
                 depends_on(f"root cxxstd={_v.value}", when=f"cxxstd={_v.value} {_v.when} ^root")
         else:
+            depends_on(f"geant4 cxxstd={_v.value}", when=f"cxxstd={_v.value} {_v.when} ^geant4")
             depends_on(f"root cxxstd={_cxxstd}", when=f"cxxstd={_cxxstd} ^root")
 
     # ACTS has been using C++17 for a while, which precludes use of old GCC
@@ -309,15 +354,15 @@ def cmake_args(self):
 
         def cmake_variant(cmake_label, spack_variant):
             enabled = spec.satisfies("+" + spack_variant)
-            return "-DACTS_BUILD_{0}={1}".format(cmake_label, enabled)
+            return f"-DACTS_BUILD_{cmake_label}={enabled}"
 
         def enable_cmake_variant(cmake_label, spack_variant):
             enabled = spec.satisfies(spack_variant)
-            return "-DACTS_ENABLE_{0}={1}".format(cmake_label, enabled)
+            return f"-DACTS_ENABLE_{cmake_label}={enabled}"
 
         def example_cmake_variant(cmake_label, spack_variant, type="BUILD"):
             enabled = spec.satisfies("+examples +" + spack_variant)
-            return "-DACTS_{0}_EXAMPLES_{1}={2}".format(type, cmake_label, enabled)
+            return f"-DACTS_{type}_EXAMPLES_{cmake_label}={enabled}"
 
         def plugin_label(plugin_name):
             if spec.satisfies("@0.33:"):
@@ -342,10 +387,12 @@ def plugin_cmake_variant(plugin_name, spack_variant):
             cmake_variant("ANALYSIS_APPS", "analysis"),
             plugin_cmake_variant("AUTODIFF", "autodiff"),
             cmake_variant("BENCHMARKS", "benchmarks"),
+            example_cmake_variant("BINARIES", "binaries"),
             plugin_cmake_variant("CUDA", "cuda"),
             plugin_cmake_variant("DD4HEP", "dd4hep"),
             example_cmake_variant("DD4HEP", "dd4hep"),
             plugin_cmake_variant("DIGITIZATION", "digitization"),
+            plugin_cmake_variant("EDM4HEP", "edm4hep"),
             example_cmake_variant("EDM4HEP", "edm4hep"),
             cmake_variant("EXAMPLES", "examples"),
             cmake_variant("FATRAS", "fatras"),
@@ -357,10 +404,12 @@ def plugin_cmake_variant(plugin_name, spack_variant):
             cmake_variant(integration_tests_label, "integration_tests"),
             plugin_cmake_variant("JSON", "json"),
             cmake_variant(legacy_plugin_label, "legacy"),
+            plugin_cmake_variant("MLPACK", "mlpack"),
             cmake_variant("ODD", "odd"),
             plugin_cmake_variant("ONNX", "onnx"),
             enable_cmake_variant("CPU_PROFILING", "profilecpu"),
             enable_cmake_variant("MEMORY_PROFILING", "profilemem"),
+            plugin_cmake_variant("PODIO", "podio"),
             example_cmake_variant("PYTHIA8", "pythia8"),
             example_cmake_variant("PYTHON_BINDINGS", "python"),
             plugin_cmake_variant("ACTSVG", "svg"),
@@ -371,7 +420,7 @@ def plugin_cmake_variant(plugin_name, spack_variant):
         ]
 
         log_failure_threshold = spec.variants["log_failure_threshold"].value
-        args.append("-DACTS_LOG_FAILURE_THRESHOLD={0}".format(log_failure_threshold))
+        args.append(f"-DACTS_LOG_FAILURE_THRESHOLD={log_failure_threshold}")
         if spec.satisfies("@19.4.0:"):
             args.append("-DACTS_ENABLE_LOG_FAILURE_THRESHOLD=ON")
 
@@ -402,11 +451,11 @@ def plugin_cmake_variant(plugin_name, spack_variant):
         if "+cuda" in spec:
             cuda_arch = spec.variants["cuda_arch"].value
             if cuda_arch != "none":
-                args.append("-DCUDA_FLAGS=-arch=sm_{0}".format(cuda_arch[0]))
+                args.append(f"-DCUDA_FLAGS=-arch=sm_{cuda_arch[0]}")
 
         if "+python" in spec:
             python = spec["python"].command.path
-            args.append("-DPython_EXECUTABLE={0}".format(python))
+            args.append(f"-DPython_EXECUTABLE={python}")
 
         args.append(self.define_from_variant("CMAKE_CXX_STANDARD", "cxxstd"))
 
diff --git a/var/spack/repos/builtin/packages/actsvg/package.py b/var/spack/repos/builtin/packages/actsvg/package.py
index b54ec6f5e23d06..ccf603ed13ea64 100644
--- a/var/spack/repos/builtin/packages/actsvg/package.py
+++ b/var/spack/repos/builtin/packages/actsvg/package.py
@@ -18,6 +18,7 @@ class Actsvg(CMakePackage):
 
     maintainers("HadrienG2", "wdconinc")
 
+    version("0.4.35", sha256="693a4cc0e702842072a478c913895ed3596350ffdfa87f5d296ddd6ea36b61c6")
     version("0.4.33", sha256="25c93b8382bdb1864b4d8de64b146fe8ea86eec84048d594c375700d2fff1d1d")
     version("0.4.30", sha256="f7ffea39b3132914fcbb0fac6ab7395bef295cd6078dfd1c2509fd2d9aab0acb")
     version("0.4.29", sha256="971f4f344c3143b654e6a86422534c6916f785f2c2c3785664c4ae7ddf2f5e4b")
diff --git a/var/spack/repos/builtin/packages/additivefoam/assets/Allwmake b/var/spack/repos/builtin/packages/additivefoam/assets/Allwmake
new file mode 100755
index 00000000000000..e0aa5c6b87a127
--- /dev/null
+++ b/var/spack/repos/builtin/packages/additivefoam/assets/Allwmake
@@ -0,0 +1,4 @@
+#!/bin/sh
+cd ${0%/*} || exit 1    # Run from this directory
+
+applications/Allwmake $targetType $*
diff --git a/var/spack/repos/builtin/packages/additivefoam/assets/applications/Allwmake b/var/spack/repos/builtin/packages/additivefoam/assets/applications/Allwmake
new file mode 100755
index 00000000000000..c0edc3142b8d10
--- /dev/null
+++ b/var/spack/repos/builtin/packages/additivefoam/assets/applications/Allwmake
@@ -0,0 +1,5 @@
+#!/bin/sh
+cd ${0%/*} || exit 1    # Run from this directory
+
+wmake libso solvers/additiveFoam/movingHeatSource
+wmake solvers/additiveFoam
diff --git a/var/spack/repos/builtin/packages/additivefoam/package.py b/var/spack/repos/builtin/packages/additivefoam/package.py
new file mode 100644
index 00000000000000..3141c4b6ee8a8b
--- /dev/null
+++ b/var/spack/repos/builtin/packages/additivefoam/package.py
@@ -0,0 +1,59 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+from spack.pkg.builtin.openfoam import add_extra_files
+
+
+class Additivefoam(Package):
+    """AdditiveFOAM is a heat and mass transfer software for Additive Manufacturing (AM)"""
+
+    homepage = "https://github.com/ORNL/AdditiveFOAM"
+    git = "https://github.com/ORNL/AdditiveFOAM.git"
+    url = "https://github.com/ORNL/AdditiveFOAM/archive/1.0.0.tar.gz"
+
+    maintainers("streeve", "colemanjs", "gknapp1")
+
+    tags = ["ecp"]
+
+    version("main", branch="main")
+    version("1.0.0", sha256="abbdf1b0230cd2f26f526be76e973f508978611f404fe8ec4ecdd7d5df88724c")
+
+    depends_on("openfoam-org@10")
+
+    common = ["spack-derived-Allwmake"]
+    assets = ["applications/Allwmake", "Allwmake"]
+
+    build_script = "./spack-derived-Allwmake"
+
+    phases = ["configure", "build", "install"]
+
+    def patch(self):
+        add_extra_files(self, self.common, self.assets)
+
+    def configure(self, spec, prefix):
+        pass
+
+    def build(self, spec, prefix):
+        """Build with Allwmake script, wrapped to source environment first."""
+        args = []
+        if self.parallel:  # Parallel build? - pass via environment
+            os.environ["WM_NCOMPPROCS"] = str(make_jobs)
+        builder = Executable(self.build_script)
+        builder(*args)
+
+    def install(self, spec, prefix):
+        """Install under the prefix directory"""
+
+        for f in ["README.md", "LICENSE"]:
+            if os.path.isfile(f):
+                install(f, join_path(self.prefix, f))
+
+        dirs = ["tutorials", "applications"]
+        for d in dirs:
+            if os.path.isdir(d):
+                install_tree(d, join_path(self.prefix, d), symlinks=True)
diff --git a/var/spack/repos/builtin/packages/adf/package.py b/var/spack/repos/builtin/packages/adf/package.py
index 908cd5351faf16..244087f1116027 100644
--- a/var/spack/repos/builtin/packages/adf/package.py
+++ b/var/spack/repos/builtin/packages/adf/package.py
@@ -19,7 +19,7 @@ class Adf(Package):
     version("2017.113", sha256="666ef15d253b74c707dd14da35e7cf283ca20e21e24ed43cb953fb9d1f2f1e15")
 
     def url_for_version(self, version):
-        return "file://{0}/adf/adf{1}.pc64_linux.openmpi.bin.tgz".format(os.getcwd(), version)
+        return f"file://{os.getcwd()}/adf/adf{version}.pc64_linux.openmpi.bin.tgz"
 
     # Licensing
     license_required = True
diff --git a/var/spack/repos/builtin/packages/adiak/package.py b/var/spack/repos/builtin/packages/adiak/package.py
index 5bc8804dea9bac..05f936e3f92c5a 100644
--- a/var/spack/repos/builtin/packages/adiak/package.py
+++ b/var/spack/repos/builtin/packages/adiak/package.py
@@ -36,8 +36,8 @@ class Adiak(CMakePackage):
     def cmake_args(self):
         args = []
         if self.spec.satisfies("+mpi"):
-            args.append("-DMPI_CXX_COMPILER=%s" % self.spec["mpi"].mpicxx)
-            args.append("-DMPI_C_COMPILER=%s" % self.spec["mpi"].mpicc)
+            args.append(f"-DMPI_CXX_COMPILER={self.spec['mpi'].mpicxx}")
+            args.append(f"-DMPI_C_COMPILER={self.spec['mpi'].mpicc}")
             args.append("-DENABLE_MPI=ON")
         else:
             args.append("-DENABLE_MPI=OFF")
diff --git a/var/spack/repos/builtin/packages/adios/package.py b/var/spack/repos/builtin/packages/adios/package.py
index 06e39234779e9e..86bc26e89e9046 100644
--- a/var/spack/repos/builtin/packages/adios/package.py
+++ b/var/spack/repos/builtin/packages/adios/package.py
@@ -65,7 +65,7 @@ class Adios(AutotoolsPackage):
 
     depends_on("mpi", when="+mpi")
     # optional transformations
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("bzip2", when="+bzip2")
     depends_on("szip", when="+szip")
     depends_on("sz@:1.4.10", when="@:1.12.0 +sz")
@@ -119,7 +119,7 @@ def validate(self, spec):
 
     def with_or_without_hdf5(self, activated):
         if activated:
-            return "--with-phdf5={0}".format(self.spec["hdf5"].prefix)
+            return f"--with-phdf5={self.spec['hdf5'].prefix}"
 
         return "--without-phdf5"
 
@@ -134,7 +134,7 @@ def configure_args(self):
 
         extra_args = [
             # required, otherwise building its python bindings will fail
-            "CFLAGS={0}".format(self.compiler.cc_pic_flag)
+            f"CFLAGS={self.compiler.cc_pic_flag}"
         ]
 
         extra_args += self.enable_or_disable("shared")
@@ -147,8 +147,13 @@ def configure_args(self):
         extra_args += self.with_or_without("mpi", activation_value="prefix")
         extra_args += self.with_or_without("infiniband")
 
+        if "+zlib" in spec:
+            extra_args.append(f"--with-zlib={spec['zlib-api'].prefix}")
+        else:
+            extra_args.append("--without-zlib")
+
         # Transforms
-        variants = ["zlib", "bzip2", "szip"]
+        variants = ["bzip2", "szip"]
         if spec.satisfies("@1.11.0:"):
             variants += ["zfp"]
         if spec.satisfies("@1.12.0:"):
diff --git a/var/spack/repos/builtin/packages/adios2/2.9.2-cmake-find-threads-package-first.patch b/var/spack/repos/builtin/packages/adios2/2.9.2-cmake-find-threads-package-first.patch
new file mode 100644
index 00000000000000..afc6808d2862a6
--- /dev/null
+++ b/var/spack/repos/builtin/packages/adios2/2.9.2-cmake-find-threads-package-first.patch
@@ -0,0 +1,36 @@
+From 80e4739fb53b0b7e02dae48b928d8b8247992763 Mon Sep 17 00:00:00 2001
+From: Vicente Adolfo Bolea Sanchez 
+Date: Thu, 2 Nov 2023 12:18:49 -0400
+Subject: [PATCH] cmake: find threads package first
+
+---
+ cmake/DetectOptions.cmake | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake
+index 3f511e02a..615995b71 100644
+--- a/cmake/DetectOptions.cmake
++++ b/cmake/DetectOptions.cmake
+@@ -67,6 +67,9 @@ function(lists_get_prefix listVars outVar)
+   set(${outVar} "${prefix}" PARENT_SCOPE)
+ endfunction()
+ 
++# Multithreading
++find_package(Threads REQUIRED)
++
+ # Blosc2
+ if(ADIOS2_USE_Blosc2 STREQUAL AUTO)
+   # Prefect CONFIG mode
+@@ -554,9 +557,6 @@ if(AWSSDK_FOUND)
+     set(ADIOS2_HAVE_AWSSDK TRUE)
+ endif()
+ 
+-# Multithreading
+-find_package(Threads REQUIRED)
+-
+ # Floating point detection
+ include(CheckTypeRepresentation)
+ 
+-- 
+2.35.3
+
diff --git a/var/spack/repos/builtin/packages/adios2/package.py b/var/spack/repos/builtin/packages/adios2/package.py
index 76dc49e6479f67..4a038ddcacafc5 100644
--- a/var/spack/repos/builtin/packages/adios2/package.py
+++ b/var/spack/repos/builtin/packages/adios2/package.py
@@ -4,17 +4,19 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import os
+import tempfile
 
 from spack.package import *
 
 
-class Adios2(CMakePackage, CudaPackage):
+class Adios2(CMakePackage, CudaPackage, ROCmPackage):
     """The Adaptable Input Output System version 2,
     developed in the Exascale Computing Program"""
 
     homepage = "https://csmd.ornl.gov/software/adios2"
     url = "https://github.com/ornladios/ADIOS2/archive/v2.8.0.tar.gz"
     git = "https://github.com/ornladios/ADIOS2.git"
+    test_requires_compiler = True
 
     maintainers("ax3l", "vicentebolea", "williamfgc")
 
@@ -22,10 +24,12 @@ class Adios2(CMakePackage, CudaPackage):
 
     version("master", branch="master")
     version(
-        "2.9.0",
-        sha256="69f98ef58c818bb5410133e1891ac192653b0ec96eb9468590140f2552b6e5d1",
+        "2.9.2",
+        sha256="78309297c82a95ee38ed3224c98b93d330128c753a43893f63bbe969320e4979",
         preferred=True,
     )
+    version("2.9.1", sha256="ddfa32c14494250ee8a48ef1c97a1bf6442c15484bbbd4669228a0f90242f4f9")
+    version("2.9.0", sha256="69f98ef58c818bb5410133e1891ac192653b0ec96eb9468590140f2552b6e5d1")
     version("2.8.3", sha256="4906ab1899721c41dd918dddb039ba2848a1fb0cf84f3a563a1179b9d6ee0d9f")
     version("2.8.2", sha256="9909f6409dc44b2c28c1fda0042dab4b711f25ec3277ef0cb6ffc40f5483910d")
     version("2.8.1", sha256="3f515b442bbd52e3189866b121613fe3b59edb8845692ea86fad83d1eba35d93")
@@ -59,11 +63,13 @@ class Adios2(CMakePackage, CudaPackage):
     variant(
         "libpressio", default=False, when="@2.8:", description="Enable LibPressio for compression"
     )
-    variant("blosc", default=True, when="@2.4:", description="Enable Blosc compression")
+    variant("blosc", default=True, when="@2.4:2.8", description="Enable Blosc compression")
+    variant("blosc2", default=True, when="@2.9:", description="Enable Blosc2 compression")
     variant("bzip2", default=True, when="@2.4:", description="Enable BZip2 compression")
     variant("zfp", default=True, description="Enable ZFP compression")
     variant("png", default=True, when="@2.4:", description="Enable PNG compression")
     variant("sz", default=True, description="Enable SZ compression")
+    variant("mgard", default=True, when="@2.8:", description="Enable MGARD compression")
 
     # Rransport engines
     variant("sst", default=True, description="Enable the SST staging engine")
@@ -74,11 +80,24 @@ class Adios2(CMakePackage, CudaPackage):
         description="Enable the DataMan engine for WAN transports",
     )
     variant("dataspaces", default=False, when="@2.5:", description="Enable support for DATASPACES")
-    variant("ssc", default=True, description="Enable the SSC staging engine")
+    variant("ssc", default=True, when="@:2.7", description="Enable the SSC staging engine")
     variant("hdf5", default=False, description="Enable the HDF5 engine")
+    variant(
+        "aws",
+        default=False,
+        when="@2.9:",
+        description="Enable support for S3 compatible storage using AWS SDK's S3 module",
+    )
+    variant(
+        "libcatalyst",
+        default=True,
+        when="@2.9:",
+        description="Enable support for in situ visualization plugin using ParaView Catalyst",
+    )
 
     # Optional language bindings, C++11 and C always provided
-    variant("cuda", default=False, when="@2.8:", description="Enable CUDA support")
+    variant("kokkos", default=False, when="@2.9:", description="Enable Kokkos support")
+    variant("sycl", default=False, when="@2.10:", description="Enable SYCL support")
     variant("python", default=False, description="Enable the Python bindings")
     variant("fortran", default=True, description="Enable the Fortran bindings")
 
@@ -92,20 +111,51 @@ class Adios2(CMakePackage, CudaPackage):
 
     depends_on("cmake@3.12.0:", type="build")
 
+    # Standalone CUDA support
+    depends_on("cuda", when="+cuda ~kokkos")
+
+    # Kokkos support
+    depends_on("kokkos@3.7: +cuda +wrapper", when="+kokkos +cuda")
+    depends_on("kokkos@3.7: +rocm", when="+kokkos +rocm")
+    depends_on("kokkos@3.7: +sycl", when="+kokkos +sycl")
+
+    # Propagate CUDA target to kokkos for +cuda
+    for cuda_arch in CudaPackage.cuda_arch_values:
+        depends_on(
+            "kokkos cuda_arch=%s" % cuda_arch, when="+kokkos +cuda cuda_arch=%s" % cuda_arch
+        )
+
+    # Propagate AMD GPU target to kokkos for +rocm
+    for amdgpu_value in ROCmPackage.amdgpu_targets:
+        depends_on(
+            "kokkos amdgpu_target=%s" % amdgpu_value,
+            when="+kokkos +rocm amdgpu_target=%s" % amdgpu_value,
+        )
+
+    conflicts("+cuda", when="@:2.7")
+    conflicts("+rocm", when="@:2.8")
+
+    conflicts("+cuda", when="+sycl")
+    conflicts("+rocm", when="+cuda")
+    conflicts("+rocm", when="+sycl")
+
+    conflicts("+rocm", when="~kokkos", msg="ADIOS2 does not support HIP without Kokkos")
+    conflicts("+sycl", when="~kokkos", msg="ADIOS2 does not support SYCL without Kokkos")
+
     for _platform in ["linux", "darwin", "cray"]:
-        depends_on("pkgconfig", type="build", when="platform=%s" % _platform)
+        depends_on("pkgconfig", type="build", when=f"platform={_platform}")
         variant(
             "pic",
             default=False,
             description="Build pic-enabled static libraries",
-            when="platform=%s" % _platform,
+            when=f"platform={_platform}",
         )
         # libffi and libfabric and not currently supported on Windows
         # see Paraview's superbuild handling of libfabric at
         # https://gitlab.kitware.com/paraview/paraview-superbuild/-/blob/master/projects/adios2.cmake#L3
-        depends_on("libffi", when="+sst platform=%s" % _platform)  # optional in DILL
+        depends_on("libffi", when=f"+sst platform={_platform}")  # optional in DILL
         depends_on(
-            "libfabric@1.6.0:", when="+sst platform=%s" % _platform
+            "libfabric@1.6.0:", when=f"+sst platform={_platform}"
         )  # optional in EVPath and SST
         # depends_on('bison', when='+sst')     # optional in FFS, broken package
         # depends_on('flex', when='+sst')      # optional in FFS, depends on BISON
@@ -114,15 +164,18 @@ class Adios2(CMakePackage, CudaPackage):
     depends_on("libzmq", when="+dataman")
     depends_on("dataspaces@1.8.0:", when="+dataspaces")
 
+    depends_on("hdf5@:1.12", when="@:2.8 +hdf5")
     depends_on("hdf5~mpi", when="+hdf5~mpi")
     depends_on("hdf5+mpi", when="+hdf5+mpi")
 
     depends_on("libpressio", when="+libpressio")
     depends_on("c-blosc", when="+blosc")
+    depends_on("c-blosc2", when="+blosc2")
     depends_on("bzip2", when="+bzip2")
     depends_on("libpng@1.6:", when="+png")
     depends_on("zfp@0.5.1:0.5", when="+zfp")
     depends_on("sz@2.0.2.0:", when="+sz")
+    depends_on("mgard", when="+mgard")
 
     extends("python", when="+python")
     depends_on("python@2.7:2.8,3.5:", when="@:2.4.0 +python", type=("build", "run"))
@@ -131,6 +184,8 @@ class Adios2(CMakePackage, CudaPackage):
     depends_on("python@3.5:", when="@2.5.0:", type="test")
     depends_on("py-numpy@1.6.1:", when="+python", type=("build", "run"))
     depends_on("py-mpi4py@2.0.0:", when="+mpi +python", type=("build", "run"))
+    depends_on("aws-sdk-cpp", when="+aws")
+    depends_on("libcatalyst@2", when="+libcatalyst")
 
     # Fix findmpi when called by dependees
     # See https://github.com/ornladios/ADIOS2/pull/1632
@@ -157,6 +212,10 @@ class Adios2(CMakePackage, CudaPackage):
         sha256="8221073d1b2f8944395a88a5d60a15c7370646b62f5fc6309867bbb6a8c2096c",
     )
 
+    # cmake: find threads package first
+    # https://github.com/ornladios/ADIOS2/pull/3893
+    patch("2.9.2-cmake-find-threads-package-first.patch", when="@2.9.2:")
+
     @when("%fj")
     def patch(self):
         """add fujitsu mpi commands #16864"""
@@ -179,7 +238,9 @@ def cmake_args(self):
         args = [
             from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
             from_variant("BUILD_SHARED_LIBS", "shared"),
+            from_variant("ADIOS2_USE_AWSSDK", "aws"),
             from_variant("ADIOS2_USE_Blosc", "blosc"),
+            from_variant("ADIOS2_USE_Blosc2", "blosc2"),
             from_variant("ADIOS2_USE_BZip2", "bzip2"),
             from_variant("ADIOS2_USE_DataMan", "dataman"),
             from_variant("ADIOS2_USE_DataSpaces", "dataspaces"),
@@ -192,8 +253,13 @@ def cmake_args(self):
             from_variant("ADIOS2_USE_SST", "sst"),
             from_variant("ADIOS2_USE_SZ", "sz"),
             from_variant("ADIOS2_USE_ZFP", "zfp"),
-            from_variant("ADIOS2_USE_CUDA", "cuda"),
+            from_variant("ADIOS2_USE_Catalyst", "libcatalyst"),
             from_variant("ADIOS2_USE_LIBPRESSIO", "libpressio"),
+            self.define("ADIOS2_USE_CUDA", self.spec.satisfies("+cuda ~kokkos")),
+            self.define("ADIOS2_USE_Kokkos", self.spec.satisfies("+kokkos")),
+            self.define("Kokkos_ENABLE_CUDA", self.spec.satisfies("+cuda +kokkos")),
+            self.define("Kokkos_ENABLE_HIP", self.spec.satisfies("+rocm")),
+            self.define("Kokkos_ENABLE_SYCL", self.spec.satisfies("+sycl")),
             self.define("BUILD_TESTING", self.run_tests),
             self.define("ADIOS2_BUILD_EXAMPLES", False),
             self.define("ADIOS2_USE_Endian_Reverse", True),
@@ -218,8 +284,16 @@ def cmake_args(self):
             args.extend(["-DCMAKE_Fortran_SUBMODULE_EXT=.smod", "-DCMAKE_Fortran_SUBMODULE_SEP=."])
 
         if "+python" in spec or self.run_tests:
-            args.append("-DPYTHON_EXECUTABLE:FILEPATH=%s" % spec["python"].command.path)
-            args.append("-DPython_EXECUTABLE:FILEPATH=%s" % spec["python"].command.path)
+            args.append(f"-DPYTHON_EXECUTABLE:FILEPATH={spec['python'].command.path}")
+            args.append(f"-DPython_EXECUTABLE:FILEPATH={spec['python'].command.path}")
+
+        # hip support
+        if "+cuda" in spec:
+            args.append(self.builder.define_cuda_architectures(self))
+
+        # hip support
+        if "+rocm" in spec:
+            args.append(self.builder.define_hip_architectures(self))
 
         return args
 
@@ -261,3 +335,58 @@ def setup_run_environment(self, env):
             env.prepend_path("HDF5_PLUGIN_PATH", os.path.dirname(all_libs[idx]))
         except ValueError:
             pass
+
+    @run_after("install")
+    def setup_install_tests(self):
+        """
+        Copy the example files after the package is installed to an
+        install test subdirectory for use during `spack test run`.
+        """
+        extra_install_tests = [join_path("testing", "install", "C")]
+        self.cache_extra_test_sources(extra_install_tests)
+
+    def test_run_executables(self):
+        """Run installed adios2 executables"""
+
+        commands_and_args = [("bpls", ["-v", "-V"]), ("adios2-config", ["-v"])]
+
+        for cmd, opts in commands_and_args:
+            with test_part(
+                self,
+                f"test_run_executables_{cmd}",
+                purpose=f"run installed adios2 executable {cmd}",
+            ):
+                exe = which(join_path(self.prefix.bin, cmd))
+                exe(*opts)
+
+    def test_examples(self):
+        """Build and run an example program"""
+        src_dir = self.test_suite.current_test_cache_dir.testing.install.C
+        test_stage_dir = self.test_suite.test_dir_for_spec(self.spec)
+
+        # Create the build tree within this spec's test stage dir so it gets
+        # cleaned up automatically
+        build_dir = tempfile.mkdtemp(dir=test_stage_dir)
+
+        std_cmake_args = []
+
+        if "+mpi" in self.spec:
+            mpi_exec = join_path(self.spec["mpi"].prefix, "bin", "mpiexec")
+            std_cmake_args.append(f"-DMPIEXEC_EXECUTABLE={mpi_exec}")
+
+        built_programs = ["adios_c_mpi_test", "adios_adios2c_test", "adios_c_test"]
+
+        with working_dir(build_dir):
+            with test_part(
+                self, "test_examples_build", purpose="build example against installed adios2"
+            ):
+                cmake(src_dir, *std_cmake_args)
+                make()
+
+            for p in built_programs:
+                exe = which(join_path(".", p))
+                if exe:
+                    with test_part(
+                        self, f"test_examples_run_{p}", purpose=f"run built adios2 example {p}"
+                    ):
+                        exe()
diff --git a/var/spack/repos/builtin/packages/adol-c/package.py b/var/spack/repos/builtin/packages/adol-c/package.py
index c493a53cbaae48..475edbd088e715 100644
--- a/var/spack/repos/builtin/packages/adol-c/package.py
+++ b/var/spack/repos/builtin/packages/adol-c/package.py
@@ -83,12 +83,12 @@ def configure_args(self):
         configure_args = []
 
         if "+boost" in spec:
-            configure_args.append("--with-boost={0}".format(spec["boost"].prefix))
+            configure_args.append(f"--with-boost={spec['boost'].prefix}")
         else:
             configure_args.append("--with-boost=no")
 
         if "+openmp" in spec:
-            configure_args.append("--with-openmp-flag={0}".format(self.compiler.openmp_flag))
+            configure_args.append(f"--with-openmp-flag={self.compiler.openmp_flag}")
 
         configure_args.extend(
             self.enable_or_disable("advanced-branching", variant="advanced_branching")
diff --git a/var/spack/repos/builtin/packages/advancecomp/package.py b/var/spack/repos/builtin/packages/advancecomp/package.py
index b3d050247bec84..176ce547c17e40 100644
--- a/var/spack/repos/builtin/packages/advancecomp/package.py
+++ b/var/spack/repos/builtin/packages/advancecomp/package.py
@@ -23,4 +23,4 @@ class Advancecomp(AutotoolsPackage):
     depends_on("automake", type="build")
     depends_on("libtool", type="build")
     depends_on("m4", type="build")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
diff --git a/var/spack/repos/builtin/packages/akantu/package.py b/var/spack/repos/builtin/packages/akantu/package.py
index 2952f9bb4a1688..bba92edcc8e396 100644
--- a/var/spack/repos/builtin/packages/akantu/package.py
+++ b/var/spack/repos/builtin/packages/akantu/package.py
@@ -65,10 +65,8 @@ def cmake_args(self):
             "-DAKANTU_HEAT_TRANSFER:BOOL=ON",
             "-DAKANTU_SOLID_MECHANICS:BOOL=ON",
             "-DAKANTU_STRUCTURAL_MECHANICS:BOOL=OFF",
-            "-DAKANTU_PARALLEL:BOOL={0}".format("ON" if spec.satisfies("+mpi") else "OFF"),
-            "-DAKANTU_PYTHON_INTERFACE:BOOL={0}".format(
-                "ON" if spec.satisfies("+python") else "OFF"
-            ),
+            f"-DAKANTU_PARALLEL:BOOL={'ON' if spec.satisfies('+mpi') else 'OFF'}",
+            f"-DAKANTU_PYTHON_INTERFACE:BOOL={'ON' if spec.satisfies('+python') else 'OFF'}",
         ]
 
         if spec.satisfies("@:3.0"):
@@ -84,14 +82,14 @@ def cmake_args(self):
         solvers = []
         if spec.satisfies("external_solvers=mumps"):
             solvers.append("Mumps")
-            args.append("-DMUMPS_DIR:PATH=${0}".format(spec["mumps"].prefix))
+            args.append(f"-DMUMPS_DIR:PATH=${spec['mumps'].prefix}")
         if spec.satisfies("external_solvers=petsc"):
             solvers.append("PETSc")
 
         if len(solvers) > 0:
             args.extend(
                 [
-                    "-DAKANTU_IMPLICIT_SOLVER:STRING={0}".format("+".join(solvers)),
+                    f"-DAKANTU_IMPLICIT_SOLVER:STRING={'+'.join(solvers)}",
                     "-DAKANTU_IMPLICIT:BOOL=ON",
                 ]
             )
diff --git a/var/spack/repos/builtin/packages/alembic/package.py b/var/spack/repos/builtin/packages/alembic/package.py
index 9a873e097ba3b9..31a653c0c8d9d5 100644
--- a/var/spack/repos/builtin/packages/alembic/package.py
+++ b/var/spack/repos/builtin/packages/alembic/package.py
@@ -25,7 +25,7 @@ class Alembic(CMakePackage):
     depends_on("openexr@2.2.0:")
     depends_on("hdf5@1.8.9:", when="+hdf5")
     depends_on("boost@1.55:")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("py-ilmbase", when="+python")
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/alglib/package.py b/var/spack/repos/builtin/packages/alglib/package.py
index f962d0fd44fc10..98ade340fd2e11 100644
--- a/var/spack/repos/builtin/packages/alglib/package.py
+++ b/var/spack/repos/builtin/packages/alglib/package.py
@@ -30,7 +30,7 @@ def edit(self, spec, prefix):
         filter_file(r"so", dso_suffix, make_file)
 
     def install(self, spec, prefix):
-        name = "libalglib.{0}".format(dso_suffix)
+        name = f"libalglib.{dso_suffix}"
         with working_dir("src"):
             mkdirp(prefix.lib)
             install(name, prefix.lib)
diff --git a/var/spack/repos/builtin/packages/alquimia/package.py b/var/spack/repos/builtin/packages/alquimia/package.py
index 0a03abc2e0e6f1..d2b33d8524473d 100644
--- a/var/spack/repos/builtin/packages/alquimia/package.py
+++ b/var/spack/repos/builtin/packages/alquimia/package.py
@@ -15,7 +15,8 @@ class Alquimia(CMakePackage):
 
     maintainers("smolins", "balay")
 
-    version("develop")
+    version("master")
+    version("1.1.0", commit="211931c3e76b1ae7cdb48c46885b248412d6fe3d")  # tag v1.1.0
     version("1.0.10", commit="b2c11b6cde321f4a495ef9fcf267cb4c7a9858a0")  # tag v.1.0.10
     version("1.0.9", commit="2ee3bcfacc63f685864bcac2b6868b48ad235225")  # tag v.1.0.9
     version("xsdk-0.6.0", commit="9a0aedd3a927d4d5e837f8fd18b74ad5a78c3821")
@@ -25,6 +26,7 @@ class Alquimia(CMakePackage):
 
     depends_on("mpi")
     depends_on("hdf5")
+    depends_on("pflotran@5.0.0", when="@1.1.0")
     depends_on("pflotran@4.0.1", when="@1.0.10")
     depends_on("pflotran@3.0.2", when="@1.0.9")
     depends_on("pflotran@xsdk-0.6.0", when="@xsdk-0.6.0")
diff --git a/var/spack/repos/builtin/packages/alsa-lib/package.py b/var/spack/repos/builtin/packages/alsa-lib/package.py
index 631e9bf6851717..46e3d2f1bfa672 100644
--- a/var/spack/repos/builtin/packages/alsa-lib/package.py
+++ b/var/spack/repos/builtin/packages/alsa-lib/package.py
@@ -30,8 +30,8 @@ def configure_args(self):
         spec = self.spec
         args = []
         if spec.satisfies("+python"):
-            args.append("--with-pythonlibs={0}".format(spec["python"].libs.ld_flags))
-            args.append("--with-pythonincludes={0}".format(spec["python"].headers.include_flags))
+            args.append(f"--with-pythonlibs={spec['python'].libs.ld_flags}")
+            args.append(f"--with-pythonincludes={spec['python'].headers.include_flags}")
         else:
             args.append("--disable-python")
         return args
diff --git a/var/spack/repos/builtin/packages/aluminum/package.py b/var/spack/repos/builtin/packages/aluminum/package.py
index d223ccba48780a..bb085f8681044b 100644
--- a/var/spack/repos/builtin/packages/aluminum/package.py
+++ b/var/spack/repos/builtin/packages/aluminum/package.py
@@ -48,7 +48,7 @@ class Aluminum(CMakePackage, CudaPackage, ROCmPackage):
         sha256="78b03e36e5422e8651f400feb4d8a527f87302db025d77aa37e223be6b9bdfc9",
         deprecated=True,
     )
-    version("1.0.0-lbann", tag="v1.0.0-lbann")
+    version("1.0.0-lbann", tag="v1.0.0-lbann", commit="40a062b1f63e84e074489c0f926f36b806c6b8f3")
     version("1.0.0", sha256="028d12e271817214db5c07c77b0528f88862139c3e442e1b12f58717290f414a")
     version(
         "0.7.0",
@@ -119,12 +119,14 @@ class Aluminum(CMakePackage, CudaPackage, ROCmPackage):
         "ofi_libfabric_plugin",
         default=spack.platforms.cray.slingshot_network(),
         when="+rccl",
+        sticky=True,
         description="Builds with support for OFI libfabric enhanced RCCL/NCCL communication lib",
     )
     variant(
         "ofi_libfabric_plugin",
         default=spack.platforms.cray.slingshot_network(),
         when="+nccl",
+        sticky=True,
         description="Builds with support for OFI libfabric enhanced RCCL/NCCL communication lib",
     )
 
diff --git a/var/spack/repos/builtin/packages/amber/package.py b/var/spack/repos/builtin/packages/amber/package.py
index cb0e552cf32431..e647a74cd42443 100644
--- a/var/spack/repos/builtin/packages/amber/package.py
+++ b/var/spack/repos/builtin/packages/amber/package.py
@@ -125,7 +125,7 @@ class Amber(Package, CudaPackage):
     variant("x11", description="Build programs that require X11", default=False)
     variant("update", description="Update the sources prior compilation", default=False)
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2")
     depends_on("flex", type="build")
     depends_on("bison", type="build")
diff --git a/var/spack/repos/builtin/packages/amd-aocl/package.py b/var/spack/repos/builtin/packages/amd-aocl/package.py
index 337162a3757f01..a1fb1cb061f9cf 100644
--- a/var/spack/repos/builtin/packages/amd-aocl/package.py
+++ b/var/spack/repos/builtin/packages/amd-aocl/package.py
@@ -17,13 +17,14 @@ class AmdAocl(BundlePackage):
     you agree to the terms and conditions of the AMD AOCL license agreement.
     You may obtain a copy of this license agreement from
     https://www.amd.com/en/developer/aocl/aocl-eula.html
-    https://www.amd.com/en/developer/aocl/aocl-4-0-eula.html
+    https://www.amd.com/en/developer/aocl/eula/aocl-4-1-eula.html
     """
 
     homepage = "https://developer.amd.com/amd-aocl/"
 
     maintainers("amd-toolchain-support")
 
+    version("4.1")
     version("4.0")
     version("3.2")
     version("3.1")
@@ -31,8 +32,7 @@ class AmdAocl(BundlePackage):
     version("2.2")
 
     variant("openmp", default=False, description="Enable OpenMP support.")
-
-    for vers in ["2.2", "3.0", "3.1", "3.2", "4.0"]:
+    for vers in ["2.2", "3.0", "3.1", "3.2", "4.0", "4.1"]:
         depends_on("amdblis@{0} threads=openmp".format(vers), when="@{0} +openmp".format(vers))
         depends_on("amdblis@{0} threads=none".format(vers), when="@{0} ~openmp".format(vers))
         depends_on("amdfftw@{0} +openmp".format(vers), when="@{0} +openmp".format(vers))
@@ -40,11 +40,13 @@ class AmdAocl(BundlePackage):
         depends_on("amdlibflame@{0}".format(vers), when="@{0}".format(vers))
         depends_on("amdlibm@{0}".format(vers), when="@{0}".format(vers))
         depends_on(
-            "amdscalapack@{0} ^amdblis@{0} threads=none".format(vers),
+            "amdscalapack@{0} ^amdblis@{0} ^amdlibflame@{0} threads=none".format(vers),
             when="@{0} ~openmp".format(vers),
         )
         depends_on(
-            "amdscalapack@{0} ^amdblis@{0} threads=openmp".format(vers),
+            "amdscalapack@{0} ^amdblis@{0} ^amdlibflame@{0} threads=openmp".format(vers),
             when="@{0} +openmp".format(vers),
         )
-        depends_on("aocl-sparse@{0}".format(vers), when="@{0}".format(vers))
+        depends_on(
+            "aocl-sparse@{0} ^amdblis@{0} ^amdlibflame@{0}".format(vers), when="@{0}".format(vers)
+        )
diff --git a/var/spack/repos/builtin/packages/amdblis/package.py b/var/spack/repos/builtin/packages/amdblis/package.py
index 1209dd8c6054ca..5d8bb376f3a9f9 100644
--- a/var/spack/repos/builtin/packages/amdblis/package.py
+++ b/var/spack/repos/builtin/packages/amdblis/package.py
@@ -5,6 +5,8 @@
 
 import os
 
+from llnl.util import tty
+
 from spack.package import *
 from spack.pkg.builtin.blis import BlisBase
 
@@ -21,17 +23,18 @@ class Amdblis(BlisBase):
     LICENSING INFORMATION: By downloading, installing and using this software,
     you agree to the terms and conditions of the AMD AOCL-BLIS license
     agreement.  You may obtain a copy of this license agreement from
-    https://www.amd.com/en/developer/aocl/blis/blis-4-0-eula.html
-    https://www.amd.com/en/developer/aocl/blis/blis-eula.html
+    https://www.amd.com/en/developer/aocl/dense/eula/blas-4-1-eula.html
+    https://www.amd.com/en/developer/aocl/dense/eula/blas-eula.html
     """
 
     _name = "amdblis"
     homepage = "https://www.amd.com/en/developer/aocl/blis.html"
     url = "https://github.com/amd/blis/archive/3.0.tar.gz"
-    git = "https://github.com/amd/blis.git"
+    git = "https://github.com/amd/blis"
 
     maintainers("amd-toolchain-support")
 
+    version("4.1", sha256="a05c6c7d359232580d1d599696053ad0beeedf50f3b88d5d22ee7d34375ab577")
     version("4.0", sha256="cddd31176834a932753ac0fc4c76332868feab3e9ac607fa197d8b44c1e74a41")
     version("3.2", sha256="5a400ee4fc324e224e12f73cc37b915a00f92b400443b15ce3350278ad46fff6")
     version("3.1", sha256="2891948925b9db99eec02a1917d9887a7bee9ad2afc5421c9ba58602a620f2bf")
@@ -40,39 +43,51 @@ class Amdblis(BlisBase):
     version("2.2", sha256="e1feb60ac919cf6d233c43c424f6a8a11eab2c62c2c6e3f2652c15ee9063c0c9")
 
     variant("ilp64", default=False, when="@3.0.1:", description="ILP64 support")
+    variant("aocl_gemm", default=False, when="@4.1:", description="aocl_gemm support")
+    variant("suphandling", default=True, description="Small Unpacked Kernel handling")
 
     def configure_args(self):
         spec = self.spec
         args = super().configure_args()
 
+        if not (
+            spec.satisfies(r"%aocc@3.2:4.1")
+            or spec.satisfies(r"%gcc@12.2:13.1")
+            or spec.satisfies(r"%clang@15:16")
+        ):
+            tty.warn(
+                "AOCL has been tested to work with the following compilers\
+                    versions - gcc@12.2:13.1, aocc@3.2:4.1, and clang@15:16\
+                    see the following aocl userguide for details: \
+                    https://www.amd.com/content/dam/amd/en/documents/developer/version-4-1-documents/aocl/aocl-4-1-user-guide.pdf"
+            )
+
         if spec.satisfies("+ilp64"):
             args.append("--blas-int-size=64")
 
+        if spec.satisfies("+aocl_gemm"):
+            args.append("-a aocl_gemm")
+            args.append("CC={0}".format(os.path.basename(spack_cc)))
+            args.append("CXX={0}".format(os.path.basename(spack_cxx)))
+
+        if spec.satisfies("+suphandling"):
+            args.append("--enable-sup-handling")
+        else:
+            args.append("--disable-sup-handling")
+
         # To enable Fortran to C calling convention for
         # complex types when compiling with aocc flang
-        if self.spec.satisfies("@3.0 %aocc"):
+        if spec.satisfies("@3.0 %aocc"):
             args.append("CFLAGS={0}".format("-DAOCL_F2C"))
             args.append("CXXFLAGS={0}".format("-DAOCL_F2C"))
-        elif self.spec.satisfies("@3.0.1: %aocc"):
+        elif spec.satisfies("@3.0.1: %aocc"):
             args.append("--complex-return=intel")
 
-        if self.spec.satisfies("@3.1:"):
+        if spec.satisfies("@3.1:"):
             args.append("--disable-aocl-dynamic")
 
         return args
 
-    def config_args(self):
-        config_args = super().config_args()
-
-        # "amdzen" - A fat binary or multiarchitecture binary
-        # support for 3.1 release onwards
-        if self.spec.satisfies("@3.1:"):
-            config_args.append("amdzen")
-        else:
-            config_args.append("auto")
-
-        return config_args
-
     @run_after("install")
     def create_symlink(self):
         with working_dir(self.prefix.lib):
diff --git a/var/spack/repos/builtin/packages/amdfftw/package.py b/var/spack/repos/builtin/packages/amdfftw/package.py
index 61752783a548cb..475327e3359d0a 100644
--- a/var/spack/repos/builtin/packages/amdfftw/package.py
+++ b/var/spack/repos/builtin/packages/amdfftw/package.py
@@ -5,6 +5,8 @@
 
 import os
 
+from llnl.util import tty
+
 from spack.package import *
 from spack.pkg.builtin.fftw import FftwBase
 
@@ -26,8 +28,8 @@ class Amdfftw(FftwBase):
     LICENSING INFORMATION: By downloading, installing and using this software,
     you agree to the terms and conditions of the AMD AOCL-FFTW license
     agreement.  You may obtain a copy of this license agreement from
-    https://www.amd.com/en/developer/aocl/fftw/fftw-libraries-4-0-eula.html
-    https://www.amd.com/en/developer/aocl/fftw/fftw-libraries-eula.html
+    https://www.amd.com/en/developer/aocl/fftw/eula/fftw-libraries-4-1-eula.html
+    https://www.amd.com/en/developer/aocl/fftw/eula/fftw-libraries-eula.html
     """
 
     _name = "amdfftw"
@@ -37,6 +39,7 @@ class Amdfftw(FftwBase):
 
     maintainers("amd-toolchain-support")
 
+    version("4.1", sha256="f1cfecfcc0729f96a5bd61c6b26f3fa43bb0662d3fff370d4f73490c60cf4e59")
     version("4.0", sha256="5f02cb05f224bd86bd88ec6272b294c26dba3b1d22c7fb298745fd7b9d2271c0")
     version("3.2", sha256="31cab17a93e03b5b606e88dd6116a1055b8f49542d7d0890dbfcca057087b8d0")
     version("3.1", sha256="3e777f3acef13fa1910db097e818b1d0d03a6a36ef41186247c6ab1ab0afc132")
@@ -51,21 +54,47 @@ class Amdfftw(FftwBase):
     variant(
         "amd-fast-planner",
         default=False,
+        when="@3.0:",
         description="Option to reduce the planning time without much "
         "tradeoff in the performance. It is supported for "
         "float and double precisions only.",
     )
-    variant("amd-top-n-planner", default=False, description="Build with amd-top-n-planner support")
     variant(
-        "amd-mpi-vader-limit", default=False, description="Build with amd-mpi-vader-limit support"
+        "amd-top-n-planner",
+        default=False,
+        when="@3.0.1: ~amd-fast-planner ~mpi ~openmp ~threads",
+        description="Build with amd-top-n-planner support",
+    )
+    variant(
+        "amd-mpi-vader-limit",
+        default=False,
+        when="@3.0.1:",
+        description="Build with amd-mpi-vader-limit support",
     )
     variant("static", default=False, description="Build with static suppport")
-    variant("amd-trans", default=False, description="Build with amd-trans suppport")
-    variant("amd-app-opt", default=False, description="Build with amd-app-opt suppport")
     variant(
-        "amd-dynamic-dispatcher",
+        "amd-trans",
+        default=False,
+        when="~mpi ~openmp ~threads",
+        description="Build with amd-trans suppport",
+    )
+    variant(
+        "amd-app-opt",
         default=False,
-        when="@3.2:",
+        when="@3.1: ~mpi",
+        description="Build with amd-app-opt suppport",
+    )
+    variant(
+        "amd-dynamic-dispatcher",
+        default=True,
+        when="@4.1: %aocc@4.1.0:",
+        description="Single portable optimized library"
+        " to execute on different x86 CPU architectures",
+    )
+    variant(
+        "amd-dynamic-dispatcher",
+        default=True,
+        when="@3.2: %gcc",
         description="Single portable optimized library"
         " to execute on different x86 CPU architectures",
     )
@@ -83,88 +112,38 @@ class Amdfftw(FftwBase):
         "+debug", when="@2.2 %aocc", msg="debug mode is not supported by AOCC clang version 2.2"
     )
     conflicts("%gcc@:7.2", when="@2.2:", msg="GCC version above 7.2 is required for AMDFFTW")
-    conflicts(
-        "+amd-fast-planner", when="@2.2", msg="amd-fast-planner is supported from 3.0 onwards"
-    )
-    conflicts(
-        "+amd-fast-planner",
-        when="precision=quad",
-        msg="Quad precision is not supported with amd-fast-planner",
-    )
-    conflicts(
-        "+amd-fast-planner",
-        when="precision=long_double",
-        msg="long_double precision is not supported with amd-fast-planner",
-    )
-    conflicts(
-        "+amd-top-n-planner",
-        when="@:3.0.0",
-        msg="amd-top-n-planner is supported from 3.0.1 onwards",
-    )
-    conflicts(
-        "+amd-top-n-planner",
-        when="precision=long_double",
-        msg="long_double precision is not supported with amd-top-n-planner",
-    )
-    conflicts(
-        "+amd-top-n-planner",
-        when="precision=quad",
-        msg="Quad precision is not supported with amd-top-n-planner",
-    )
-    conflicts(
-        "+amd-top-n-planner",
-        when="+amd-fast-planner",
-        msg="amd-top-n-planner cannot be used with amd-fast-planner",
-    )
-    conflicts(
-        "+amd-top-n-planner", when="+threads", msg="amd-top-n-planner works only for single thread"
-    )
-    conflicts(
-        "+amd-top-n-planner", when="+mpi", msg="mpi thread is not supported with amd-top-n-planner"
-    )
-    conflicts(
-        "+amd-top-n-planner",
-        when="+openmp",
-        msg="openmp thread is not supported with amd-top-n-planner",
-    )
-    conflicts(
-        "+amd-mpi-vader-limit",
-        when="@:3.0.0",
-        msg="amd-mpi-vader-limit is supported from 3.0.1 onwards",
-    )
+
+    with when("+amd-fast-planner"):
+        conflicts("precision=quad", msg="Quad precision is not supported with amd-fast-planner")
+        conflicts(
+            "precision=long_double",
+            msg="long_double precision is not supported with amd-fast-planner",
+        )
+
+    with when("+amd-top-n-planner"):
+        conflicts("precision=quad", msg="Quad precision is not supported with amd-top-n-planner")
+        conflicts(
+            "precision=long_double",
+            msg="long_double precision is not supported with amd-top-n-planner",
+        )
+
     conflicts(
         "+amd-mpi-vader-limit",
         when="precision=quad",
         msg="Quad precision is not supported with amd-mpi-vader-limit",
     )
-    conflicts("+amd-trans", when="+threads", msg="amd-trans works only for single thread")
-    conflicts("+amd-trans", when="+mpi", msg="mpi thread is not supported with amd-trans")
-    conflicts("+amd-trans", when="+openmp", msg="openmp thread is not supported with amd-trans")
-    conflicts(
-        "+amd-trans",
-        when="precision=long_double",
-        msg="long_double precision is not supported with amd-trans",
-    )
-    conflicts(
-        "+amd-trans", when="precision=quad", msg="Quad precision is not supported with amd-trans"
-    )
-    conflicts("+amd-app-opt", when="@:3.0.1", msg="amd-app-opt is supported from 3.1 onwards")
-    conflicts("+amd-app-opt", when="+mpi", msg="mpi thread is not supported with amd-app-opt")
-    conflicts(
-        "+amd-app-opt",
-        when="precision=long_double",
-        msg="long_double precision is not supported with amd-app-opt",
-    )
-    conflicts(
-        "+amd-app-opt",
-        when="precision=quad",
-        msg="Quad precision is not supported with amd-app-opt",
-    )
-    conflicts(
-        "+amd-dynamic-dispatcher",
-        when="%aocc",
-        msg="dynamic-dispatcher is not supported by AOCC clang compiler",
-    )
+
+    with when("+amd-trans"):
+        conflicts(
+            "precision=long_double", msg="long_double precision is not supported with amd-trans"
+        )
+        conflicts("precision=quad", msg="Quad precision is not supported with amd-trans")
+
+    with when("+amd-app-opt"):
+        conflicts(
+            "precision=long_double", msg="long_double precision is not supported with amd-app-opt"
+        )
+        conflicts("precision=quad", msg="Quad precision is not supported with amd-app-opt")
 
     def configure(self, spec, prefix):
         """Configure function"""
@@ -174,7 +153,7 @@ def configure(self, spec, prefix):
         # Dynamic dispatcher builds a single portable optimized library
         # that can execute on different x86 CPU architectures.
         # It is supported for GCC compiler and Linux based systems only.
-        if "+amd-dynamic-dispatcher" in self.spec:
+        if "+amd-dynamic-dispatcher" in spec:
             options.append("--enable-dynamic-dispatcher")
 
         # Check if compiler is AOCC
@@ -183,6 +162,18 @@ def configure(self, spec, prefix):
             options.append("FC={0}".format(os.path.basename(spack_fc)))
             options.append("F77={0}".format(os.path.basename(spack_fc)))
 
+        if not (
+            spec.satisfies(r"%aocc@3.2:4.1")
+            or spec.satisfies(r"%gcc@12.2:13.1")
+            or spec.satisfies(r"%clang@15:16")
+        ):
+            tty.warn(
+                "AOCL has been tested to work with the following compilers\
+                    versions - gcc@12.2:13.1, aocc@3.2:4.1, and clang@15:16\
+                    see the following aocl userguide for details: \
+                    https://www.amd.com/content/dam/amd/en/documents/developer/version-4-1-documents/aocl/aocl-4-1-user-guide.pdf"
+            )
+
         if "+debug" in spec:
             options.append("--enable-debug")
 
@@ -219,6 +210,10 @@ def configure(self, spec, prefix):
         # float and double precisions are supported
         simd_features = ["sse2", "avx", "avx2", "avx512"]
 
+        # "avx512" is supported from amdfftw 4.0 version onwards
+        if "@2.2:3.2" in self.spec:
+            simd_features.remove("avx512")
+
         simd_options = []
         for feature in simd_features:
             msg = "--enable-{0}" if feature in spec.target else "--disable-{0}"
diff --git a/var/spack/repos/builtin/packages/amdlibflame/package.py b/var/spack/repos/builtin/packages/amdlibflame/package.py
index 571fd9babf1730..e1b96e042e02a9 100644
--- a/var/spack/repos/builtin/packages/amdlibflame/package.py
+++ b/var/spack/repos/builtin/packages/amdlibflame/package.py
@@ -3,6 +3,9 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 # ----------------------------------------------------------------------------\
+import os
+
+from llnl.util import tty
 
 from spack.package import *
 from spack.pkg.builtin.libflame import LibflameBase
@@ -33,17 +36,18 @@ class Amdlibflame(LibflameBase):
     LICENSING INFORMATION: By downloading, installing and using this software,
     you agree to the terms and conditions of the AMD AOCL-libFLAME license
     agreement.  You may obtain a copy of this license agreement from
-    https://www.amd.com/en/developer/aocl/blis/libflame-4-0-eula.html
-    https://www.amd.com/en/developer/aocl/blis/libflame-eula.html
+    https://www.amd.com/en/developer/aocl/dense/eula-libflame/libflame-4-1-eula.html
+    https://www.amd.com/en/developer/aocl/dense/eula-libflame/libflame-eula.html
     """
 
     _name = "amdlibflame"
     homepage = "https://www.amd.com/en/developer/aocl/blis.html#libflame"
     url = "https://github.com/amd/libflame/archive/3.0.tar.gz"
-    git = "https://github.com/amd/libflame.git"
+    git = "https://github.com/amd/amdlibflame"
 
     maintainers("amd-toolchain-support")
 
+    version("4.1", sha256="8aed69c60d11cc17e058cabcb8a931cee4f343064ade3e73d3392b7214624b61")
     version("4.0", sha256="bcb05763aa1df1e88f0da5e43ff86d956826cbea1d9c5ff591d78a3e091c66a4")
     version("3.2", sha256="6b5337fb668b82d0ed0a4ab4b5af4e2f72e4cedbeeb4a8b6eb9a3ef057fb749a")
     version("3.1", sha256="4520fb93fcc89161f65a40810cae0fa1f87cecb242da4a69655f502545a53426")
@@ -55,34 +59,64 @@ class Amdlibflame(LibflameBase):
 
     conflicts("+ilp64", when="@:3.0.0", msg="ILP64 is supported from 3.0.1 onwards")
     conflicts("threads=pthreads", msg="pthread is not supported")
-    conflicts("threads=openmp", msg="openmp is not supported")
+    conflicts("threads=openmp", when="@:3", msg="openmp is not supported by amdlibflame < 4.0")
 
     patch("aocc-2.2.0.patch", when="@:2", level=1)
     patch("cray-compiler-wrapper.patch", when="@:3.0.0", level=1)
+    patch("supermat.patch", when="@4.0:4.1", level=1)
 
     provides("flame@5.2", when="@2:")
 
     depends_on("python+pythoncmd", type="build")
     depends_on("gmake@4:", when="@3.0.1,3.1:", type="build")
+    depends_on("aocl-utils", type=("build"), when="@4.1: ")
 
     @property
     def lapack_libs(self):
         """find lapack_libs function"""
-        shared = True if "+shared" in self.spec else False
-        return find_libraries("libflame", root=self.prefix, shared=shared, recursive=True)
+        return find_libraries(
+            "libflame", root=self.prefix, shared="+shared" in self.spec, recursive=True
+        )
+
+    @property
+    def libs(self):
+        """find libflame libs function"""
+        return find_libraries(
+            "libflame", root=self.prefix, shared="+shared" in self.spec, recursive=True
+        )
+
+    def flag_handler(self, name, flags):
+        if name == "cflags":
+            if self.spec.satisfies("%clang@16:") or self.spec.satisfies("%aocc@4.1.0:"):
+                flags.append("-Wno-error=incompatible-function-pointer-types")
+                flags.append("-Wno-implicit-function-declaration")
+                flags.append("-Wno-sometimes-uninitialized")
+        return (flags, None, None)
 
     def configure_args(self):
         """configure_args function"""
         args = super().configure_args()
 
+        if not (
+            self.spec.satisfies(r"%aocc@3.2:4.1")
+            or self.spec.satisfies(r"%gcc@12.2:13.1")
+            or self.spec.satisfies(r"%clang@15:16")
+        ):
+            tty.warn(
+                "AOCL has been tested to work with the following compilers\
+                    versions - gcc@12.2:13.1, aocc@3.2:4.1, and clang@15:16\
+                    see the following aocl userguide for details: \
+                    https://www.amd.com/content/dam/amd/en/documents/developer/version-4-1-documents/aocl/aocl-4-1-user-guide.pdf"
+            )
+
         # From 3.2 version, amd optimized flags are encapsulated under:
-        # enable-amd-flags for gcc compiler
-        # enable-amd-aocc-flags for aocc compiler
+        # enable-amd-aocc-flags for AOCC compiler
+        # enable-amd-flags for all other compilers
         if "@3.2:" in self.spec:
-            if "%gcc" in self.spec:
-                args.append("--enable-amd-flags")
             if "%aocc" in self.spec:
                 args.append("--enable-amd-aocc-flags")
+            else:
+                args.append("--enable-amd-flags")
 
         if "@:3.1" in self.spec:
             args.append("--enable-external-lapack-interfaces")
@@ -101,8 +135,20 @@ def configure_args(self):
         if "@3.0.1: +ilp64" in self.spec:
             args.append("--enable-ilp64")
 
+        if "@4.1:" in self.spec:
+            args.append("CFLAGS=-I{0}".format(self.spec["aocl-utils"].prefix.include))
+            aocl_utils_lib_path = os.path.join(
+                self.spec["aocl-utils"].prefix.lib, "libaoclutils.a"
+            )
+            args.append("LIBAOCLUTILS_LIBRARY_PATH={0}".format(aocl_utils_lib_path))
+
         return args
 
+    @when("@4.1:")
+    def build(self, spec, prefix):
+        aocl_utils_lib_path = os.path.join(self.spec["aocl-utils"].prefix.lib, "libaoclutils.a")
+        make("all", "LIBAOCLUTILS_LIBRARY_PATH={0}".format(aocl_utils_lib_path))
+
     @run_after("build")
     @on_package_attributes(run_tests=True)
     def check(self):
diff --git a/var/spack/repos/builtin/packages/amdlibflame/supermat.patch b/var/spack/repos/builtin/packages/amdlibflame/supermat.patch
new file mode 100644
index 00000000000000..374ffa3dc34e44
--- /dev/null
+++ b/var/spack/repos/builtin/packages/amdlibflame/supermat.patch
@@ -0,0 +1,11 @@
+diff --git a/src/map/lapack2flamec/FLA_getrf.c b/src/map/lapack2flamec/FLA_getrf.c
+index af70857e..1ffc63a1 100644
+--- a/src/map/lapack2flamec/FLA_getrf.c
++++ b/src/map/lapack2flamec/FLA_getrf.c
+@@ -232,6 +232,7 @@ extern fla_context global_context;
+ 
+ #else /* FLA_ENABLE_SUPERMATRIX */
+ 
++#define LAPACK_getrf_body_s LAPACK_getrf_body
+ #define LAPACK_getrf_body_d LAPACK_getrf_body
+ #define LAPACK_getrf_body_z LAPACK_getrf_body
diff --git a/var/spack/repos/builtin/packages/amdlibm/package.py b/var/spack/repos/builtin/packages/amdlibm/package.py
index 30bf0708ae86ec..9a988b49624b84 100644
--- a/var/spack/repos/builtin/packages/amdlibm/package.py
+++ b/var/spack/repos/builtin/packages/amdlibm/package.py
@@ -5,6 +5,8 @@
 
 import os
 
+from llnl.util import tty
+
 from spack.package import *
 
 
@@ -19,7 +21,7 @@ class Amdlibm(SConsPackage):
     LICENSING INFORMATION: By downloading, installing and using this software,
     you agree to the terms and conditions of the AMD AOCL-FFTW license
     agreement.  You may obtain a copy of this license agreement from
-    https://www.amd.com/en/developer/aocl/libm/libm-4-0-eula.html
+    https://www.amd.com/en/developer/aocl/libm/eula/libm-4-1-eula.html
     https://www.amd.com/en/developer/aocl/libm/libm-eula.html
     """
 
@@ -29,6 +31,7 @@ class Amdlibm(SConsPackage):
     url = "https://github.com/amd/aocl-libm-ose/archive/refs/tags/3.0.tar.gz"
     maintainers("amd-toolchain-support")
 
+    version("4.1", sha256="5bbbbc6bc721d9a775822eab60fbc11eb245e77d9f105b4fcb26a54d01456122")
     version("4.0", sha256="038c1eab544be77598eccda791b26553d3b9e2ee4ab3f5ad85fdd2a77d015a7d")
     version("3.2", sha256="c75b287c38a3ce997066af1f5c8d2b19fc460d5e56678ea81f3ac33eb79ec890")
     version("3.1", sha256="dee487cc2d89c2dc93508be2c67592670ffc1d02776c017e8907317003f48845")
@@ -40,20 +43,36 @@ class Amdlibm(SConsPackage):
     # Mandatory dependencies
     depends_on("python@3.6.1:", type=("build", "run"))
     depends_on("scons@3.1.2:", type=("build"))
+    depends_on("aocl-utils", type=("build"), when="@4.1: ")
     depends_on("mpfr", type=("link"))
 
     patch("0001-libm-ose-Scripts-cleanup-pyc-files.patch", when="@2.2")
     patch("0002-libm-ose-prevent-log-v3.c-from-building.patch", when="@2.2")
 
     conflicts("%gcc@:9.1.0", msg="Minimum supported GCC version is 9.2.0")
-    conflicts("%gcc@12.2.0:", msg="Maximum supported GCC version is 12.1.0")
-    conflicts("%clang@9:", msg="Minimum supported Clang version is 9.0.0")
+    conflicts("%gcc@13.2.0:", msg="Maximum supported GCC version is 13.1.0")
+    conflicts("%clang@9.0:16.0", msg="supported Clang version is from 9 to 16")
     conflicts("%aocc@3.2.0", msg="dependency on python@3.6.2")
 
     def build_args(self, spec, prefix):
         """Setting build arguments for amdlibm"""
         args = ["--prefix={0}".format(prefix)]
 
+        if self.spec.satisfies("@4.1: "):
+            args.append("--aocl_utils_install_path={0}".format(self.spec["aocl-utils"].prefix))
+
+        if not (
+            self.spec.satisfies(r"%aocc@3.2:4.1")
+            or self.spec.satisfies(r"%gcc@12.2:13.1")
+            or self.spec.satisfies(r"%clang@15:16")
+        ):
+            tty.warn(
+                "AOCL has been tested to work with the following compilers\
+                    versions - gcc@12.2:13.1, aocc@3.2:4.1, and clang@15:16\
+                    see the following aocl userguide for details: \
+                    https://www.amd.com/content/dam/amd/en/documents/developer/version-4-1-documents/aocl/aocl-4-1-user-guide.pdf"
+            )
+
         # we are circumventing the use of
         # Spacks compiler wrappers because
         # SCons wipes out all environment variables.
diff --git a/var/spack/repos/builtin/packages/amdscalapack/package.py b/var/spack/repos/builtin/packages/amdscalapack/package.py
index 3da1a8a03be7b6..0213da5c97dcca 100644
--- a/var/spack/repos/builtin/packages/amdscalapack/package.py
+++ b/var/spack/repos/builtin/packages/amdscalapack/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+from llnl.util import tty
+
 from spack.package import *
 from spack.pkg.builtin.netlib_scalapack import ScalapackBase
 
@@ -20,16 +22,17 @@ class Amdscalapack(ScalapackBase):
     LICENSING INFORMATION: By downloading, installing and using this software,
     you agree to the terms and conditions of the AMD AOCL-ScaLAPACK license
     agreement.  You may obtain a copy of this license agreement from
-    https://www.amd.com/en/developer/aocl/scalapack/scalapack-libraries-4-0-eula.html
-    https://www.amd.com/en/developer/aocl/scalapack/scalapack-libraries-eula.html
+    https://www.amd.com/en/developer/aocl/scalapack/eula/scalapack-libraries-4-1-eula.html
+    https://www.amd.com/en/developer/aocl/scalapack/eula/scalapack-libraries-eula.html
     """
 
     _name = "amdscalapack"
     homepage = "https://www.amd.com/en/developer/aocl/scalapack.html"
-    git = "https://github.com/amd/scalapack.git"
+    git = "https://github.com/amd/aocl-scalapack"
 
     maintainers("amd-toolchain-support")
 
+    version("4.1", sha256="b2e51c3604e5869d1faaef2e52c92071fcb3de1345aebb2ea172206622067ad9")
     version("4.0", sha256="f02913b5984597b22cdb9a36198ed61039a1bf130308e778dc31b2a7eb88b33b")
     version("3.2", sha256="9e00979bb1be39d627bdacb01774bc043029840d542fafc934d16fec3e3b0892")
     version("3.1", sha256="4c2ee2c44644a0feec0c6fc1b1a413fa9028f14d7035d43a398f5afcfdbacb98")
@@ -52,9 +55,24 @@ def cmake_args(self):
         args = super().cmake_args()
         spec = self.spec
 
+        if not (
+            spec.satisfies(r"%aocc@3.2:4.1")
+            or spec.satisfies(r"%gcc@12.2:13.1")
+            or spec.satisfies(r"%clang@15:16")
+        ):
+            tty.warn(
+                "AOCL has been tested to work with the following compilers\
+                    versions - gcc@12.2:13.1, aocc@3.2:4.1, and clang@15:16\
+                    see the following aocl userguide for details: \
+                    https://www.amd.com/content/dam/amd/en/documents/developer/version-4-1-documents/aocl/aocl-4-1-user-guide.pdf"
+            )
+
         if spec.satisfies("%gcc@10:"):
             args.extend(["-DCMAKE_Fortran_FLAGS={0}".format("-fallow-argument-mismatch")])
 
+        if spec.satisfies("%clang@16:"):
+            args.extend(["-DCMAKE_Fortran_FLAGS={0}".format("-cpp -fno-implicit-none")])
+
         if spec.satisfies("@2.2"):
             args.extend(
                 [
@@ -69,6 +87,16 @@ def cmake_args(self):
         # -DUSE_F2C:BOOL=ON
         args.extend([self.define("USE_F2C", spec.satisfies("@:3.0"))])
 
+        if self.spec.satisfies("%clang@16:") or self.spec.satisfies("%aocc@4.1.0:"):
+            c_flags = []
+            c_flags.append("-Wno-implicit-function-declaration")
+            c_flags.append("-Wno-deprecated-non-prototype")
+            c_flags.append("-Wno-incompatible-pointer-types")
+            args.append(self.define("CMAKE_C_FLAGS", " ".join(c_flags)))
+
+        # link libflame library
+        args.extend(["-DLAPACK_LIBRARIES={0}".format(self.spec["lapack"].libs)])
+
         args.extend(
             [
                 "-DLAPACK_FOUND=true",
diff --git a/var/spack/repos/builtin/packages/amdsmi/package.py b/var/spack/repos/builtin/packages/amdsmi/package.py
new file mode 100644
index 00000000000000..5c293799b80a2e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/amdsmi/package.py
@@ -0,0 +1,49 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class Amdsmi(CMakePackage):
+    """The AMD System Management Interface Library, or AMD SMI library,
+    is a C library for Linux that provides a user space interface for
+    applications to monitor and control AMD device."""
+
+    homepage = "https://github.com/RadeonOpenCompute/amdsmi"
+    url = "https://github.com/RadeonOpenCompute/amdsmi/archive/refs/tags/rocm-5.6.0.tar.gz"
+
+    tags = ["rocm"]
+    maintainers("srekolam", "renjithravindrankannath")
+    libraries = ["libamd_smi"]
+
+    version("5.6.0", sha256="595c9d6d79d9071290b2f19ab4ef9222c8d2983b4322b3143fcd9d0b1ce0f6d8")
+    version("5.5.1", sha256="b794c7fd562fd92f2c9f2bbdc2d5dded7486101fcd4598f2e8c3484c9a939281")
+    version("5.5.0", sha256="dcfbd96e93afcf86b1261464e008e9ef7e521670871a1885e6eaffc7cdc8f555")
+
+    depends_on("cmake@3.11:", type="build")
+    depends_on("python@3.6:", type="run")
+    depends_on("py-virtualenv", type="build")
+    depends_on("llvm@14:", type="build")
+    depends_on("pkgconfig", type="build")
+    depends_on("libdrm", type="build")
+    depends_on("py-pyyaml", type="build")
+
+    @classmethod
+    def determine_version(cls, lib):
+        match = re.search(r"lib\S*\.so\.\d+\.\d+\.(\d)(\d\d)(\d\d)", lib)
+        if match:
+            ver = "{0}.{1}.{2}".format(
+                int(match.group(1)), int(match.group(2)), int(match.group(3))
+            )
+        else:
+            ver = None
+        return ver
+
+    def cmake_args(self):
+        args = []
+        args.append(self.define("BUILD_TESTS", "ON"))
+        args.append("-DCMAKE_INSTALL_LIBDIR=lib")
+        return args
diff --git a/var/spack/repos/builtin/packages/amg/package.py b/var/spack/repos/builtin/packages/amg/package.py
deleted file mode 100644
index a0293523a9a6d1..00000000000000
--- a/var/spack/repos/builtin/packages/amg/package.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-from spack.package import *
-
-
-class Amg(MakefilePackage):
-    """AMG is a parallel algebraic multigrid solver for linear systems arising
-    from problems on unstructured grids.  The driver provided with AMG
-    builds linear systems for various 3-dimensional problems.
-    """
-
-    tags = ["proxy-app", "ecp-proxy-app"]
-
-    homepage = "https://computing.llnl.gov/projects/co-design/amg2013"
-    git = "https://github.com/LLNL/AMG.git"
-
-    version("develop", branch="master")
-    version("1.2", tag="1.2")
-    version("1.1", tag="1.1")
-    version("1.0", tag="1.0")
-
-    variant("openmp", default=True, description="Build with OpenMP support")
-    variant("optflags", default=False, description="Additional optimizations")
-    variant("int64", default=False, description="Use 64-bit integers for global variables")
-
-    depends_on("mpi")
-
-    @property
-    def build_targets(self):
-        targets = []
-
-        include_cflags = ["-DTIMER_USE_MPI"]
-        include_lflags = ["-lm"]
-
-        if "+openmp" in self.spec:
-            include_cflags.append("-DHYPRE_USING_OPENMP")
-            include_cflags.append(self.compiler.openmp_flag)
-            include_lflags.append(self.compiler.openmp_flag)
-            if "+optflags" in self.spec:
-                include_cflags.append("-DHYPRE_USING_PERSISTENT_COMM")
-                include_cflags.append("-DHYPRE_HOPSCOTCH")
-
-        if "+int64" in self.spec:
-            include_cflags.append("-DHYPRE_BIGINT")
-
-        targets.append("INCLUDE_CFLAGS={0}".format(" ".join(include_cflags)))
-        targets.append("INCLUDE_LFLAGS={0}".format(" ".join(include_lflags)))
-        targets.append("CC={0}".format(self.spec["mpi"].mpicc))
-
-        return targets
-
-    def install(self, spec, prefix):
-        mkdirp(prefix.bin)
-        install("test/amg", prefix.bin)
-        install_tree("docs", prefix.docs)
diff --git a/var/spack/repos/builtin/packages/amg2013/package.py b/var/spack/repos/builtin/packages/amg2013/package.py
index c1bb45976e0c61..def0b495a83aac 100644
--- a/var/spack/repos/builtin/packages/amg2013/package.py
+++ b/var/spack/repos/builtin/packages/amg2013/package.py
@@ -7,29 +7,23 @@
 
 
 class Amg2013(MakefilePackage):
-    """AMG2013 is a parallel algebraic multigrid solver for linear
-    systems arising from problems on unstructured grids.
-    It has been derived directly from the BoomerAMG solver in the
-    hypre library, a large linear solver library that is being developed
-    in the Center for Applied Scientific Computing (CASC) at LLNL.
+    """AMG is a parallel algebraic multigrid solver for linear systems arising
+    from problems on unstructured grids.  The driver provided with AMG
+    builds linear systems for various 3-dimensional problems.
     """
 
-    tags = ["proxy-app"]
+    tags = ["proxy-app", "ecp-proxy-app"]
+
     homepage = "https://computing.llnl.gov/projects/co-design/amg2013"
-    url = "https://computing.llnl.gov/projects/co-design/download/amg2013.tgz"
+    git = "https://github.com/LLNL/AMG.git"
 
-    version(
-        "master",
-        sha256="b03771d84a04e3dbbbe32ba5648cd7b789e5853b938dd501e17d23d43f13c50f",
-        url="https://computing.llnl.gov/projects/co-design/download/amg2013.tgz",
-    )
+    version("develop", branch="master")
+    version("1.2", tag="1.2", commit="3ada8a128e311543e84d9d66344ece77924127a8")
+    version("1.1", tag="1.1", commit="09fe8a78baf6ba5eaef7d2804f7b653885d60fee")
+    version("1.0", tag="1.0", commit="f5b864708ca3ef48a86e1e46fcb812cbbfa80c51")
 
     variant("openmp", default=True, description="Build with OpenMP support")
-    variant(
-        "assumedpartition",
-        default=False,
-        description="Use assumed partition (for thousands of processors)",
-    )
+    variant("optflags", default=False, description="Additional optimizations")
     variant("int64", default=False, description="Use 64-bit integers for global variables")
 
     depends_on("mpi")
@@ -45,22 +39,20 @@ def build_targets(self):
             include_cflags.append("-DHYPRE_USING_OPENMP")
             include_cflags.append(self.compiler.openmp_flag)
             include_lflags.append(self.compiler.openmp_flag)
-
-        if "+assumedpartition" in self.spec:
-            include_cflags.append("-DHYPRE_NO_GLOBAL_PARTITION")
+            if "+optflags" in self.spec:
+                include_cflags.append("-DHYPRE_USING_PERSISTENT_COMM")
+                include_cflags.append("-DHYPRE_HOPSCOTCH")
 
         if "+int64" in self.spec:
-            include_cflags.append("-DHYPRE_LONG_LONG")
+            include_cflags.append("-DHYPRE_BIGINT")
 
-        targets.append("INCLUDE_CFLAGS={0}".format(" ".join(include_cflags)))
-        targets.append("INCLUDE_LFLAGS={0}".format(" ".join(include_lflags)))
-        targets.append("CC={0}".format(self.spec["mpi"].mpicc))
+        targets.append(f"INCLUDE_CFLAGS={' '.join(include_cflags)}")
+        targets.append(f"INCLUDE_LFLAGS={' '.join(include_lflags)}")
+        targets.append(f"CC={self.spec['mpi'].mpicc}")
 
         return targets
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
-        install("test/amg2013", prefix.bin)
+        install("test/amg", prefix.bin)
         install_tree("docs", prefix.docs)
-        install("COPYRIGHT", prefix.docs)
-        install("COPYING.LESSER", prefix.docs)
diff --git a/var/spack/repos/builtin/packages/amg2023/package.py b/var/spack/repos/builtin/packages/amg2023/package.py
new file mode 100644
index 00000000000000..a2e8b676e9a9c9
--- /dev/null
+++ b/var/spack/repos/builtin/packages/amg2023/package.py
@@ -0,0 +1,51 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Amg2023(CMakePackage, CudaPackage, ROCmPackage):
+    """AMG2023 is a parallel algebraic multigrid solver for linear systems
+    arising from problems on unstructured grids. The driver provided here
+    builds linear systems for various 3-dimensional problems. It requires
+    an installation of hypre-2.27.0 or higher.
+    """
+
+    tags = ["benchmark"]
+    homepage = "https://github.com/LLNL/AMG2023"
+    git = "https://github.com/LLNL/AMG2023.git"
+
+    version("develop", branch="main")
+    version("cmake-build", git="https://github.com/dyokelson/AMG2023.git", branch="cmake")
+
+    variant("mpi", default=True, description="Enable MPI support")
+    variant("openmp", default=False, description="Enable OpenMP support")
+    variant("caliper", default=False, description="Enable Caliper monitoring")
+
+    depends_on("mpi", when="+mpi")
+    depends_on("hypre+mpi", when="+mpi")
+    requires("+mpi", when="^hypre+mpi")
+    depends_on("caliper", when="+caliper")
+    depends_on("adiak", when="+caliper")
+    depends_on("hypre+caliper", when="+caliper")
+    depends_on("hypre@2.27.0:")
+    depends_on("hypre+cuda", when="+cuda")
+    requires("+cuda", when="^hypre+cuda")
+    depends_on("hypre+rocm", when="+rocm")
+    requires("+rocm", when="^hypre+rocm")
+
+    def cmake_args(self):
+        cmake_options = []
+        cmake_options.append(self.define_from_variant("AMG_WITH_CALIPER", "caliper"))
+        cmake_options.append(self.define_from_variant("AMG_WITH_OMP", "openmp"))
+        cmake_options.append(self.define("HYPRE_PREFIX", self.spec["hypre"].prefix))
+        if self.spec["hypre"].satisfies("+cuda"):
+            cmake_options.append("-DAMG_WITH_CUDA=ON")
+        if self.spec["hypre"].satisfies("+rocm"):
+            cmake_options.append("-DAMG_WITH_HIP=ON")
+        if self.spec["hypre"].satisfies("+mpi"):
+            cmake_options.append("-DAMG_WITH_MPI=ON")
+
+        return cmake_options
diff --git a/var/spack/repos/builtin/packages/amp/package.py b/var/spack/repos/builtin/packages/amp/package.py
index 0a03f76a4dc67c..f0ec4071ce14a4 100644
--- a/var/spack/repos/builtin/packages/amp/package.py
+++ b/var/spack/repos/builtin/packages/amp/package.py
@@ -48,7 +48,7 @@ class Amp(CMakePackage):
     depends_on("silo", when="+silo")
     depends_on("sundials", when="+sundials")
     depends_on("trilinos", when="+trilinos")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
 
     # MPI related dependencies
     depends_on("mpi", when="+mpi")
@@ -101,6 +101,10 @@ def cmake_args(self):
             ]
         )
 
+        if "+zlib" in spec:
+            tpl_list.append("ZLIB")
+            options.append(self.define("TPL_ZLIB_INSTALL_DIR", spec["zlib-api"].prefix))
+
         for vname in (
             "boost",
             "hdf5",
@@ -110,13 +114,10 @@ def cmake_args(self):
             "silo",
             "sundials",
             "trilinos",
-            "zlib",
         ):
             if "+" + vname in spec:
                 tpl_list.append(vname.upper())
-                options.append(
-                    self.define("TPL_{0}_INSTALL_DIR".format(vname.upper()), spec[vname].prefix)
-                )
+                options.append(self.define(f"TPL_{vname.upper()}_INSTALL_DIR", spec[vname].prefix))
 
         if "+netcdf" in spec:
             tpl_list.append("NETCDF")
diff --git a/var/spack/repos/builtin/packages/amqp-cpp/package.py b/var/spack/repos/builtin/packages/amqp-cpp/package.py
index 363ad0b546e1fa..522547277609c8 100644
--- a/var/spack/repos/builtin/packages/amqp-cpp/package.py
+++ b/var/spack/repos/builtin/packages/amqp-cpp/package.py
@@ -27,7 +27,7 @@ class AmqpCpp(CMakePackage):
     )
     variant("shared", default=True, description="Build as a shared library (static by default)")
 
-    conflicts("tcp", when="platform=darwin", msg="TCP module requires Linux")
+    conflicts("+tcp", when="platform=darwin", msg="TCP module requires Linux")
 
     depends_on("cmake@3.5:", type="build")
     depends_on("openssl@1.1.1:", when="+tcp", type=("build", "link", "run"))
diff --git a/var/spack/repos/builtin/packages/amrex/package.py b/var/spack/repos/builtin/packages/amrex/package.py
index 5439bc02680d90..8f14d508ad0085 100644
--- a/var/spack/repos/builtin/packages/amrex/package.py
+++ b/var/spack/repos/builtin/packages/amrex/package.py
@@ -24,6 +24,10 @@ class Amrex(CMakePackage, CudaPackage, ROCmPackage):
     maintainers("WeiqunZhang", "asalmgren", "etpalmer63")
 
     version("develop", branch="development")
+    version("23.11", sha256="49b9fea10cd2a2b6cb0fedf7eac8f7889eacc68a05ae5ac7c5702bc0eb1b3848")
+    version("23.10", sha256="3c85aa0ad5f96303e797960a6e0aa37c427f6483f39cdd61dbc2f7ca16357714")
+    version("23.09", sha256="1a539c2628041b17ad910afd9270332060251c8e346b1482764fdb87a4f25053")
+    version("23.08", sha256="a83b7249d65ad8b6ac1881377e5f814b6db8ed8410ea5562b8ae9d4ed1f37c29")
     version("23.07", sha256="4edb991da51bcaad040f852e42c82834d8605301aa7eeb01cd1512d389a58d90")
     version("23.06", sha256="3bddcb07cce3e65e06cac35005c30820d311ce47ae54b46e4af333fa272b236b")
     version("23.05", sha256="a4bf5ad5322e706b9fae46ff52043e2cca5ddba81479647816251e9ab21c0027")
@@ -74,7 +78,22 @@ class Amrex(CMakePackage, CudaPackage, ROCmPackage):
     version("18.09.1", sha256="a065ee4d1d98324b6c492ae20ea63ba12a4a4e23432bf5b3fe9788d44aa4398e")
 
     # Config options
-    variant("dimensions", default="3", description="Dimensionality", values=("1", "2", "3"))
+    variant(
+        "dimensions",
+        default="3",
+        values=("1", "2", "3"),
+        multi=False,
+        description="Dimensionality",
+        when="@:23.05",
+    )
+    variant(
+        "dimensions",
+        default="1,2,3",
+        values=("1", "2", "3"),
+        multi=True,
+        description="Dimensionality",
+        when="@23.06:",
+    )
     variant("shared", default=False, description="Build shared library")
     variant("mpi", default=True, description="Build with MPI support")
     variant("openmp", default=False, description="Build with OpenMP support")
@@ -137,12 +156,12 @@ class Amrex(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("cmake@3.22:", type="build", when="+sycl")
     depends_on("hdf5@1.10.4: +mpi", when="+hdf5")
     depends_on("rocrand", type="build", when="+rocm")
+    depends_on("hiprand", type="build", when="+rocm")
     depends_on("rocprim", type="build", when="@21.05: +rocm")
     depends_on("hypre@2.18.2:", type="link", when="@:21.02 +hypre")
     depends_on("hypre@2.19.0:", type="link", when="@21.03: ~cuda +hypre")
     depends_on("hypre@2.20.0:", type="link", when="@21.03: +cuda +hypre")
     depends_on("petsc", type="link", when="+petsc")
-    depends_on("intel-oneapi-compilers@2023.0.0:", type="build", when="@23.01: +sycl")
     depends_on("intel-oneapi-mkl", type=("build", "link"), when="+sycl")
 
     # these versions of gcc have lambda function issues
@@ -232,6 +251,8 @@ def get_cuda_arch_string(self, values):
     #
     @when("@20.12:,develop")
     def cmake_args(self):
+        if self.spec.satisfies("@23.01: +sycl") and not self.spec.satisfies("%oneapi@2023.0.0:"):
+            raise InstallError("amrex +sycl requires %oneapi@2023.0.0:")
         args = [
             "-DUSE_XSDK_DEFAULTS=ON",
             self.define_from_variant("AMReX_SPACEDIM", "dimensions"),
diff --git a/var/spack/repos/builtin/packages/amrvis/package.py b/var/spack/repos/builtin/packages/amrvis/package.py
index 88be340a383095..ba25d7dff9f65f 100644
--- a/var/spack/repos/builtin/packages/amrvis/package.py
+++ b/var/spack/repos/builtin/packages/amrvis/package.py
@@ -18,7 +18,7 @@ class Amrvis(MakefilePackage):
 
     maintainers("etpalmer63")
 
-    version("main", tag="main")
+    version("main", branch="main")
 
     variant(
         "dims",
diff --git a/var/spack/repos/builtin/packages/ams/package.py b/var/spack/repos/builtin/packages/ams/package.py
new file mode 100644
index 00000000000000..ba75a25e63f6c3
--- /dev/null
+++ b/var/spack/repos/builtin/packages/ams/package.py
@@ -0,0 +1,153 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Ams(CMakePackage, CudaPackage):
+    """AMS Autonomous Multiscale Framework."""
+
+    homepage = "https://github.com/LLNL/AMS"
+    git = "git@github.com:LLNL/AMS.git"
+
+    maintainers("koparasy", "lpottier")
+
+    version("develop", branch="develop", submodules=False)
+    version(
+        "07.25.23-alpha",
+        tag="07.25.23-alpha",
+        commit="3aa8421f1f1ce1ae448d017214c602b9def19c90",
+        submodules=False,
+    )
+
+    variant(
+        "faiss",
+        default=False,
+        description="Build with FAISS index as uncertainty quantification module",
+    )
+    variant(
+        "caliper", default=False, description="Build with caliper for gather performance counters"
+    )
+    variant("torch", default=False, description="Use torch for surrogate models")
+    variant("mpi", default=False, description="Enable MPI support")
+    variant("examples", default=False, description="Enable examples")
+    variant("redis", default=False, description="Enable redis database")
+    variant("hdf5", default=False, description="Enable HDF5 data storage")
+    variant("rabbitmq", default=False, description="Enable RabbitMQ as data broker")
+    variant(
+        "verbose",
+        default=False,
+        description="Enable AMSLib verbose output (controlled by environment variable)",
+    )
+
+    depends_on("umpire")
+    depends_on("mpi", when="+mpi")
+
+    depends_on("caliper+cuda", when="+caliper +cuda")
+    depends_on("faiss+cuda", when="+faiss +cuda")
+    depends_on("mfem+cuda", when="+examples +cuda")
+    depends_on("py-torch+cuda", when="+torch +cuda")
+
+    depends_on("py-torch~cuda", when="+torch ~cuda")
+    depends_on("caliper ~cuda", when="+caliper ~cuda")
+    depends_on("faiss ~cuda", when="+faiss ~cuda")
+    depends_on("mfem ~cuda", when="+examples ~cuda")
+
+    depends_on("redis-plus-plus", when="+redis")
+    depends_on("hdf5", when="+hdf5")
+    depends_on("amqp-cpp +tcp", when="+rabbitmq")
+
+    with when("+cuda"):
+        cuda_archs = CudaPackage.cuda_arch_values
+        with when("+examples"):
+            depends_on("mfem+cuda")
+            for sm_ in cuda_archs:
+                depends_on(
+                    "mfem +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)
+                )
+
+        with when("+torch"):
+            depends_on("py-torch+cuda")
+            for sm_ in cuda_archs:
+                depends_on(
+                    "py-torch +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)
+                )
+
+        with when("+caliper"):
+            depends_on("caliper+cuda", when="+caliper")
+            for sm_ in cuda_archs:
+                depends_on(
+                    "caliper +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)
+                )
+
+        depends_on("umpire+cuda")
+        for sm_ in cuda_archs:
+            depends_on("umpire +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_))
+
+        with when("+faiss"):
+            depends_on("faiss+cuda", when="+faiss")
+            for sm_ in cuda_archs:
+                depends_on(
+                    "umpire +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)
+                )
+
+    def cmake_args(self):
+        spec = self.spec
+        args = []
+        args.append("-DUMPIRE_DIR={0}".format(spec["umpire"].prefix))
+        args.append("-DWITH_MPI={0}".format("On" if "+mpi" in spec else "Off"))
+
+        args.append(
+            "-DWITH_DB={0}".format(
+                "On" if ("+redis" in spec or "hdf5" in spec or "+rabbitmq" in spec) else "Off"
+            )
+        )
+
+        if "+verbose" in spec:
+            args.append("-DWITH_AMS_DEBUG=On")
+
+        if "+hdf5" in spec:
+            args.append("-DWITH_HDF5=On")
+            args.append("-DHDF5_Dir={0}".format(spec["hdf5"].prefix))
+
+        if "+cuda" in spec:
+            args.append("-DWITH_CUDA=On")
+            cuda_arch = spec.variants["cuda_arch"].value[0]
+            args.append("-DAMS_CUDA_ARCH={0}".format(cuda_arch))
+
+        if "+caliper" in spec:
+            args.append("-DWITH_CALIPER=On")
+            args.append("-DCALIPER_DIR={0}/share/cmake/caliper".format(spec["caliper"].prefix))
+        else:
+            args.append("-DWITH_CALIPER=Off")
+
+        if "+faiss" in spec:
+            args.append("-DWITH_FAISS=On")
+            args.append("-DFAISS_DIR={0}".format(spec["faiss"].prefix))
+        else:
+            args.append("-DWITH_FAISS=Off")
+
+        if "+torch" in spec:
+            args.append("-DWITH_TORCH=On")
+            args.append(
+                "-DTorch_DIR={0}/lib/python{1}/site-packages"
+                "/torch/share/cmake/Torch".format(
+                    spec["py-torch"].prefix, spec["python"].version.up_to(2)
+                )
+            )
+
+        if "+redis" in spec:
+            args.append("-DWITH_REDIS=On")
+            args.append("-DREDIS_PLUS_PLUS_DIR={0}".format(spec["redis-plus-plus"].prefix))
+
+        if "+rabbitmq" in spec:
+            args.append("-DWITH_RMQ=On")
+            args.append("-Damqpcpp_DIR={0}/cmake".format(spec["amqp-cpp"].prefix))
+
+        if "+examples" in spec:
+            args.append("-DWITH_EXAMPLES=On")
+            args.append("-DMFEM_DIR={0}".format(spec["mfem"].prefix))
+
+        return args
diff --git a/var/spack/repos/builtin/packages/anaconda3/package.py b/var/spack/repos/builtin/packages/anaconda3/package.py
index 4c0196fe9092b1..c405e9d9034197 100644
--- a/var/spack/repos/builtin/packages/anaconda3/package.py
+++ b/var/spack/repos/builtin/packages/anaconda3/package.py
@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import platform
 from os.path import split
 
 from spack.package import *
@@ -22,172 +23,199 @@ class Anaconda3(Package):
 
     maintainers("ajkotobi")
 
-    version(
-        "2022.10",
-        sha256="e7ecbccbc197ebd7e1f211c59df2e37bc6959d081f2235d387e08c9026666acd",
-        expand=False,
-    )
-    version(
-        "2022.05",
-        sha256="a7c0afe862f6ea19a596801fc138bde0463abcbce1b753e8d5c474b506a2db2d",
-        expand=False,
-    )
-    version(
-        "2021.11",
-        sha256="fedf9e340039557f7b5e8a8a86affa9d299f5e9820144bd7b92ae9f7ee08ac60",
-        expand=False,
-    )
+    if platform.machine() == "ppc64le":
+        version(
+            "2023.09-0",
+            sha256="5ea1ed9808af95eb2655fe6a4ffdb66bea66ecd1d053fc2ee69eacc7685ef665",
+            expand=False,
+        )
+    elif platform.machine() == "aarch64":
+        version(
+            "2023.09-0",
+            sha256="69ee26361c1ec974199bce5c0369e3e9a71541de7979d2b9cfa4af556d1ae0ea",
+            expand=False,
+        )
+    elif platform.machine() == "x86_64":
+        version(
+            "2023.09-0",
+            sha256="6c8a4abb36fbb711dc055b7049a23bbfd61d356de9468b41c5140f8a11abd851",
+            expand=False,
+        )
+        version(
+            "2023.07-2",
+            sha256="589fb34fe73bc303379abbceba50f3131254e85ce4e7cd819ba4276ba29cad16",
+            expand=False,
+        )
+        version(
+            "2022.10",
+            sha256="e7ecbccbc197ebd7e1f211c59df2e37bc6959d081f2235d387e08c9026666acd",
+            expand=False,
+        )
+        version(
+            "2022.05",
+            sha256="a7c0afe862f6ea19a596801fc138bde0463abcbce1b753e8d5c474b506a2db2d",
+            expand=False,
+        )
+        version(
+            "2021.11",
+            sha256="fedf9e340039557f7b5e8a8a86affa9d299f5e9820144bd7b92ae9f7ee08ac60",
+            expand=False,
+        )
 
-    version(
-        "2021.05",
-        sha256="2751ab3d678ff0277ae80f9e8a74f218cfc70fe9a9cdc7bb1c137d7e47e33d53",
-        expand=False,
-    )
-    version(
-        "2020.11",
-        sha256="cf2ff493f11eaad5d09ce2b4feaa5ea90db5174303d5b3fe030e16d29aeef7de",
-        expand=False,
-    )
-    version(
-        "2020.07",
-        sha256="38ce717758b95b3bd0b1797cc6ccfb76f29a90c25bdfa50ee45f11e583edfdbf",
-        expand=False,
-    )
-    version(
-        "2020.02",
-        sha256="2b9f088b2022edb474915d9f69a803d6449d5fdb4c303041f60ac4aefcc208bb",
-        expand=False,
-    )
-    version(
-        "2019.10",
-        sha256="46d762284d252e51cd58a8ca6c8adc9da2eadc82c342927b2f66ed011d1d8b53",
-        expand=False,
-    )
-    version(
-        "2019.07",
-        sha256="69581cf739365ec7fb95608eef694ba959d7d33b36eb961953f2b82cb25bdf5a",
-        expand=False,
-    )
-    version(
-        "2019.03",
-        sha256="45c851b7497cc14d5ca060064394569f724b67d9b5f98a926ed49b834a6bb73a",
-        expand=False,
-    )
-    version(
-        "2018.12",
-        sha256="1019d0857e5865f8a6861eaf15bfe535b87e92b72ce4f531000dc672be7fce00",
-        expand=False,
-    )
-    version(
-        "5.3.1",
-        sha256="d4c4256a8f46173b675dd6a62d12f566ed3487f932bab6bb7058f06c124bcc27",
-        expand=False,
-    )
-    version(
-        "5.3.0",
-        sha256="cfbf5fe70dd1b797ec677e63c61f8efc92dad930fd1c94d60390bb07fdc09959",
-        expand=False,
-    )
-    version(
-        "5.2.0",
-        sha256="09f53738b0cd3bb96f5b1bac488e5528df9906be2480fe61df40e0e0d19e3d48",
-        expand=False,
-    )
-    version(
-        "5.1.0",
-        sha256="7e6785caad25e33930bc03fac4994a434a21bc8401817b7efa28f53619fa9c29",
-        expand=False,
-    )
-    version(
-        "5.0.1",
-        sha256="55e4db1919f49c92d5abbf27a4be5986ae157f074bf9f8238963cd4582a4068a",
-        expand=False,
-    )
-    version(
-        "5.0.0.1",
-        sha256="092c92427f44687d789a41922ce8426fbdc3c529cc9d6d4ee6de5b62954b93b2",
-        expand=False,
-    )
-    version(
-        "5.0.0",
-        sha256="67f5c20232a3e493ea3f19a8e273e0618ab678fa14b03b59b1783613062143e9",
-        expand=False,
-    )
-    version(
-        "4.4.0",
-        sha256="3301b37e402f3ff3df216fe0458f1e6a4ccbb7e67b4d626eae9651de5ea3ab63",
-        expand=False,
-    )
-    version(
-        "4.3.1",
-        sha256="4447b93d2c779201e5fb50cfc45de0ec96c3804e7ad0fe201ab6b99f73e90302",
-        expand=False,
-    )
-    version(
-        "4.3.0",
-        sha256="e9169c3a5029aa820393ac92704eb9ee0701778a085ca7bdc3c57b388ac1beb6",
-        expand=False,
-    )
-    version(
-        "4.2.0",
-        sha256="73b51715a12b6382dd4df3dd1905b531bd6792d4aa7273b2377a0436d45f0e78",
-        expand=False,
-    )
-    version(
-        "4.1.1",
-        sha256="4f5c95feb0e7efeadd3d348dcef117d7787c799f24b0429e45017008f3534e55",
-        expand=False,
-    )
-    version(
-        "4.1.0",
-        sha256="11d32cf4026603d3b327dc4299863be6b815905ff51a80329085e1bb9f96c8bd",
-        expand=False,
-    )
-    version(
-        "4.0.0",
-        sha256="36a558a1109868661a5735f5f32607643f6dc05cf581fefb1c10fb8abbe22f39",
-        expand=False,
-    )
-    version(
-        "2.5.0",
-        sha256="addadcb927f15cb0b5b6e36890563d3352a8ff6a901ea753d389047d274a29a9",
-        expand=False,
-    )
-    version(
-        "2.4.1",
-        sha256="0735e69199fc37135930ea2fd4fb6ad0adef215a2a7ba9fd6b0a0a4daaadb1cf",
-        expand=False,
-    )
-    version(
-        "2.4.0",
-        sha256="fb4e480059e991f2fa632b5a9bcdd284c7f0677814cd719c11d524453f96a40d",
-        expand=False,
-    )
-    version(
-        "2.3.0",
-        sha256="3be5410b2d9db45882c7de07c554cf4f1034becc274ec9074b23fd37a5c87a6f",
-        expand=False,
-    )
-    version(
-        "2.2.0",
-        sha256="4aac68743e7706adb93f042f970373a6e7e087dbf4b02ac467c94ca4ce33d2d1",
-        expand=False,
-    )
-    version(
-        "2.1.0",
-        sha256="af3225ccbe8df0ffb918939e009aa57740e35058ebf9dfcf5fec794a77556c3c",
-        expand=False,
-    )
-    version(
-        "2.0.1",
-        sha256="3c3b834793e461f3316ad1d9a9178c67859a9d74aaf7bcade076f04134dd1e26",
-        expand=False,
-    )
-    version(
-        "2.0.0",
-        sha256="57ce4f97e300cf94c5724f72d992e9eecef708fdaa13bc672ae9779773056540",
-        expand=False,
-    )
+        version(
+            "2021.05",
+            sha256="2751ab3d678ff0277ae80f9e8a74f218cfc70fe9a9cdc7bb1c137d7e47e33d53",
+            expand=False,
+        )
+        version(
+            "2020.11",
+            sha256="cf2ff493f11eaad5d09ce2b4feaa5ea90db5174303d5b3fe030e16d29aeef7de",
+            expand=False,
+        )
+        version(
+            "2020.07",
+            sha256="38ce717758b95b3bd0b1797cc6ccfb76f29a90c25bdfa50ee45f11e583edfdbf",
+            expand=False,
+        )
+        version(
+            "2020.02",
+            sha256="2b9f088b2022edb474915d9f69a803d6449d5fdb4c303041f60ac4aefcc208bb",
+            expand=False,
+        )
+        version(
+            "2019.10",
+            sha256="46d762284d252e51cd58a8ca6c8adc9da2eadc82c342927b2f66ed011d1d8b53",
+            expand=False,
+        )
+        version(
+            "2019.07",
+            sha256="69581cf739365ec7fb95608eef694ba959d7d33b36eb961953f2b82cb25bdf5a",
+            expand=False,
+        )
+        version(
+            "2019.03",
+            sha256="45c851b7497cc14d5ca060064394569f724b67d9b5f98a926ed49b834a6bb73a",
+            expand=False,
+        )
+        version(
+            "2018.12",
+            sha256="1019d0857e5865f8a6861eaf15bfe535b87e92b72ce4f531000dc672be7fce00",
+            expand=False,
+        )
+        version(
+            "5.3.1",
+            sha256="d4c4256a8f46173b675dd6a62d12f566ed3487f932bab6bb7058f06c124bcc27",
+            expand=False,
+        )
+        version(
+            "5.3.0",
+            sha256="cfbf5fe70dd1b797ec677e63c61f8efc92dad930fd1c94d60390bb07fdc09959",
+            expand=False,
+        )
+        version(
+            "5.2.0",
+            sha256="09f53738b0cd3bb96f5b1bac488e5528df9906be2480fe61df40e0e0d19e3d48",
+            expand=False,
+        )
+        version(
+            "5.1.0",
+            sha256="7e6785caad25e33930bc03fac4994a434a21bc8401817b7efa28f53619fa9c29",
+            expand=False,
+        )
+        version(
+            "5.0.1",
+            sha256="55e4db1919f49c92d5abbf27a4be5986ae157f074bf9f8238963cd4582a4068a",
+            expand=False,
+        )
+        version(
+            "5.0.0.1",
+            sha256="092c92427f44687d789a41922ce8426fbdc3c529cc9d6d4ee6de5b62954b93b2",
+            expand=False,
+        )
+        version(
+            "5.0.0",
+            sha256="67f5c20232a3e493ea3f19a8e273e0618ab678fa14b03b59b1783613062143e9",
+            expand=False,
+        )
+        version(
+            "4.4.0",
+            sha256="3301b37e402f3ff3df216fe0458f1e6a4ccbb7e67b4d626eae9651de5ea3ab63",
+            expand=False,
+        )
+        version(
+            "4.3.1",
+            sha256="4447b93d2c779201e5fb50cfc45de0ec96c3804e7ad0fe201ab6b99f73e90302",
+            expand=False,
+        )
+        version(
+            "4.3.0",
+            sha256="e9169c3a5029aa820393ac92704eb9ee0701778a085ca7bdc3c57b388ac1beb6",
+            expand=False,
+        )
+        version(
+            "4.2.0",
+            sha256="73b51715a12b6382dd4df3dd1905b531bd6792d4aa7273b2377a0436d45f0e78",
+            expand=False,
+        )
+        version(
+            "4.1.1",
+            sha256="4f5c95feb0e7efeadd3d348dcef117d7787c799f24b0429e45017008f3534e55",
+            expand=False,
+        )
+        version(
+            "4.1.0",
+            sha256="11d32cf4026603d3b327dc4299863be6b815905ff51a80329085e1bb9f96c8bd",
+            expand=False,
+        )
+        version(
+            "4.0.0",
+            sha256="36a558a1109868661a5735f5f32607643f6dc05cf581fefb1c10fb8abbe22f39",
+            expand=False,
+        )
+        version(
+            "2.5.0",
+            sha256="addadcb927f15cb0b5b6e36890563d3352a8ff6a901ea753d389047d274a29a9",
+            expand=False,
+        )
+        version(
+            "2.4.1",
+            sha256="0735e69199fc37135930ea2fd4fb6ad0adef215a2a7ba9fd6b0a0a4daaadb1cf",
+            expand=False,
+        )
+        version(
+            "2.4.0",
+            sha256="fb4e480059e991f2fa632b5a9bcdd284c7f0677814cd719c11d524453f96a40d",
+            expand=False,
+        )
+        version(
+            "2.3.0",
+            sha256="3be5410b2d9db45882c7de07c554cf4f1034becc274ec9074b23fd37a5c87a6f",
+            expand=False,
+        )
+        version(
+            "2.2.0",
+            sha256="4aac68743e7706adb93f042f970373a6e7e087dbf4b02ac467c94ca4ce33d2d1",
+            expand=False,
+        )
+        version(
+            "2.1.0",
+            sha256="af3225ccbe8df0ffb918939e009aa57740e35058ebf9dfcf5fec794a77556c3c",
+            expand=False,
+        )
+        version(
+            "2.0.1",
+            sha256="3c3b834793e461f3316ad1d9a9178c67859a9d74aaf7bcade076f04134dd1e26",
+            expand=False,
+        )
+        version(
+            "2.0.0",
+            sha256="57ce4f97e300cf94c5724f72d992e9eecef708fdaa13bc672ae9779773056540",
+            expand=False,
+        )
+
+    def url_for_version(self, version):
+        url = "https://repo.anaconda.com/archive/Anaconda3-{0}-Linux-{1}.sh"
+        return url.format(version, platform.machine())
 
     def install(self, spec, prefix):
         dir, anaconda_script = split(self.stage.archive_file)
diff --git a/var/spack/repos/builtin/packages/angsd/package.py b/var/spack/repos/builtin/packages/angsd/package.py
index ce505bb83cb4b7..4e8caed26ba27f 100644
--- a/var/spack/repos/builtin/packages/angsd/package.py
+++ b/var/spack/repos/builtin/packages/angsd/package.py
@@ -27,7 +27,7 @@ class Angsd(MakefilePackage):
     depends_on("htslib")
     conflicts("^htslib@1.6:", when="@0.919")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("lzma")
     depends_on("curl")
 
diff --git a/var/spack/repos/builtin/packages/anicalculator/package.py b/var/spack/repos/builtin/packages/anicalculator/package.py
index b14e05a67020fe..9d002975d98485 100644
--- a/var/spack/repos/builtin/packages/anicalculator/package.py
+++ b/var/spack/repos/builtin/packages/anicalculator/package.py
@@ -19,7 +19,7 @@ class Anicalculator(Package):
     https://spack.readthedocs.io/en/latest/mirrors.html"""
 
     homepage = "https://ani.jgi.doe.gov/html/download.php?"
-    url = "file://{0}/ANIcalculator_v1.tgz".format(os.getcwd())
+    url = f"file://{os.getcwd()}/ANIcalculator_v1.tgz"
     manual_download = True
 
     version("1", sha256="236596a9a204cbcad162fc66be3506b2530b1f48f4f84d9647ccec3ca7483a43")
diff --git a/var/spack/repos/builtin/packages/ants/package.py b/var/spack/repos/builtin/packages/ants/package.py
index 71e8c5f36a90a4..8e5001dddeb49c 100644
--- a/var/spack/repos/builtin/packages/ants/package.py
+++ b/var/spack/repos/builtin/packages/ants/package.py
@@ -25,7 +25,7 @@ class Ants(CMakePackage):
     version("2.3.5", sha256="2fddfd5f274a47f1c383e734a7e763b627c4a8383d2d3b9971561f335016bb0a")
     version("2.2.0", sha256="62f8f9ae141cb45025f4bb59277c053acf658d4a3ba868c9e0f609af72e66b4a")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     variant("minc", default=True, description="Build ITK with MINC support")
 
diff --git a/var/spack/repos/builtin/packages/aocc/package.py b/var/spack/repos/builtin/packages/aocc/package.py
index ad56d0f0f01b27..229a37bb5375c5 100644
--- a/var/spack/repos/builtin/packages/aocc/package.py
+++ b/var/spack/repos/builtin/packages/aocc/package.py
@@ -33,6 +33,11 @@ class Aocc(Package):
 
     maintainers("amd-toolchain-support")
 
+    version(
+        ver="4.1.0",
+        sha256="5b04bfdb751c68dfb9470b34235d76efa80a6b662a123c3375b255982cb52acd",
+        url="https://download.amd.com/developer/eula/aocc/aocc-4-1/aocc-compiler-4.1.0.tar",
+    )
     version(
         ver="4.0.0",
         sha256="2729ec524cbc927618e479994330eeb72df5947e90cfcc49434009eee29bf7d4",
@@ -48,7 +53,7 @@ class Aocc(Package):
     license_url = "https://www.amd.com/en/developer/aocc/aocc-compiler/eula.html"
 
     depends_on("libxml2")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("ncurses")
     depends_on("libtool")
     depends_on("texinfo")
diff --git a/var/spack/repos/builtin/packages/aocl-sparse/package.py b/var/spack/repos/builtin/packages/aocl-sparse/package.py
index 365f6b47be336b..7adf7edaef6a1f 100644
--- a/var/spack/repos/builtin/packages/aocl-sparse/package.py
+++ b/var/spack/repos/builtin/packages/aocl-sparse/package.py
@@ -5,6 +5,8 @@
 
 import os
 
+from llnl.util import tty
+
 from spack.package import *
 
 
@@ -17,29 +19,24 @@ class AoclSparse(CMakePackage):
     LICENSING INFORMATION: By downloading, installing and using this software,
     you agree to the terms and conditions of the AMD AOCL-Sparse license agreement.
     You may obtain a copy of this license agreement from
-    https://www.amd.com/en/developer/aocl/sparse/sparse-libraries-4-0-eula.html
-    https://www.amd.com/en/developer/aocl/sparse/sparse-libraries-eula.html
+    https://www.amd.com/en/developer/aocl/sparse/eula/sparse-libraries-4-1-eula.html
+    https://www.amd.com/en/developer/aocl/sparse/eula/sparse-libraries-eula.html
     """
 
     _name = "aocl-sparse"
     homepage = "https://www.amd.com/en/developer/aocl/sparse.html"
     url = "https://github.com/amd/aocl-sparse/archive/3.0.tar.gz"
-    git = "https://github.com/amd/aocl-sparse.git"
+    git = "https://github.com/amd/aocl-sparse"
 
     maintainers("amd-toolchain-support")
 
+    version("4.1", sha256="35ef437210bc25fdd802b462eaca830bfd928f962569b91b592f2866033ef2bb")
     version("4.0", sha256="68524e441fdc7bb923333b98151005bed39154d9f4b5e8310b5c37de1d69c2c3")
     version("3.2", sha256="db7d681a8697d6ef49acf3e97e8bec35b048ce0ad74549c3b738bbdff496618f")
     version("3.1", sha256="8536f06095c95074d4297a3d2910654085dd91bce82e116c10368a9f87e9c7b9")
     version("3.0", sha256="1d04ba16e04c065051af916b1ed9afce50296edfa9b1513211a7378e1d6b952e")
     version("2.2", sha256="33c2ed6622cda61d2613ee63ff12c116a6cd209c62e54307b8fde986cd65f664")
 
-    variant(
-        "build_type",
-        default="Release",
-        description="CMake build type",
-        values=("Debug", "Release"),
-    )
     variant("shared", default=True, description="Build shared library")
     variant("ilp64", default=False, description="Build with ILP64 support")
     variant("examples", default=False, description="Build sparse examples")
@@ -52,9 +49,11 @@ class AoclSparse(CMakePackage):
         description="Enable experimental AVX512 support",
     )
 
+    depends_on("amdblis", when="@4.1:")
+    depends_on("amdlibflame", when="@4.1:")
     depends_on("boost", when="+benchmarks")
     depends_on("boost", when="@2.2")
-    depends_on("cmake@3.5:", type="build")
+    depends_on("cmake@3.11:", type="build")
 
     @property
     def build_directory(self):
@@ -76,17 +75,19 @@ def cmake_args(self):
         """Runs ``cmake`` in the build directory"""
         spec = self.spec
 
-        args = [
-            "../..",
-            "-DCMAKE_INSTALL_PREFIX:PATH={0}".format(spec.prefix),
-            "-DCMAKE_CXX_COMPILER={0}".format(os.path.basename(spack_cxx)),
-        ]
-
-        if spec.variants["build_type"].value == "Debug":
-            args.append("-DCMAKE_BUILD_TYPE=Debug")
-        else:
-            args.append("-DCMAKE_BUILD_TYPE=Release")
+        if not (
+            spec.satisfies(r"%aocc@3.2:4.1")
+            or spec.satisfies(r"%gcc@12.2:13.1")
+            or spec.satisfies(r"%clang@15:16")
+        ):
+            tty.warn(
+                "AOCL has been tested to work with the following compilers\
+                    versions - gcc@12.2:13.1, aocc@3.2:4.1, and clang@15:16\
+                    see the following aocl userguide for details: \
+                    https://www.amd.com/content/dam/amd/en/documents/developer/version-4-1-documents/aocl/aocl-4-1-user-guide.pdf"
+            )
 
+        args = []
         args.append(self.define_from_variant("BUILD_SHARED_LIBS", "shared"))
         args.append(self.define_from_variant("BUILD_CLIENTS_SAMPLES", "examples"))
         args.append(self.define_from_variant("BUILD_CLIENTS_TESTS", "unit_tests"))
@@ -96,6 +97,16 @@ def cmake_args(self):
         if spec.satisfies("@3.0:"):
             args.append(self.define_from_variant("BUILD_ILP64", "ilp64"))
 
+        if self.spec.satisfies("@4.0:"):
+            args.append("-DAOCL_BLIS_LIB=" + str(self.spec["amdblis"].libs))
+            args.append(
+                "-DAOCL_BLIS_INCLUDE_DIR={0}/blis".format(self.spec["amdblis"].prefix.include)
+            )
+            args.append("-DAOCL_LIBFLAME=" + str(self.spec["amdlibflame"].libs))
+            args.append(
+                "-DAOCL_LIBFLAME_INCLUDE_DIR={0}".format(self.spec["amdlibflame"].prefix.include)
+            )
+
         return args
 
     @run_after("build")
diff --git a/var/spack/repos/builtin/packages/aocl-utils/package.py b/var/spack/repos/builtin/packages/aocl-utils/package.py
new file mode 100644
index 00000000000000..cc0e8ff4d61853
--- /dev/null
+++ b/var/spack/repos/builtin/packages/aocl-utils/package.py
@@ -0,0 +1,64 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from llnl.util import tty
+
+from spack.package import *
+
+
+class AoclUtils(CMakePackage):
+    """The library AOCL-Utils is intended to provide a uniform interface to all
+    AOCL libraries to access CPU features, especially for AMD CPUs. The library
+    provides the following features:
+    * Core details
+    * Flags available/usable
+    * ISA available/usable
+    * Topology about L1/L2/L3
+    AOCL-Utils is designed to be integrated into other AMD AOCL libraries. Each
+    project has their own mechanism to identify CPU and provide necessary
+    features such as "dynamic dispatch".The main purpose of this library is to
+    provide a centralized mechanism to update/validate and provide information
+    to the users of this library.
+
+    LICENSING INFORMATION: By downloading, installing and using this software,
+    you agree to the terms and conditions of the AMD AOCL-Utils license
+    agreement. You may obtain a copy of this license agreement from
+    https://www.amd.com/en/developer/aocl/utils/utils-eula/utils-eula-4-1.html
+    """
+
+    _name = "aocl-utils"
+    homepage = "https://www.amd.com/en/developer/aocl/utils.html"
+    url = "https://github.com/amd/aocl-utils/archive/refs/tags/4.1.tar.gz"
+    git = "https://github.com/amd/aocl-utils"
+
+    maintainers("amd-toolchain-support")
+
+    version("4.1", sha256="a2f271f5eef07da366dae421af3c89286ebb6239047a31a46451758d4a06bc85")
+
+    variant("doc", default=False, description="enable documentation")
+    variant("tests", default=False, description="enable testing")
+    variant("examples", default=False, description="enable examples")
+
+    depends_on("doxygen", when="+doc")
+
+    def cmake_args(self):
+        if not (
+            self.spec.satisfies(r"%aocc@3.2:4.1")
+            or self.spec.satisfies(r"%gcc@12.2:13.1")
+            or self.spec.satisfies(r"%clang@15:16")
+        ):
+            tty.warn(
+                "AOCL has been tested to work with the following compilers\
+                    versions - gcc@12.2:13.1, aocc@3.2:4.1, and clang@15:16\
+                    see the following aocl userguide for details: \
+                    https://www.amd.com/content/dam/amd/en/documents/developer/version-4-1-documents/aocl/aocl-4-1-user-guide.pdf"
+            )
+
+        args = []
+        args.append(self.define_from_variant("ALCI_DOCS", "doc"))
+        args.append(self.define_from_variant("ALCI_TESTS", "tests"))
+        args.append(self.define_from_variant("ALCI_EXAMPLES", "examples"))
+
+        return args
diff --git a/var/spack/repos/builtin/packages/ape/package.py b/var/spack/repos/builtin/packages/ape/package.py
index 9468a7f9a69567..39b833ab0e10d6 100644
--- a/var/spack/repos/builtin/packages/ape/package.py
+++ b/var/spack/repos/builtin/packages/ape/package.py
@@ -23,9 +23,9 @@ def install(self, spec, prefix):
         args = []
         args.extend(
             [
-                "--prefix=%s" % prefix,
-                "--with-gsl-prefix=%s" % spec["gsl"].prefix,
-                "--with-libxc-prefix=%s" % spec["libxc"].prefix,
+                f"--prefix={prefix}",
+                f"--with-gsl-prefix={spec['gsl'].prefix}",
+                f"--with-libxc-prefix={spec['libxc'].prefix}",
             ]
         )
 
diff --git a/var/spack/repos/builtin/packages/apex/package.py b/var/spack/repos/builtin/packages/apex/package.py
index f1590023bf12c0..bf6ea039acbf2f 100644
--- a/var/spack/repos/builtin/packages/apex/package.py
+++ b/var/spack/repos/builtin/packages/apex/package.py
@@ -18,6 +18,7 @@ class Apex(CMakePackage):
 
     version("develop", branch="develop")
     version("master", branch="master")
+    version("2.6.3", sha256="7fef12937d3bd1271a01abe44cb931b1d63823fb5c74287a332f3012ed7297d5")
     version("2.6.2", sha256="0c3ec26631db7925f50cf4e8920a778b57d11913f239a0eb964081f925129725")
     version("2.6.1", sha256="511dbab0af541489052a3d6379c48f9577e51654491d3b2c8545020e9d29fb29")
     version("2.6.0", sha256="25b4f6afd1083475dc6680b5da87759c62d31fcf368996185573694fc40d5317")
@@ -92,7 +93,7 @@ class Apex(CMakePackage):
     variant("examples", default=False, description="Build Examples")
 
     # Dependencies
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("cmake@3.10.0:", type="build")
     depends_on("binutils@2.33:+libiberty+headers", when="+binutils")
     depends_on("gettext", when="+binutils ^binutils+nls")
@@ -115,6 +116,9 @@ class Apex(CMakePackage):
     conflicts("+jemalloc", when="+gperftools")
     conflicts("+plugins", when="~activeharmony")
 
+    # https://github.com/UO-OACISS/apex/pull/177#issuecomment-1726322959
+    conflicts("+openmp", when="%gcc")
+
     # Patches
 
     # This patch ensures that the missing dependency_tree.hpp header is
diff --git a/var/spack/repos/builtin/packages/apptainer/package.py b/var/spack/repos/builtin/packages/apptainer/package.py
index 600f37d210e5fa..d1b9ed71e39365 100644
--- a/var/spack/repos/builtin/packages/apptainer/package.py
+++ b/var/spack/repos/builtin/packages/apptainer/package.py
@@ -53,3 +53,8 @@ def config_options(self):
         if spec.satisfies("@1.1.0: +suid"):
             options.append("--with-suid")
         return options
+
+    def flag_handler(self, name, flags):
+        # Certain go modules this build pulls in cannot be built with anything
+        # other than -O0. Best to just discard any injected flags.
+        return (None, flags, None)
diff --git a/var/spack/repos/builtin/packages/apr-util/package.py b/var/spack/repos/builtin/packages/apr-util/package.py
index 316d6b605a3572..dc0fad53d225d9 100644
--- a/var/spack/repos/builtin/packages/apr-util/package.py
+++ b/var/spack/repos/builtin/packages/apr-util/package.py
@@ -32,20 +32,21 @@ class AprUtil(AutotoolsPackage):
     depends_on("postgresql", when="+pgsql")
     depends_on("sqlite", when="+sqlite")
     depends_on("unixodbc", when="+odbc")
+    depends_on("pkgconfig", type="build", when="+crypto ^openssl~shared")
 
     @property
     def libs(self):
         return find_libraries(
-            ["libaprutil-{0}".format(self.version.up_to(1))], root=self.prefix, recursive=True
+            [f"libaprutil-{self.version.up_to(1)}"], root=self.prefix, recursive=True
         )
 
     def configure_args(self):
         spec = self.spec
 
         args = [
-            "--with-apr={0}".format(spec["apr"].prefix),
-            "--with-expat={0}".format(spec["expat"].prefix),
-            "--with-iconv={0}".format(spec["iconv"].prefix),
+            f"--with-apr={spec['apr'].prefix}",
+            f"--with-expat={spec['expat'].prefix}",
+            f"--with-iconv={spec['iconv'].prefix}",
             # TODO: Add support for the following database managers
             "--without-ndbm",
             "--without-berkeley-db",
@@ -54,41 +55,39 @@ def configure_args(self):
         ]
 
         if "+crypto" in spec:
-            args.extend(["--with-crypto", "--with-openssl={0}".format(spec["openssl"].prefix)])
+            args.extend(["--with-crypto", f"--with-openssl={spec['openssl'].prefix}"])
         else:
             args.append("--without-crypto")
 
         if "+gdbm" in spec:
-            args.append("--with-gdbm={0}".format(spec["gdbm"].prefix))
+            args.append(f"--with-gdbm={spec['gdbm'].prefix}")
         else:
             args.append("--without-gdbm")
 
         if "+pgsql" in spec:
-            args.append("--with-pgsql={0}".format(spec["postgresql"].prefix))
+            args.append(f"--with-pgsql={spec['postgresql'].prefix}")
         else:
             args.append("--without-pgsql")
 
         if "+sqlite" in spec:
             if spec.satisfies("^sqlite@3.0:3"):
-                args.extend(
-                    ["--with-sqlite3={0}".format(spec["sqlite"].prefix), "--without-sqlite2"]
-                )
+                args.extend([f"--with-sqlite3={spec['sqlite'].prefix}", "--without-sqlite2"])
             elif spec.satisfies("^sqlite@2.0:2"):
-                args.extend(
-                    ["--with-sqlite2={0}".format(spec["sqlite"].prefix), "--without-sqlite3"]
-                )
+                args.extend([f"--with-sqlite2={spec['sqlite'].prefix}", "--without-sqlite3"])
         else:
             args.extend(["--without-sqlite2", "--without-sqlite3"])
 
         if "+odbc" in spec:
-            args.append("--with-odbc={0}".format(spec["unixodbc"].prefix))
+            args.append(f"--with-odbc={spec['unixodbc'].prefix}")
         else:
             args.append("--without-odbc")
 
-        if "+crypto" in spec:
-            if spec["openssl"].satisfies("~shared"):
-                zlibs = self.spec["zlib"].libs
-                args.append("LIBS={0} {1}".format(zlibs.ld_flags, zlibs.link_flags))
+        if spec.satisfies("+crypto ^openssl~shared"):
+            # Need pkg-config to get zlib and -ldl flags
+            # (see https://dev.apr.apache.narkive.com/pNnO9F1S/configure-bug-openssl)
+            pkgconf = which("pkg-config")
+            ssl_libs = pkgconf("--libs", "--static", "openssl", output=str)
+            args.append(f"LIBS={ssl_libs}")
 
         return args
 
diff --git a/var/spack/repos/builtin/packages/apr/package.py b/var/spack/repos/builtin/packages/apr/package.py
index de82ee5817c18a..45de21e3ee02af 100644
--- a/var/spack/repos/builtin/packages/apr/package.py
+++ b/var/spack/repos/builtin/packages/apr/package.py
@@ -26,5 +26,5 @@ class Apr(AutotoolsPackage):
     @property
     def libs(self):
         return find_libraries(
-            ["libapr-{0}".format(self.version.up_to(1))], root=self.prefix, recursive=True
+            [f"libapr-{self.version.up_to(1)}"], root=self.prefix, recursive=True
         )
diff --git a/var/spack/repos/builtin/packages/aragorn/package.py b/var/spack/repos/builtin/packages/aragorn/package.py
index dc55dc52bb84be..8ac7894192f457 100644
--- a/var/spack/repos/builtin/packages/aragorn/package.py
+++ b/var/spack/repos/builtin/packages/aragorn/package.py
@@ -31,7 +31,7 @@ class Aragorn(Package):
 
     # fix checksum error
     def url_for_version(self, version):
-        return "http://www.ansikte.se/ARAGORN/Downloads/aragorn{0}.c".format(version)
+        return f"http://www.ansikte.se/ARAGORN/Downloads/aragorn{version}.c"
 
     def install(self, spec, prefix):
         cc = Executable(spack_cc)
diff --git a/var/spack/repos/builtin/packages/arbor/package.py b/var/spack/repos/builtin/packages/arbor/package.py
index 41020a3eb31b82..1215ee4fc50076 100644
--- a/var/spack/repos/builtin/packages/arbor/package.py
+++ b/var/spack/repos/builtin/packages/arbor/package.py
@@ -13,26 +13,32 @@ class Arbor(CMakePackage, CudaPackage):
     homepage = "https://arbor-sim.org"
     git = "https://github.com/arbor-sim/arbor.git"
     url = "https://github.com/arbor-sim/arbor/releases/download/v0.8.1/arbor-v0.8.1-full.tar.gz"
-    maintainers("bcumming", "brenthuisman", "haampie", "schmitts")
+    maintainers = ["thorstenhater", "brenthuisman"]
 
-    version("master", branch="master", submodules=True)
+    version("master", branch="master")
+    version("develop")
+    version(
+        "0.9.0",
+        sha256="5f9740955c821aca81e23298c17ad64f33f635756ad9b4a0c1444710f564306a",
+        url="https://github.com/arbor-sim/arbor/releases/download/v0.9.0/arbor-v0.9.0-full.tar.gz",
+    )
     version(
         "0.8.1",
         sha256="caebf96676ace6a9c50436541c420ca4bb53f0639dcab825de6fa370aacf6baa",
         url="https://github.com/arbor-sim/arbor/releases/download/v0.8.1/arbor-v0.8.1-full.tar.gz",
     )
     version(
-        "0.8",
+        "0.8.0",
         sha256="18df5600308841616996a9de93b55a105be0f59692daa5febd3a65aae5bc2c5d",
         url="https://github.com/arbor-sim/arbor/releases/download/v0.8/arbor-v0.8-full.tar.gz",
     )
     version(
-        "0.7",
+        "0.7.0",
         sha256="c3a6b7193946aee882bb85f9c38beac74209842ee94e80840968997ba3b84543",
         url="https://github.com/arbor-sim/arbor/releases/download/v0.7/arbor-v0.7-full.tar.gz",
     )
     version(
-        "0.6",
+        "0.6.0",
         sha256="4cd333b18effc8833428ddc0b99e7dc976804771bc85da90034c272c7019e1e8",
         url="https://github.com/arbor-sim/arbor/releases/download/v0.6/arbor-v0.6-full.tar.gz",
     )
@@ -68,8 +74,9 @@ class Arbor(CMakePackage, CudaPackage):
     # misc dependencies
     depends_on("fmt@7.1:", when="@0.5.3:")  # required by the modcc compiler
     depends_on("fmt@9.1:", when="@0.7.1:")
+    depends_on("googletest@1.12.1", when="@0.7.1:")
     depends_on("pugixml@1.11:", when="@0.7.1:")
-    depends_on("nlohmann-json")
+    depends_on("nlohmann-json@3.11.2")
     depends_on("random123")
     with when("+cuda"):
         depends_on("cuda@10:")
@@ -84,9 +91,9 @@ class Arbor(CMakePackage, CudaPackage):
     depends_on("python@3.7:", when="+python", type=("build", "run"))
     depends_on("py-numpy", when="+python", type=("build", "run"))
     with when("+python"):
-        depends_on("py-pybind11@2.6:", type=("build"))
-        depends_on("py-pybind11@2.8.1:", when="@0.5.3:", type=("build"))
-        depends_on("py-pybind11@2.10.1:", when="@0.7.1:", type=("build"))
+        depends_on("py-pybind11@2.6:", type="build")
+        depends_on("py-pybind11@2.8.1:", when="@0.5.3:", type="build")
+        depends_on("py-pybind11@2.10.1:", when="@0.7.1:", type="build")
 
     # sphinx based documentation
     depends_on("python@3.7:", when="+doc", type="build")
@@ -117,3 +124,8 @@ def cmake_args(self):
         args.append("-DARB_CXX_FLAGS_TARGET=" + opt_flags)
 
         return args
+
+    @run_after("install", when="+python")
+    @on_package_attributes(run_tests=True)
+    def install_test(self):
+        python("-c", "import arbor")
diff --git a/var/spack/repos/builtin/packages/arborx/package.py b/var/spack/repos/builtin/packages/arborx/package.py
index 6284bef236ce1d..e766d8897b3b15 100644
--- a/var/spack/repos/builtin/packages/arborx/package.py
+++ b/var/spack/repos/builtin/packages/arborx/package.py
@@ -21,6 +21,7 @@ class Arborx(CMakePackage, CudaPackage, ROCmPackage):
     test_requires_compiler = True
 
     version("master", branch="master")
+    version("1.4.1", sha256="2ca828ef6615859654b233a7df17017e7cfd904982b80026ec7409eb46b77a95")
     version("1.4", sha256="803a1018a6305cf3fea161172b3ada49537f59261279d91c2abbcce9492ee7af")
     version("1.3", sha256="3f1e17f029a460ab99f8396e2772cec908eefc4bf3868c8828907624a2d0ce5d")
     version("1.2", sha256="ed1939110b2330b7994dcbba649b100c241a2353ed2624e627a200a398096c20")
@@ -90,6 +91,7 @@ class Arborx(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("trilinos@13.2.0:", when="@1.2+trilinos")
     depends_on("trilinos@13.4.0:", when="@1.3+trilinos")
     depends_on("trilinos@14.0.0:", when="@1.4:+trilinos")
+    patch("trilinos14.0-kokkos-major-version.patch", when="@1.4+trilinos ^trilinos@14.0.0")
     conflicts("~serial", when="+trilinos")
     conflicts("+cuda", when="+trilinos")
 
diff --git a/var/spack/repos/builtin/packages/arborx/trilinos14.0-kokkos-major-version.patch b/var/spack/repos/builtin/packages/arborx/trilinos14.0-kokkos-major-version.patch
new file mode 100644
index 00000000000000..6d3924b04212b4
--- /dev/null
+++ b/var/spack/repos/builtin/packages/arborx/trilinos14.0-kokkos-major-version.patch
@@ -0,0 +1,16 @@
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -1,7 +1,12 @@
+ cmake_minimum_required(VERSION 3.16)
+ project(ArborX CXX)
+
+-find_package(Kokkos 3.7.01 REQUIRED CONFIG)
++# Workaround to compile against Trilinos 14.0. That version of Trilinos does not
++# set the compatibility mode for Kokkos correctly.
++find_package(Kokkos QUIET 3.7.01 CONFIG)
++if (NOT Kokkos_FOUND)
++  find_package(Kokkos 4.0.00 REQUIRED CONFIG)
++endif()
+ message(STATUS "Found Kokkos: ${Kokkos_DIR} (version \"${Kokkos_VERSION}\")")
+ if(Kokkos_ENABLE_CUDA)
+   kokkos_check(OPTIONS CUDA_LAMBDA)
diff --git a/var/spack/repos/builtin/packages/archer/package.py b/var/spack/repos/builtin/packages/archer/package.py
index 8492eebc96b26c..52011bebd46f26 100644
--- a/var/spack/repos/builtin/packages/archer/package.py
+++ b/var/spack/repos/builtin/packages/archer/package.py
@@ -38,9 +38,9 @@ def patch(self):
 
     def cmake_args(self):
         return [
-            "-DCMAKE_C_COMPILER=clang",
-            "-DCMAKE_CXX_COMPILER=clang++",
-            "-DOMP_PREFIX:PATH=%s" % self.spec["llvm-openmp-ompt"].prefix,
+            self.define("CMAKE_C_COMPILER", "clang"),
+            self.define("CMAKE_CXX_COMPILER", "clang++"),
+            self.define("OMP_PREFIX:PATH", self.spec["llvm-openmp-ompt"].prefix),
         ]
 
     @run_after("install")
@@ -56,7 +56,7 @@ def test_run_parallel_example(self):
             raise SkipTest("Parallel test directory does not exist")
 
         test_exe = "parallel-simple"
-        test_src = "{0}.c".format(test_exe)
+        test_src = f"{test_exe}.c"
         with working_dir(test_dir):
             clang = which("clang-archer")
             clang("-o", test_exe, test_src)
diff --git a/var/spack/repos/builtin/packages/argon2/package.py b/var/spack/repos/builtin/packages/argon2/package.py
index c9762b6bc978cb..d41ba9761c1366 100644
--- a/var/spack/repos/builtin/packages/argon2/package.py
+++ b/var/spack/repos/builtin/packages/argon2/package.py
@@ -20,4 +20,4 @@ class Argon2(MakefilePackage):
     version("20161029", sha256="fe0049728b946b58b94cc6db89b34e2d050c62325d16316a534d2bedd78cd5e7")
 
     def install(self, spec, prefix):
-        make("PREFIX={0}".format(prefix), "install", "LIBRARY_REL=lib")
+        make(f"PREFIX={prefix}", "install", "LIBRARY_REL=lib")
diff --git a/var/spack/repos/builtin/packages/aria2/package.py b/var/spack/repos/builtin/packages/aria2/package.py
index af6de10a6f9040..176fe9048747e2 100644
--- a/var/spack/repos/builtin/packages/aria2/package.py
+++ b/var/spack/repos/builtin/packages/aria2/package.py
@@ -20,6 +20,6 @@ class Aria2(AutotoolsPackage):
     depends_on("libxml2")
     depends_on("libssh2")
     depends_on("libgcrypt")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("c-ares")
     depends_on("sqlite")
diff --git a/var/spack/repos/builtin/packages/arm-forge/package.py b/var/spack/repos/builtin/packages/arm-forge/package.py
index 952b6f42b1bdde..16d842bd1e6e77 100644
--- a/var/spack/repos/builtin/packages/arm-forge/package.py
+++ b/var/spack/repos/builtin/packages/arm-forge/package.py
@@ -22,7 +22,7 @@ class ArmForge(Package):
     # TODO: this mess should be fixed as soon as a way to parametrize/constrain
     #       versions (and checksums) based on the target platform shows up
 
-    if platform.machine() == "aarch64":
+    if platform.machine() in ["aarch64", "arm64"]:
         version(
             "22.1.3", sha256="131884f998b82673e885a7b42cc883210e3a0229b50af374092140cdfd42a408"
         )
diff --git a/var/spack/repos/builtin/packages/armadillo/package.py b/var/spack/repos/builtin/packages/armadillo/package.py
index 9d83de741a8efb..78794086f91b7c 100644
--- a/var/spack/repos/builtin/packages/armadillo/package.py
+++ b/var/spack/repos/builtin/packages/armadillo/package.py
@@ -66,14 +66,14 @@ def cmake_args(self):
 
         return [
             # ARPACK support
-            "-DARPACK_LIBRARY={0}".format(spec["arpack-ng"].libs.joined(";")),
+            self.define("ARPACK_LIBRARY", spec["arpack-ng"].libs.joined(";")),
             # BLAS support
-            "-DBLAS_LIBRARY={0}".format(spec["blas"].libs.joined(";")),
+            self.define("BLAS_LIBRARY", spec["blas"].libs.joined(";")),
             # LAPACK support
-            "-DLAPACK_LIBRARY={0}".format(spec["lapack"].libs.joined(";")),
+            self.define("LAPACK_LIBRARY", spec["lapack"].libs.joined(";")),
             # SuperLU support
-            "-DSuperLU_INCLUDE_DIR={0}".format(spec["superlu"].prefix.include),
-            "-DSuperLU_LIBRARY={0}".format(spec["superlu"].libs.joined(";")),
+            self.define("SuperLU_INCLUDE_DIR", spec["superlu"].prefix.include),
+            self.define("SuperLU_LIBRARY", spec["superlu"].libs.joined(";")),
             # HDF5 support
-            "-DDETECT_HDF5={0}".format("ON" if "+hdf5" in spec else "OFF"),
+            self.define("DETECT_HDF5", "ON" if spec.satisfies("+hdf5") else "OFF"),
         ]
diff --git a/var/spack/repos/builtin/packages/armcomputelibrary/package.py b/var/spack/repos/builtin/packages/armcomputelibrary/package.py
index f7af790e07a5f2..991e430ef6e280 100644
--- a/var/spack/repos/builtin/packages/armcomputelibrary/package.py
+++ b/var/spack/repos/builtin/packages/armcomputelibrary/package.py
@@ -42,12 +42,12 @@ class Armcomputelibrary(SConsPackage):
 
     phases = ["build"]
 
-    variant("build_type", default="release", values=("release", "debug"))
+    variant("build_type", default="release", values=("release", "debug"), description="Build type")
     variant(
         "threads",
         default="cppthreads",
         values=("cppthreads", "openmp"),
-        description="Enable C++11 threads/OpenMP backend. OpenMP backend only"
+        description="Enable C++11 threads/OpenMP backend. OpenMP backend only "
         "works when building with g++ and not clang++.",
     )
     variant(
diff --git a/var/spack/repos/builtin/packages/armpl-gcc/package.py b/var/spack/repos/builtin/packages/armpl-gcc/package.py
index 22f8521d925169..f0157ae551ffe1 100644
--- a/var/spack/repos/builtin/packages/armpl-gcc/package.py
+++ b/var/spack/repos/builtin/packages/armpl-gcc/package.py
@@ -31,11 +31,62 @@
     "rhel8": "RHEL-8",
     "rhel9": "RHEL-9",
     "rocky8": "RHEL-8",
+    "rocky9": "RHEL-9",
     "amzn2": "AmazonLinux-2",
     "amzn2023": "AmazonLinux-2023",
 }
 
 _versions = {
+    "23.10_gcc-12.2": {
+        "RHEL-7": ("e5e2c69ad281a676f2a06c835fbf31d4f9fdf46aa3f3f7c8aafff46985f64902"),
+        "RHEL-8": ("cc0f3572ead93d1e31797b7a39a40cff3414878df9bd24a452bf4877dc35ca4c"),
+        "RHEL-9": ("18c75f57333031e454921cc3f4f22fd567e5a701424ff9ac219bbfe9955a8a96"),
+        "SLES-15": ("e1e891eceaffedecf7351e2c499ef2b49a36c9af29174b366ff470d0a568c18f"),
+        "Ubuntu-20.04": ("976424875c52c2062fc76cbc5d527ee82413cdc0432d7c59f423295a3b0cc612"),
+        "Ubuntu-22.04": ("6dd778edf55e13e8b766d75c340f0259f6cb507a93966d76d188b8b3943c769b"),
+        "AmazonLinux-2": ("423ac3df262b5fcca6cea480503b693306c970dd8e8e05c753ece92446ac7fee"),
+        "AmazonLinux-2023": ("acadf3b6cde866cb41f7363b290a646a492769aaa5819d4c0d60df89913342a9"),
+    },
+    "23.10_gcc-11.3": {
+        "RHEL-7": ("b2afbdc056ae01fb5c71935448b19300ef368962a94ae76b8811f1d328c723c2"),
+        "RHEL-8": ("79b83a8a2c46b949896b3964c761cbd0b66c37826996afb62c466af5fb420bc2"),
+        "RHEL-9": ("7a84f561bcf941bb25123b3ef730b4c02616bc51215933870677163e78af38e3"),
+        "SLES-15": ("9243c405d092d3eabff112ccabc300e96f13c3d2c5c319df04d7093bb6f535a2"),
+        "Ubuntu-20.04": ("a16df088ef9303040d92b017b233c6e4c6f0300d09c2ad0a66c0318831bf009c"),
+        "Ubuntu-22.04": ("fabda66dc6388fa8c094443fa53deece5590db66caaa6a1e39e99e64d5bb0709"),
+        "AmazonLinux-2": ("db5d039fa1d07695a71b8733584d878bb778d41bc0ecc3e19059b75cffdcf8cd"),
+        "AmazonLinux-2023": ("977fd465702f086a69e3f7fc28f2bcb6c79a7af381dc7d865345115b26f4631f"),
+    },
+    "23.10_gcc-10.4": {
+        "RHEL-7": ("3c8bad3af82a76ca1a45705afd47028cc26c7093377a554e692e1cd6f61cb304"),
+        "RHEL-8": ("381afae0e3e94aa91029f571de0e51c2342e50b4f855db7a9b9ca66e16e26276"),
+        "SLES-15": ("226e9519407331b4ad5ded8699cd15f1d9b845843304bbf21f47009a399fe2a0"),
+        "Ubuntu-20.04": ("45de59f795ad9026a838ab611b03b1644169a034ce59d6cca2c7940850fa17ad"),
+        "AmazonLinux-2": ("637b51da12548dc66da9132328fe2ea39ba0736af66fb30332ca8eeb540e3373"),
+    },
+    "23.10_gcc-9.3": {
+        "RHEL-7": ("6fc2e3319b83ea2b1bf8d98ec43f614b937bb5f23d15aefe9e9171c882d24a60"),
+        "RHEL-8": ("1a05548a7051d1df42280fdcfcffeaf89d519aa7978bffd29171da60fdbccecf"),
+        "SLES-15": ("389ddd34e1299e4d942864f63f236158a81ce4190f59af512a1bea3221153bfe"),
+        "Ubuntu-20.04": ("a1a221859b5f0962df3a0c6ce31669827bff0bfffb185b80429620f14b40f4f4"),
+        "AmazonLinux-2": ("2eef9b28e95e75f0040eb61c9e1b406ec4d0b81cce3e95a652029aa0898733a0"),
+    },
+    "23.10_gcc-8.2": {
+        "RHEL-7": ("d6596721e74e7bdc8d9ce7b8b2a4c5ab2bd430f3ca69b9ec84f587f1aa181083"),
+        "RHEL-8": ("004aed52003e19a6c14df303456318e486ad783eb543b79285c7953a23722a4a"),
+        "SLES-15": ("12c638c0cc5bdc220699499ec6bb160a7b889f105901f4354bd2748a77d25c8e"),
+        "AmazonLinux-2": ("d039134236cda298cd0920c3c5b017eeef83fcab82949221dc7deb081026252f"),
+    },
+    "23.10_gcc-7.5": {
+        "RHEL-7": ("1a0ca860c168987d174923dfc7800e10521303914793162a8bae2b2cd3f68203"),
+        "AmazonLinux-2": ("58b201a6bbe7ee10563d8d42b32a77c4b15c57b4e81abb35d24b8c3fc9cff4d9"),
+    },
+    "23.10_flang-new_clang_17": {
+        "macOS": ("baf09cd6d1d1b7c780b8b31cfe1dd709596b182dc714127fbc9f23007ff9e23a")
+    },
+    "23.06_flang-new_clang_16": {
+        "macOS": ("232f5e89e0f1f4777480c64a790e477dfd2f423d3cf5704a116a2736f36250ea")
+    },
     "23.04.1_gcc-12.2": {
         "RHEL-7": ("789cc093cb7e0d9294aff0fdf94b74987435a09cdff4c1b7118a03350548d03c"),
         "RHEL-8": ("1b668baec6d3df2d48c5aedc70baa6a9b638983b94bf2cd58d378859a1da49f0"),
@@ -177,20 +228,28 @@
 
 
 def get_os(ver):
-    spack_os = spack.platforms.host().default_os
+    platform = spack.platforms.host()
+    if platform.name == "darwin":
+        return "macOS"
     if ver.startswith("22."):
-        return _os_map_before_23.get(spack_os, "")
+        return _os_map_before_23.get(platform.default_os, "")
     else:
-        return _os_map.get(spack_os, "RHEL-7")
+        return _os_map.get(platform.default_os, "RHEL-7")
 
 
 def get_package_url(version):
     base_url = "https://developer.arm.com/-/media/Files/downloads/hpc/arm-performance-libraries/"
     armpl_version = version.split("_")[0]
     armpl_version_dashed = armpl_version.replace(".", "-")
-    gcc_version = version.split("_")[1]
+    compiler_version = version.split("_", 1)[1]
     os = get_os(armpl_version)
-    filename = "arm-performance-libraries_" + armpl_version + "_" + os + "_" + gcc_version + ".tar"
+    if os == "macOS":
+        if armpl_version.startswith("23.06"):
+            return f"{base_url}{armpl_version_dashed}/armpl_{armpl_version}_{compiler_version}.dmg"
+        else:
+            filename = f"arm-performance-libraries_{armpl_version}_macOS.dmg"
+            return f"{base_url}{armpl_version_dashed}/macos/{filename}"
+    filename = f"arm-performance-libraries_{armpl_version}_{os}_{compiler_version}.tar"
     os_short = ""
     if armpl_version.startswith("22.0."):
         os_short = os.replace("-", "")
@@ -198,7 +257,7 @@ def get_package_url(version):
         os_short = os.split(".")[0].lower()
         if "amazonlinux" in os_short:
             os_short = os_short.replace("amazonlinux", "al")
-    return base_url + armpl_version_dashed + "/" + os_short + "/" + filename
+    return f"{base_url}{armpl_version_dashed}/{os_short}/{filename}"
 
 
 def get_armpl_prefix(spec):
@@ -215,16 +274,26 @@ class ArmplGcc(Package):
     maintainers("annop-w")
 
     for ver, packages in _versions.items():
-        key = "{0}".format(get_os(ver))
+        key = get_os(ver)
         sha256sum = packages.get(key)
         url = get_package_url(ver)
         if sha256sum:
-            version(ver, sha256=sha256sum, url=url)
+            extension = os.path.splitext(url)[1]
+            # Don't attempt to expand .dmg files
+            expand = extension != ".dmg"
+            version(ver, sha256=sha256sum, url=url, extension=extension, expand=expand)
 
     conflicts("target=x86:", msg="Only available on Aarch64")
     conflicts("target=ppc64:", msg="Only available on Aarch64")
     conflicts("target=ppc64le:", msg="Only available on Aarch64")
 
+    conflicts("%gcc@:11", when="@23.10_gcc-12.2")
+    conflicts("%gcc@:10", when="@23.10_gcc-11.3")
+    conflicts("%gcc@:9", when="@23.10_gcc-10.4")
+    conflicts("%gcc@:8", when="@23.10_gcc-9.3")
+    conflicts("%gcc@:7", when="@23.10_gcc-8.2")
+    conflicts("%gcc@:6", when="@23.10_gcc-7.5")
+
     conflicts("%gcc@:11", when="@23.04.1_gcc-12.2")
     conflicts("%gcc@:10", when="@23.04.1_gcc-11.3")
     conflicts("%gcc@:9", when="@23.04.1_gcc-10.2")
@@ -266,17 +335,29 @@ class ArmplGcc(Package):
 
     # Run the installer with the desired install directory
     def install(self, spec, prefix):
+        if spec.platform == "darwin":
+            hdiutil = which("hdiutil")
+            # Mount image
+            mountpoint = os.path.join(self.stage.path, "mount")
+            hdiutil("attach", "-mountpoint", mountpoint, self.stage.archive_file)
+            try:
+                # Run installer
+                exe_name = f"armpl_{spec.version.string}_install.sh"
+                installer = Executable(os.path.join(mountpoint, exe_name))
+                installer("-y", f"--install_dir={prefix}")
+            finally:
+                # Unmount image
+                hdiutil("detach", mountpoint)
+            return
         if self.compiler.name != "gcc":
             raise spack.error.SpackError(("Only compatible with GCC.\n"))
 
         with when("@:22"):
-            armpl_version = "{}".format(spec.version.up_to(3)).split("_")[0]
+            armpl_version = spec.version.up_to(3).string.split("_")[0]
         with when("@23:"):
-            armpl_version = "{}".format(spec.version).split("_")[0]
+            armpl_version = spec.version.string.split("_")[0]
 
-        exe = Executable(
-            "./arm-performance-libraries_{0}_{1}.sh".format(armpl_version, get_os(armpl_version))
-        )
+        exe = Executable(f"./arm-performance-libraries_{armpl_version}_{get_os(armpl_version)}.sh")
         exe("--accept", "--force", "--install-to", prefix)
 
     @property
@@ -330,14 +411,22 @@ def headers(self):
 
     def setup_run_environment(self, env):
         armpl_dir = get_armpl_prefix(self.spec)
-        env.prepend_path("LD_LIBRARY_PATH", join_path(armpl_dir, "lib"))
+        if self.spec.platform == "darwin":
+            env.prepend_path("DYLD_LIBRARY_PATH", join_path(armpl_dir, "lib"))
+        else:
+            env.prepend_path("LD_LIBRARY_PATH", join_path(armpl_dir, "lib"))
 
     @run_after("install")
     def check_install(self):
         armpl_dir = get_armpl_prefix(self.spec)
         armpl_example_dir = join_path(armpl_dir, "examples")
         # run example makefile
-        make("-C", armpl_example_dir, "ARMPL_DIR=" + armpl_dir)
+        if self.spec.platform == "darwin":
+            # Fortran examples on MacOS requires flang-new which is
+            # not commonly installed, so only run the C examples.
+            make("-C", armpl_example_dir, "ARMPL_DIR=" + armpl_dir, "c_examples")
+        else:
+            make("-C", armpl_example_dir, "ARMPL_DIR=" + armpl_dir)
         # clean up
         make("-C", armpl_example_dir, "ARMPL_DIR=" + armpl_dir, "clean")
 
diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py
index 92176069c1f19a..c50b90d6d10f45 100644
--- a/var/spack/repos/builtin/packages/arpack-ng/package.py
+++ b/var/spack/repos/builtin/packages/arpack-ng/package.py
@@ -150,14 +150,14 @@ def configure_args(self):
         options = (
             self.enable_or_disable("mpi")
             + [
-                "--with-blas={0}".format(spec["blas"].libs.ld_flags),
-                "--with-lapack={0}".format(spec["lapack"].libs.ld_flags),
+                f"--with-blas={spec['blas'].libs.ld_flags}",
+                f"--with-lapack={spec['lapack'].libs.ld_flags}",
             ]
             + self.enable_or_disable("shared")
         )
 
         if "+mpi" in spec:
-            options.append("F77={0}".format(spec["mpi"].mpif77))
+            options.append(f"F77={spec['mpi'].mpif77}")
 
         return options
 
diff --git a/var/spack/repos/builtin/packages/arrayfire/package.py b/var/spack/repos/builtin/packages/arrayfire/package.py
index 70cdfc67b200ab..9befc618417835 100644
--- a/var/spack/repos/builtin/packages/arrayfire/package.py
+++ b/var/spack/repos/builtin/packages/arrayfire/package.py
@@ -79,7 +79,7 @@ def cmake_args(self):
             ]
             args.append(self.define("CUDA_architecture_build_targets", arch_list))
 
-        if "^mkl" in self.spec:
+        if self.spec["blas"].name in INTEL_MATH_LIBRARIES:
             if self.version >= Version("3.8.0"):
                 args.append(self.define("AF_COMPUTE_LIBRARY", "Intel-MKL"))
             else:
diff --git a/var/spack/repos/builtin/packages/arrow/package.py b/var/spack/repos/builtin/packages/arrow/package.py
index 3281affb1a9b79..8a499fbec2709e 100644
--- a/var/spack/repos/builtin/packages/arrow/package.py
+++ b/var/spack/repos/builtin/packages/arrow/package.py
@@ -56,8 +56,9 @@ class Arrow(CMakePackage, CudaPackage):
     depends_on("utf8proc@2.7.0: +shared", when="+gandiva")
     depends_on("utf8proc@2.7.0: +shared", when="+python")
     depends_on("xsimd@8.1.0:", when="@9.0.0:")
-    depends_on("zlib+pic", when="+zlib @9:")
-    depends_on("zlib+pic", when="@:8")
+    depends_on("zlib-api", when="+zlib @9:")
+    depends_on("zlib-api", when="@:8")
+    conflicts("^zlib~pic")
     depends_on("zstd", when="+zstd @9:")
     depends_on("zstd", when="@:8")
 
@@ -147,14 +148,23 @@ def cmake_args(self):
         args.append(self.define_from_variant("ARROW_WITH_ZLIB", "zlib"))
         args.append(self.define_from_variant("ARROW_WITH_ZSTD", "zstd"))
 
-        with when("@:8"):
-            dep_list = ("flatbuffers", "rapidjson", "zlib", "zstd")
+        if not self.spec.dependencies("re2"):
+            args.append(self.define("ARROW_WITH_RE2", False))
+        if not self.spec.dependencies("utf8proc"):
+            args.append(self.define("ARROW_WITH_UTF8PROC", False))
+
+        if self.spec.satisfies("@:8"):
+            args.extend(
+                [
+                    self.define("FLATBUFFERS_HOME", self.spec["flatbuffers"].prefix),
+                    self.define("RAPIDJSON_HOME", self.spec["rapidjson"].prefix),
+                    self.define("ZSTD_HOME", self.spec["zstd"].prefix),
+                    self.define("ZLIB_HOME", self.spec["zlib-api"].prefix),
+                    self.define("ZLIB_LIBRARIES", self.spec["zlib-api"].libs),
+                ]
+            )
 
             if self.spec.satisfies("+snappy"):
-                dep_list.append("snappy")
-
-            for dep in dep_list:
-                args.append("-D{0}_HOME={1}".format(dep.upper(), self.spec[dep].prefix))
-            args.append("-DZLIB_LIBRARIES={0}".format(self.spec["zlib"].libs))
+                args.append(self.define("SNAPPY_HOME", self.spec["snappy"].prefix))
 
         return args
diff --git a/var/spack/repos/builtin/packages/ascent/package.py b/var/spack/repos/builtin/packages/ascent/package.py
index 394014b52f28ee..566be6b632de3f 100644
--- a/var/spack/repos/builtin/packages/ascent/package.py
+++ b/var/spack/repos/builtin/packages/ascent/package.py
@@ -52,19 +52,37 @@ class Ascent(CMakePackage, CudaPackage):
 
     version("develop", branch="develop", submodules=True)
 
-    version("0.9.2", tag="v0.9.2", submodules=True, preferred=True)
+    version(
+        "0.9.2",
+        tag="v0.9.2",
+        commit="b842516d12640e4a0d9433a18c7249440ef6fc3d",
+        submodules=True,
+        preferred=True,
+    )
 
-    version("0.9.1", tag="v0.9.1", submodules=True)
+    version(
+        "0.9.1", tag="v0.9.1", commit="027a2fe184f65a4923817a8cdfed0b0c61c2c75a", submodules=True
+    )
 
-    version("0.9.0", tag="v0.9.0", submodules=True)
+    version(
+        "0.9.0", tag="v0.9.0", commit="a31c88c579c8d0026e0025de8bace0cf22f6305b", submodules=True
+    )
 
-    version("0.8.0", tag="v0.8.0", submodules=True)
+    version(
+        "0.8.0", tag="v0.8.0", commit="08504374908518e013d7fe8d8882cfb1c2378e3b", submodules=True
+    )
 
-    version("0.7.1", tag="v0.7.1", submodules=True)
+    version(
+        "0.7.1", tag="v0.7.1", commit="79d35b2f48e92eb151313f0217e9bd7c15779582", submodules=True
+    )
 
-    version("0.7.0", tag="v0.7.0", submodules=True)
+    version(
+        "0.7.0", tag="v0.7.0", commit="cfed1b0a469e4dcc970fd7e0bcd78b522d97ea53", submodules=True
+    )
 
-    version("0.6.0", tag="v0.6.0", submodules=True)
+    version(
+        "0.6.0", tag="v0.6.0", commit="9ade37b0a9ea495e45adb25cda7498c0bf9465c5", submodules=True
+    )
 
     ###########################################################################
     # package variants
@@ -187,7 +205,7 @@ class Ascent(CMakePackage, CudaPackage):
     depends_on("vtk-m+fpic", when="@0.8.0: +vtkh")
     depends_on("vtk-m~shared+fpic", when="@0.8.0: +vtkh~shared")
     # Ascent defaults to C++11
-    depends_on("kokkos std=11", when="+vtkh ^vtk-m +kokkos")
+    depends_on("kokkos cxxstd=11", when="+vtkh ^vtk-m +kokkos")
 
     #######################
     # VTK-h
diff --git a/var/spack/repos/builtin/packages/asdcplib/package.py b/var/spack/repos/builtin/packages/asdcplib/package.py
index 7ca3b2f9f1b786..5aec849ee59480 100644
--- a/var/spack/repos/builtin/packages/asdcplib/package.py
+++ b/var/spack/repos/builtin/packages/asdcplib/package.py
@@ -27,6 +27,6 @@ class Asdcplib(AutotoolsPackage):
     def configure_args(self):
         spec = self.spec
 
-        args = ["--with-openssl={0}".format(spec["openssl"].prefix)]
+        args = [f"--with-openssl={spec['openssl'].prefix}"]
 
         return args
diff --git a/var/spack/repos/builtin/packages/asdf-cxx/package.py b/var/spack/repos/builtin/packages/asdf-cxx/package.py
index 840d89579948d9..01e83e1923070e 100644
--- a/var/spack/repos/builtin/packages/asdf-cxx/package.py
+++ b/var/spack/repos/builtin/packages/asdf-cxx/package.py
@@ -47,4 +47,4 @@ class AsdfCxx(CMakePackage):
     depends_on("swig @3.0.0:3", type="build")
     # Neither earlier nor later versions of yaml-cpp work
     depends_on("yaml-cpp @0.6.3")
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/aspa/package.py b/var/spack/repos/builtin/packages/aspa/package.py
index 6bfbad1d3926e2..8219a46b004bd3 100644
--- a/var/spack/repos/builtin/packages/aspa/package.py
+++ b/var/spack/repos/builtin/packages/aspa/package.py
@@ -35,12 +35,12 @@ def build_targets(self):
         targets = [
             "--directory=exec",
             "--file=Makefile",
-            "LIBS={0} {1} {2}".format(
+            "LIBS={} {} {}".format(
                 self.spec["lapack"].libs.ld_flags,
                 self.spec["blas"].libs.ld_flags,
                 self.spec["hdf5"].libs.ld_flags,
             ),
-            "CXX={0}".format(self.spec["mpi"].mpicxx),
+            f"CXX={self.spec['mpi'].mpicxx}",
         ]
         return targets
 
diff --git a/var/spack/repos/builtin/packages/aspcud/package.py b/var/spack/repos/builtin/packages/aspcud/package.py
index fbaef453e990e4..8233dcaba0e058 100644
--- a/var/spack/repos/builtin/packages/aspcud/package.py
+++ b/var/spack/repos/builtin/packages/aspcud/package.py
@@ -28,11 +28,9 @@ class Aspcud(CMakePackage):
     depends_on("clingo")
 
     def cmake_args(self):
-        spec = self.spec
-        gringo_path = join_path(spec["clingo"].prefix.bin, "gringo")
-        clasp_path = join_path(spec["clingo"].prefix.bin, "clasp")
-        args = [
-            "-DASPCUD_GRINGO_PATH={0}".format(gringo_path),
-            "-DASPCUD_CLASP_PATH={0}".format(clasp_path),
+        gringo_path = join_path(self.spec["clingo"].prefix.bin, "gringo")
+        clasp_path = join_path(self.spec["clingo"].prefix.bin, "clasp")
+        return [
+            self.define("ASPCUD_GRINGO_PATH", gringo_path),
+            self.define("ASPCUD_CLASP_PATH", clasp_path),
         ]
-        return args
diff --git a/var/spack/repos/builtin/packages/aspera-cli/package.py b/var/spack/repos/builtin/packages/aspera-cli/package.py
index afa63b4c33dc8f..91aa1e19e47818 100644
--- a/var/spack/repos/builtin/packages/aspera-cli/package.py
+++ b/var/spack/repos/builtin/packages/aspera-cli/package.py
@@ -29,7 +29,7 @@ def install(self, spec, prefix):
         # Update destination path
         filter_file(
             "INSTALL_DIR=~/.aspera",
-            "INSTALL_DIR=%s" % prefix,
+            f"INSTALL_DIR={prefix}",
             runfile,
             string=True,
             stop_at="__ARCHIVE_FOLLOWS__",
diff --git a/var/spack/repos/builtin/packages/assimp/package.py b/var/spack/repos/builtin/packages/assimp/package.py
index 58a4377ce818fd..96dda57b817d75 100644
--- a/var/spack/repos/builtin/packages/assimp/package.py
+++ b/var/spack/repos/builtin/packages/assimp/package.py
@@ -17,6 +17,7 @@ class Assimp(CMakePackage):
     maintainers("wdconinc")
 
     version("master", branch="master")
+    version("5.3.1", sha256="a07666be71afe1ad4bc008c2336b7c688aca391271188eb9108d0c6db1be53f1")
     version("5.2.5", sha256="b5219e63ae31d895d60d98001ee5bb809fb2c7b2de1e7f78ceeb600063641e1a")
     version("5.2.4", sha256="6a4ff75dc727821f75ef529cea1c4fc0a7b5fc2e0a0b2ff2f6b7993fe6cb54ba")
     version("5.2.3", sha256="b20fc41af171f6d8f1f45d4621f18e6934ab7264e71c37cd72fd9832509af2a8")
@@ -36,7 +37,7 @@ class Assimp(CMakePackage):
     variant("shared", default=True, description="Enables the build of shared libraries")
 
     depends_on("pkgconfig", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def patch(self):
         filter_file("-Werror", "", "code/CMakeLists.txt")
diff --git a/var/spack/repos/builtin/packages/astral/package.py b/var/spack/repos/builtin/packages/astral/package.py
index 3afa3691ba0b0f..af196ab9c391c4 100644
--- a/var/spack/repos/builtin/packages/astral/package.py
+++ b/var/spack/repos/builtin/packages/astral/package.py
@@ -31,7 +31,7 @@ def install(self, spec, prefix):
         make()
         mkdirp(prefix.bin)
         install_tree("lib", prefix.tools.lib)
-        jar_file = "astral.{v}.jar".format(v=self.version)
+        jar_file = f"astral.{self.version}.jar"
         install(jar_file, prefix.tools)
 
         script_sh = join_path(os.path.dirname(__file__), "astral.sh")
diff --git a/var/spack/repos/builtin/packages/astyle/package.py b/var/spack/repos/builtin/packages/astyle/package.py
index 951661004b724a..ef4fe29378ffa0 100644
--- a/var/spack/repos/builtin/packages/astyle/package.py
+++ b/var/spack/repos/builtin/packages/astyle/package.py
@@ -30,11 +30,11 @@ def build_directory(self):
 
     def edit(self, spec, prefix):
         makefile = join_path(self.build_directory, "Makefile")
-        filter_file(r"^CXX\s*=.*", "CXX=%s" % spack_cxx, makefile)
+        filter_file(r"^CXX\s*=.*", f"CXX={spack_cxx}", makefile)
         # If the group is not a user account, the installation will fail,
         # so remove the -o $ (USER) -g $ (USER) parameter.
         filter_file(r"^INSTALL=.*", "INSTALL=install", makefile)
 
     @property
     def install_targets(self):
-        return ["install", "prefix={0}".format(self.prefix)]
+        return ["install", f"prefix={self.prefix}"]
diff --git a/var/spack/repos/builtin/packages/at-spi2-atk/package.py b/var/spack/repos/builtin/packages/at-spi2-atk/package.py
index 27e875f9f65c30..6e2f492112ce25 100644
--- a/var/spack/repos/builtin/packages/at-spi2-atk/package.py
+++ b/var/spack/repos/builtin/packages/at-spi2-atk/package.py
@@ -27,4 +27,4 @@ class AtSpi2Atk(MesonPackage):
     def url_for_version(self, version):
         """Handle gnome's version-based custom URLs."""
         url = "http://ftp.gnome.org/pub/gnome/sources/at-spi2-atk"
-        return url + "/%s/at-spi2-atk-%s.tar.xz" % (version.up_to(2), version)
+        return url + f"/{version.up_to(2)}/at-spi2-atk-{version}.tar.xz"
diff --git a/var/spack/repos/builtin/packages/at-spi2-core/package.py b/var/spack/repos/builtin/packages/at-spi2-core/package.py
index f1bf5a61c86fca..ec8cbd5e23c68b 100644
--- a/var/spack/repos/builtin/packages/at-spi2-core/package.py
+++ b/var/spack/repos/builtin/packages/at-spi2-core/package.py
@@ -45,7 +45,7 @@ def patch(self):
     def url_for_version(self, version):
         """Handle gnome's version-based custom URLs."""
         url = "http://ftp.gnome.org/pub/gnome/sources/at-spi2-core"
-        return url + "/%s/at-spi2-core-%s.tar.xz" % (version.up_to(2), version)
+        return url + f"/{version.up_to(2)}/at-spi2-core-{version}.tar.xz"
 
     def setup_run_environment(self, env):
         env.prepend_path("GI_TYPELIB_PATH", join_path(self.prefix.lib, "girepository-1.0"))
diff --git a/var/spack/repos/builtin/packages/atk/package.py b/var/spack/repos/builtin/packages/atk/package.py
index 41dec1c587080d..52849669ecca01 100644
--- a/var/spack/repos/builtin/packages/atk/package.py
+++ b/var/spack/repos/builtin/packages/atk/package.py
@@ -43,7 +43,7 @@ class Atk(Package):
     def url_for_version(self, version):
         """Handle gnome's version-based custom URLs."""
         url = "http://ftp.gnome.org/pub/gnome/sources/atk"
-        return url + "/%s/atk-%s.tar.xz" % (version.up_to(2), version)
+        return url + f"/{version.up_to(2)}/atk-{version}.tar.xz"
 
     def setup_run_environment(self, env):
         env.prepend_path("GI_TYPELIB_PATH", join_path(self.prefix.lib, "girepository-1.0"))
@@ -64,7 +64,7 @@ def install(self, spec, prefix):
 
     @when("@:2.27")
     def install(self, spec, prefix):
-        configure("--prefix={0}".format(prefix))
+        configure(f"--prefix={prefix}")
         make()
         if self.run_tests:
             make("check")
diff --git a/var/spack/repos/builtin/packages/atlas/package.py b/var/spack/repos/builtin/packages/atlas/package.py
index d23a16cb1ba16b..ecc6379b570c62 100644
--- a/var/spack/repos/builtin/packages/atlas/package.py
+++ b/var/spack/repos/builtin/packages/atlas/package.py
@@ -109,11 +109,11 @@ def install(self, spec, prefix):
 
         # Lapack resource to provide full lapack build. Note that
         # ATLAS only provides a few LAPACK routines natively.
-        options.append("--with-netlib-lapack-tarfile=%s" % self.stage[1].archive_file)
+        options.append(f"--with-netlib-lapack-tarfile={self.stage[1].archive_file}")
 
         with working_dir("spack-build", create=True):
             configure = Executable("../configure")
-            configure("--prefix=%s" % prefix, *options)
+            configure(f"--prefix={prefix}", *options)
             make()
             make("check")
             make("ptcheck")
@@ -147,7 +147,7 @@ def install_test(self):
         source_file = join_path(os.path.dirname(self.module.__file__), "test_cblas_dgemm.c")
         blessed_file = join_path(os.path.dirname(self.module.__file__), "test_cblas_dgemm.output")
 
-        include_flags = ["-I%s" % self.spec.prefix.include]
+        include_flags = [f"-I{self.spec.prefix.include}"]
         link_flags = self.spec["atlas"].libs.ld_flags.split()
 
         output = compile_c_and_execute(source_file, include_flags, link_flags)
diff --git a/var/spack/repos/builtin/packages/atmi/package.py b/var/spack/repos/builtin/packages/atmi/package.py
index be8664fcd34420..8d57098d3cac7b 100644
--- a/var/spack/repos/builtin/packages/atmi/package.py
+++ b/var/spack/repos/builtin/packages/atmi/package.py
@@ -15,11 +15,12 @@ class Atmi(CMakePackage):
 
     homepage = "https://github.com/RadeonOpenCompute/atmi"
     git = "https://github.com/RadeonOpenCompute/atmi.git"
-    url = "https://github.com/RadeonOpenCompute/atmi/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/atmi/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
-
+    version("5.5.1", sha256="6b3ee68433506315b55d093a4b47463916874fb6f3f602098eaff2ec283e69ab")
+    version("5.5.0", sha256="b8bfd32e5c386f5169da62172964343f9b7fad207e0e74dd1093c7acf06d9811")
     version("5.4.3", sha256="243aae6614e5bd136a099102957a6d65a01434b620291349613ad63701868ef8")
     version("5.4.0", sha256="b5cce10d7099fecbb40a0d9c2f29a7675315471fe145212b375e37e4c8ba5618")
     version("5.3.3", sha256="cc1144e4939cea2944f6c72a21406b9dc5b56d933696494074c280df7469834a")
@@ -100,13 +101,6 @@ class Atmi(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3:", type="build")
     depends_on("rsync")
 
@@ -134,11 +128,16 @@ class Atmi(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
     ]:
         depends_on("comgr@" + ver, type="link", when="@" + ver)
         depends_on("hsa-rocr-dev@" + ver, type="link", when="@" + ver)
         depends_on("elf", type="link", when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     root_cmakelists_dir = "src"
 
     patch("0001-Remove-relative-link-paths-to-external-libraries.patch", when="@3.5.0")
diff --git a/var/spack/repos/builtin/packages/atom-dft/package.py b/var/spack/repos/builtin/packages/atom-dft/package.py
index 9015067428fb15..3f8c5e7756303a 100644
--- a/var/spack/repos/builtin/packages/atom-dft/package.py
+++ b/var/spack/repos/builtin/packages/atom-dft/package.py
@@ -24,8 +24,8 @@ def edit(self, spec, prefix):
     @property
     def build_targets(self):
         return [
-            "XMLF90_ROOT=%s" % self.spec["xmlf90"].prefix,
-            "GRIDXC_ROOT=%s" % self.spec["libgridxc"].prefix,
+            f"XMLF90_ROOT={self.spec['xmlf90'].prefix}",
+            f"GRIDXC_ROOT={self.spec['libgridxc'].prefix}",
             "FC=fc",
         ]
 
diff --git a/var/spack/repos/builtin/packages/atompaw/package.py b/var/spack/repos/builtin/packages/atompaw/package.py
index 7cc4b4d417fc0f..f0ea750583910a 100644
--- a/var/spack/repos/builtin/packages/atompaw/package.py
+++ b/var/spack/repos/builtin/packages/atompaw/package.py
@@ -49,8 +49,8 @@ def configure_args(self):
         spec = self.spec
         linalg = spec["lapack"].libs + spec["blas"].libs
         return [
-            "--with-linalg-libs=%s" % linalg.ld_flags,
+            f"--with-linalg-libs={linalg.ld_flags}",
             "--enable-libxc",
-            "--with-libxc-incs=-I%s" % spec["libxc"].prefix.include,
-            "--with-libxc-libs=-L%s -lxcf90 -lxc" % spec["libxc"].prefix.lib,
+            f"--with-libxc-incs=-I{spec['libxc'].prefix.include}",
+            f"--with-libxc-libs=-L{spec['libxc'].prefix.lib} -lxcf90 -lxc",
         ]
diff --git a/var/spack/repos/builtin/packages/atop/package.py b/var/spack/repos/builtin/packages/atop/package.py
index f2dd7d3929f60f..670b7e37c7538d 100644
--- a/var/spack/repos/builtin/packages/atop/package.py
+++ b/var/spack/repos/builtin/packages/atop/package.py
@@ -18,7 +18,7 @@ class Atop(Package):
     version("2.2.6", sha256="d0386840ee4df36e5d0ad55f144661b434d9ad35d94deadc0405b514485db615")
     version("2.2-3", sha256="c785b8a2355be28b3de6b58a8ea4c4fcab8fadeaa57a99afeb03c66fac8e055d")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("ncurses")
 
     def setup_build_environment(self, env):
diff --git a/var/spack/repos/builtin/packages/audacious/package.py b/var/spack/repos/builtin/packages/audacious/package.py
index 4cce09e0fcf23f..1d6634780b5870 100644
--- a/var/spack/repos/builtin/packages/audacious/package.py
+++ b/var/spack/repos/builtin/packages/audacious/package.py
@@ -28,7 +28,7 @@ class Audacious(AutotoolsPackage):
 
     def patch(self):
         search_path_args = " ".join(self.autoreconf_search_path_args)
-        search_path_str = "-I m4 {0}".format(search_path_args)
+        search_path_str = f"-I m4 {search_path_args}"
         filter_file("-I m4", search_path_str, "autogen.sh")
 
     def autoreconf(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/augustus/package.py b/var/spack/repos/builtin/packages/augustus/package.py
index 5aad91293444c5..2c5cfa5c0d531e 100644
--- a/var/spack/repos/builtin/packages/augustus/package.py
+++ b/var/spack/repos/builtin/packages/augustus/package.py
@@ -45,7 +45,7 @@ class Augustus(MakefilePackage):
     # for instance depends_on('boost +filesystem')
     # See https://github.com/spack/spack/pull/22303 for reference
     depends_on(Boost.with_default_variants)
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("htslib")
     depends_on("bcftools")
     depends_on("samtools")
@@ -69,7 +69,7 @@ def edit(self, spec, prefix):
                 filter_file("g++", spack_cxx, "makefile", string=True)
                 filter_file(
                     "g++ -I/usr/include/boost",
-                    "{0} -I{1}".format(spack_cxx, self.spec["boost"].prefix.include),
+                    f"{spack_cxx} -I{self.spec['boost'].prefix.include}",
                     "src/subdir.mk",
                     string=True,
                 )
@@ -101,7 +101,7 @@ def edit(self, spec, prefix):
 
         with working_dir(join_path("auxprogs", "filterBam", "src")):
             makefile = FileFilter("Makefile")
-            makefile.filter("BAMTOOLS = .*", "BAMTOOLS = {0}".format(bamtools))
+            makefile.filter("BAMTOOLS = .*", f"BAMTOOLS = {bamtools}")
             makefile.filter("INCLUDES = *", "INCLUDES = -I$(BAMTOOLS)/include/bamtools ")
             if "bamtools@2.5:" in spec:
                 makefile.filter(
@@ -113,32 +113,30 @@ def edit(self, spec, prefix):
                 )
         with working_dir(join_path("auxprogs", "bam2hints")):
             makefile = FileFilter("Makefile")
-            makefile.filter("/usr/include/bamtools", "{0}/include/bamtools".format(bamtools))
+            makefile.filter("/usr/include/bamtools", f"{bamtools}/include/bamtools")
             if "bamtools@2.5:" in spec:
                 makefile.filter(
-                    "LIBS = -lbamtools -lz",
-                    "LIBS = {0}/lib64" "/libbamtools.a -lz".format(bamtools),
+                    "LIBS = -lbamtools -lz", f"LIBS = {bamtools}/lib64/libbamtools.a -lz"
                 )
             if "bamtools@:2.4" in spec:
                 makefile.filter(
-                    "LIBS = -lbamtools -lz",
-                    "LIBS = {0}/lib/bamtools" "/libbamtools.a -lz".format(bamtools),
+                    "LIBS = -lbamtools -lz", f"LIBS = {bamtools}/lib/bamtools/libbamtools.a -lz"
                 )
 
         if self.version < Version("3.4.0"):
             with working_dir(join_path("auxprogs", "bam2wig")):
                 makefile = FileFilter("Makefile")
                 # point tools to spack installations
-                makefile.filter("BCFTOOLS=.*$", "BCFTOOLS={0}/include".format(bcftools))
-                makefile.filter("SAMTOOLS=.*$", "SAMTOOLS={0}/include".format(samtools))
-                makefile.filter("HTSLIB=.*$", "HTSLIB={0}/include".format(htslib))
+                makefile.filter("BCFTOOLS=.*$", f"BCFTOOLS={bcftools}/include")
+                makefile.filter("SAMTOOLS=.*$", f"SAMTOOLS={samtools}/include")
+                makefile.filter("HTSLIB=.*$", f"HTSLIB={htslib}/include")
 
                 # fix bad linking dirs
                 makefile.filter("$(SAMTOOLS)/libbam.a", "$(SAMTOOLS)/../lib/libbam.a", string=True)
                 makefile.filter("$(HTSLIB)/libhts.a", "$(HTSLIB)/../lib/libhts.a", string=True)
             with working_dir(join_path("auxprogs", "checkTargetSortedness")):
                 makefile = FileFilter("Makefile")
-                makefile.filter("SAMTOOLS.*=.*$", "SAMTOOLS={0}/include".format(samtools))
+                makefile.filter("SAMTOOLS.*=.*$", f"SAMTOOLS={samtools}/include")
                 makefile.filter("LIBS=-lbam", "LIBS=$(SAMTOOLS)/../lib/libbam.a", string=True)
         else:
             mysql = self.spec["mysql-client"].prefix
@@ -147,12 +145,12 @@ def edit(self, spec, prefix):
 
             with working_dir("src"):
                 makefile = FileFilter("Makefile")
-                makefile.filter(r"/usr/include/mysql\+\+", "{0}/include/mysql++".format(mysqlpp))
+                makefile.filter(r"/usr/include/mysql\+\+", f"{mysqlpp}/include/mysql++")
                 if "^mariadb-c-client" in spec:
-                    makefile.filter("/usr/include/mysql", "{0}/include/mariadb".format(mysql))
+                    makefile.filter("/usr/include/mysql", f"{mysql}/include/mariadb")
                 else:
-                    makefile.filter("/usr/include/mysql", "{0}/include/mysql".format(mysql))
-                makefile.filter("/usr/include/lpsolve", "{0}/include/lpsolve".format(lpsolve))
+                    makefile.filter("/usr/include/mysql", f"{mysql}/include/mysql")
+                makefile.filter("/usr/include/lpsolve", f"{lpsolve}/include/lpsolve")
 
     def install(self, spec, prefix):
         install_tree("bin", join_path(self.spec.prefix, "bin"))
@@ -163,12 +161,12 @@ def install(self, spec, prefix):
     def filter_sbang(self):
         with working_dir(self.prefix.scripts):
             pattern = "^#!.*"
-            repl = "#!{0}".format(self.spec["perl"].command.path)
+            repl = f"#!{self.spec['perl'].command.path}"
             files = glob.glob("*.pl")
             for file in files:
                 filter_file(pattern, repl, *files, backup=False)
 
-            repl = "#!{0}".format(self.spec["python"].command.path)
+            repl = f"#!{self.spec['python'].command.path}"
             files = glob.glob("*.py")
             for file in files:
                 filter_file(pattern, repl, *files, backup=False)
diff --git a/var/spack/repos/builtin/packages/authd/package.py b/var/spack/repos/builtin/packages/authd/package.py
index b2cee813c1eae0..dbb290839bc916 100644
--- a/var/spack/repos/builtin/packages/authd/package.py
+++ b/var/spack/repos/builtin/packages/authd/package.py
@@ -20,4 +20,4 @@ def setup_run_environment(self, env):
         env.prepend_path("PATH", self.prefix.sbin)
 
     def install(self, spec, prefix):
-        make("prefix={0}".format(prefix), "install")
+        make(f"prefix={prefix}", "install")
diff --git a/var/spack/repos/builtin/packages/autodiff/package.py b/var/spack/repos/builtin/packages/autodiff/package.py
index 693fb5640e4af0..ec4c09bad5b39a 100644
--- a/var/spack/repos/builtin/packages/autodiff/package.py
+++ b/var/spack/repos/builtin/packages/autodiff/package.py
@@ -29,12 +29,8 @@ class Autodiff(CMakePackage):
     version("0.6.4", sha256="cfe0bb7c0de10979caff9d9bfdad7e6267faea2b8d875027397486b47a7edd75")
     version("0.5.13", sha256="a73dc571bcaad6b44f74865fed51af375f5a877db44321b5568d94a4358b77a1")
 
-    variant(
-        "python", default="False", description="Enable the compilation of the python bindings."
-    )
-    variant(
-        "examples", default="False", description="Enable the compilation of the example files."
-    )
+    variant("python", default=False, description="Enable the compilation of the python bindings.")
+    variant("examples", default=False, description="Enable the compilation of the example files.")
 
     depends_on("cmake@3.0:", type="build")
     depends_on("cmake@3.22:", when="@0.6.8", type="build")
diff --git a/var/spack/repos/builtin/packages/autodock-vina/package.py b/var/spack/repos/builtin/packages/autodock-vina/package.py
index dadf50ea054ab3..8ca01804d8c0dc 100644
--- a/var/spack/repos/builtin/packages/autodock-vina/package.py
+++ b/var/spack/repos/builtin/packages/autodock-vina/package.py
@@ -44,10 +44,10 @@ def edit(self, spec, prefix):
         with working_dir(self.build_directory):
             makefile = FileFilter("Makefile")
             makefile.filter(
-                "BOOST_INCLUDE = .*", "BOOST_INCLUDE = %s" % self.spec["boost"].prefix.include
+                "BOOST_INCLUDE = .*", f"BOOST_INCLUDE = {self.spec['boost'].prefix.include}"
             )
             makefile.filter("C_PLATFORM=.*", "C_PLATFORM=-pthread")
-            makefile.filter("GPP=.*", "GPP=%s" % spack_cxx)
+            makefile.filter("GPP=.*", f"GPP={spack_cxx}")
 
     def build(self, spec, prefix):
         with working_dir(self.build_directory):
diff --git a/var/spack/repos/builtin/packages/autogen/package.py b/var/spack/repos/builtin/packages/autogen/package.py
index 2ecc434106eec3..54b088beb599c6 100644
--- a/var/spack/repos/builtin/packages/autogen/package.py
+++ b/var/spack/repos/builtin/packages/autogen/package.py
@@ -36,7 +36,7 @@ def configure_args(self):
         ]
 
         if "+xml" in spec:
-            args.append("--with-libxml2={0}".format(spec["libxml2"].prefix))
+            args.append(f"--with-libxml2={spec['libxml2'].prefix}")
         else:
             args.append("--without-libxml2")
 
diff --git a/var/spack/repos/builtin/packages/avizo/package.py b/var/spack/repos/builtin/packages/avizo/package.py
index 42c201e780ed0a..43364919cd757f 100644
--- a/var/spack/repos/builtin/packages/avizo/package.py
+++ b/var/spack/repos/builtin/packages/avizo/package.py
@@ -24,25 +24,25 @@ class Avizo(Package):
     version(
         "2020.1",
         sha256="9321aaa276567eebf116e268353c33a4c930d768d22793f921338e1d8cefe991",
-        url="file://{0}/Avizo-20201-Linux64-gcc48.bin".format(os.getcwd()),
+        url=f"file://{os.getcwd()}/Avizo-20201-Linux64-gcc48.bin",
         expand=False,
     )
     version(
         "2019.4",
         sha256="a637720535bcbe254ab56368004a9544c64ec36186373fa24f26cee279685248",
-        url="file://{0}/Avizo-20194-Linux64-gcc48.bin".format(os.getcwd()),
+        url=f"file://{os.getcwd()}/Avizo-20194-Linux64-gcc48.bin",
         expand=False,
     )
     version(
         "2019.3",
         sha256="be109df81e2f7238f234862367841dae05e76cc62218c1f36b1d9bc9514ce5f7",
-        url="file://{0}/Avizo-20193-Linux64-gcc48.bin".format(os.getcwd()),
+        url=f"file://{os.getcwd()}/Avizo-20193-Linux64-gcc48.bin",
         expand=False,
     )
     version(
         "9.7.0",
         sha256="9c9b9e81957387f4218df0c5adbb80717e9ae80ab3ca6ff8da523f7f499dcc5b",
-        url="file://{0}/Avizo-970-Linux64-gcc44.bin".format(os.getcwd()),
+        url=f"file://{os.getcwd()}/Avizo-970-Linux64-gcc44.bin",
         expand=False,
     )
 
@@ -67,15 +67,11 @@ def setup_run_environment(self, env):
     def install(self, spec, prefix):
         ver = self.version.joined
         sh = which("sh")
-        sh(
-            "Avizo-{0}-Linux64-gcc{1}.bin".format(ver, self.gcc_ver[self.version.string]),
-            "--noexec",
-            "--keep",
-        )
+        sh(f"Avizo-{ver}-Linux64-gcc{self.gcc_ver[self.version.string]}.bin", "--noexec", "--keep")
 
         with working_dir("Avizo"):
             avizo_tar = tarfile.open(
-                name="Avizo-{0}-Linux64-gcc{1}.tar.bz2".format(
+                name="Avizo-{}-Linux64-gcc{}.tar.bz2".format(
                     self.version, self.gcc_ver[self.version.string]
                 )
             )
diff --git a/var/spack/repos/builtin/packages/aws-ofi-rccl/package.py b/var/spack/repos/builtin/packages/aws-ofi-rccl/package.py
index 76fab3443965cc..d21a1600d615e5 100644
--- a/var/spack/repos/builtin/packages/aws-ofi-rccl/package.py
+++ b/var/spack/repos/builtin/packages/aws-ofi-rccl/package.py
@@ -35,12 +35,12 @@ class AwsOfiRccl(AutotoolsPackage):
     # To enable this plug-in to work with RCCL add it to the LD_LIBRARY_PATH
     def setup_run_environment(self, env):
         aws_ofi_rccl_home = self.spec["aws-ofi-rccl"].prefix
-        env.append_path("LD_LIBRARY_PATH", aws_ofi_rccl_home.lib)
+        env.prepend_path("LD_LIBRARY_PATH", aws_ofi_rccl_home.lib)
 
     # To enable this plug-in to work with RCCL add it to the LD_LIBRARY_PATH
     def setup_dependent_run_environment(self, env, dependent_spec):
         aws_ofi_rccl_home = self.spec["aws-ofi-rccl"].prefix
-        env.append_path("LD_LIBRARY_PATH", aws_ofi_rccl_home.lib)
+        env.prepend_path("LD_LIBRARY_PATH", aws_ofi_rccl_home.lib)
 
     def configure_args(self):
         spec = self.spec
diff --git a/var/spack/repos/builtin/packages/aws-sdk-cpp/package.py b/var/spack/repos/builtin/packages/aws-sdk-cpp/package.py
index 84a4224dbd99db..114cb4b2b41e53 100644
--- a/var/spack/repos/builtin/packages/aws-sdk-cpp/package.py
+++ b/var/spack/repos/builtin/packages/aws-sdk-cpp/package.py
@@ -18,11 +18,39 @@ class AwsSdkCpp(CMakePackage):
     homepage = "https://github.com/aws/aws-sdk-cpp"
     git = "https://github.com/aws/aws-sdk-cpp.git"
 
-    version("1.10.32", tag="1.10.32", submodules=True)
-    version("1.9.247", tag="1.9.247", submodules=True)
+    version(
+        "1.11.144",
+        tag="1.11.144",
+        commit="498339d47146f8e3ee9ae0fad3fcda5acbaea1e6",
+        submodules=True,
+    )
+    version(
+        "1.10.57",
+        tag="1.10.57",
+        commit="777e0dd4b90eda8bf65bc4b5fce2f90febdbd2b8",
+        submodules=True,
+    )
+    version(
+        "1.10.32",
+        tag="1.10.32",
+        commit="40594bc05a3f68fe14f2f1a54c5c3e6752b1e290",
+        submodules=True,
+    )
+    version(
+        "1.9.379",
+        tag="1.9.379",
+        commit="94d02db44730b0a5cef98ce33deedf43f5333700",
+        submodules=True,
+    )
+    version(
+        "1.9.247",
+        tag="1.9.247",
+        commit="4bf33aa0172704eb5fc1da8086686670711bd801",
+        submodules=True,
+    )
 
     depends_on("cmake@3.1:", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("curl")
 
     # https://github.com/aws/aws-sdk-cpp/issues/1816
@@ -31,3 +59,10 @@ class AwsSdkCpp(CMakePackage):
         sha256="ba86e0556322604fb4b70e2dd4f4fb874701868b07353fc1d5c329d90777bf45",
         when="@1.9.247",
     )
+
+    def cmake_args(self):
+        return [
+            self.define("BUILD_ONLY", ("s3", "transfer")),
+            self.define("ENABLE_TESTING", False),
+            self.define("AUTORUN_UNIT_TESTS", False),
+        ]
diff --git a/var/spack/repos/builtin/packages/awscli/package.py b/var/spack/repos/builtin/packages/awscli/package.py
index 3a667f19c4ad54..dc024638bd9576 100644
--- a/var/spack/repos/builtin/packages/awscli/package.py
+++ b/var/spack/repos/builtin/packages/awscli/package.py
@@ -42,5 +42,5 @@ class Awscli(PythonPackage):
     depends_on("py-rsa@3.1.2:4.7", when="@1.27:", type=("build", "run"))
     depends_on("py-rsa@3.1.2:3.5.0", when="@1.16", type=("build", "run"))
 
-    depends_on("py-s3transfer@0.6.0:0.6", when="@1.27:", type=("build", "run"))
-    depends_on("py-s3transfer@0.2.0:0.2", when="@1.16", type=("build", "run"))
+    depends_on("py-s3transfer@0.6", when="@1.27:", type=("build", "run"))
+    depends_on("py-s3transfer@0.2", when="@1.16", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/axl/package.py b/var/spack/repos/builtin/packages/axl/package.py
index 0576583a3a776c..169f7afaacbd70 100644
--- a/var/spack/repos/builtin/packages/axl/package.py
+++ b/var/spack/repos/builtin/packages/axl/package.py
@@ -44,7 +44,7 @@ class Axl(CMakePackage):
     )
 
     depends_on("kvtree")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     depends_on("kvtree@main", when="@main")
     depends_on("kvtree@:1.2.0", when="@:0.5.0")
diff --git a/var/spack/repos/builtin/packages/axom/package.py b/var/spack/repos/builtin/packages/axom/package.py
index 4e7e3115912a73..ab3ca1d9c00109 100644
--- a/var/spack/repos/builtin/packages/axom/package.py
+++ b/var/spack/repos/builtin/packages/axom/package.py
@@ -40,16 +40,18 @@ class Axom(CachedCMakePackage, CudaPackage, ROCmPackage):
 
     version("main", branch="main")
     version("develop", branch="develop")
-    version("0.7.0", tag="v0.7.0")
-    version("0.6.1", tag="v0.6.1")
-    version("0.6.0", tag="v0.6.0")
-    version("0.5.0", tag="v0.5.0")
-    version("0.4.0", tag="v0.4.0")
-    version("0.3.3", tag="v0.3.3")
-    version("0.3.2", tag="v0.3.2")
-    version("0.3.1", tag="v0.3.1")
-    version("0.3.0", tag="v0.3.0")
-    version("0.2.9", tag="v0.2.9")
+    version("0.8.1", tag="v0.8.1", commit="0da8a5b1be596887158ac2fcd321524ba5259e15")
+    version("0.8.0", tag="v0.8.0", commit="71fab3262eb7e1aa44a04c21d072b77f06362f7b")
+    version("0.7.0", tag="v0.7.0", commit="ea5158191181c137117ae37959879bdc8b107f35")
+    version("0.6.1", tag="v0.6.1", commit="ee240d3963d7879ae0e9c392902195bd7b04e37d")
+    version("0.6.0", tag="v0.6.0", commit="65287dc00bc7c271a08cb86c632f5909c30e3506")
+    version("0.5.0", tag="v0.5.0", commit="db137349b3e28617c3e0570dbd18e4a91654da98")
+    version("0.4.0", tag="v0.4.0", commit="38c0d7495ece35a30fca5f5b578b8f9d54346bd2")
+    version("0.3.3", tag="v0.3.3", commit="f0539ef0525469ffda054d86144f310c15b4f9e0")
+    version("0.3.2", tag="v0.3.2", commit="c446b496e20e6118b8cba7e80f1f84c76a49e463")
+    version("0.3.1", tag="v0.3.1", commit="cbefc0457a229d8acfb70622360d0667e90e50a2")
+    version("0.3.0", tag="v0.3.0", commit="20068ccab4b4f70055918b4f17960ec3ed6dbce8")
+    version("0.2.9", tag="v0.2.9", commit="9e9a54ede3326817c05f35922738516e43b5ec3d")
 
     # https://github.com/spack/spack/issues/31829
     patch("examples-oneapi.patch", when="@0.6.1 +examples %oneapi")
@@ -163,6 +165,8 @@ class Axom(CachedCMakePackage, CudaPackage, ROCmPackage):
     conflicts("+openmp", when="+rocm")
     conflicts("+cuda", when="+rocm")
 
+    conflicts("^blt@:0.3.6", when="+rocm")
+
     def flag_handler(self, name, flags):
         if self.spec.satisfies("%cce") and name == "fflags":
             flags.append("-ef")
diff --git a/var/spack/repos/builtin/packages/bacio/package.py b/var/spack/repos/builtin/packages/bacio/package.py
index 50170278dcc094..f872fc339127c3 100644
--- a/var/spack/repos/builtin/packages/bacio/package.py
+++ b/var/spack/repos/builtin/packages/bacio/package.py
@@ -15,7 +15,7 @@ class Bacio(CMakePackage):
     url = "https://github.com/NOAA-EMC/NCEPLIBS-bacio/archive/refs/tags/v2.4.1.tar.gz"
     git = "https://github.com/NOAA-EMC/NCEPLIBS-bacio"
 
-    maintainers("t-brown", "edwardhartnett", "AlexanderRichert-NOAA", "Hang-Lei-NOAA")
+    maintainers("edwardhartnett", "AlexanderRichert-NOAA", "Hang-Lei-NOAA")
 
     version("develop", branch="develop")
     version("2.6.0", sha256="03fef581e1bd3710fb8d2f2659a6c3e01a0437c1350ba53958d2ff1ffef47bcb")
@@ -43,3 +43,7 @@ def cmake_args(self):
     def patch(self):
         if self.spec.satisfies("@2.4.1"):
             filter_file(".+", "2.4.1", "VERSION")
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/bam-readcount/package.py b/var/spack/repos/builtin/packages/bam-readcount/package.py
index 8a0ff30849afb2..b895bfcbb18197 100644
--- a/var/spack/repos/builtin/packages/bam-readcount/package.py
+++ b/var/spack/repos/builtin/packages/bam-readcount/package.py
@@ -12,4 +12,8 @@ class BamReadcount(CMakePackage):
     homepage = "https://github.com/genome/bam-readcount"
     url = "https://github.com/genome/bam-readcount/archive/v0.8.0.tar.gz"
 
+    version("1.0.1", sha256="8ebf84d9efee0f2d3b43f0452dbf16b27337c960e25128f6a7173119e62588b8")
     version("0.8.0", sha256="4f4dd558e3c6bfb24d6a57ec441568f7524be6639b24f13ea6f2bb350c7ea65f")
+
+    def setup_build_environment(self, env):
+        env.append_flags("CFLAGS", self.compiler.cc_pic_flag)
diff --git a/var/spack/repos/builtin/packages/bamdst/package.py b/var/spack/repos/builtin/packages/bamdst/package.py
index 398c579fdf8666..fdd01f2b65450a 100644
--- a/var/spack/repos/builtin/packages/bamdst/package.py
+++ b/var/spack/repos/builtin/packages/bamdst/package.py
@@ -14,7 +14,7 @@ class Bamdst(MakefilePackage):
 
     version("master", git="https://github.com/shiquan/bamdst.git")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     parallel = False
 
diff --git a/var/spack/repos/builtin/packages/bamtools/package.py b/var/spack/repos/builtin/packages/bamtools/package.py
index 42bf32444f5b32..c45a07e61ef244 100644
--- a/var/spack/repos/builtin/packages/bamtools/package.py
+++ b/var/spack/repos/builtin/packages/bamtools/package.py
@@ -22,7 +22,7 @@ class Bamtools(CMakePackage):
     version("2.3.0", sha256="288046e6d5d41afdc5fce8608c5641cf2b8e670644587c1315b90bbe92f039af")
     version("2.2.3", sha256="92ddef44801a1f8f01ce1a397f83e0f8b5e1ae8ad92c620f2dafaaf8d54cf178")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("jsoncpp")
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/bamutil/package.py b/var/spack/repos/builtin/packages/bamutil/package.py
index ddc2ce9f146f00..ffa9c2a2695bb5 100644
--- a/var/spack/repos/builtin/packages/bamutil/package.py
+++ b/var/spack/repos/builtin/packages/bamutil/package.py
@@ -24,7 +24,7 @@ class Bamutil(MakefilePackage):
         url="https://genome.sph.umich.edu/w/images/7/70/BamUtilLibStatGen.1.0.13.tgz",
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("git", type="build", when="@1.0.15:")
 
     patch("libstatgen-issue-9.patch", when="@1.0.13")
diff --git a/var/spack/repos/builtin/packages/bart/package.py b/var/spack/repos/builtin/packages/bart/package.py
index cc371f4f5c31da..9fa0baa01833d4 100644
--- a/var/spack/repos/builtin/packages/bart/package.py
+++ b/var/spack/repos/builtin/packages/bart/package.py
@@ -48,7 +48,7 @@ def edit(self, spec, prefix):
         if spec["blas"].name == "openblas":
             env["OPENBLAS"] = "1"
 
-        if "^mkl" in spec:
+        elif spec["blas"].name in INTEL_MATH_LIBRARIES:
             env["MKL"] = "1"
             env["MKL_BASE"] = spec["mkl"].prefix.mkl
         else:
diff --git a/var/spack/repos/builtin/packages/batchedblas/package.py b/var/spack/repos/builtin/packages/batchedblas/package.py
index c44b50bc81e349..712f270e8cf8fc 100644
--- a/var/spack/repos/builtin/packages/batchedblas/package.py
+++ b/var/spack/repos/builtin/packages/batchedblas/package.py
@@ -23,7 +23,7 @@ class Batchedblas(MakefilePackage):
     def edit(self, spec, prefix):
         CCFLAGS = [self.compiler.openmp_flag, "-I./", "-O3"]
         BLAS = ["-lm", spec["blas"].libs.ld_flags]
-        if not spec.satisfies("^mkl"):
+        if spec["blas"].name not in INTEL_MATH_LIBRARIES:
             CCFLAGS.append("-D_CBLAS_")
         if spec.satisfies("%intel"):
             CCFLAGS.extend(["-Os"])
diff --git a/var/spack/repos/builtin/packages/bats/package.py b/var/spack/repos/builtin/packages/bats/package.py
index 05299a0a30660a..9370ed834e118a 100644
--- a/var/spack/repos/builtin/packages/bats/package.py
+++ b/var/spack/repos/builtin/packages/bats/package.py
@@ -9,11 +9,16 @@
 class Bats(Package):
     """Bats is a TAP-compliant testing framework for Bash."""
 
-    homepage = "https://github.com/sstephenson/bats"
-    url = "https://github.com/sstephenson/bats/archive/v0.4.0.tar.gz"
+    homepage = "https://github.com/bats-core/bats-core"
+    url = "https://github.com/bats-core/bats-core/archive/refs/tags/v1.10.0.tar.gz"
 
-    version("0.4.0", sha256="480d8d64f1681eee78d1002527f3f06e1ac01e173b761bc73d0cf33f4dc1d8d7")
+    version("1.10.0", sha256="a1a9f7875aa4b6a9480ca384d5865f1ccf1b0b1faead6b47aa47d79709a5c5fd")
+    version(
+        "0.4.0",
+        sha256="480d8d64f1681eee78d1002527f3f06e1ac01e173b761bc73d0cf33f4dc1d8d7",
+        url="https://github.com/sstephenson/bats/archive/v0.4.0.tar.gz",
+    )
 
     def install(self, spec, prefix):
         bash = which("bash")
-        bash("install.sh", prefix)
+        bash("./install.sh", prefix)
diff --git a/var/spack/repos/builtin/packages/bazel/package.py b/var/spack/repos/builtin/packages/bazel/package.py
index a2cc4d96d205f9..705eef43b94064 100644
--- a/var/spack/repos/builtin/packages/bazel/package.py
+++ b/var/spack/repos/builtin/packages/bazel/package.py
@@ -479,13 +479,14 @@ class Bazel(Package):
     # https://blog.bazel.build/2021/05/21/bazel-4-1.html
     conflicts("platform=darwin target=aarch64:", when="@:4.0")
 
-    # patches for compiling bazel-4.1:4 with gcc-11
+    # patches for compiling various older bazels which had ICWYU
+    # violations revealed by (but not unique to) GCC 11 header changes.
     # these are derived from
     # https://gitlab.alpinelinux.org/alpine/aports/-/merge_requests/29084/
-    patch("gcc11_1.patch", when="@0.3.2:4%gcc@11:")
-    patch("gcc11_2.patch", when="@0.3.2:4%gcc@11:")
-    patch("gcc11_3.patch", when="@0.3:4%gcc@11:")
-    patch("gcc11_4.patch", when="@4.1:4%gcc@11:")
+    patch("gcc11_1.patch", when="@0.3.2:4")
+    patch("gcc11_2.patch", when="@0.3.2:4")
+    patch("gcc11_3.patch", when="@0.3:4")
+    patch("gcc11_4.patch", when="@4.1:4")
 
     # bazel-4.0.0 does not compile with gcc-11
     # newer versions of grpc and abseil dependencies are needed but are not in
@@ -519,14 +520,7 @@ def setup_build_environment(self, env):
             # Spack's logs don't handle colored output well
             "--color=no --host_javabase=@local_jdk//:jdk"
             # Enable verbose output for failures
-            " --verbose_failures"
-            # Ask bazel to explain what it's up to
-            # Needs a filename as argument
-            " --explain=explainlogfile.txt"
-            # Increase verbosity of explanation,
-            " --verbose_explanations"
-            # Show (formatted) subcommands being executed
-            " --subcommands=pretty_print" " --jobs={0}".format(make_jobs),
+            " --verbose_failures --jobs={0}".format(make_jobs),
         )
 
     @run_before("install")
diff --git a/var/spack/repos/builtin/packages/bbcp/package.py b/var/spack/repos/builtin/packages/bbcp/package.py
index 27ad399e46558e..fc329aa0660c76 100644
--- a/var/spack/repos/builtin/packages/bbcp/package.py
+++ b/var/spack/repos/builtin/packages/bbcp/package.py
@@ -17,7 +17,7 @@ class Bbcp(Package):
     # Stanford's git server does not support "smart https" shallow clones
     version("master", branch="master", get_full_repo=True)
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("openssl")
     depends_on("libnsl")
 
diff --git a/var/spack/repos/builtin/packages/bcl2fastq2/cxxConfigure-aarch64.patch b/var/spack/repos/builtin/packages/bcl2fastq2/cxxConfigure-aarch64.patch
new file mode 100644
index 00000000000000..490b9fcd078c93
--- /dev/null
+++ b/var/spack/repos/builtin/packages/bcl2fastq2/cxxConfigure-aarch64.patch
@@ -0,0 +1,11 @@
+--- a/src/cmake/cxxConfigure.cmake      2017-06-22 12:14:50.000000000 -0500
++++ b/src/cmake/cxxConfigure.cmake      2023-09-25 11:50:55.819690648 -0500
+@@ -118,7 +118,7 @@
+ set(BCL2FASTQ_DEP_LIB ${BCL2FASTQ_DEP_LIB} "${LIBEXSLT_LIBRARIES}" "${LIBXSLT_LIBRARIES}" "${LIBXML2_LIBRARIES}")
+
+ #set (CMAKE_CXX_FLAGS "$ENV{CXX_FLAGS} $ENV{CXXFLAGS} -fopenmp -msse2 -Werror -Wall -Wextra -Wunused -Wno-long-long -Wsign-compare -Wpointer-arith" CACHE STRING "g++ flags" FORCE)
+-set (CMAKE_CXX_FLAGS "$ENV{CXX_FLAGS} $ENV{CXXFLAGS} -std=c++11 -fopenmp -msse2 -Wall -Wextra -Wunused -Wno-long-long -Wsign-compare -Wpointer-arith" CACHE STRING "g++ flags" FORCE)
++set (CMAKE_CXX_FLAGS "$ENV{CXX_FLAGS} $ENV{CXXFLAGS} -std=c++11 -fopenmp -Wall -Wextra -Wunused -Wno-long-long -Wsign-compare -Wpointer-arith" CACHE STRING "g++ flags" FORCE)
+ #set (CMAKE_CXX_FLAGS_DEBUG "-O0 -g -pg -fprofile-arcs -ftest-coverage -D_GLIBCXX_DEBUG" CACHE STRING "g++ flags" FORCE)
+ set (CMAKE_CXX_FLAGS_DEBUG "-O0 -std=c++11 -g -pg -fprofile-arcs -ftest-coverage" CACHE STRING "g++ flags" FORCE)
+ #set (CMAKE_CXX_FLAGS_DEBUG "-std=c++11 -O0 -g -pg -fprofile-arcs -ftest-coverage" CACHE STRING "g++ flags" FORCE)
diff --git a/var/spack/repos/builtin/packages/bcl2fastq2/package.py b/var/spack/repos/builtin/packages/bcl2fastq2/package.py
index 9b49f6073c13c8..65695f04f43c56 100644
--- a/var/spack/repos/builtin/packages/bcl2fastq2/package.py
+++ b/var/spack/repos/builtin/packages/bcl2fastq2/package.py
@@ -41,7 +41,7 @@ class Bcl2fastq2(Package):
     depends_on("libxml2@2.7.8")
     depends_on("libxslt@1.1.26~crypto")
     depends_on("libgcrypt")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # Their cmake macros don't set the flag when they find a library
     # that makes them happy.
@@ -49,6 +49,8 @@ class Bcl2fastq2(Package):
     # After finding the libxslt bits, cmake still needs to wire in the
     # libexslt bits.
     patch("cxxConfigure-cmake.patch")
+    # -msse2 isn't valid for arm
+    patch("cxxConfigure-aarch64.patch", when="target=aarch64:")
 
     root_cmakelists_dir = "src"
 
diff --git a/var/spack/repos/builtin/packages/bear/package.py b/var/spack/repos/builtin/packages/bear/package.py
index 18240c7f330954..becb364a2c2d99 100644
--- a/var/spack/repos/builtin/packages/bear/package.py
+++ b/var/spack/repos/builtin/packages/bear/package.py
@@ -23,10 +23,10 @@ class Bear(CMakePackage):
     version("2.0.4", sha256="33ea117b09068aa2cd59c0f0f7535ad82c5ee473133779f1cc20f6f99793a63e")
 
     depends_on("pkgconfig", when="@3:")
-    depends_on("fmt", when="@3.0.0:")
-    depends_on("grpc", when="@3.0.0:")
+    depends_on("fmt@8", when="@3.0.0:")
+    depends_on("grpc +shared", when="@3.0.0:")
     depends_on("nlohmann-json", when="@3.0.0:")
-    depends_on("spdlog", when="@3.0.0:")
+    depends_on("spdlog +fmt_external", when="@3.0.0:")
     depends_on("cmake@2.8:", type="build")
     depends_on("python", type="build")
     depends_on("googletest", type="test", when="@3:")
diff --git a/var/spack/repos/builtin/packages/beast2/package.py b/var/spack/repos/builtin/packages/beast2/package.py
index 5fc0cfb2fefbbb..25b729455cd7a9 100644
--- a/var/spack/repos/builtin/packages/beast2/package.py
+++ b/var/spack/repos/builtin/packages/beast2/package.py
@@ -30,9 +30,12 @@ class Beast2(Package):
     version("2.5.2", sha256="2feb2281b4f7cf8f7de1a62de50f52a8678ed0767fc72f2322e77dde9b8cd45f")
     version("2.4.6", sha256="84029c5680cc22f95bef644824130090f5f12d3d7f48d45cb4efc8e1d6b75e93")
 
+    variant("beagle", default=True, description="Build with libbeagle support")
+
     depends_on("java")
     depends_on("java@17:", when="@2.7.0:")
     depends_on("javafx", when="@2.7.0:")
+    depends_on("libbeagle", type="run", when="+beagle")
 
     def patch(self):
         # handle javafx stuff
diff --git a/var/spack/repos/builtin/packages/beatnik/package.py b/var/spack/repos/builtin/packages/beatnik/package.py
new file mode 100644
index 00000000000000..9afa9afa3825ce
--- /dev/null
+++ b/var/spack/repos/builtin/packages/beatnik/package.py
@@ -0,0 +1,90 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Beatnik(CMakePackage, CudaPackage, ROCmPackage):
+    """Fluid interface model solver based on Pandya and Shkoller's Z-Model formulation."""
+
+    homepage = "https://github.com/CUP-ECS/beatnik"
+    git = "https://github.com/CUP-ECS/beatnik.git"
+
+    maintainers("patrickb314", "JStewart28")
+
+    version("1.0", commit="ae31ef9cb44678d5ace77994b45b0778defa3d2f")
+    version("develop", branch="develop")
+    version("main", branch="main")
+
+    # Variants are primarily backends to build on GPU systems and pass the right
+    # informtion to the packages we depend on
+    variant("cuda", default=False, description="Use CUDA support from subpackages")
+    variant("openmp", default=False, description="Use OpenMP support from subpackages")
+
+    # Dependencies for all Beatnik versions
+    depends_on("mpi")
+    depends_on("mpi +cuda", when="+cuda")
+    depends_on("mpi +rocm", when="+rocm")
+
+    # Kokkos dependencies
+    depends_on("kokkos @4:")
+    depends_on("kokkos +cuda +cuda_lambda +cuda_constexpr", when="+cuda")
+    depends_on("kokkos +rocm", when="+rocm")
+    depends_on("kokkos +wrapper", when="%gcc+cuda")
+
+    # Cabana dependencies
+    depends_on("cabana @0.6.0 +grid +heffte +silo +hdf5 +mpi")
+    depends_on("cabana +cuda", when="+cuda")
+    depends_on("cabana +rocm", when="+rocm")
+
+    # Silo dependencies
+    depends_on("silo @4.11:")
+    depends_on("silo @4.11.1:", when="%cce")  # Eariler silo versions have trouble cce
+
+    # Heffte dependencies - We always require FFTW so that there's a host
+    # backend even when we're compiling for GPUs
+    depends_on("heffte +fftw")
+    depends_on("heffte +cuda", when="+cuda")
+    depends_on("heffte +rocm", when="+rocm")
+
+    # If we're using CUDA or ROCM, require MPIs be GPU-aware
+    conflicts("mpich ~cuda", when="+cuda")
+    conflicts("mpich ~rocm", when="+rocm")
+    conflicts("openmpi ~cuda", when="+cuda")
+    conflicts("^intel-mpi")  # Heffte won't build with intel MPI because of needed C++ MPI support
+    conflicts("^spectrum-mpi", when="^cuda@11.3:")  # cuda-aware spectrum is broken with cuda 11.3:
+
+    # Propagate CUDA and AMD GPU targets to cabana
+    for cuda_arch in CudaPackage.cuda_arch_values:
+        depends_on("cabana cuda_arch=%s" % cuda_arch, when="+cuda cuda_arch=%s" % cuda_arch)
+    for amdgpu_value in ROCmPackage.amdgpu_targets:
+        depends_on(
+            "cabana +rocm amdgpu_target=%s" % amdgpu_value,
+            when="+rocm amdgpu_target=%s" % amdgpu_value,
+        )
+
+    # CMake specific build functions
+    def cmake_args(self):
+        args = []
+
+        # Use hipcc as the c compiler if we are compiling for rocm. Doing it this way
+        # keeps the wrapper insted of changeing CMAKE_CXX_COMPILER keeps the spack wrapper
+        # and the rpaths it sets for us from the underlying spec.
+        if "+rocm" in self.spec:
+            env["SPACK_CXX"] = self.spec["hip"].hipcc
+
+        # If we're building with cray mpich, we need to make sure we get the GTL library for
+        # gpu-aware MPI, since cabana and beatnik require it
+        if self.spec.satisfies("+rocm ^cray-mpich"):
+            gtl_dir = join_path(self.spec["cray-mpich"].prefix, "..", "..", "..", "gtl", "lib")
+            args.append(
+                "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath={0} -L{0} -lmpi_gtl_hsa".format(gtl_dir)
+            )
+        elif self.spec.satisfies("+cuda ^cray-mpich"):
+            gtl_dir = join_path(self.spec["cray-mpich"].prefix, "..", "..", "..", "gtl", "lib")
+            args.append(
+                "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath={0} -L{0} -lmpi_gtl_cuda".format(gtl_dir)
+            )
+        return args
diff --git a/var/spack/repos/builtin/packages/bedtools2/package.py b/var/spack/repos/builtin/packages/bedtools2/package.py
index 5313c7bbdbdc87..9a3fb8bec4941d 100644
--- a/var/spack/repos/builtin/packages/bedtools2/package.py
+++ b/var/spack/repos/builtin/packages/bedtools2/package.py
@@ -24,7 +24,7 @@ class Bedtools2(Package):
     version("2.25.0", sha256="159122afb9978015f7ec85d7b17739b01415a5738086b20a48147eeefcf08cfb")
     version("2.23.0", sha256="9dacaa561d11ce9835d1d51e5aeb092bcbe117b7119663ec9a671abac6a36056")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2", when="@2.29:")
     depends_on("xz", when="@2.29:")
     depends_on("python", type="build")
diff --git a/var/spack/repos/builtin/packages/benchmark/package.py b/var/spack/repos/builtin/packages/benchmark/package.py
index 9026d3d1c14928..fe0b286352fcd1 100644
--- a/var/spack/repos/builtin/packages/benchmark/package.py
+++ b/var/spack/repos/builtin/packages/benchmark/package.py
@@ -16,7 +16,16 @@ class Benchmark(CMakePackage):
     # first properly installed CMake config packages in
     # 1.2.0 release: https://github.com/google/benchmark/issues/363
     version("main", branch="main")
+    version("1.8.3", sha256="6bc180a57d23d4d9515519f92b0c83d61b05b5bab188961f36ac7b06b0d9e9ce")
+    version("1.8.2", sha256="2aab2980d0376137f969d92848fbb68216abb07633034534fc8c65cc4e7a0e93")
+    version("1.8.1", sha256="e9ff65cecfed4f60c893a1e8a1ba94221fad3b27075f2f80f47eb424b0f8c9bd")
+    version("1.8.0", sha256="ea2e94c24ddf6594d15c711c06ccd4486434d9cf3eca954e2af8a20c88f9f172")
+    version("1.7.1", sha256="6430e4092653380d9dc4ccb45a1e2dc9259d581f4866dc0759713126056bc1d7")
+    version("1.7.0", sha256="3aff99169fa8bdee356eaa1f691e835a6e57b1efeadb8a0f9f228531158246ac")
+    version("1.6.2", sha256="a9f77e6188c1cd4ebedfa7538bf5176d6acc72ead6f456919e5f464ef2f06158")
+    version("1.6.1", sha256="6132883bc8c9b0df5375b16ab520fac1a85dc9e4cf5be59480448ece74b278d4")
     version("1.6.0", sha256="1f71c72ce08d2c1310011ea6436b31e39ccab8c2db94186d26657d41747c85d6")
+    version("1.5.6", sha256="789f85b4810d13ff803834ea75999e41b326405d83d6a538baf01499eda96102")
     version("1.5.5", sha256="3bff5f237c317ddfd8d5a9b96b3eede7c0802e799db520d38ce756a2a46a18a0")
     version("1.5.4", sha256="e3adf8c98bb38a198822725c0fc6c0ae4711f16fbbf6aeb311d5ad11e5a081b5")
     version("1.5.0", sha256="3c6a165b6ecc948967a1ead710d4a181d7b0fbcaa183ef7ea84604994966221a")
diff --git a/var/spack/repos/builtin/packages/berkeley-db/package.py b/var/spack/repos/builtin/packages/berkeley-db/package.py
index ed5337b64f2ceb..1f14d0ef2a84dc 100644
--- a/var/spack/repos/builtin/packages/berkeley-db/package.py
+++ b/var/spack/repos/builtin/packages/berkeley-db/package.py
@@ -32,7 +32,7 @@ class BerkeleyDb(AutotoolsPackage):
     )
     version("5.3.28", sha256="e0a992d740709892e81f9d93f06daf305cf73fb81b545afe72478043172c3628")
 
-    variant("docs", default=False)
+    variant("docs", default=False, description="Build documentation")
     variant("cxx", default=True, description="Build with C++ API")
     variant("stl", default=True, description="Build with C++ STL API")
 
diff --git a/var/spack/repos/builtin/packages/bfs/package.py b/var/spack/repos/builtin/packages/bfs/package.py
new file mode 100644
index 00000000000000..f90c882648c1a7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/bfs/package.py
@@ -0,0 +1,26 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Bfs(MakefilePackage):
+    """A breadth-first version of the UNIX find command."""
+
+    homepage = "https://github.com/tavianator/bfs"
+    url = "https://github.com/tavianator/bfs/archive/refs/tags/3.0.1.tar.gz"
+
+    maintainers("alecbcs")
+
+    version("3.0.2", sha256="d3456a9aeecc031064db0dbe012e55a11eb97be88d0ab33a90e570fe66457f92")
+    version("3.0.1", sha256="a38bb704201ed29f4e0b989fb2ab3791ca51c3eff90acfc31fff424579bbf962")
+
+    depends_on("acl", when="platform=linux")
+    depends_on("attr", when="platform=linux")
+    depends_on("libcap", when="platform=linux")
+    depends_on("oniguruma")
+
+    def install(self, spec, prefix):
+        make("install", f"PREFIX={prefix}")
diff --git a/var/spack/repos/builtin/packages/binder/package.py b/var/spack/repos/builtin/packages/binder/package.py
index 835541e6647d44..7ae8fe7f515ffc 100644
--- a/var/spack/repos/builtin/packages/binder/package.py
+++ b/var/spack/repos/builtin/packages/binder/package.py
@@ -22,10 +22,10 @@ class Binder(CMakePackage):
     maintainers("lyskov", "kliegeois")
 
     version("master", branch="master")
-    version("1.3.0", tag="v1.3.0")
-    version("1.2.0", tag="v1.2.0")
-    version("1.1.0", tag="v1.0.0")
-    version("1.0.0", tag="v1.0.0")
+    version("1.3.0", tag="v1.3.0", commit="e9b55985af297ca161d615058e4a5da07c22bc77")
+    version("1.2.0", tag="v1.2.0", commit="90cf5b31b6f4ecad3fe87518ca2b949dc9e8ed1a")
+    version("1.1.0", tag="v1.0.0", commit="3de7949343197295250f988716d511a264b21324")
+    version("1.0.0", tag="v1.0.0", commit="3de7949343197295250f988716d511a264b21324")
 
     # Add dependencies
     depends_on("llvm+clang+llvm_dylib@7.0:9")
diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py
index e182d2236e942d..30aee917e0125f 100644
--- a/var/spack/repos/builtin/packages/binutils/package.py
+++ b/var/spack/repos/builtin/packages/binutils/package.py
@@ -21,6 +21,7 @@ class Binutils(AutotoolsPackage, GNUMirrorPackage):
 
     executables = ["^nm$", "^readelf$"]
 
+    version("2.41", sha256="a4c4bec052f7b8370024e60389e194377f3f48b56618418ea51067f67aaab30b")
     version("2.40", sha256="f8298eb153a4b37d112e945aa5cb2850040bcf26a3ea65b5a715c83afe05e48a")
     version("2.39", sha256="da24a84fef220102dd24042df06fdea851c2614a5377f86effa28f33b7b16148")
     version("2.38", sha256="070ec71cf077a6a58e0b959f05a09a35015378c2d8a51e90f3aeabfe30590ef8")
@@ -95,13 +96,7 @@ class Binutils(AutotoolsPackage, GNUMirrorPackage):
         when="@2.37:",
     )
     variant("ld", default=False, description="Enable ld.")
-    # When you build binutils with ~ld and +gas and load it in your PATH, you
-    # may end up with incompatibilities between a potentially older system ld
-    # and a recent assembler. For instance the linker on ubuntu 16.04 from
-    # binutils 2.26 and the assembler from binutils 2.36.1 will result in:
-    # "unable to initialize decompress status for section .debug_info"
-    # when compiling with debug symbols on gcc.
-    variant("gas", default=False, when="+ld", description="Enable as assembler.")
+    variant("gas", default=False, description="Enable as assembler.")
     variant("interwork", default=False, description="Enable interwork.")
     variant("gprofng", default=False, description="Enable gprofng.", when="@2.39:")
     variant(
@@ -130,7 +125,7 @@ class Binutils(AutotoolsPackage, GNUMirrorPackage):
     # pkg-config is used to find zstd in gas/configure
     depends_on("pkgconfig", type="build")
     depends_on("zstd@1.4.0:", when="@2.40:")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     depends_on("diffutils", type="build")
     depends_on("gettext", when="+nls")
@@ -161,6 +156,14 @@ class Binutils(AutotoolsPackage, GNUMirrorPackage):
         "~lto", when="+pgo", msg="Profile-guided optimization enables link-time optimization"
     )
 
+    # When you build binutils with ~ld and +gas and load it in your PATH, you
+    # may end up with incompatibilities between a potentially older system ld
+    # and a recent assembler. For instance the linker on ubuntu 16.04 from
+    # binutils 2.26 and the assembler from binutils 2.36.1 will result in:
+    # "unable to initialize decompress status for section .debug_info"
+    # when compiling with debug symbols on gcc.
+    conflicts("+gas", "~ld", msg="Assembler not always compatible with system ld")
+
     @classmethod
     def determine_version(cls, exe):
         output = Executable(exe)("--version", output=str, error=str)
diff --git a/var/spack/repos/builtin/packages/bioawk/package.py b/var/spack/repos/builtin/packages/bioawk/package.py
index 21b1d08e198b3a..6754a660b1c036 100644
--- a/var/spack/repos/builtin/packages/bioawk/package.py
+++ b/var/spack/repos/builtin/packages/bioawk/package.py
@@ -17,11 +17,14 @@ class Bioawk(MakefilePackage):
 
     version("1.0", sha256="316a6561dda41e8327b85106db3704e94e23d7a89870392d19ef8559f7859e2d")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bison", type=("build"))
 
     parallel = False
 
+    def build(self, spec, prefix):
+        make("CC={0}".format(spack_cc))
+
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
         install("bioawk", prefix.bin)
diff --git a/var/spack/repos/builtin/packages/biobloom/package.py b/var/spack/repos/builtin/packages/biobloom/package.py
index a24f8cec9556c8..aacf157428fedf 100644
--- a/var/spack/repos/builtin/packages/biobloom/package.py
+++ b/var/spack/repos/builtin/packages/biobloom/package.py
@@ -19,7 +19,7 @@ class Biobloom(AutotoolsPackage):
     depends_on("boost+exception+math+serialization+container")
     depends_on("sdsl-lite")
     depends_on("sparsehash")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def configure_args(self):
         # newer versions of sdsl-lite introduce tolerable warnings
diff --git a/var/spack/repos/builtin/packages/bioconductor-dupradar/package.py b/var/spack/repos/builtin/packages/bioconductor-dupradar/package.py
new file mode 100644
index 00000000000000..d92d43d27763f0
--- /dev/null
+++ b/var/spack/repos/builtin/packages/bioconductor-dupradar/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class BioconductorDupradar(RPackage):
+    """Assessment of duplication rates in RNA-Seq datasets"""
+
+    homepage = "https://bioconductor.org/packages/3.16/bioc/html/dupRadar.html"
+    url = "https://bioconductor.org/packages/release/bioc/src/contrib/dupRadar_1.30.0.tar.gz"
+    maintainers("pabloaledo")
+
+    bioc = "dupradar"
+
+    version("1.30.0", sha256="a299d7a4578047dfc19237e34255b0f50f70ce41d29762ef9f5a7741ba35aa3d")
+
+    depends_on("r-kernsmooth")
+    depends_on("subread")
+    depends_on("bioconductor-rsubread")
diff --git a/var/spack/repos/builtin/packages/bioconductor-ebseq/package.py b/var/spack/repos/builtin/packages/bioconductor-ebseq/package.py
index b27636dd16c40b..9a14a95c924092 100644
--- a/var/spack/repos/builtin/packages/bioconductor-ebseq/package.py
+++ b/var/spack/repos/builtin/packages/bioconductor-ebseq/package.py
@@ -19,6 +19,7 @@ class BioconductorEbseq(RPackage):
 
     homepage = "https://www.biostat.wisc.edu/~kendzior/EBSEQ/"
     url = "https://bioconductor.org/packages/release/bioc/src/contrib/EBSeq_1.40.0.tar.gz"
+    maintainers("pabloaledo")
 
     bioc = "ebseq"
 
diff --git a/var/spack/repos/builtin/packages/bioconductor-rsubread/package.py b/var/spack/repos/builtin/packages/bioconductor-rsubread/package.py
new file mode 100644
index 00000000000000..657473676ae6e9
--- /dev/null
+++ b/var/spack/repos/builtin/packages/bioconductor-rsubread/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class BioconductorRsubread(RPackage):
+    """Mapping, quantification and variant analysis of sequencing data"""
+
+    homepage = "https://bioconductor.org/packages/3.16/bioc/html/Rsubread.html"
+    url = "https://bioconductor.org/packages/release/bioc/src/contrib/Rsubread_2.14.2.tar.gz"
+
+    bioc = "rsubread"
+
+    depends_on("r-matrix")
+    depends_on("zlib-api")
+
+    version("2.14.2", sha256="ac8be0fad0eb2743443e3a60a9a94eec78c746638aaccca70e7166d034dcebb5")
diff --git a/var/spack/repos/builtin/packages/bioconductor-tximeta/package.py b/var/spack/repos/builtin/packages/bioconductor-tximeta/package.py
new file mode 100644
index 00000000000000..fd1206a2c020ef
--- /dev/null
+++ b/var/spack/repos/builtin/packages/bioconductor-tximeta/package.py
@@ -0,0 +1,39 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class BioconductorTximeta(RPackage):
+    """Transcript Quantification Import with Automatic Metadata
+
+    Transcript quantification import from Salmon and alevin with automatic
+    attachment of transcript ranges and release information, and other associated
+    metadata. De novo transcriptomes can be linked to the appropriate sources with
+    linkedTxomes and shared for computational reproducibility."""
+
+    homepage = "https://bioconductor.org/packages/release/bioc/html/tximeta.html"
+    url = "https://bioconductor.org/packages/release/bioc/src/contrib/tximeta_1.18.1.tar.gz"
+
+    bioc = "tximeta"
+
+    version("1.18.1", sha256="ee486fc4b2352e2998a3c0c2064449ebcf09b5815f982597ea58311dc8064408")
+
+    depends_on("r", type=("build", "run"))
+    depends_on("r-annotationdbi")
+    depends_on("r-annotationhub")
+    depends_on("r-biocfilecache")
+    depends_on("r-biostrings")
+    depends_on("r-ensembldb")
+    depends_on("r-genomeinfodb")
+    depends_on("r-genomicfeatures")
+    depends_on("r-genomicranges")
+    depends_on("r-iranges")
+    depends_on("r-s4vectors")
+    depends_on("r-summarizedexperiment")
+    depends_on("r-tximport")
+    depends_on("r-jsonlite")
+    depends_on("r-matrix")
+    depends_on("r-tibble")
diff --git a/var/spack/repos/builtin/packages/bison/package.py b/var/spack/repos/builtin/packages/bison/package.py
index afb4086191103e..1ad363f747a667 100644
--- a/var/spack/repos/builtin/packages/bison/package.py
+++ b/var/spack/repos/builtin/packages/bison/package.py
@@ -53,6 +53,7 @@ class Bison(AutotoolsPackage, GNUMirrorPackage):
 
     provides("yacc")
 
+    depends_on("gettext", when="@3.4:")
     depends_on("diffutils", type="build")
     depends_on("m4", type=("build", "run"))
     depends_on("perl", type="build")
diff --git a/var/spack/repos/builtin/packages/bitgroomingz/package.py b/var/spack/repos/builtin/packages/bitgroomingz/package.py
index 0343a2674ac55c..f501e70bac1ea3 100644
--- a/var/spack/repos/builtin/packages/bitgroomingz/package.py
+++ b/var/spack/repos/builtin/packages/bitgroomingz/package.py
@@ -19,7 +19,7 @@ class Bitgroomingz(CMakePackage):
 
     variant("shared", default=True, description="build shared libs")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def cmake_args(self):
         args = []
diff --git a/var/spack/repos/builtin/packages/blaspp/package.py b/var/spack/repos/builtin/packages/blaspp/package.py
index e82255238534f0..d43ab4de6bb269 100644
--- a/var/spack/repos/builtin/packages/blaspp/package.py
+++ b/var/spack/repos/builtin/packages/blaspp/package.py
@@ -19,6 +19,12 @@ class Blaspp(CMakePackage, CudaPackage, ROCmPackage):
     maintainers("teonnik", "Sely85", "G-Ragghianti", "mgates3")
 
     version("master", branch="master")
+    version(
+        "2023.08.25", sha256="1d9c7227a6d8776944aa866592142b7b51c6e4ba5529d168eb8ae2b329c47401"
+    )
+    version(
+        "2023.06.00", sha256="e261ad6cde2aa4f97e985aa9067f4c40d2aaa856904bb78007a3dcadf1378ae9"
+    )
     version(
         "2022.07.00", sha256="566bd644f0364caffde6669e0f86514658eb06ca3d252a4fe67203921a875481"
     )
@@ -40,11 +46,14 @@ class Blaspp(CMakePackage, CudaPackage, ROCmPackage):
 
     variant("openmp", default=True, description="Use OpenMP internally.")
     variant("shared", default=True, description="Build shared libraries")
+    variant("sycl", default=False, description="Build support for the SYCL backend")
 
     depends_on("cmake@3.15.0:", type="build")
     depends_on("blas")
     depends_on("llvm-openmp", when="%apple-clang +openmp")
     depends_on("rocblas", when="+rocm")
+    depends_on("intel-oneapi-mkl", when="+sycl")
+    depends_on("intel-oneapi-mkl threads=openmp", when="+sycl")
 
     # only supported with clingo solver: virtual dependency preferences
     # depends_on('openblas threads=openmp', when='+openmp ^openblas')
@@ -57,7 +66,13 @@ class Blaspp(CMakePackage, CudaPackage, ROCmPackage):
     conflicts(
         "+rocm", when="@:2020.10.02", msg="ROCm support requires BLAS++ 2021.04.00 or greater"
     )
-    conflicts("+rocm", when="+cuda", msg="BLAS++ can only support one GPU backend at a time")
+    backend_msg = "BLAS++ supports only one GPU backend at a time"
+    conflicts("+rocm", when="+cuda", msg=backend_msg)
+    conflicts("+rocm", when="+sycl", msg=backend_msg)
+    conflicts("+cuda", when="+sycl", msg=backend_msg)
+    conflicts("+sycl", when="@:2023.06.00", msg="SYCL support requires BLAS++ version 2023.08.25")
+
+    requires("%oneapi", when="+sycl", msg="blaspp+sycl must be compiled with %oneapi")
 
     def cmake_args(self):
         spec = self.spec
@@ -68,6 +83,8 @@ def cmake_args(self):
                 backend = "cuda"
             if "+rocm" in spec:
                 backend = "hip"
+            if "+sycl" in spec:
+                backend = "sycl"
             backend_config = "-Dgpu_backend=%s" % backend
 
         args = [
@@ -85,8 +102,8 @@ def cmake_args(self):
 
     def check(self):
         # If the tester fails to build, ensure that the check() fails.
-        if os.path.isfile(join_path(self.build_directory, "test", "tester")):
-            with working_dir(self.build_directory):
+        if os.path.isfile(join_path(self.builder.build_directory, "test", "tester")):
+            with working_dir(self.builder.build_directory):
                 make("check")
         else:
             raise Exception("The tester was not built!")
diff --git a/var/spack/repos/builtin/packages/blast-plus/package.py b/var/spack/repos/builtin/packages/blast-plus/package.py
index 1e934b93fbed5b..8febe7d5f3a4dd 100644
--- a/var/spack/repos/builtin/packages/blast-plus/package.py
+++ b/var/spack/repos/builtin/packages/blast-plus/package.py
@@ -14,6 +14,7 @@ class BlastPlus(AutotoolsPackage):
 
     maintainers("weijianwen")
 
+    version("2.14.1", sha256="712c2dbdf0fb13cc1c2d4f4ef5dd1ce4b06c3b57e96dfea8f23e6e99f5b1650e")
     version("2.13.0", sha256="89553714d133daf28c477f83d333794b3c62e4148408c072a1b4620e5ec4feb2")
     version("2.12.0", sha256="fda3c9c9d488cad6c1880a98a236d842bcf3610e3e702af61f7a48cf0a714b88")
     version("2.11.0", sha256="d88e1858ae7ce553545a795a2120e657a799a6d334f2a07ef0330cc3e74e1954")
@@ -69,7 +70,7 @@ def patch(self):
     # depends_on('hdf5', when='+hdf5')
     depends_on("gnutls", when="+gnutls")
     depends_on("openssl", when="+openssl")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("bzip2", when="+bzip2")
     depends_on("lzo", when="+lzo")
     depends_on("pcre", when="+pcre")
@@ -121,7 +122,7 @@ def configure_args(self):
         #     config_args.append('--without-hdf5')
 
         if "+zlib" in spec:
-            config_args.append("--with-z={0}".format(self.spec["zlib"].prefix))
+            config_args.append("--with-z={0}".format(self.spec["zlib-api"].prefix))
         else:
             config_args.append("--without-z")
 
diff --git a/var/spack/repos/builtin/packages/bohrium/package.py b/var/spack/repos/builtin/packages/bohrium/package.py
index 1ea1e303212295..6cb3fbfc670775 100644
--- a/var/spack/repos/builtin/packages/bohrium/package.py
+++ b/var/spack/repos/builtin/packages/bohrium/package.py
@@ -95,7 +95,7 @@ class Bohrium(CMakePackage, CudaPackage):
     depends_on("py-pip", type="build", when="+python")
     depends_on("py-wheel", type="build", when="+python")
 
-    depends_on("zlib", when="+proxy")
+    depends_on("zlib-api", when="+proxy")
 
     depends_on("libsigsegv")
 
diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py
index c8c11ba452c54b..d236c5c5524c72 100644
--- a/var/spack/repos/builtin/packages/boost/package.py
+++ b/var/spack/repos/builtin/packages/boost/package.py
@@ -233,7 +233,7 @@ def libs(self):
 
     depends_on("mpi", when="+mpi")
     depends_on("bzip2", when="+iostreams")
-    depends_on("zlib", when="+iostreams")
+    depends_on("zlib-api", when="+iostreams")
     depends_on("zstd", when="+iostreams")
     depends_on("xz", when="+iostreams")
     depends_on("py-numpy", when="+numpy", type=("build", "run"))
@@ -434,6 +434,12 @@ def url_for_version(self, version):
 
         return url.format(version.dotted, version.underscored)
 
+    def flag_handler(self, name, flags):
+        if name == "cxxflags":
+            if self.spec.satisfies("@1.79.0 %oneapi"):
+                flags.append("-Wno-error=enum-constexpr-conversion")
+        return (flags, None, None)
+
     def determine_toolset(self, spec):
         toolsets = {
             "g++": "gcc",
@@ -492,7 +498,9 @@ def determine_bootstrap_options(self, spec, with_libs, options):
         with open("user-config.jam", "w") as f:
             # Boost may end up using gcc even though clang+gfortran is set in
             # compilers.yaml. Make sure this does not happen:
-            f.write("using {0} : : {1} ;\n".format(boost_toolset_id, spack_cxx))
+            if not spec.satisfies("platform=windows"):
+                # Skip this on Windows since we don't have a cl.exe wrapper in spack
+                f.write("using {0} : : {1} ;\n".format(boost_toolset_id, spack_cxx))
 
             if "+mpi" in spec:
                 # Use the correct mpi compiler.  If the compiler options are
@@ -530,9 +538,9 @@ def determine_b2_options(self, spec, options):
                     "-s",
                     "BZIP2_LIBPATH=%s" % spec["bzip2"].prefix.lib,
                     "-s",
-                    "ZLIB_INCLUDE=%s" % spec["zlib"].prefix.include,
+                    "ZLIB_INCLUDE=%s" % spec["zlib-api"].prefix.include,
                     "-s",
-                    "ZLIB_LIBPATH=%s" % spec["zlib"].prefix.lib,
+                    "ZLIB_LIBPATH=%s" % spec["zlib-api"].prefix.lib,
                     "-s",
                     "LZMA_INCLUDE=%s" % spec["xz"].prefix.include,
                     "-s",
@@ -578,7 +586,7 @@ def determine_b2_options(self, spec, options):
 
         options.extend(["link=%s" % ",".join(link_types), "--layout=%s" % layout])
 
-        if not spec.satisfies("@:1.75 %intel"):
+        if not spec.satisfies("@:1.75 %intel") and not spec.satisfies("platform=windows"):
             # When building any version >= 1.76, the toolset must be specified.
             # Earlier versions could not specify Intel as the toolset
             # as that was considered to be redundant/conflicting with
@@ -645,7 +653,7 @@ def determine_b2_options(self, spec, options):
         return threading_opts
 
     def add_buildopt_symlinks(self, prefix):
-        with working_dir(prefix.lib):
+        with working_dir(prefix.lib, create=True):
             for lib in os.listdir(os.curdir):
                 if os.path.isfile(lib):
                     prefix, remainder = lib.split(".", 1)
@@ -698,12 +706,15 @@ def install(self, spec, prefix):
         # to make Boost find the user-config.jam
         env["BOOST_BUILD_PATH"] = self.stage.source_path
 
-        bootstrap = Executable("./bootstrap.sh")
-
         bootstrap_options = ["--prefix=%s" % prefix]
         self.determine_bootstrap_options(spec, with_libs, bootstrap_options)
 
-        bootstrap(*bootstrap_options)
+        if self.spec.satisfies("platform=windows"):
+            bootstrap = Executable("cmd.exe")
+            bootstrap("/c", ".\\bootstrap.bat", *bootstrap_options)
+        else:
+            bootstrap = Executable("./bootstrap.sh")
+            bootstrap(*bootstrap_options)
 
         # strip the toolchain to avoid double include errors (intel) or
         # user-config being overwritten (again intel, but different boost version)
@@ -715,6 +726,8 @@ def install(self, spec, prefix):
 
         # b2 used to be called bjam, before 1.47 (sigh)
         b2name = "./b2" if spec.satisfies("@1.47:") else "./bjam"
+        if self.spec.satisfies("platform=windows"):
+            b2name = "b2.exe" if spec.satisfies("@1.47:") else "bjam.exe"
 
         b2 = Executable(b2name)
         jobs = make_jobs
@@ -722,11 +735,14 @@ def install(self, spec, prefix):
         if jobs > 64 and spec.satisfies("@:1.58"):
             jobs = 64
 
-        b2_options = [
-            "-j",
-            "%s" % jobs,
-            "--user-config=%s" % os.path.join(self.stage.source_path, "user-config.jam"),
-        ]
+        # Windows just wants a b2 call with no args
+        b2_options = []
+        if not self.spec.satisfies("platform=windows"):
+            path_to_config = "--user-config=%s" % os.path.join(
+                self.stage.source_path, "user-config.jam"
+            )
+            b2_options = ["-j", "%s" % jobs]
+            b2_options.append(path_to_config)
 
         threading_opts = self.determine_b2_options(spec, b2_options)
 
@@ -738,8 +754,11 @@ def install(self, spec, prefix):
 
         # In theory it could be done on one call but it fails on
         # Boost.MPI if the threading options are not separated.
-        for threading_opt in threading_opts:
-            b2("install", "threading=%s" % threading_opt, *b2_options)
+        if not self.spec.satisfies("platform=windows"):
+            for threading_opt in threading_opts:
+                b2("install", "threading=%s" % threading_opt, *b2_options)
+        else:
+            b2("install", *b2_options)
 
         if "+multithreaded" in spec and "~taggedlayout" in spec:
             self.add_buildopt_symlinks(prefix)
diff --git a/var/spack/repos/builtin/packages/botan/package.py b/var/spack/repos/builtin/packages/botan/package.py
index 0e32aba7480827..0967c8b032d7ca 100644
--- a/var/spack/repos/builtin/packages/botan/package.py
+++ b/var/spack/repos/builtin/packages/botan/package.py
@@ -14,6 +14,9 @@ class Botan(MakefilePackage):
 
     maintainers("aumuell")
 
+    version("3.2.0", sha256="049c847835fcf6ef3a9e206b33de05dd38999c325e247482772a5598d9e5ece3")
+    version("3.1.1", sha256="30c84fe919936a98fef5331f246c62aa2c0e4d2085b2d4511207f6a20afa3a6b")
+    version("3.1.0", sha256="4e18e755a8bbc6bf96fac916fbf072ecd06740c72a72017c27162e4c0b4725fe")
     version("3.0.0", sha256="5da552e00fa1c047a90c22eb5f0247ec27e7432b68b78e10a7ce0955269ccad7")
     version("2.19.3", sha256="dae047f399c5a47f087db5d3d9d9e8f11ae4985d14c928d71da1aff801802d55")
     version("2.19.2", sha256="3af5f17615c6b5cd8b832d269fb6cb4d54ec64f9eb09ddbf1add5093941b4d75")
@@ -41,6 +44,8 @@ class Botan(MakefilePackage):
     depends_on("python", type="build")
     depends_on("py-sphinx@1.2:", type="build", when="+doc")
 
+    conflicts("%gcc@:10", when="@3:")
+
     def edit(self, spec, prefix):
         configure = Executable("./configure.py")
         configure(*self.configure_args())
diff --git a/var/spack/repos/builtin/packages/bowtie/package.py b/var/spack/repos/builtin/packages/bowtie/package.py
index 6bb94444602252..55e6f53cb52b1a 100644
--- a/var/spack/repos/builtin/packages/bowtie/package.py
+++ b/var/spack/repos/builtin/packages/bowtie/package.py
@@ -48,7 +48,7 @@ class Bowtie(MakefilePackage):
     variant("tbb", default=False, description="Use Intel thread building block")
 
     depends_on("tbb", when="+tbb")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # See: https://github.com/BenLangmead/bowtie/issues/87, a
     # different fix is in the FreeBSD ports/package tree
diff --git a/var/spack/repos/builtin/packages/bowtie2/package.py b/var/spack/repos/builtin/packages/bowtie2/package.py
index a7fa9d84e04aec..4a26602a742c01 100644
--- a/var/spack/repos/builtin/packages/bowtie2/package.py
+++ b/var/spack/repos/builtin/packages/bowtie2/package.py
@@ -29,7 +29,7 @@ class Bowtie2(MakefilePackage):
     depends_on("readline", when="@2.3.1:")
     depends_on("perl", type="run")
     depends_on("python", type="run")
-    depends_on("zlib", when="@2.3.1:")
+    depends_on("zlib-api", when="@2.3.1:")
     depends_on("simde", when="@2.4.0: target=aarch64:", type="link")
     depends_on("simde", when="@2.4.0: target=ppc64le:", type="link")
 
diff --git a/var/spack/repos/builtin/packages/boxlib/package.py b/var/spack/repos/builtin/packages/boxlib/package.py
index 6f36545ed289fc..aa82407333a665 100644
--- a/var/spack/repos/builtin/packages/boxlib/package.py
+++ b/var/spack/repos/builtin/packages/boxlib/package.py
@@ -32,6 +32,7 @@ def cmake_args(self):
         options.extend(
             [
                 "-DBL_SPACEDIM=%d" % int(spec.variants["dims"].value),
+                "-DBL_USE_PARTICLES=1",
                 "-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON",
                 "-DENABLE_FBASELIB=ON",
                 "-DCMAKE_C_COMPILER=%s" % spec["mpi"].mpicc,
diff --git a/var/spack/repos/builtin/packages/brahma/package.py b/var/spack/repos/builtin/packages/brahma/package.py
new file mode 100644
index 00000000000000..3932de204f7c92
--- /dev/null
+++ b/var/spack/repos/builtin/packages/brahma/package.py
@@ -0,0 +1,28 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Brahma(CMakePackage):
+    """Interceptor library for I/O calls using Gotcha"""
+
+    homepage = "https://github.com/hariharan-devarajan/brahma"
+    git = "https://github.com/hariharan-devarajan/brahma.git"
+    maintainers("hariharan-devarajan")
+
+    version("develop", branch="dev")
+    version("master", branch="master")
+    version("0.0.1", tag="v0.0.1", commit="15156036f14e36511dfc3f3751dc953540526a2b")
+
+    variant("mpi", default=False, description="Enable MPI support")
+    depends_on("cpp-logger@0.0.1")
+    depends_on("gotcha@develop")
+    depends_on("catch2@3.0.1")
+
+    depends_on("mpi", when="+mpi")
+
+    def cmake_args(self):
+        return [self.define_from_variant("BUILD_WITH_MPI", "mpi")]
diff --git a/var/spack/repos/builtin/packages/breakdancer/package.py b/var/spack/repos/builtin/packages/breakdancer/package.py
index 82ed40a244b982..743e7ed81ca27a 100644
--- a/var/spack/repos/builtin/packages/breakdancer/package.py
+++ b/var/spack/repos/builtin/packages/breakdancer/package.py
@@ -28,7 +28,7 @@ class Breakdancer(CMakePackage):
         preferred=True,
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     depends_on("ncurses", type="link")
 
diff --git a/var/spack/repos/builtin/packages/breseq/package.py b/var/spack/repos/builtin/packages/breseq/package.py
index 16da2454f7660d..865c1f333dc3d2 100644
--- a/var/spack/repos/builtin/packages/breseq/package.py
+++ b/var/spack/repos/builtin/packages/breseq/package.py
@@ -23,7 +23,7 @@ class Breseq(AutotoolsPackage):
     depends_on("automake", type="build")
     depends_on("libtool", type="build")
     depends_on("m4", type="build")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     depends_on("bedtools2", type="run")
     depends_on("r", type="run")
@@ -32,5 +32,5 @@ class Breseq(AutotoolsPackage):
     conflicts("%clang@:3.3")
 
     def setup_build_environment(self, env):
-        env.set("LDFLAGS", "-L{0}".format(self.spec["zlib"].prefix.lib))
-        env.set("CFLAGS", "-I{0}".format(self.spec["zlib"].prefix.include))
+        env.set("LDFLAGS", "-L{0}".format(self.spec["zlib-api"].prefix.lib))
+        env.set("CFLAGS", "-I{0}".format(self.spec["zlib-api"].prefix.include))
diff --git a/var/spack/repos/builtin/packages/bricks/package.py b/var/spack/repos/builtin/packages/bricks/package.py
index be5023304e59c1..ccbaccdf51cde4 100644
--- a/var/spack/repos/builtin/packages/bricks/package.py
+++ b/var/spack/repos/builtin/packages/bricks/package.py
@@ -24,6 +24,7 @@ class Bricks(CMakePackage):
     maintainers("ztuowen", "drhansj")
 
     version("r0.1", branch="r0.1")
+    version("2023.08.25", commit="d81725055c117c4b63a1b3835c6b634768b5bea7")  # no official release
 
     variant("cuda", default=False, description="Build bricks with CUDA enabled")
 
diff --git a/var/spack/repos/builtin/packages/brunsli/package.py b/var/spack/repos/builtin/packages/brunsli/package.py
index ac5ebd82b43e07..126da9bbae69fa 100644
--- a/var/spack/repos/builtin/packages/brunsli/package.py
+++ b/var/spack/repos/builtin/packages/brunsli/package.py
@@ -12,7 +12,7 @@ class Brunsli(CMakePackage):
     homepage = "https://github.com/google/brunsli"
     git = "https://github.com/google/brunsli.git"
 
-    version("0.1", tag="v0.1", submodules=True)
+    version("0.1", tag="v0.1", commit="8a0e9b8ca2e3e089731c95a1da7ce8a3180e667c", submodules=True)
 
     depends_on("cmake@3.1:", type="build")
 
diff --git a/var/spack/repos/builtin/packages/bufr/package.py b/var/spack/repos/builtin/packages/bufr/package.py
index 3182dc77529c27..aa7c43229e1d36 100644
--- a/var/spack/repos/builtin/packages/bufr/package.py
+++ b/var/spack/repos/builtin/packages/bufr/package.py
@@ -18,9 +18,11 @@ class Bufr(CMakePackage):
 
     homepage = "https://noaa-emc.github.io/NCEPLIBS-bufr"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-bufr/archive/refs/tags/v12.0.1.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-bufr"
 
-    maintainers("t-brown", "AlexanderRichert-NOAA", "edwardhartnett", "Hang-Lei-NOAA", "jbathegit")
+    maintainers("AlexanderRichert-NOAA", "edwardhartnett", "Hang-Lei-NOAA", "jbathegit")
 
+    version("develop", branch="develop")
     version("12.0.1", sha256="525f26238dba6511a453fc71cecc05f59e4800a603de2abbbbfb8cbb5adf5708")
     version("12.0.0", sha256="d01c02ea8e100e51fd150ff1c4a1192ca54538474acb1b7f7a36e8aeab76ee75")
     version("11.7.1", sha256="6533ce6eaa6b02c0cb5424cfbc086ab120ccebac3894980a4daafd4dfadd71f8")
@@ -29,34 +31,30 @@ class Bufr(CMakePackage):
     version("11.5.0", sha256="d154839e29ef1fe82e58cf20232e9f8a4f0610f0e8b6a394b7ca052e58f97f43")
     version("11.4.0", sha256="946482405e675b99e8e0c221d137768f246076f5e9ba92eed6cae47fb68b7a26")
 
-    # tar file name depends on version
-    def url_for_version(self, version):
-        url = "https://github.com/NOAA-EMC/NCEPLIBS-bufr/archive/refs/tags"
-        if version >= Version("12.0.1"):
-            url += "/v{0}.tar.gz".format(version)
-        else:
-            url += "/bufr_v{0}.tar.gz".format(version)
-        return url
-
     # Patch to not add "-c" to ranlib flags when using llvm-ranlib on Apple systems
     patch("cmakelists-apple-llvm-ranlib.patch", when="@11.5.0:11.6.0")
     # C test does not explicity link to -lm causing DSO error when building shared libs
     patch("c-tests-libm.patch", when="@11.5.0:11.7.0")
     # Patch to identify Python version correctly
-    patch("python-version.patch", when="+python @:12.0.0")
+    patch("python-version.patch", when="@11.5:12.0.0 +python")
 
-    variant("python", default=False, description="Enable Python interface?")
+    variant("python", default=False, description="Enable Python interface")
     variant("shared", default=True, description="Build shared libraries", when="@11.5:")
-    variant("tests", default=False, description="Build tests")
 
     extends("python", when="+python")
 
     depends_on("python@3:", type=("build", "run"), when="+python")
     depends_on("py-setuptools", type="build", when="+python")
-    depends_on("py-numpy", type="build", when="+python")
+    depends_on("py-numpy", type=("build", "run"), when="+python")
     depends_on("py-pip", type="build", when="+python")
     depends_on("py-wheel", type="build", when="+python")
 
+    def url_for_version(self, version):
+        pre = "bufr_" if version < Version("12.0.1") else ""
+        return (
+            f"https://github.com/NOAA-EMC/NCEPLIBS-bufr/archive/refs/tags/{pre}v{version}.tar.gz"
+        )
+
     # Need to make the lines shorter at least on some systems
     def patch(self):
         with when("@:11.7.1"):
@@ -66,7 +64,7 @@ def cmake_args(self):
         args = [
             self.define_from_variant("ENABLE_PYTHON", "python"),
             self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
-            self.define_from_variant("BUILD_TESTS", "tests"),
+            self.define("BUILD_TESTS", self.run_tests),
         ]
 
         return args
@@ -107,7 +105,7 @@ def _setup_bufr_environment(self, env, suffix):
 
         if self.spec.satisfies("+python"):
             pyver = self.spec["python"].version.up_to(2)
-            pydir = os.path.join(os.path.dirname(lib[0]), "python%s/site-packages" % pyver)
+            pydir = join_path(os.path.dirname(lib[0]), f"python{pyver}", "site-packages")
             env.prepend_path("PYTHONPATH", pydir)
 
     def setup_run_environment(self, env):
@@ -116,3 +114,8 @@ def setup_run_environment(self, env):
             suffixes += ["8", "d"]
         for suffix in suffixes:
             self._setup_bufr_environment(env, suffix)
+
+    def check(self):
+        if self.spec.satisfies("~python"):
+            with working_dir(self.builder.build_directory):
+                make("test")
diff --git a/var/spack/repos/builtin/packages/busybox/package.py b/var/spack/repos/builtin/packages/busybox/package.py
index 4b74b34611d5a1..a71e28907fae8b 100644
--- a/var/spack/repos/builtin/packages/busybox/package.py
+++ b/var/spack/repos/builtin/packages/busybox/package.py
@@ -22,8 +22,8 @@ class Busybox(MakefilePackage):
 
     def build(self, spec, prefix):
         make("defconfig")
-        make()
+        make("CC={0}".format(spack_cc))
 
     def install(self, spec, prefix):
-        make("install")
+        make("install", "CC={0}".format(spack_cc))
         install_tree(".", prefix)
diff --git a/var/spack/repos/builtin/packages/butterflypack/package.py b/var/spack/repos/builtin/packages/butterflypack/package.py
index 848dbcdfebabef..746fa32082d894 100644
--- a/var/spack/repos/builtin/packages/butterflypack/package.py
+++ b/var/spack/repos/builtin/packages/butterflypack/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+from platform import machine
+
 from spack.package import *
 
 
@@ -26,6 +28,7 @@ class Butterflypack(CMakePackage):
     maintainers("liuyangzhuan")
 
     version("master", branch="master")
+    version("2.4.0", sha256="12d04e7101b2c8292b5c62d9f42b5cd1e8a3c5af639d2665596e3e4255fd0804")
     version("2.2.2", sha256="73f67073e4291877f1eee19483a8a7b3c761eaf79a75805d52105ceedead85ea")
     version("2.2.1", sha256="4cedc2896a6b368773ce4f9003aa2c0230baf56a4464a6b899a155e01406a232")
     version("2.2.0", sha256="1ce5b8461b3c4f488cee6396419e8a6f0a1bcf95254f24d7c27bfa53b391c30b")
@@ -74,7 +77,7 @@ def cmake_args(self):
         args.append("-Denable_openmp=%s" % ("ON" if "+openmp" in spec else "OFF"))
         if "%cce" in spec:
             # Assume the proper Cray CCE module (cce) is loaded:
-            craylibs_path = env["CRAYLIBS_" + env["MACHTYPE"].capitalize()]
+            craylibs_path = env["CRAYLIBS_" + machine().upper()]
             env.setdefault("LDFLAGS", "")
             env["LDFLAGS"] += " -Wl,-rpath," + craylibs_path
 
diff --git a/var/spack/repos/builtin/packages/bwa/package.py b/var/spack/repos/builtin/packages/bwa/package.py
index 53019f4a731836..49e7f40269b69f 100644
--- a/var/spack/repos/builtin/packages/bwa/package.py
+++ b/var/spack/repos/builtin/packages/bwa/package.py
@@ -23,7 +23,7 @@ class Bwa(Package):
         url="https://github.com/lh3/bwa/archive/0.7.12.tar.gz",
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("sse2neon", when="target=aarch64:")
 
     patch(
@@ -33,7 +33,7 @@ class Bwa(Package):
     )
 
     def install(self, spec, prefix):
-        zlib_inc_path = spec["zlib"].prefix.include
+        zlib_inc_path = spec["zlib-api"].prefix.include
         if platform.machine() == "aarch64":
             sse2neon_inc_path = spec["sse2neon"].prefix.include
             filter_file(
@@ -43,7 +43,7 @@ def install(self, spec, prefix):
             )
         else:
             filter_file(r"^INCLUDES=", "INCLUDES=-I%s" % zlib_inc_path, "Makefile")
-        filter_file(r"^LIBS=", "LIBS=-L%s " % spec["zlib"].prefix.lib, "Makefile")
+        filter_file(r"^LIBS=", "LIBS=-L%s " % spec["zlib-api"].prefix.lib, "Makefile")
         # use spack C compiler
         filter_file("^CC=.*", "CC={0}".format(spack_cc), "Makefile")
         # fix gcc 10+ errors
diff --git a/var/spack/repos/builtin/packages/byte-unixbench/package.py b/var/spack/repos/builtin/packages/byte-unixbench/package.py
index cb90ea5611676c..6cd3ec223bd6ec 100644
--- a/var/spack/repos/builtin/packages/byte-unixbench/package.py
+++ b/var/spack/repos/builtin/packages/byte-unixbench/package.py
@@ -16,6 +16,10 @@ class ByteUnixbench(MakefilePackage):
 
     build_directory = "UnixBench"
 
+    @property
+    def build_targets(self):
+        return [f"CC={spack_cc}"]
+
     def install(self, spec, prefix):
         with working_dir(self.build_directory):
             install_tree(".", prefix)
diff --git a/var/spack/repos/builtin/packages/c-blosc/package.py b/var/spack/repos/builtin/packages/c-blosc/package.py
index dbb97965082fc6..31de7ef7ae35b1 100644
--- a/var/spack/repos/builtin/packages/c-blosc/package.py
+++ b/var/spack/repos/builtin/packages/c-blosc/package.py
@@ -15,6 +15,7 @@ class CBlosc(CMakePackage):
     homepage = "https://www.blosc.org"
     url = "https://github.com/Blosc/c-blosc/archive/v1.11.1.tar.gz"
 
+    version("1.21.5", sha256="32e61961bbf81ffea6ff30e9d70fca36c86178afd3e3cfa13376adec8c687509")
     version("1.21.4", sha256="e72bd03827b8564bbb3dc3ea0d0e689b4863871ce3861d946f2efd7a186ecf3e")
     version("1.21.2", sha256="e5b4ddb4403cbbad7aab6e9ff55762ef298729c8a793c6147160c771959ea2aa")
     version("1.21.1", sha256="f387149eab24efa01c308e4cba0f59f64ccae57292ec9c794002232f7903b55b")
@@ -34,7 +35,7 @@ class CBlosc(CMakePackage):
 
     depends_on("cmake@2.8.10:", type="build")
     depends_on("snappy")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("zstd")
     depends_on("lz4")
 
diff --git a/var/spack/repos/builtin/packages/c-blosc2/package.py b/var/spack/repos/builtin/packages/c-blosc2/package.py
index 8b824ab2580425..4b745f426f1093 100644
--- a/var/spack/repos/builtin/packages/c-blosc2/package.py
+++ b/var/spack/repos/builtin/packages/c-blosc2/package.py
@@ -11,12 +11,20 @@ class CBlosc2(CMakePackage):
     other bells and whistles"""
 
     homepage = "https://www.blosc.org/"
-    url = "https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.0.1.tar.gz"
+    url = "https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.10.2.tar.gz"
     git = "https://github.com/Blosc/c-blosc2.git"
 
     maintainers("ax3l", "robert-mijakovic")
 
     version("develop", branch="master")
+    version("2.11.1", sha256="1e9923e0f026eb6e6caee608b4b9a523837806076fc79409055a6386cf5de1ea")
+    version("2.10.5", sha256="a88f94bf839c1371aab8207a6a43698ceb92c72f65d0d7fe5b6e59f24c138b4d")
+    # 2.10.2+ fixes regressions with external dependencies
+    version("2.10.2", sha256="069785bc14c006c7dab40ea0c620bdf3eb8752663fd55c706d145bceabc2a31d")
+    # 2.10.1+ adds Blosc2 CMake CONFIG files
+    version("2.10.1", sha256="1dd65be2d76eee205c06e8812cc1360448620eee5e368b25ade4ea310654cd01")
+    version("2.10.0", sha256="cb7f7c0c62af78982140ecff21a2f3ca9ce6a0a1c02e314fcdce1a98da0fe231")
+    version("2.9.3", sha256="1f36b7d79d973505582b9a804803b640dcc0425af3d5e676070847ac4eb38176")
     version("2.2.0", sha256="66f9977de26d6bc9ea1c0e623d873c3225e4fff709aa09b3335fd09d41d57c0e")
     version("2.1.1", sha256="a8cbedefb8ed3b83629534ec8b4a822ffcdb1576e13dfb93107361551d32e6e9")
     version("2.1.0", sha256="a9570f4101654c2fe2a706adcab0821cdbcf7523fcb6602407dda21b67fdacfd")
@@ -32,13 +40,15 @@ class CBlosc2(CMakePackage):
     variant("zlib", default=True, description="support for ZLIB")
     variant("zstd", default=True, description="support for ZSTD")
 
-    depends_on("cmake@2.8.10:", type="build")
+    depends_on("cmake@3.16.3:", type="build")
     depends_on("lizard", when="+lizard")
     depends_on("lz4", when="+lz4")
     depends_on("snappy", when="+snappy")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("zstd", when="+zstd")
 
+    conflicts("%oneapi")
+
     def cmake_args(self):
         spec = self.spec
 
diff --git a/var/spack/repos/builtin/packages/ca-certificates-mozilla/package.py b/var/spack/repos/builtin/packages/ca-certificates-mozilla/package.py
index 21d108c824877b..8bc297fc91f422 100644
--- a/var/spack/repos/builtin/packages/ca-certificates-mozilla/package.py
+++ b/var/spack/repos/builtin/packages/ca-certificates-mozilla/package.py
@@ -14,6 +14,11 @@ class CaCertificatesMozilla(Package):
 
     maintainers("haampie")
 
+    version(
+        "2023-05-30",
+        sha256="5fadcae90aa4ae041150f8e2d26c37d980522cdb49f923fc1e1b5eb8d74e71ad",
+        expand=False,
+    )
     version(
         "2023-01-10",
         sha256="fb1ecd641d0a02c01bc9036d513cb658bbda62a75e246bedbc01764560a639f0",
diff --git a/var/spack/repos/builtin/packages/cabana/package.py b/var/spack/repos/builtin/packages/cabana/package.py
index cebc2d4aaff134..ae4d4e9fca35aa 100644
--- a/var/spack/repos/builtin/packages/cabana/package.py
+++ b/var/spack/repos/builtin/packages/cabana/package.py
@@ -7,18 +7,19 @@
 from spack.pkg.builtin.kokkos import Kokkos
 
 
-class Cabana(CMakePackage):
+class Cabana(CMakePackage, CudaPackage, ROCmPackage):
     """The Exascale Co-Design Center for Particle Applications Toolkit"""
 
     homepage = "https://github.com/ECP-copa/Cabana"
     git = "https://github.com/ECP-copa/Cabana.git"
-    url = "https://github.com/ECP-copa/Cabana/archive/0.5.0.tar.gz"
+    url = "https://github.com/ECP-copa/Cabana/archive/0.6.0.tar.gz"
 
     maintainers("junghans", "streeve", "sslattery")
 
     tags = ["e4s", "ecp"]
 
     version("master", branch="master")
+    version("0.6.0", sha256="a88a3f80215998169cdbd37661c0c0af57e344af74306dcd2b61983d7c69e6e5")
     version("0.5.0", sha256="b7579d44e106d764d82b0539285385d28f7bbb911a572efd05c711b28b85d8b1")
     version("0.4.0", sha256="c347d23dc4a5204f9cc5906ccf3454f0b0b1612351bbe0d1c58b14cddde81e85")
     version("0.3.0", sha256="fb67ab9aaf254b103ae0eb5cc913ddae3bf3cd0cf6010e9686e577a2981ca84f")
@@ -37,15 +38,18 @@ class Cabana(CMakePackage):
     variant("heffte", default=False, description="Build with heFFTe support")
     variant("hypre", default=False, description="Build with HYPRE support")
     variant("silo", default=False, description="Build with SILO support")
-    variant("cajita", default=False, description="Build Cajita subpackage")
+    variant("hdf5", default=False, description="Build with HDF5 support")
+    variant("cajita", default=False, description="Build Cajita subpackage (Grid in 0.6:)")
+    variant("grid", default=False, description="Build Grid subpackage")
     variant("testing", default=False, description="Build unit tests")
     variant("examples", default=False, description="Build tutorial examples")
     variant("performance_testing", default=False, description="Build performance tests")
 
     depends_on("cmake@3.9:", type="build", when="@:0.4.0")
     depends_on("cmake@3.16:", type="build", when="@0.5.0:")
-    depends_on("googletest", type="build", when="testing")
-    _versions = {":0.2": "-legacy", "0.3:": "@3.1:", "0.4:": "@3.2:", "master": "@3.4:"}
+
+    depends_on("googletest", type="test", when="+testing")
+    _versions = {":0.2": "-legacy", "0.3:": "@3.1:", "0.4:": "@3.2:", "0.6:": "@3.7:"}
     for _version in _versions:
         _kk_version = _versions[_version]
         for _backend in _kokkos_backends:
@@ -53,29 +57,63 @@ class Cabana(CMakePackage):
                 _kk_spec = "kokkos-legacy+pthreads"
             elif _kk_version == "-legacy" and _backend not in ["serial", "openmp", "cuda"]:
                 continue
+            # Handled separately by Cuda/ROCmPackage below
+            elif _backend == "cuda" or _backend == "hip":
+                continue
             else:
                 _kk_spec = "kokkos{0}+{1}".format(_kk_version, _backend)
             depends_on(_kk_spec, when="@{0}+{1}".format(_version, _backend))
+
+    # Propagate cuda architectures down to Kokkos and optional submodules
+    for arch in CudaPackage.cuda_arch_values:
+        cuda_dep = "+cuda cuda_arch={0}".format(arch)
+        depends_on("kokkos {0}".format(cuda_dep), when=cuda_dep)
+        depends_on("heffte {0}".format(cuda_dep), when="+heffte {0}".format(cuda_dep))
+        depends_on("arborx {0}".format(cuda_dep), when="+arborx {0}".format(cuda_dep))
+        depends_on("hypre {0}".format(cuda_dep), when="+hypre {0}".format(cuda_dep))
+
+    for arch in ROCmPackage.amdgpu_targets:
+        rocm_dep = "+rocm amdgpu_target={0}".format(arch)
+        depends_on("kokkos {0}".format(rocm_dep), when=rocm_dep)
+        depends_on("heffte {0}".format(rocm_dep), when="+heffte {0}".format(rocm_dep))
+        depends_on("arborx {0}".format(rocm_dep), when="+arborx {0}".format(rocm_dep))
+        depends_on("hypre {0}".format(rocm_dep), when="+hypre {0}".format(rocm_dep))
+
+    conflicts("+cuda", when="cuda_arch=none")
+    conflicts("+rocm", when="amdgpu_target=none")
+
+    depends_on("kokkos+cuda_lambda", when="+cuda")
+
+    # Dependencies for subpackages
     depends_on("arborx", when="@0.3.0:+arborx")
     depends_on("hypre-cmake@2.22.0:", when="@0.4.0:+hypre")
     depends_on("hypre-cmake@2.22.1:", when="@0.5.0:+hypre")
-    # Heffte pinned at 2.x.0 because its cmakefiles can't roll forward
-    # compatibilty to later minor versions.
     depends_on("heffte@2.0.0", when="@0.4.0+heffte")
-    depends_on("heffte@2.1.0", when="@0.5.0:+heffte")
+    depends_on("heffte@2.1.0", when="@0.5.0+heffte")
+    depends_on("heffte@2.3.0:", when="@0.6.0:+heffte")
     depends_on("silo", when="@0.5.0:+silo")
+    depends_on("hdf5", when="@0.6.0:+hdf5")
     depends_on("mpi", when="+mpi")
 
+    # Cabana automatically builds HDF5 support with newer cmake versions
+    # in version 0.6.0. This is fixed post-0.6
+    conflicts("~hdf5", when="@0.6.0 ^cmake@:3.26")
+
+    # Cajita support requires MPI
     conflicts("+cajita ~mpi")
+    conflicts("+grid ~mpi")
 
+    # Conflict variants only available in newer versions of cabana
     conflicts("+rocm", when="@:0.2.0")
     conflicts("+sycl", when="@:0.3.0")
+    conflicts("+silo", when="@:0.3.0")
+    conflicts("+hdf5", when="@:0.5.0")
 
     def cmake_args(self):
         options = [self.define_from_variant("BUILD_SHARED_LIBS", "shared")]
 
         enable = ["CAJITA", "TESTING", "EXAMPLES", "PERFORMANCE_TESTING"]
-        require = ["ARBORX", "HEFFTE", "HYPRE", "SILO"]
+        require = ["ARBORX", "HEFFTE", "HYPRE", "SILO", "HDF5"]
 
         # These variables were removed in 0.3.0 (where backends are
         # automatically used from Kokkos)
@@ -87,9 +125,24 @@ def cmake_args(self):
         else:
             require += ["MPI"]
 
+        # Cajita was renamed Grid in 0.6
+        if self.spec.satisfies("@0.6.0:"):
+            enable += ["GRID"]
+
         for category, cname in zip([enable, require], ["ENABLE", "REQUIRE"]):
             for var in category:
                 cbn_option = "Cabana_{0}_{1}".format(cname, var)
                 options.append(self.define_from_variant(cbn_option, var.lower()))
 
+        # Only enable user-requested options.
+        for var in require:
+            enabled_var = "+{0}".format(var.lower())
+            if enabled_var not in self.spec:
+                cbn_disable = "CMAKE_DISABLE_FIND_PACKAGE_{0}".format(var)
+                options.append(self.define(cbn_disable, "ON"))
+
+        # Use hipcc for HIP.
+        if "+rocm" in self.spec:
+            options.append(self.define("CMAKE_CXX_COMPILER", self.spec["hip"].hipcc))
+
         return options
diff --git a/var/spack/repos/builtin/packages/cairo/package.py b/var/spack/repos/builtin/packages/cairo/package.py
index f960374d4b9cbd..530b4d49042d2c 100644
--- a/var/spack/repos/builtin/packages/cairo/package.py
+++ b/var/spack/repos/builtin/packages/cairo/package.py
@@ -40,7 +40,7 @@ class Cairo(AutotoolsPackage):
     variant("png", default=False, description="Enable cairo's PNG functions feature")
     variant("svg", default=False, description="Enable cairo's SVN functions feature")
     variant("shared", default=True, description="Build shared libraries")
-    variant("pic", default=False, description="Enable position-independent code (PIC)")
+    variant("pic", default=True, description="Enable position-independent code (PIC)")
 
     depends_on("libx11", when="+X")
     depends_on("libxext", when="+X")
@@ -63,6 +63,7 @@ class Cairo(AutotoolsPackage):
 
     conflicts("+png", when="platform=darwin")
     conflicts("+svg", when="platform=darwin")
+    conflicts("+shared~pic")
 
     # patch from https://gitlab.freedesktop.org/cairo/cairo/issues/346
     patch("fontconfig.patch", when="@1.16.0:1.17.2")
@@ -89,10 +90,12 @@ def configure_args(self):
         args.extend(self.enable_or_disable("shared"))
         args.extend(self.with_or_without("pic"))
 
-        if self.spec.satisfies("+ft"):
-            if self.spec["freetype"].satisfies("~shared"):
-                args.append("LDFLAGS=%s" % self.spec["bzip2"].libs.search_flags)
-                args.append("LIBS=%s" % self.spec["bzip2"].libs.link_flags)
+        if self.spec.satisfies("+ft ^freetype~shared"):
+            pkgconf = which("pkg-config")
+            ldflags = pkgconf("--libs-only-L", "--static", "freetype2", output=str)
+            libs = pkgconf("--libs-only-l", "--static", "freetype2", output=str)
+            args.append(f"LDFLAGS={ldflags}")
+            args.append(f"LIBS={libs}")
 
         return args
 
diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py
index 143ee2253dfc85..66bd9cc84cd0ad 100644
--- a/var/spack/repos/builtin/packages/caliper/package.py
+++ b/var/spack/repos/builtin/packages/caliper/package.py
@@ -18,7 +18,7 @@ class Caliper(CMakePackage, CudaPackage, ROCmPackage):
 
     homepage = "https://github.com/LLNL/Caliper"
     git = "https://github.com/LLNL/Caliper.git"
-    url = "https://github.com/LLNL/Caliper/archive/v2.9.0.tar.gz"
+    url = "https://github.com/LLNL/Caliper/archive/v2.10.0.tar.gz"
     tags = ["e4s", "radiuss"]
 
     maintainers("daboehme")
@@ -26,20 +26,52 @@ class Caliper(CMakePackage, CudaPackage, ROCmPackage):
     test_requires_compiler = True
 
     version("master", branch="master")
+    version("2.10.0", sha256="14c4fb5edd5e67808d581523b4f8f05ace8549698c0e90d84b53171a77f58565")
+    version("2.9.1", sha256="4771d630de505eff9227e0ec498d0da33ae6f9c34df23cb201b56181b8759e9e")
     version("2.9.0", sha256="507ea74be64a2dfd111b292c24c4f55f459257528ba51a5242313fa50978371f")
     version("2.8.0", sha256="17807b364b5ac4b05997ead41bd173e773f9a26ff573ff2fe61e0e70eab496e4")
-    version("2.7.0", sha256="b3bf290ec2692284c6b4f54cc0c507b5700c536571d3e1a66e56626618024b2b")
-    version("2.6.0", sha256="6efcd3e4845cc9a6169e0d934840766b12182c6d09aa3ceca4ae776e23b6360f")
-    version("2.5.0", sha256="d553e60697d61c53de369b9ca464eb30710bda90fba9671201543b64eeac943c")
-    version("2.4.0", tag="v2.4.0")
-    version("2.3.0", tag="v2.3.0")
-    version("2.2.0", tag="v2.2.0")
-    version("2.1.1", tag="v2.1.1")
-    version("2.0.1", tag="v2.0.1")
-    version("1.9.1", tag="v1.9.1")
-    version("1.9.0", tag="v1.9.0")
-    version("1.8.0", tag="v1.8.0")
-    version("1.7.0", tag="v1.7.0")
+    version(
+        "2.7.0",
+        sha256="b3bf290ec2692284c6b4f54cc0c507b5700c536571d3e1a66e56626618024b2b",
+        deprecated=True,
+    )
+    version(
+        "2.6.0",
+        sha256="6efcd3e4845cc9a6169e0d934840766b12182c6d09aa3ceca4ae776e23b6360f",
+        deprecated=True,
+    )
+    version(
+        "2.5.0",
+        sha256="d553e60697d61c53de369b9ca464eb30710bda90fba9671201543b64eeac943c",
+        deprecated=True,
+    )
+    version(
+        "2.4.0", tag="v2.4.0", commit="30577b4b8beae104b2b35ed487fec52590a99b3d", deprecated=True
+    )
+    version(
+        "2.3.0", tag="v2.3.0", commit="9fd89bb0120750d1f9dfe37bd963e24e478a2a20", deprecated=True
+    )
+    version(
+        "2.2.0", tag="v2.2.0", commit="c408e9b3642c7aa80eff37b0826d819c57e7bc04", deprecated=True
+    )
+    version(
+        "2.1.1", tag="v2.1.1", commit="0593b0e01c1d8d3e50c990399cc0fee403485599", deprecated=True
+    )
+    version(
+        "2.0.1", tag="v2.0.1", commit="4d7ff46381c53a461e62edd949e2d9dea9db7b08", deprecated=True
+    )
+    version(
+        "1.9.1", tag="v1.9.1", commit="cfc1defbbee20b50dd3e3477badd09a92b1df970", deprecated=True
+    )
+    version(
+        "1.9.0", tag="v1.9.0", commit="8356e747349b285aa621c5b74e71559f0babc4a1", deprecated=True
+    )
+    version(
+        "1.8.0", tag="v1.8.0", commit="117c1ef596b617dc71407b8b67eebef094a654f8", deprecated=True
+    )
+    version(
+        "1.7.0", tag="v1.7.0", commit="898277c93d884d4e7ca1ffcf3bbea81d22364f26", deprecated=True
+    )
 
     is_linux = sys.platform.startswith("linux")
     variant("shared", default=True, description="Build shared libraries")
@@ -58,6 +90,8 @@ class Caliper(CMakePackage, CudaPackage, ROCmPackage):
     variant("sampler", default=is_linux, description="Enable sampling support on Linux")
     variant("sosflow", default=False, description="Enable SOSflow support")
     variant("fortran", default=False, description="Enable Fortran support")
+    variant("variorum", default=False, description="Enable Variorum support")
+    variant("kokkos", default=True, when="@2.3.0:", description="Enable Kokkos profiling support")
 
     depends_on("adiak@0.1:0", when="@2.2: +adiak")
 
@@ -69,6 +103,7 @@ class Caliper(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("mpi", when="+mpi")
     depends_on("unwind@1.2:1", when="+libunwind")
     depends_on("elfutils", when="+libdw")
+    depends_on("variorum", when="+variorum")
 
     depends_on("sosflow@spack", when="@1.0:1+sosflow")
 
@@ -83,7 +118,10 @@ class Caliper(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("+rocm+cuda")
 
     patch("for_aarch64.patch", when="target=aarch64:")
-    patch("sampler-service-missing-libunwind-include-dir.patch", when="@2.9.0 +libunwind +sampler")
+    patch(
+        "sampler-service-missing-libunwind-include-dir.patch",
+        when="@2.9.0:2.9.1 +libunwind +sampler",
+    )
 
     def cmake_args(self):
         spec = self.spec
@@ -106,6 +144,8 @@ def cmake_args(self):
             self.define_from_variant("WITH_NVTX", "cuda"),
             self.define_from_variant("WITH_ROCTRACER", "rocm"),
             self.define_from_variant("WITH_ROCTX", "rocm"),
+            self.define_from_variant("WITH_VARIORUM", "variorum"),
+            self.define_from_variant("WITH_KOKKOS", "kokkos"),
         ]
 
         if "+papi" in spec:
@@ -116,6 +156,8 @@ def cmake_args(self):
             args.append("-DLIBPFM_INSTALL=%s" % spec["libpfm4"].prefix)
         if "+sosflow" in spec:
             args.append("-DSOS_PREFIX=%s" % spec["sosflow"].prefix)
+        if "+variorum" in spec:
+            args.append("-DVARIORUM_PREFIX=%s" % spec["variorum"].prefix)
 
         # -DWITH_CALLPATH was renamed -DWITH_LIBUNWIND in 2.5
         callpath_flag = "LIBUNWIND" if spec.satisfies("@2.5:") else "CALLPATH"
diff --git a/var/spack/repos/builtin/packages/camp/package.py b/var/spack/repos/builtin/packages/camp/package.py
index 9f27592772fc13..a1bdc830d36eab 100644
--- a/var/spack/repos/builtin/packages/camp/package.py
+++ b/var/spack/repos/builtin/packages/camp/package.py
@@ -53,7 +53,9 @@ class Camp(CMakePackage, CudaPackage, ROCmPackage):
 
     depends_on("cub", when="+cuda")
 
-    depends_on("blt")
+    depends_on("blt", type="build")
+
+    conflicts("^blt@:0.3.6", when="+rocm")
 
     def cmake_args(self):
         spec = self.spec
diff --git a/var/spack/repos/builtin/packages/canu/package.py b/var/spack/repos/builtin/packages/canu/package.py
index fc174af777fccd..2f138736b9a7bf 100644
--- a/var/spack/repos/builtin/packages/canu/package.py
+++ b/var/spack/repos/builtin/packages/canu/package.py
@@ -30,7 +30,7 @@ class Canu(MakefilePackage):
     depends_on("perl", type="run")
     # version guessed from include date of vendored boost
     depends_on("boost@1.60.0:+graph")
-    conflicts("boost@1.70.1:", when="@:2.0")
+    conflicts("^boost@1.70.1:", when="@:2.0")
 
     build_directory = "src"
     build_targets = ["clean"]
diff --git a/var/spack/repos/builtin/packages/capnproto/package.py b/var/spack/repos/builtin/packages/capnproto/package.py
index e8d95b7eed45c8..cec409eed9c5f9 100644
--- a/var/spack/repos/builtin/packages/capnproto/package.py
+++ b/var/spack/repos/builtin/packages/capnproto/package.py
@@ -34,7 +34,7 @@ class Capnproto(AutotoolsPackage):
     version("0.5.1.1", sha256="caf308e92683b278bc6c568d4fb5558eca78180cac1eb4a3db15d435bf25116f")
     version("0.4.1.2", sha256="6376c1910e9bc9d09dc46d53b063c5bdcb5cdf066a8210e9fffe299fb863f0d9")
 
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("openssl", when="+tls")
 
     depends_on("autoconf", type="build", when="build_system=autotools")
diff --git a/var/spack/repos/builtin/packages/care/package.py b/var/spack/repos/builtin/packages/care/package.py
index 4f257baf0f0c92..683b480088e25a 100644
--- a/var/spack/repos/builtin/packages/care/package.py
+++ b/var/spack/repos/builtin/packages/care/package.py
@@ -17,8 +17,12 @@ class Care(CMakePackage, CudaPackage, ROCmPackage):
 
     version("develop", branch="develop", submodules="True")
     version("master", branch="main", submodules="True")
-    version("0.3.0", tag="v0.3.0", submodules="True")
-    version("0.2.0", tag="v0.2.0", submodules="True")
+    version(
+        "0.3.0", tag="v0.3.0", commit="5e2b69b2836c9f2215207ca9a36a690cb77eea33", submodules="True"
+    )
+    version(
+        "0.2.0", tag="v0.2.0", commit="30135e03b14b1dc753634e9147dafede0663906f", submodules="True"
+    )
 
     variant("openmp", default=False, description="Build Shared Libs")
     variant(
@@ -34,6 +38,7 @@ class Care(CMakePackage, CudaPackage, ROCmPackage):
 
     depends_on("blt@0.4.0:", type="build", when="@0.3.1:")
     depends_on("blt@:0.3.6", type="build", when="@:0.3.0")
+    conflicts("^blt@:0.3.6", when="+rocm")
 
     depends_on("camp")
     depends_on("umpire@develop")
diff --git a/var/spack/repos/builtin/packages/catch2/package.py b/var/spack/repos/builtin/packages/catch2/package.py
index bb731a43f1a223..82eb629bb82661 100644
--- a/var/spack/repos/builtin/packages/catch2/package.py
+++ b/var/spack/repos/builtin/packages/catch2/package.py
@@ -19,6 +19,7 @@ class Catch2(CMakePackage):
     version("develop", branch="devel")
 
     # Releases
+    version("3.4.0", sha256="122928b814b75717316c71af69bd2b43387643ba076a6ec16e7882bfb2dfacbb")
     version("3.3.2", sha256="8361907f4d9bff3ae7c1edb027f813659f793053c99b67837a0c0375f065bae2")
     version("3.3.1", sha256="d90351cdc55421f640c553cfc0875a8c834428679444e8062e9187d05b18aace")
     version("3.3.0", sha256="fe2f29a54ca775c2dd04bb97ffb79d398e6210e3caa174348b5cd3b7e4ca887d")
@@ -104,6 +105,11 @@ class Catch2(CMakePackage):
     version("1.3.5", sha256="f15730d81b4173fb860ce3561768de7d41bbefb67dc031d7d1f5ae2c07f0a472")
     version("1.3.0", sha256="245f6ee73e2fea66311afa1da59e5087ddab8b37ce64994ad88506e8af28c6ac")
 
+    variant(
+        "pic", when="@3: ~shared", default=True, description="Build with position-independent code"
+    )
+    variant("shared", when="@3:", default=False, description="Build shared library")
+
     def cmake_args(self):
         spec = self.spec
         args = []
@@ -112,6 +118,10 @@ def cmake_args(self):
             args.append("-DNO_SELFTEST={0}".format("OFF" if self.run_tests else "ON"))
         elif spec.satisfies("@2.1.1:"):
             args.append(self.define("BUILD_TESTING", self.run_tests))
+        if spec.satisfies("@3:"):
+            args.append(self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"))
+            args.append(self.define_from_variant("BUILD_SHARED_LIBS", "shared"))
+
         return args
 
     @when("@:1.6")
diff --git a/var/spack/repos/builtin/packages/cbflib/package.py b/var/spack/repos/builtin/packages/cbflib/package.py
index f0b83a07a1e2c1..4ec7b02a33d27f 100644
--- a/var/spack/repos/builtin/packages/cbflib/package.py
+++ b/var/spack/repos/builtin/packages/cbflib/package.py
@@ -43,6 +43,9 @@ def edit(self, spec, prefix):
         mf.filter(r"^C\+\+.+", "C++ = {0}".format(spack_cxx))
         mf.filter("gfortran", spack_fc)
         mf.filter(r"^INSTALLDIR .+", "INSTALLDIR = {0}".format(prefix))
+        real_version = Version(self.compiler.get_real_version())
+        if real_version >= Version("10"):
+            mf.filter(r"^F90FLAGS[ \t]*=[ \t]*(.+)", "F90FLAGS = \\1 -fallow-invalid-boz")
 
     def build(self, spec, prefix):
         pass
diff --git a/var/spack/repos/builtin/packages/ccache/package.py b/var/spack/repos/builtin/packages/ccache/package.py
index a852f54d8ca534..06b1eb09685026 100644
--- a/var/spack/repos/builtin/packages/ccache/package.py
+++ b/var/spack/repos/builtin/packages/ccache/package.py
@@ -21,6 +21,7 @@ class Ccache(CMakePackage):
 
     executables = ["^ccache$"]
 
+    version("4.8.2", sha256="75eef15b8b9da48db9c91e1d0ff58b3645fc70c0e4ca2ef1b6825a12f21f217d")
     version("4.8.1", sha256="869903c1891beb8bee87f1ec94d8a0dad18c2add4072c456acbc85cdfc23ca63")
     version("4.8", sha256="ac4b01748fd59cfe07e070c34432b91bdd0fd8640e1e653a80b01d6a523186b0")
     version("4.7.4", sha256="dc283906b73bd7c461178ca472a459e9d86b5523405035921bd8204e77620264")
@@ -59,7 +60,7 @@ class Ccache(CMakePackage):
 
     depends_on("gperf", when="@:3")
     depends_on("libxslt", when="@:3")
-    depends_on("zlib", when="@:3")
+    depends_on("zlib-api", when="@:3")
 
     depends_on("zstd", when="@4.0:")
 
diff --git a/var/spack/repos/builtin/packages/cctools/package.py b/var/spack/repos/builtin/packages/cctools/package.py
index 14d107051a7a36..a8e990b783ed45 100644
--- a/var/spack/repos/builtin/packages/cctools/package.py
+++ b/var/spack/repos/builtin/packages/cctools/package.py
@@ -39,7 +39,7 @@ class Cctools(AutotoolsPackage):
     depends_on("gettext")  # Corrects python linking of -lintl flag.
     depends_on("swig")
     # depends_on('xrootd')
-    depends_on("zlib")
+    depends_on("zlib-api")
     patch("arm.patch", when="target=aarch64:")
     patch("cctools_7.0.18.python.patch", when="@7.0.18")
     patch("cctools_6.1.1.python.patch", when="@6.1.1")
@@ -90,7 +90,9 @@ def configure_args(self):
             args.append("--with-{0}-path=no".format(p))
 
         # point these bits at the Spack installations
-        for p in ["openssl", "perl", "readline", "swig", "zlib"]:
+        for p in ["openssl", "perl", "readline", "swig"]:
             args.append("--with-{0}-path={1}".format(p, self.spec[p].prefix))
 
+        args.append("--with-zlib-path={0}".format(self.spec["zlib-api"].prefix))
+
         return args
diff --git a/var/spack/repos/builtin/packages/cdbfasta/package.py b/var/spack/repos/builtin/packages/cdbfasta/package.py
index 3349b481c0e68e..adac5de886c965 100644
--- a/var/spack/repos/builtin/packages/cdbfasta/package.py
+++ b/var/spack/repos/builtin/packages/cdbfasta/package.py
@@ -14,7 +14,7 @@ class Cdbfasta(MakefilePackage):
 
     version("2017-03-16", commit="b3e481fe02dfbc767a3842bcb1b687c60376a5e8")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/cdhit/package.py b/var/spack/repos/builtin/packages/cdhit/package.py
index ce45ba1b0bcf59..fb9db33107d8c3 100644
--- a/var/spack/repos/builtin/packages/cdhit/package.py
+++ b/var/spack/repos/builtin/packages/cdhit/package.py
@@ -25,7 +25,7 @@ class Cdhit(MakefilePackage):
 
     depends_on("perl", type=("build", "run"))
     depends_on("perl-text-nsp", type="run")
-    depends_on("zlib", when="+zlib", type="link")
+    depends_on("zlib-api", when="+zlib", type="link")
 
     def patch(self):
         for f in glob("*.pl"):
diff --git a/var/spack/repos/builtin/packages/cdo/package.py b/var/spack/repos/builtin/packages/cdo/package.py
index 2f2c3e08ab94eb..051a499d45793e 100644
--- a/var/spack/repos/builtin/packages/cdo/package.py
+++ b/var/spack/repos/builtin/packages/cdo/package.py
@@ -20,6 +20,11 @@ class Cdo(AutotoolsPackage):
 
     maintainers("skosukhin", "Try2Code")
 
+    version(
+        "2.2.2",
+        sha256="419c77315244019af41a296c05066f474cccbf94debfaae9e2106da51bc7c937",
+        url="https://code.mpimet.mpg.de/attachments/download/28882/cdo-2.2.2.tar.gz",
+    )
     version(
         "2.2.0",
         sha256="679c8d105706caffcba0960ec5ddc4a1332c1b40c52f82c3937356999d8fadf2",
@@ -167,6 +172,9 @@ class Cdo(AutotoolsPackage):
     # We also need the backend of netcdf to be thread safe.
     depends_on("hdf5+threadsafe", when="+netcdf +openmp")
 
+    # Same in case hdf5 is used in the frontend
+    depends_on("hdf5+threadsafe", when="+hdf5")
+
     depends_on("grib-api", when="grib2=grib-api")
     depends_on("eccodes", when="grib2=eccodes")
 
@@ -222,12 +230,11 @@ def yes_or_prefix(spec_name):
             config_args.append("--without-netcdf")
 
         if self.spec.variants["grib2"].value == "eccodes":
+            config_args.append("--without-grib_api")
             eccodes_spec = self.spec["eccodes"]
             if self.spec.satisfies("@1.9:") and eccodes_spec.satisfies("+shared"):
                 config_args.append("--with-eccodes=" + yes_or_prefix("eccodes"))
-                config_args.append("--without-grib_api")
             else:
-                config_args.append("--with-grib_api=yes")
                 eccodes_libs = eccodes_spec.libs
                 flags["LIBS"].append(eccodes_libs.link_flags)
                 if not is_system_path(eccodes_spec.prefix):
diff --git a/var/spack/repos/builtin/packages/cdt/package.py b/var/spack/repos/builtin/packages/cdt/package.py
new file mode 100644
index 00000000000000..559fd82ee53ada
--- /dev/null
+++ b/var/spack/repos/builtin/packages/cdt/package.py
@@ -0,0 +1,47 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Cdt(CMakePackage):
+    """CDT is a C++ library for generating constraint or conforming Delaunay triangulations."""
+
+    homepage = "https://artem-ogre.github.io/CDT"
+    url = "https://github.com/artem-ogre/CDT/archive/refs/tags/1.3.0.tar.gz"
+
+    maintainers("jcortial-safran")
+
+    version("1.3.0", sha256="7e8feadf9534cf79f9bf188365510fd6bc68ea997758e1c68d1569f98da924da")
+
+    variant(
+        "boost",
+        default=False,
+        description="Use Boost as a fallback for features missing in C++98 and performance tweaks",
+    )
+    variant(
+        "64_bit_index",
+        default=False,
+        description="Use 64bits to store vertex/triangle index types. Otherwise 32bits are used.",
+    )
+    variant(
+        "compiled",
+        default=False,
+        description="Instantiate templates for float and double and compiled into a library",
+    )
+    variant("shared", default=True, description="Compile as a shared library")
+
+    depends_on("cmake@3.4:", type="build")
+    depends_on("boost", when="+boost")
+
+    root_cmakelists_dir = "CDT"
+
+    def cmake_args(self):
+        return [
+            self.define_from_variant("CDT_USE_BOOST", "boost"),
+            self.define_from_variant("CDT_USE_64_BIT_INDEX_TYPE", "64_bit_index"),
+            self.define_from_variant("CDT_USE_AS_COMPILED_LIBRARY", "compiled"),
+            self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
+        ]
diff --git a/var/spack/repos/builtin/packages/celeritas/package.py b/var/spack/repos/builtin/packages/celeritas/package.py
index 3ec70ad3b1b125..83690862bfdf9f 100644
--- a/var/spack/repos/builtin/packages/celeritas/package.py
+++ b/var/spack/repos/builtin/packages/celeritas/package.py
@@ -17,6 +17,8 @@ class Celeritas(CMakePackage, CudaPackage, ROCmPackage):
 
     maintainers("sethrj")
 
+    version("0.3.2", sha256="65a33de2518716638375df259d9dfc4d68b821ba1110f56b24c823ef5c5df249")
+    version("0.3.1", sha256="0f1effab306856d66f5079e8cadcb63e8c1f8a79245b94bf44b89251b3fb0cf0")
     version("0.3.0", sha256="f9620b6bcd8c9b5324ef215f8e44461f915c3fff47bf85ae442c9dafacaa79ac")
     version("0.2.2", sha256="ba5e341d636e00e3d7dbac13a2016b97014917489f46b8b387a2adf9d9563872")
     version(
@@ -81,9 +83,10 @@ class Celeritas(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("cmake@3.22:", type="build", when="+rocm")
 
     depends_on("nlohmann-json")
-    depends_on("geant4@10.7:11.0", when="@:0.2.0 +geant4")
+    depends_on("geant4@10.5:", when="@0.3.1: +geant4")
+    depends_on("geant4@10.6:", when="@0.3.0 +geant4")
     depends_on("geant4@10.6:11.0", when="@0.2.1:0.2 +geant4")
-    depends_on("geant4@10.6:", when="@0.3: +geant4")
+    depends_on("geant4@10.7:11.0", when="@:0.2.0 +geant4")
     depends_on("hepmc3", when="+hepmc3")
     depends_on("root", when="+root")
     depends_on("swig", when="+swig")
diff --git a/var/spack/repos/builtin/packages/cepgen/package.py b/var/spack/repos/builtin/packages/cepgen/package.py
index f3268dbb78a159..5058964d46bfad 100644
--- a/var/spack/repos/builtin/packages/cepgen/package.py
+++ b/var/spack/repos/builtin/packages/cepgen/package.py
@@ -14,12 +14,16 @@ class Cepgen(CMakePackage):
 
     tags = ["hep"]
 
+    version("1.1.0", sha256="2a4eaed161f007269516cbfb6e90421e657ab1922d4509de0165f08dde91bf3d")
     version(
         "1.0.2patch1", sha256="333bba0cb1965a98dec127e00c150eab1a515cd348a90f7b1d66d5cd8d206d21"
     )
 
     generator("ninja")
 
+    depends_on("cmake@3.5:", type="build", when="@1.0:")
+    depends_on("cmake@3.20:", type="build", when="@1.1:")
+
     depends_on("gsl")
     depends_on("openblas")
     depends_on("hepmc")
diff --git a/var/spack/repos/builtin/packages/ceres-solver/package.py b/var/spack/repos/builtin/packages/ceres-solver/package.py
index 30ee0f19709137..c9cdecc07e1f4f 100644
--- a/var/spack/repos/builtin/packages/ceres-solver/package.py
+++ b/var/spack/repos/builtin/packages/ceres-solver/package.py
@@ -17,6 +17,7 @@ class CeresSolver(CMakePackage):
     homepage = "http://ceres-solver.org"
     url = "http://ceres-solver.org/ceres-solver-1.12.0.tar.gz"
 
+    version("2.2.0", sha256="48b2302a7986ece172898477c3bcd6deb8fb5cf19b3327bc49969aad4cede82d")
     version("2.0.0", sha256="10298a1d75ca884aa0507d1abb0e0f04800a92871cd400d4c361b56a777a7603")
     version("1.14.0", sha256="4744005fc3b902fed886ea418df70690caa8e2ff6b5a90f3dd88a3d291ef8e8e")
     version("1.12.0", sha256="745bfed55111e086954126b748eb9efe20e30be5b825c6dec3c525cf20afc895")
@@ -25,12 +26,26 @@ class CeresSolver(CMakePackage):
     variant("shared", default=True, description="Build shared libraries")
     variant("examples", default=False, description="Build examples")
 
+    depends_on("cmake@2.8.0:", type="build", when="@1.12.0:1.14.0")
+    depends_on("cmake@3.5:", type="build", when="@2.0.0")
+    depends_on("cmake@3.16:3.27", type="build", when="@2.2.0")
     depends_on("eigen@3:")
+    depends_on("eigen@3.3:", when="@2.0.0:")
     depends_on("lapack")
-    depends_on("glog")
+    depends_on("glog@0.3.5:")
+    depends_on("suite-sparse", when="+suitesparse")
 
     def cmake_args(self):
-        args = ["-DCXSPARSE=OFF", "-DEIGENSPARSE=ON", "-DLAPACK=ON", "-DSCHUR_SPECIALIZATIONS=OFF"]
+        args = []
+        if self.spec.satisfies("@:2.0.0"):
+            args.extend(
+                [
+                    "-DCXSPARSE=OFF",
+                    "-DEIGENSPARSE=ON",
+                    "-DLAPACK=ON",
+                    "-DSCHUR_SPECIALIZATIONS=OFF",
+                ]
+            )
 
         if "+suitesparse" in self.spec:
             args.append("-DSUITESPARSE=ON")
diff --git a/var/spack/repos/builtin/packages/cernlib/package.py b/var/spack/repos/builtin/packages/cernlib/package.py
index 4888a591012038..bf6fc22115c4e8 100644
--- a/var/spack/repos/builtin/packages/cernlib/package.py
+++ b/var/spack/repos/builtin/packages/cernlib/package.py
@@ -20,6 +20,8 @@ class Cernlib(CMakePackage):
         sha256="733d148415ef78012ff81f21922d3bf641be7514b0242348dd0200cf1b003e46",
     )
 
+    variant("shared", default=True, description="Build shared libraries")
+
     depends_on("freetype")
     depends_on("motif")
     depends_on("libnsl")
@@ -35,5 +37,5 @@ def patch(self):
         filter_file("crypto", "crypt", "packlib/CMakeLists.txt")
 
     def cmake_args(self):
-        args = ["-DCERNLIB_BUILD_SHARED:BOOL=ON"]
+        args = [self.define_from_variant("CERNLIB_BUILD_SHARED", "shared")]
         return args
diff --git a/var/spack/repos/builtin/packages/cfitsio/package.py b/var/spack/repos/builtin/packages/cfitsio/package.py
index 1cca3d6e929498..708461b2e2d35f 100644
--- a/var/spack/repos/builtin/packages/cfitsio/package.py
+++ b/var/spack/repos/builtin/packages/cfitsio/package.py
@@ -14,6 +14,7 @@ class Cfitsio(AutotoolsPackage):
     homepage = "https://heasarc.gsfc.nasa.gov/fitsio/"
     url = "https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio-3.49.tar.gz"
 
+    version("4.3.0", sha256="fdadc01d09cf9f54253802c5ec87eb10de51ce4130411415ae88c30940621b8b")
     version("4.2.0", sha256="eba53d1b3f6e345632bb09a7b752ec7ced3d63ec5153a848380f3880c5d61889")
     version("4.1.0", sha256="b367c695d2831958e7166921c3b356d5dfa51b1ecee505b97416ba39d1b6c17a")
     version("4.0.0", sha256="b2a8efba0b9f86d3e1bd619f662a476ec18112b4f27cc441cc680a4e3777425e")
diff --git a/var/spack/repos/builtin/packages/cgal/package.py b/var/spack/repos/builtin/packages/cgal/package.py
index fa66fc4d375451..ecdd759d1770e5 100644
--- a/var/spack/repos/builtin/packages/cgal/package.py
+++ b/var/spack/repos/builtin/packages/cgal/package.py
@@ -60,7 +60,7 @@ class Cgal(CMakePackage):
 
     # Required for CGAL_ImageIO
     # depends_on('opengl', when='+imageio') # not yet in Spack
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # Optional to build CGAL_Qt5 (demos)
     # depends_on('opengl', when='+demos')   # not yet in Spack
diff --git a/var/spack/repos/builtin/packages/cgns/package.py b/var/spack/repos/builtin/packages/cgns/package.py
index 8c43a2fe0925cb..563dffd2875772 100644
--- a/var/spack/repos/builtin/packages/cgns/package.py
+++ b/var/spack/repos/builtin/packages/cgns/package.py
@@ -46,6 +46,7 @@ class Cgns(CMakePackage):
     variant("legacy", default=False, description="Enable legacy options")
     variant("mem_debug", default=False, description="Enable memory debugging option")
     variant("tools", default=False, description="Enable CGNS tools")
+    variant("pic", default=False, description="Produce position-independent code")
 
     depends_on("cmake@3.12:", when="@4.3:", type="build")
     depends_on("cmake@3.8:", when="@4.2:", type="build")
@@ -62,6 +63,8 @@ class Cgns(CMakePackage):
     depends_on("libxmu", when="+tools")
     depends_on("libsm", when="+tools")
 
+    conflicts("~pic", when="+fortran", msg="+pic required when +fortran")
+
     # patch for error undefined reference to `matherr, see
     # https://bugs.gentoo.org/662210
     patch("no-matherr.patch", when="@:3.3.1 +tools")
@@ -83,6 +86,8 @@ def cmake_args(self):
                 self.define_from_variant("CGNS_ENABLE_BASE_SCOPE", "base_scope"),
                 self.define_from_variant("CGNS_ENABLE_LEGACY", "legacy"),
                 self.define_from_variant("CGNS_ENABLE_MEM_DEBUG", "mem_debug"),
+                self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
+                self.define_from_variant("CGNS_ENABLE_64BIT", "int64"),
             ]
         )
 
@@ -91,11 +96,10 @@ def cmake_args(self):
                 [
                     "-DCMAKE_C_COMPILER=%s" % spec["mpi"].mpicc,
                     "-DCMAKE_CXX_COMPILER=%s" % spec["mpi"].mpicxx,
-                    "-DCMAKE_Fortran_COMPILER=%s" % spec["mpi"].mpifc,
                 ]
             )
-
-        options.append(self.define_from_variant("CGNS_ENABLE_64BIT", "int64"))
+            if "+fortran" in spec:
+                options.append(self.define("CMAKE_Fortran_COMPILER", spec["mpi"].mpifc))
 
         if "+hdf5" in spec:
             options.extend(
diff --git a/var/spack/repos/builtin/packages/chaco/package.py b/var/spack/repos/builtin/packages/chaco/package.py
index e78ef4f70e3641..a778dc817ead4f 100644
--- a/var/spack/repos/builtin/packages/chaco/package.py
+++ b/var/spack/repos/builtin/packages/chaco/package.py
@@ -12,7 +12,7 @@ class Chaco(CMakePackage):
     homepage = "https://gitlab.com/truchas/tpl-forks/chaco"
     git = "https://gitlab.com/truchas/tpl-forks/chaco.git"
 
-    maintainers("pbrady")
+    maintainers("pbrady", "zjibben")
 
     version("develop", branch="truchas")
     version("2020-07-16", commit="92a877b381933d12b02507413897f696d81b4682", preferred=True)
diff --git a/var/spack/repos/builtin/packages/chai/package.py b/var/spack/repos/builtin/packages/chai/package.py
index ea81e86d74e5d3..9cde5d950733ca 100644
--- a/var/spack/repos/builtin/packages/chai/package.py
+++ b/var/spack/repos/builtin/packages/chai/package.py
@@ -21,18 +21,43 @@ class Chai(CachedCMakePackage, CudaPackage, ROCmPackage):
 
     version("develop", branch="develop", submodules=False)
     version("main", branch="main", submodules=False)
-    version("2022.03.0", tag="v2022.03.0", submodules=False)
-    version("2.4.0", tag="v2.4.0", submodules=True)
-    version("2.3.0", tag="v2.3.0", submodules=True)
-    version("2.2.2", tag="v2.2.2", submodules=True)
-    version("2.2.1", tag="v2.2.1", submodules=True)
-    version("2.2.0", tag="v2.2.0", submodules=True)
-    version("2.1.1", tag="v2.1.1", submodules=True)
-    version("2.1.0", tag="v2.1.0", submodules=True)
-    version("2.0.0", tag="v2.0.0", submodules=True)
-    version("1.2.0", tag="v1.2.0", submodules=True)
-    version("1.1.0", tag="v1.1.0", submodules=True)
-    version("1.0", tag="v1.0", submodules=True)
+    version(
+        "2022.03.0",
+        tag="v2022.03.0",
+        commit="f0b809de1ac194376866b3ac0f5933d4146ec09e",
+        submodules=False,
+    )
+    version(
+        "2.4.0", tag="v2.4.0", commit="77d22da28187245a2c5454cf471c0c352bd98ad7", submodules=True
+    )
+    version(
+        "2.3.0", tag="v2.3.0", commit="42f3fbcc0b966227b40b4467dc919a4c24f07196", submodules=True
+    )
+    version(
+        "2.2.2", tag="v2.2.2", commit="56e75fc0f805b2746f3992af0c00c474513e3b24", submodules=True
+    )
+    version(
+        "2.2.1", tag="v2.2.1", commit="c912f583828ea5963850816e3e232cc45608ccf7", submodules=True
+    )
+    version(
+        "2.2.0", tag="v2.2.0", commit="18536c61a4817db6b3b3025f35e2dd3ae532330c", submodules=True
+    )
+    version(
+        "2.1.1", tag="v2.1.1", commit="496911e00d15c350560860f7964cd5fb5ab7f515", submodules=True
+    )
+    version(
+        "2.1.0", tag="v2.1.0", commit="fff02768068a64970b34760a1041585319edee87", submodules=True
+    )
+    version(
+        "2.0.0", tag="v2.0.0", commit="63139cf45443b1266950826b165e042c7679b557", submodules=True
+    )
+    version(
+        "1.2.0", tag="v1.2.0", commit="7bb5bc12e4508db45910d8e2b98444687da7ebf6", submodules=True
+    )
+    version(
+        "1.1.0", tag="v1.1.0", commit="907d5f40d653a73955387067799913397807adf3", submodules=True
+    )
+    version("1.0", tag="v1.0", commit="501a098ad879dc8deb4a74fcfe8c08c283a10627", submodules=True)
 
     variant("enable_pick", default=False, description="Enable pick method")
     variant("shared", default=True, description="Build Shared Libs")
@@ -52,6 +77,7 @@ class Chai(CachedCMakePackage, CudaPackage, ROCmPackage):
     depends_on("blt@0.4.1:", type="build", when="@2.4.0:")
     depends_on("blt@0.4.0:", type="build", when="@2.3.0")
     depends_on("blt@0.3.6:", type="build", when="@:2.2.2")
+    conflicts("^blt@:0.3.6", when="+rocm")
 
     depends_on("umpire")
     depends_on("umpire@2022.03.0:", when="@2022.03.0:")
diff --git a/var/spack/repos/builtin/packages/chameleon/package.py b/var/spack/repos/builtin/packages/chameleon/package.py
index 6b6e9af0cc8706..7d6239ffb09bce 100644
--- a/var/spack/repos/builtin/packages/chameleon/package.py
+++ b/var/spack/repos/builtin/packages/chameleon/package.py
@@ -11,11 +11,12 @@ class Chameleon(CMakePackage, CudaPackage):
     """Dense Linear Algebra for Scalable Multi-core Architectures and GPGPUs"""
 
     homepage = "https://gitlab.inria.fr/solverstack/chameleon"
-    url = "https://gitlab.inria.fr/solverstack/chameleon/uploads/b299d6037d7636c6be16108c89bc2aab/chameleon-1.1.0.tar.gz"
+    url = "https://gitlab.inria.fr/api/v4/projects/616/packages/generic/source/v1.2.0/chameleon-1.2.0.tar.gz"
     git = "https://gitlab.inria.fr/solverstack/chameleon.git"
     maintainers("fpruvost")
 
     version("master", branch="master", submodules=True)
+    version("1.2.0", sha256="b8988ecbff19c603ae9f61441653c21bba18d040bee9bb83f7fc9077043e50b4")
     version("1.1.0", sha256="e64d0438dfaf5effb3740e53f3ab017d12744b85a138b2ef702a81df559126df")
 
     # cmake's specific
@@ -48,6 +49,7 @@ class Chameleon(CMakePackage, CudaPackage):
     depends_on("pkgconfig", type="build")
 
     with when("runtime=starpu"):
+        depends_on("starpu@1.3", when="@1.1.0")
         depends_on("starpu")
         depends_on("starpu~mpi", when="~mpi")
         depends_on("starpu+mpi", when="+mpi")
diff --git a/var/spack/repos/builtin/packages/chaparral/package.py b/var/spack/repos/builtin/packages/chaparral/package.py
index a999949501efab..eabee9836a8860 100644
--- a/var/spack/repos/builtin/packages/chaparral/package.py
+++ b/var/spack/repos/builtin/packages/chaparral/package.py
@@ -12,7 +12,7 @@ class Chaparral(CMakePackage):
     homepage = "https://gitlab.com/truchas/tpl-forks/chaparral"
     git = "https://gitlab.com/truchas/tpl-forks/chaparral.git"
 
-    maintainers("pbrady")
+    maintainers("pbrady", "zjibben")
 
     version("develop", branch="truchas")
     version("2020-08-28", commit="c8a190bb74ef33ad8b2f7b67d20590f393fde32a", preferred=True)
diff --git a/var/spack/repos/builtin/packages/charliecloud/package.py b/var/spack/repos/builtin/packages/charliecloud/package.py
index ca3865874a531b..88cca7d6e2a88a 100644
--- a/var/spack/repos/builtin/packages/charliecloud/package.py
+++ b/var/spack/repos/builtin/packages/charliecloud/package.py
@@ -17,10 +17,32 @@ class Charliecloud(AutotoolsPackage):
     tags = ["e4s"]
 
     version("master", branch="master")
-    version("0.33", sha256="ed2bd3589d1e5f7b33a1542c887d69856f6d7d57a6ec8ef5b8e9335eda48a045")
-    version("0.32", sha256="47826b14966c400b250c35ff28a903f8e5b5e12d9e2a2b473e0f00f4e8393c47")
-    version("0.31", sha256="7305c3d9010386c1b96fb95297feccb5c9d7ff82a3377d1d98eb8faef76bced9")
-    version("0.30", sha256="97d45b25c9f813d8bae79b16de49503a165bc94c05dd2166975154d9b6ac78e9")
+    version("0.35", sha256="042f5be5ed8eda95f45230b4647510780142a50adb4e748be57e8dd8926b310e")
+    version(
+        "0.34",
+        deprecated=True,
+        sha256="034080c162949f4344ae1011cda026d4bb3ecd5cdb53135ac06d236f87e3b27d",
+    )
+    version(
+        "0.33",
+        deprecated=True,
+        sha256="ed2bd3589d1e5f7b33a1542c887d69856f6d7d57a6ec8ef5b8e9335eda48a045",
+    )
+    version(
+        "0.32",
+        deprecated=True,
+        sha256="47826b14966c400b250c35ff28a903f8e5b5e12d9e2a2b473e0f00f4e8393c47",
+    )
+    version(
+        "0.31",
+        deprecated=True,
+        sha256="7305c3d9010386c1b96fb95297feccb5c9d7ff82a3377d1d98eb8faef76bced9",
+    )
+    version(
+        "0.30",
+        deprecated=True,
+        sha256="97d45b25c9f813d8bae79b16de49503a165bc94c05dd2166975154d9b6ac78e9",
+    )
     version(
         "0.29",
         deprecated=True,
diff --git a/var/spack/repos/builtin/packages/charmpp/package.py b/var/spack/repos/builtin/packages/charmpp/package.py
index 91876fbb13b906..43ef0d5bbc0ad2 100644
--- a/var/spack/repos/builtin/packages/charmpp/package.py
+++ b/var/spack/repos/builtin/packages/charmpp/package.py
@@ -187,9 +187,7 @@ def charmarch(self):
             ("linux", "ppc", "pami"): "pami-linux-ppc64le",
             ("linux", "ppc", "verbs"): "verbs-linux-ppc64le",
             ("linux", "arm", "netlrts"): "netlrts-linux-arm7",
-            ("linux", "arm", "multicore"): "multicore-arm7",
             ("linux", "aarch64", "netlrts"): "netlrts-linux-arm8",
-            ("linux", "aarch64", "multicore"): "multicore-arm8",
             ("win", "x86_64", "mpi"): "mpi-win-x86_64",
             ("win", "x86_64", "multicore"): "multicore-win-x86_64",
             ("win", "x86_64", "netlrts"): "netlrts-win-x86_64",
@@ -203,10 +201,22 @@ def charmarch(self):
             versions.update({("linux", "i386", "multicore"): "multicore-linux"})
             versions.update({("linux", "i386", "netlrts"): "netlrts-linux"})
             versions.update({("linux", "i386", "uth"): "uth-linux"})
+            versions.update(
+                {
+                    ("linux", "arm", "multicore"): "multicore-arm7",
+                    ("linux", "aarch64", "multicore"): "multicore-arm8",
+                }
+            )
         else:
             versions.update({("linux", "i386", "mpi"): "mpi-linux-i386"})
             versions.update({("linux", "i386", "multicore"): "multicore-linux-i386"})
             versions.update({("linux", "i386", "netlrts"): "netlrts-linux-i386"})
+            versions.update(
+                {
+                    ("linux", "arm", "multicore"): "multicore-linux-arm7",
+                    ("linux", "aarch64", "multicore"): "multicore-linux-arm8",
+                }
+            )
 
         if (plat, mach, comm) not in versions:
             raise InstallError(
diff --git a/var/spack/repos/builtin/packages/chatterbug/package.py b/var/spack/repos/builtin/packages/chatterbug/package.py
index 70b9f552623b34..d92da3cda2d94d 100644
--- a/var/spack/repos/builtin/packages/chatterbug/package.py
+++ b/var/spack/repos/builtin/packages/chatterbug/package.py
@@ -19,7 +19,7 @@ class Chatterbug(MakefilePackage):
     git = "https://github.com/LLNL/chatterbug.git"
 
     version("develop", branch="master")
-    version("1.0", tag="v1.0")
+    version("1.0", tag="v1.0", commit="ee1b13c634943dbe32ac22f5e2154b00eab8c574")
 
     variant("scorep", default=False, description="Build with Score-P tracing")
 
diff --git a/var/spack/repos/builtin/packages/chexmix/chexmix.sh b/var/spack/repos/builtin/packages/chexmix/chexmix.sh
new file mode 100644
index 00000000000000..b83bce6e5865d1
--- /dev/null
+++ b/var/spack/repos/builtin/packages/chexmix/chexmix.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+# convenience wrapper for the chexmix jar file
+java -jar chexmix.jar "$@"
diff --git a/var/spack/repos/builtin/packages/chexmix/package.py b/var/spack/repos/builtin/packages/chexmix/package.py
new file mode 100644
index 00000000000000..6654102f4f11bf
--- /dev/null
+++ b/var/spack/repos/builtin/packages/chexmix/package.py
@@ -0,0 +1,49 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+
+
+class Chexmix(Package):
+    """ChExMix: the ChIP-exo mixture model. ChExMix aims to characterize protein-DNA binding
+    subtypes in ChIP-exo experiments."""
+
+    homepage = "http://mahonylab.org/software/chexmix"
+    url = "https://github.com/seqcode/chexmix/releases/download/v0.51/chexmix.v0.51.jar"
+
+    # The naming of the .jar files is inconsistent on GitHub. Until this is resolved, downloads
+    # that don't match the above url spec will need to be renamed - see rename_versions().
+    version(
+        "0.52",
+        url="https://github.com/seqcode/chexmix/releases/download/v0.52/chexmix.v0.52.public.jar",
+        sha256="7f856921b6071092cfcf226e4f99d9ab80587cf05502b41d00b8e5e16ccbfcdd",
+        expand=False,
+    )
+
+    depends_on("java@8:", type="run")
+    depends_on("meme", type="run")
+
+    @run_before("install")
+    def rename_versions(self):
+        if self.version == Version("0.52"):
+            os.rename("chexmix.v0.52.public.jar", "chexmix.v0.52.jar")
+
+    def install(self, spec, prefix):
+        # install .jar file
+        mkdirp(prefix.bin)
+        jar_file = "chexmix.v{0}.jar".format(self.version)
+        install(jar_file, prefix.bin)
+        # create a helper script to launch the .jar
+        script_sh = join_path(os.path.dirname(__file__), "chexmix.sh")
+        script = prefix.bin.chexmix
+        install(script_sh, script)
+        set_executable(script)
+        # set the helper script to explicitly point to java and the .jar file
+        java = self.spec["java"].prefix.bin.java
+        kwargs = {"ignore_absent": False, "backup": False, "string": False}
+        filter_file("^java", java, script, **kwargs)
+        filter_file("chexmix.jar", join_path(prefix.bin, jar_file), script, **kwargs)
diff --git a/var/spack/repos/builtin/packages/chgcentre/main.patch b/var/spack/repos/builtin/packages/chgcentre/main.patch
new file mode 100644
index 00000000000000..fbc7cd064e000f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/chgcentre/main.patch
@@ -0,0 +1,11 @@
+--- a/main.cpp	2022-05-02 15:00:00 +0800
++++ b/main.cpp	2022-05-02 15:01:00 +0800
+@@ -415,7 +415,7 @@
+ 												MDirection::Ref(MDirection::J2000))();
+ 	std::vector uvws(antennas.size());
+ 	MEpoch time(MVEpoch(-1.0));
+-	for(unsigned row=0; row!=std::min(set.nrow(),50u); ++row)
++	for(unsigned row=0; row!=std::min(static_cast(set.nrow()),50u); ++row)
+ 	{
+ 		if(fieldIdCol(row) == fieldIndex)
+ 		{
diff --git a/var/spack/repos/builtin/packages/chgcentre/package.py b/var/spack/repos/builtin/packages/chgcentre/package.py
index ebd8242bea6773..78d8ceaea7a659 100644
--- a/var/spack/repos/builtin/packages/chgcentre/package.py
+++ b/var/spack/repos/builtin/packages/chgcentre/package.py
@@ -18,3 +18,9 @@ class Chgcentre(CMakePackage):
 
     depends_on("casacore")
     depends_on("gsl")
+
+    # this patch is required to fix a programming error that is not acceptable by
+    # latest compilers. In particular, the `std::min` function was given `int` and
+    # `unsigned int` arguments. The `int` argument is explicitly casted to `unsigned int`.
+    # This patch was created by the staff at the Pawsey Supercomputing Research Centre.
+    patch("main.patch")
diff --git a/var/spack/repos/builtin/packages/chombo/package.py b/var/spack/repos/builtin/packages/chombo/package.py
index d0e4f9a3017be6..9ee703bd9043e8 100644
--- a/var/spack/repos/builtin/packages/chombo/package.py
+++ b/var/spack/repos/builtin/packages/chombo/package.py
@@ -20,8 +20,8 @@ class Chombo(MakefilePackage):
     tags = ["ecp", "ecp-apps"]
 
     # Use whatever path Brian V. and Terry L. agreed upon, but preserve version
-    version("3.2", commit="71d856c")
-    version("develop", tag="master")
+    version("3.2", commit="71d856c2f469e96755a606db1e5151067da0f54a")
+    version("develop", branch="master")
 
     variant("mpi", default=True, description="Enable MPI parallel support")
     variant("hdf5", default=True, description="Enable HDF5 support")
diff --git a/var/spack/repos/builtin/packages/citcoms/package.py b/var/spack/repos/builtin/packages/citcoms/package.py
index 9b334fc6b188f0..6ebe2584f28c2e 100644
--- a/var/spack/repos/builtin/packages/citcoms/package.py
+++ b/var/spack/repos/builtin/packages/citcoms/package.py
@@ -26,7 +26,7 @@ class Citcoms(AutotoolsPackage):
 
     # Required dependencies
     depends_on("mpi")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("automake", when="@master", type="build")
     depends_on("autoconf", when="@master", type="build")
     depends_on("libtool", when="@master", type="build")
diff --git a/var/spack/repos/builtin/packages/clamav/package.py b/var/spack/repos/builtin/packages/clamav/package.py
index 232e317df43439..a93511f0a6f2e6 100644
--- a/var/spack/repos/builtin/packages/clamav/package.py
+++ b/var/spack/repos/builtin/packages/clamav/package.py
@@ -20,7 +20,7 @@ class Clamav(AutotoolsPackage):
     depends_on("openssl")
     depends_on("pcre")
     depends_on("yara")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2")
     depends_on("curl", type="link")
 
@@ -31,7 +31,7 @@ def configure_args(self):
             "--with-libjson=%s" % spec["json-c"].prefix,
             "--with-openssl=%s" % spec["openssl"].prefix,
             "--with-pcre=%s" % spec["pcre"].prefix,
-            "--with-zlib=%s" % spec["zlib"].prefix,
+            "--with-zlib=%s" % spec["zlib-api"].prefix,
             "--with-bzip2=%s" % spec["bzip2"].prefix,
         ]
         return args
diff --git a/var/spack/repos/builtin/packages/claw/package.py b/var/spack/repos/builtin/packages/claw/package.py
index e67283d6c10490..bceab8345a62d6 100644
--- a/var/spack/repos/builtin/packages/claw/package.py
+++ b/var/spack/repos/builtin/packages/claw/package.py
@@ -17,15 +17,31 @@ class Claw(CMakePackage):
     git = "https://github.com/claw-project/claw-compiler.git"
     maintainers("clementval", "skosukhin")
 
-    version("2.0.3", tag="v2.0.3", submodules=True)
-    version("2.0.2", tag="v2.0.2", submodules=True)
-    version("2.0.1", tag="v2.0.1", submodules=True)
-    version("2.0", tag="v2.0", submodules=True)
-    version("1.2.3", tag="v1.2.3", submodules=True)
-    version("1.2.2", tag="v1.2.2", submodules=True)
-    version("1.2.1", tag="v1.2.1", submodules=True)
-    version("1.2.0", tag="v1.2.0", submodules=True)
-    version("1.1.0", tag="v1.1.0", submodules=True)
+    version(
+        "2.0.3", tag="v2.0.3", commit="4d8bc7a794af3651b8b61501388fc00096b23a85", submodules=True
+    )
+    version(
+        "2.0.2", tag="v2.0.2", commit="8c012d58484d8caf79a4fe45597dc74b4367421c", submodules=True
+    )
+    version(
+        "2.0.1", tag="v2.0.1", commit="f5acc929df74ce66a328aa4eda9cc9664f699b91", submodules=True
+    )
+    version("2.0", tag="v2.0", commit="53e705b8bfce40a5c5636e8194a7622e337cf4f5", submodules=True)
+    version(
+        "1.2.3", tag="v1.2.3", commit="eaf5e5fb39150090e51bec1763170ce5c5355198", submodules=True
+    )
+    version(
+        "1.2.2", tag="v1.2.2", commit="fc27a267eef9f412dd6353dc0b358a05b3fb3e16", submodules=True
+    )
+    version(
+        "1.2.1", tag="v1.2.1", commit="939989ab52edb5c292476e729608725654d0a59a", submodules=True
+    )
+    version(
+        "1.2.0", tag="v1.2.0", commit="fc9c50fe02be97b910ff9c7015064f89be88a3a2", submodules=True
+    )
+    version(
+        "1.1.0", tag="v1.1.0", commit="16b165a443b11b025a77cad830b1280b8c9bcf01", submodules=True
+    )
 
     depends_on("cmake@3.0:", type="build")
     depends_on("ant@1.9:", type="build")
diff --git a/var/spack/repos/builtin/packages/clhep/package.py b/var/spack/repos/builtin/packages/clhep/package.py
index 6089a530541f7d..43165ef8b7292d 100644
--- a/var/spack/repos/builtin/packages/clhep/package.py
+++ b/var/spack/repos/builtin/packages/clhep/package.py
@@ -53,13 +53,16 @@ class Clhep(CMakePackage):
     variant(
         "cxxstd",
         default="11",
-        values=("11", "14", "17"),
+        values=(
+            "11",
+            "14",
+            conditional("17", when="@2.3.4.3:"),
+            conditional("20", when="@2.4.6.4:"),
+        ),
         multi=False,
         description="Use the specified C++ standard when building.",
     )
 
-    conflicts("cxxstd=17", when="@:2.3.4.2")
-
     depends_on("cmake@2.8.12.2:", when="@2.2.0.4:2.3.0.0", type="build")
     depends_on("cmake@3.2:", when="@2.3.0.1:", type="build")
 
diff --git a/var/spack/repos/builtin/packages/clingo-bootstrap/mimalloc-pre-5.5.0.patch b/var/spack/repos/builtin/packages/clingo-bootstrap/mimalloc-pre-5.5.0.patch
new file mode 100644
index 00000000000000..54491c8c1a3c59
--- /dev/null
+++ b/var/spack/repos/builtin/packages/clingo-bootstrap/mimalloc-pre-5.5.0.patch
@@ -0,0 +1,39 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index f11e6e2..209970b 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -164,6 +164,7 @@ if (CLINGO_BUILD_WITH_LUA)
+         set_property(TARGET Lua::Lua PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${LUA_INCLUDE_DIR}")
+     endif()
+ endif()
++find_package(mimalloc REQUIRED)
+ find_package(BISON "2.5")
+ find_package(RE2C "0.13")
+ if (PYCLINGO_USE_CFFI AND Python_Development_FOUND)
+diff --git a/libclingo/CMakeLists.txt b/libclingo/CMakeLists.txt
+index 83acc22..51d5762 100644
+--- a/libclingo/CMakeLists.txt
++++ b/libclingo/CMakeLists.txt
+@@ -50,7 +50,7 @@ else()
+ endif()
+ 
+ add_library(libclingo ${clingo_lib_type} ${header} ${source})
+-target_link_libraries(libclingo PRIVATE libgringo libclasp)
++target_link_libraries(libclingo PRIVATE mimalloc-static libgringo libclasp)
+ target_include_directories(libclingo
+     PUBLIC
+     "$"
+diff --git a/libclingo/src/clingo_app.cc b/libclingo/src/clingo_app.cc
+index 3e4d14c..fcfc9ea 100644
+--- a/libclingo/src/clingo_app.cc
++++ b/libclingo/src/clingo_app.cc
+@@ -27,6 +27,9 @@
+ #include 
+ #include 
+ 
++#include 
++
++
+ namespace Gringo {
+ 
+ // {{{ declaration of ClingoApp
diff --git a/var/spack/repos/builtin/packages/clingo-bootstrap/mimalloc.patch b/var/spack/repos/builtin/packages/clingo-bootstrap/mimalloc.patch
new file mode 100644
index 00000000000000..b1de6845916a97
--- /dev/null
+++ b/var/spack/repos/builtin/packages/clingo-bootstrap/mimalloc.patch
@@ -0,0 +1,39 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 7fbe16bc..78539519 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -224,6 +224,7 @@ if (CLINGO_BUILD_WITH_LUA)
+         set_property(TARGET Lua::Lua PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${LUA_INCLUDE_DIR}")
+     endif()
+ endif()
++find_package(mimalloc REQUIRED)
+ find_package(BISON "2.5")
+ find_package(RE2C "0.101")
+ if (Python_Development_FOUND)
+diff --git a/libclingo/CMakeLists.txt b/libclingo/CMakeLists.txt
+index 1d70ba56..de2f2766 100644
+--- a/libclingo/CMakeLists.txt
++++ b/libclingo/CMakeLists.txt
+@@ -51,7 +51,7 @@ endif()
+ 
+ add_library(libclingo ${clingo_lib_type})
+ target_sources(libclingo ${clingo_private_scope_} ${header} ${source})
+-target_link_libraries(libclingo ${clingo_private_scope_} libgringo libclasp)
++target_link_libraries(libclingo ${clingo_private_scope_} mimalloc-static libgringo libclasp)
+ target_include_directories(libclingo
+     ${clingo_public_scope_}
+     "$"
+diff --git a/libclingo/src/clingo_app.cc b/libclingo/src/clingo_app.cc
+index 13980efa..3c3b404b 100644
+--- a/libclingo/src/clingo_app.cc
++++ b/libclingo/src/clingo_app.cc
+@@ -26,6 +26,9 @@
+ #include 
+ #include 
+ 
++#include 
++
++
+ namespace Gringo {
+ 
+ // {{{ declaration of ClingoApp
diff --git a/var/spack/repos/builtin/packages/clingo-bootstrap/package.py b/var/spack/repos/builtin/packages/clingo-bootstrap/package.py
index 0e77dde84bafeb..65535f330abfc2 100644
--- a/var/spack/repos/builtin/packages/clingo-bootstrap/package.py
+++ b/var/spack/repos/builtin/packages/clingo-bootstrap/package.py
@@ -2,8 +2,14 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import glob
+import os
+
+import spack.paths
+import spack.user_environment
 from spack.package import *
 from spack.pkg.builtin.clingo import Clingo
+from spack.util.environment import EnvironmentModifications
 
 
 class ClingoBootstrap(Clingo):
@@ -13,34 +19,66 @@ class ClingoBootstrap(Clingo):
 
     variant("build_type", default="Release", values=("Release",), description="CMake build type")
 
-    variant("static_libstdcpp", default=False, description="Require a static version of libstdc++")
+    variant(
+        "static_libstdcpp",
+        default=False,
+        when="platform=linux",
+        description="Require a static version of libstdc++",
+    )
+
+    variant(
+        "optimized",
+        default=False,
+        description="Enable a series of Spack-specific optimizations (PGO, LTO, mimalloc)",
+    )
+
+    variant(
+        "force_setuptools",
+        default=False,
+        description="Force a dependency on setuptools to help the old concretizer",
+    )
+    depends_on("py-setuptools", type="build", when="+force_setuptools")
+
+    # Enable LTO
+    conflicts("~ipo", when="+optimized")
+
+    with when("+optimized platform=linux"):
+        # Statically linked. Don't use ~override so we don't duplicate malloc/free, they
+        # get resolved to Python's libc's malloc in our case anyway.
+        depends_on("mimalloc +ipo libs=static ~override", type="build")
+        conflicts("~static_libstdcpp", msg="Custom allocator requires static libstdc++")
+        # Override new/delete with mimalloc.
+        patch("mimalloc.patch", when="@5.5.0:")
+        patch("mimalloc-pre-5.5.0.patch", when="@:5.4")
+        # ensure we hide libstdc++ with custom operator new/delete symbols
+        patch("version-script.patch")
 
     # CMake at version 3.16.0 or higher has the possibility to force the
     # Python interpreter, which is crucial to build against external Python
     # in environment where more than one interpreter is in the same prefix
     depends_on("cmake@3.16.0:", type="build")
 
-    # On Linux we bootstrap with GCC
-    for compiler_spec in [c for c in spack.compilers.supported_compilers() if c != "gcc"]:
-        conflicts(
-            "%{0}".format(compiler_spec),
-            when="platform=linux",
-            msg="GCC is required to bootstrap clingo on Linux",
-        )
-        conflicts(
-            "%{0}".format(compiler_spec),
-            when="platform=cray",
-            msg="GCC is required to bootstrap clingo on Cray",
-        )
+    # On Linux we bootstrap with GCC or clang
+    requires(
+        "%gcc",
+        "%clang",
+        when="platform=linux",
+        msg="GCC or clang are required to bootstrap clingo on Linux",
+    )
+    requires(
+        "%gcc",
+        "%clang",
+        when="platform=cray",
+        msg="GCC or clang are required to bootstrap clingo on Cray",
+    )
     conflicts("%gcc@:5", msg="C++14 support is required to bootstrap clingo")
 
     # On Darwin we bootstrap with Apple Clang
-    for compiler_spec in [c for c in spack.compilers.supported_compilers() if c != "apple-clang"]:
-        conflicts(
-            "%{0}".format(compiler_spec),
-            when="platform=darwin",
-            msg="Apple-clang is required to bootstrap clingo on MacOS",
-        )
+    requires(
+        "%apple-clang",
+        when="platform=darwin",
+        msg="Apple-clang is required to bootstrap clingo on MacOS",
+    )
 
     # Clingo needs the Python module to be usable by Spack
     conflicts("~python", msg="Python support is required to bootstrap Spack")
@@ -51,28 +89,70 @@ def cmake_py_shared(self):
 
     def cmake_args(self):
         args = super().cmake_args()
-        args.extend(
-            [
-                # Avoid building the clingo executable
-                self.define("CLINGO_BUILD_APPS", "OFF")
-            ]
-        )
+        args.append(self.define("CLINGO_BUILD_APPS", False))
         return args
 
-    def setup_build_environment(self, env):
-        opts = None
-        if "%apple-clang platform=darwin" in self.spec:
-            opts = "-mmacosx-version-min=10.13"
-        elif "%gcc" in self.spec:
-            if "+static_libstdcpp" in self.spec:
-                # This is either linux or cray
-                opts = "-static-libstdc++ -static-libgcc -Wl,--exclude-libs,ALL"
-        elif "platform=windows" in self.spec:
-            pass
+    @run_before("cmake", when="+optimized")
+    def pgo_train(self):
+        if self.spec.compiler.name == "clang":
+            llvm_profdata = which("llvm-profdata", required=True)
+        elif self.spec.compiler.name == "apple-clang":
+            llvm_profdata = Executable(
+                Executable("xcrun")("-find", "llvm-profdata", output=str).strip()
+            )
+
+        # First configure with PGO flags, and do build apps.
+        reports = os.path.abspath("reports")
+        sources = os.path.abspath(self.root_cmakelists_dir)
+        cmake_options = self.std_cmake_args + self.cmake_args() + [sources]
+
+        # Set PGO training flags.
+        generate_mods = EnvironmentModifications()
+        generate_mods.append_flags("CFLAGS", "-fprofile-generate={}".format(reports))
+        generate_mods.append_flags("CXXFLAGS", "-fprofile-generate={}".format(reports))
+        generate_mods.append_flags("LDFLAGS", "-fprofile-generate={} --verbose".format(reports))
+
+        with working_dir(self.build_directory, create=True):
+            cmake(*cmake_options, sources, extra_env=generate_mods)
+            make()
+            make("install")
+
+        # Clean the reports dir.
+        rmtree(reports, ignore_errors=True)
+
+        # Run spack solve --fresh hdf5 with instrumented clingo.
+        python_runtime_env = EnvironmentModifications()
+        for s in self.spec.traverse(deptype=("run", "link"), order="post"):
+            python_runtime_env.extend(spack.user_environment.environment_modifications_for_spec(s))
+        python_runtime_env.unset("SPACK_ENV")
+        python_runtime_env.unset("SPACK_PYTHON")
+        self.spec["python"].command(
+            spack.paths.spack_script, "solve", "--fresh", "hdf5", extra_env=python_runtime_env
+        )
+
+        # Clean the build dir.
+        rmtree(self.build_directory, ignore_errors=True)
+
+        if self.spec.compiler.name in ("clang", "apple-clang"):
+            # merge reports
+            use_report = join_path(reports, "merged.prof")
+            raw_files = glob.glob(join_path(reports, "*.profraw"))
+            llvm_profdata("merge", "--output={}".format(use_report), *raw_files)
+            use_flag = "-fprofile-instr-use={}".format(use_report)
         else:
-            msg = 'unexpected compiler for spec "{0}"'.format(self.spec)
-            raise RuntimeError(msg)
+            use_flag = "-fprofile-use={}".format(reports)
 
-        if opts:
-            env.set("CXXFLAGS", opts)
-            env.set("LDFLAGS", opts)
+        # Set PGO use flags for next cmake phase.
+        use_mods = EnvironmentModifications()
+        use_mods.append_flags("CFLAGS", use_flag)
+        use_mods.append_flags("CXXFLAGS", use_flag)
+        use_mods.append_flags("LDFLAGS", use_flag)
+        cmake.add_default_envmod(use_mods)
+
+    def setup_build_environment(self, env):
+        if "%apple-clang" in self.spec:
+            env.append_flags("CFLAGS", "-mmacosx-version-min=10.13")
+            env.append_flags("CXXFLAGS", "-mmacosx-version-min=10.13")
+            env.append_flags("LDFLAGS", "-mmacosx-version-min=10.13")
+        elif self.spec.compiler.name in ("gcc", "clang") and "+static_libstdcpp" in self.spec:
+            env.append_flags("LDFLAGS", "-static-libstdc++ -static-libgcc -Wl,--exclude-libs,ALL")
diff --git a/var/spack/repos/builtin/packages/clingo-bootstrap/version-script.patch b/var/spack/repos/builtin/packages/clingo-bootstrap/version-script.patch
new file mode 100644
index 00000000000000..df9d90cd65fc9a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/clingo-bootstrap/version-script.patch
@@ -0,0 +1,48 @@
+From 59859b8896e527bbd4a727beb798776d2716a8b3 Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels 
+Date: Thu, 10 Aug 2023 18:53:17 +0200
+Subject: [PATCH] version script
+
+---
+ libclingo/CMakeLists.txt | 12 ++++++++++++
+ libclingo/clingo.map     |  4 ++++
+ 2 files changed, 16 insertions(+)
+ create mode 100644 libclingo/clingo.map
+
+diff --git a/libclingo/CMakeLists.txt b/libclingo/CMakeLists.txt
+index 1d70ba56..0fd3bf49 100644
+--- a/libclingo/CMakeLists.txt
++++ b/libclingo/CMakeLists.txt
+@@ -58,6 +58,18 @@ target_include_directories(libclingo
+     "$")
+ target_compile_definitions(libclingo ${clingo_private_scope_} CLINGO_BUILD_LIBRARY)
+ 
++# Hide private symbols on Linux.
++include(CheckCSourceCompiles)
++file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/version.map" "{ global: f; local: *;};")
++set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})
++set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} "-Wl,--version-script='${CMAKE_CURRENT_BINARY_DIR}/version.map'")
++check_c_source_compiles("void f(void) {} int main(void) {return 0;}" HAVE_LD_VERSION_SCRIPT)
++set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})
++file(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/version.map")
++if(HAVE_LD_VERSION_SCRIPT)
++set_target_properties(libclingo PROPERTIES LINK_FLAGS "-Wl,--version-script='${CMAKE_CURRENT_SOURCE_DIR}/clingo.map'")
++endif()
++
+ if (NOT CLINGO_BUILD_SHARED)
+     target_compile_definitions(libclingo PUBLIC CLINGO_NO_VISIBILITY)
+ endif()
+diff --git a/libclingo/clingo.map b/libclingo/clingo.map
+new file mode 100644
+index 00000000..a665456c
+--- /dev/null
++++ b/libclingo/clingo.map
+@@ -0,0 +1,4 @@
++{
++    global: clingo_*; gringo_*; g_clingo_*;
++    local: *;
++};
+\ No newline at end of file
+-- 
+2.39.2
+
diff --git a/var/spack/repos/builtin/packages/clingo/package.py b/var/spack/repos/builtin/packages/clingo/package.py
index babf556cc6396e..f64dc6e8121dcd 100644
--- a/var/spack/repos/builtin/packages/clingo/package.py
+++ b/var/spack/repos/builtin/packages/clingo/package.py
@@ -42,6 +42,7 @@ class Clingo(CMakePackage):
     # See https://github.com/potassco/clingo/blob/v5.5.2/INSTALL.md
     depends_on("cmake@3.1:", type="build")
     depends_on("cmake@3.18:", type="build", when="@5.5:")
+    depends_on("py-setuptools", when="@5.6.2:", type="build")
 
     depends_on("doxygen", type="build", when="+docs")
 
@@ -68,6 +69,12 @@ class Clingo(CMakePackage):
     patch("size-t.patch", when="%msvc")
     patch("vs2022.patch", when="%msvc@19.30:")
 
+    # TODO: Simplify this after Spack 0.21 release. The old concretizer has problems with
+    # py-setuptools ^python@3.6, so we only apply the distutils -> setuptools patch for Python 3.12
+    with when("@:5.6.1 ^python@3.12:"):
+        patch("setuptools-2.patch")
+        depends_on("py-setuptools", type="build")
+
     def patch(self):
         # Doxygen is optional but can't be disabled with a -D, so patch
         # it out if it's really supposed to be disabled
@@ -120,6 +127,11 @@ def cmake_args(self):
         else:
             args += ["-DCLINGO_BUILD_WITH_PYTHON=OFF"]
 
+        # Use LTO also for non-Intel compilers please. This can be removed when they
+        # bump cmake_minimum_required to VERSION 3.9.
+        if "+ipo" in self.spec:
+            args.append("-DCMAKE_POLICY_DEFAULT_CMP0069=NEW")
+
         return args
 
     def win_add_library_dependent(self):
diff --git a/var/spack/repos/builtin/packages/clingo/setuptools-2.patch b/var/spack/repos/builtin/packages/clingo/setuptools-2.patch
new file mode 100644
index 00000000000000..6c6377936fb420
--- /dev/null
+++ b/var/spack/repos/builtin/packages/clingo/setuptools-2.patch
@@ -0,0 +1,8 @@
+diff --git a/cmake/python-site.py b/cmake/python-site.py
+--- a/cmake/python-site.py
++++ b/cmake/python-site.py
+@@ -1,3 +1,4 @@
++import setuptools  # makes import distutils work
+ from distutils.sysconfig import get_python_lib, get_config_vars
+ import sys
+ if sys.argv[1] == "prefix":
\ No newline at end of file
diff --git a/var/spack/repos/builtin/packages/cloverleaf-ref/package.py b/var/spack/repos/builtin/packages/cloverleaf-ref/package.py
new file mode 100644
index 00000000000000..70c0daaf9154cd
--- /dev/null
+++ b/var/spack/repos/builtin/packages/cloverleaf-ref/package.py
@@ -0,0 +1,114 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class CloverleafRef(MakefilePackage):
+    """Proxy Application. CloverLeaf_ref is a miniapp that solves the
+    compressible Euler equations on a Cartesian grid,
+    using an explicit, second-order accurate method.
+    """
+
+    homepage = "https://github.com/UK-MAC/CloverLeaf_ref"
+    url = "https://github.com/UK-MAC/CloverLeaf_ref/archive/refs/tags/v1.3.tar.gz"
+    git = "https://github.com/UK-MAC/CloverLeaf_ref.git"
+
+    maintainers("amd-toolchain-support")
+
+    version("master", branch="master")
+    version(
+        "1.3", sha256="fdff193286a00672bb931baa50d424a2cc19fb5817b62436804eced637e12430"
+    )  # commit "0ddf495"
+    version(
+        "1.1", sha256="0ac87accf81d85b959e5da839e6b0659afb3a2840a13f5da113a1c34eeb87942"
+    )  # commit "5667c3a"
+
+    variant(
+        "ieee", default=False, description="Build with IEEE754 compliant floating point operations"
+    )
+    variant("debug", default=False, description="Build with DEBUG flags")
+
+    depends_on("mpi")
+
+    # Cloverleaf_ref Makefile contains some but not all required options for each compiler.
+    # This package.py inserts what is needed for Intel, AOCC, and LLVM compilers.
+    @property
+    def build_targets(self):
+        targets = ["--directory=./"]
+
+        targets.append("MPI_COMPILER={0}".format(self.spec["mpi"].mpifc))
+        targets.append("C_MPI_COMPILER={0}".format(self.spec["mpi"].mpicc))
+
+        if "+debug" in self.spec:
+            targets.append("DEBUG=1")
+        if "+ieee" in self.spec:
+            targets.append("IEEE=1")
+
+        # Work around for bug in Makefiles for versions 1.3 and 1.1 (mis-defined as -openmp)
+        if self.spec.satisfies("%intel"):
+            targets.append("COMPILER=INTEL")
+            targets.append("OMP_INTEL=-qopenmp")
+
+        # Work around for missing AOCC compilers option in Makefiles for versions 1.3 and 1.1
+        elif self.spec.satisfies("%aocc"):
+            targets.append("COMPILER=AOCC")
+            targets.append("OMP_AOCC=-fopenmp")
+
+            if self.spec.satisfies("+ieee"):
+                targets.append("I3E_AOCC=-ffp-model=precise")
+                if self.spec.satisfies("%aocc@:4.0.0"):
+                    targets.append("I3E_AOCC+=-Kieee")
+
+            # logic for Debug build: no optimizatrion and debug symbols
+            if self.spec.satisfies("+debug"):
+                targets.append("FLAGS_AOCC=-O0 -g -Wall -Wextra -fsanitize=address")
+                targets.append("CFLAGS_AOCC=-O0 -g -Wall -Wextra -fsanitize=address")
+            else:
+                targets.append("CFLAGS_AOCC=-O3 -fnt-store=aggressive")
+                targets.append("FLAGS_AOCC=-O3 -fnt-store=aggressive")
+
+        # Work around for missing CLANG entries in Makefiles for master branch (commit:0fdb917),
+        # and for versions 1.3 and 1.1
+        elif self.spec.satisfies("%clang"):
+            targets.append("COMPILER=CLANG")
+            targets.append("OMP_CLANG=-fopenmp")
+
+            if self.spec.satisfies("+ieee"):
+                targets.append("I3E_CLANG=-ffp-model=precise")
+
+            # logic for Debug build: no optimizatrion and debug symbols
+            if self.spec.satisfies("+debug"):
+                targets.append("FLAGS_CLANG=-O0 -g")
+                targets.append("CFLAGS_CLANG=-O0 -g")
+            else:
+                targets.append("FLAGS_CLANG=-O3")
+                targets.append("CFLAGS_CLANG=-O3")
+
+        elif self.spec.satisfies("%gcc"):
+            targets.append("COMPILER=GNU")
+
+        elif self.spec.satisfies("%cce"):
+            targets.append("COMPILER=CRAY")
+            targets.append("OMP_CRAY=-fopenmp")
+
+        elif self.spec.satisfies("%pgi"):
+            targets.append("COMPILER=PGI")
+
+        elif self.spec.satisfies("%xl"):
+            targets.append("COMPILER=XLF")
+
+        else:
+            raise ValueError("Compiler {} not supported".format(self.spec.compiler.name))
+
+        return targets
+
+    def install(self, spec, prefix):
+        mkdirp(prefix.bin)
+        mkdirp(prefix.doc.tests)
+
+        install("./clover_leaf", prefix.bin)
+        install("./clover.in", prefix.bin)
+        install("./*.in", prefix.doc.tests)
diff --git a/var/spack/repos/builtin/packages/cloverleaf/package.py b/var/spack/repos/builtin/packages/cloverleaf/package.py
index 847f87b700d373..e30667b37a3af0 100644
--- a/var/spack/repos/builtin/packages/cloverleaf/package.py
+++ b/var/spack/repos/builtin/packages/cloverleaf/package.py
@@ -19,7 +19,7 @@ class Cloverleaf(MakefilePackage):
 
     tags = ["proxy-app"]
 
-    version("master", tag="master", submodules=True)
+    version("master", branch="master", submodules=True)
     version("1.1", sha256="de87f7ee6b917e6b3d243ccbbe620370c62df890e3ef7bdbab46569b57be132f")
 
     variant(
diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py
index 19c9854f963d38..16f3fc8d618ccd 100644
--- a/var/spack/repos/builtin/packages/cmake/package.py
+++ b/var/spack/repos/builtin/packages/cmake/package.py
@@ -27,6 +27,15 @@ class Cmake(Package):
     executables = ["^cmake[0-9]*$"]
 
     version("master", branch="master")
+    version("3.27.7", sha256="08f71a106036bf051f692760ef9558c0577c42ac39e96ba097e7662bd4158d8e")
+    version("3.27.6", sha256="ef3056df528569e0e8956f6cf38806879347ac6de6a4ff7e4105dc4578732cfb")
+    version("3.27.4", sha256="0a905ca8635ca81aa152e123bdde7e54cbe764fdd9a70d62af44cad8b92967af")
+    version("3.27.3", sha256="66afdc0f181461b70b6fedcde9ecc4226c5cd184e7203617c83b7d8e47f49521")
+    version("3.27.2", sha256="798e50085d423816fe96c9ef8bee5e50002c9eca09fed13e300de8a91d35c211")
+    version("3.27.1", sha256="b1a6b0135fa11b94476e90f5b32c4c8fad480bf91cf22d0ded98ce22c5132004")
+    version("3.27.0", sha256="aaeddb6b28b993d0a6e32c88123d728a17561336ab90e0bf45032383564d3cb8")
+    version("3.26.5", sha256="c0970b1e44a7fbca4322997ce05dac521b04748fe424922152faf22d20782bf9")
+    version("3.26.4", sha256="313b6880c291bd4fe31c0aa51d6e62659282a521e695f30d5cc0d25abbd5c208")
     version("3.26.3", sha256="bbd8d39217509d163cb544a40d6428ac666ddc83e22905d3e52c925781f0f659")
     version("3.26.2", sha256="d54f25707300064308ef01d4d21b0f98f508f52dda5d527d882b9d88379f89a8")
     version("3.26.1", sha256="f29964290ad3ced782a1e58ca9fda394a82406a647e24d6afd4e6c32e42c412f")
@@ -186,7 +195,7 @@ class Cmake(Package):
     # a build dependency, and its libs will not interfere with others in
     # the build.
     variant("ownlibs", default=True, description="Use CMake-provided third-party libraries")
-    variant("qt", default=False, description="Enables the build of cmake-gui")
+    variant("owncurl", default=False, description="Use CMake-provided curl library")
     variant(
         "doc",
         default=False,
@@ -217,11 +226,11 @@ class Cmake(Package):
     # provide Spack's TLS libs anyways, which is not flexible, and actually
     # leads to issues where we have to keep track of the vendored curl version
     # and its conflicts with OpenSSL.
-    depends_on("curl")
+    depends_on("curl", when="~owncurl")
 
     # When using curl, cmake defaults to using system zlib too, probably because
     # curl already depends on zlib. Therefore, also unconditionaly depend on zlib.
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     with when("~ownlibs"):
         depends_on("expat")
@@ -232,14 +241,19 @@ class Cmake(Package):
         depends_on("libuv@1.10.0:1.10", when="@3.11.0:3.11")
         depends_on("libuv@1.10.0:", when="@3.12.0:")
         depends_on("rhash", when="@3.8.0:")
+        depends_on("jsoncpp build_system=meson", when="@3.2:")
 
-    depends_on("qt", when="+qt")
     depends_on("ncurses", when="+ncurses")
 
     with when("+doc"):
         depends_on("python@2.7.11:", type="build")
         depends_on("py-sphinx", type="build")
 
+    for plat in ["darwin", "linux", "cray"]:
+        with when("~ownlibs platform=%s" % plat):
+            depends_on("openssl")
+            depends_on("openssl@:1.0", when="@:3.6.9")
+
     # TODO: update curl package to build with Windows SSL implementation
     # at which point we can build with +ownlibs on Windows
     conflicts("~ownlibs", when="platform=windows")
@@ -277,8 +291,6 @@ class Cmake(Package):
         when="@3.19.0:3.19",
     )
 
-    conflicts("+qt", when="^qt@5.4.0")  # qt-5.4.0 has broken CMake modules
-
     # https://gitlab.kitware.com/cmake/cmake/issues/18166
     conflicts("%intel", when="@3.11.0:3.11.4")
     conflicts("%intel@:14", when="@3.14:", msg="Intel 14 has immature C++11 support")
@@ -351,18 +363,14 @@ def bootstrap_args(self):
                 # Build and link to the Spack-installed third-party libraries
                 args.append("--system-libs")
 
-                if spec.satisfies("@3.2:"):
-                    # jsoncpp requires CMake to build
-                    # use CMake-provided library to avoid circular dependency
-                    args.append("--no-system-jsoncpp")
+                # cppdap is a CMake package, avoid circular dependency
+                if spec.satisfies("@3.27:"):
+                    args.append("--no-system-cppdap")
 
             # Whatever +/~ownlibs, use system curl.
-            args.append("--system-curl")
-
-            if "+qt" in spec:
-                args.append("--qt-gui")
-            else:
-                args.append("--no-qt-gui")
+            if "~owncurl" in spec:
+                args.append("--system-curl")
+            args.append("--no-qt-gui")
 
             if "+doc" in spec:
                 args.append("--sphinx-html")
@@ -394,18 +402,6 @@ def bootstrap_args(self):
             ]
         )
 
-        if self.spec.satisfies("^openssl~shared"):
-            args.append(
-                "-DOPENSSL_CRYPTO_LIBRARY={0};{1}".format(
-                    find_libraries("libcrypto", self.spec["openssl"].prefix.lib, shared=False),
-                    find_libraries(
-                        "libz",
-                        self.spec["zlib"].prefix.lib,
-                        shared=self.spec["zlib"].satisfies("+shared"),
-                    ),
-                )
-            )
-
         return args
 
     def cmake_bootstrap(self):
diff --git a/var/spack/repos/builtin/packages/cmdlime/package.py b/var/spack/repos/builtin/packages/cmdlime/package.py
new file mode 100644
index 00000000000000..3f1045c8cea4bf
--- /dev/null
+++ b/var/spack/repos/builtin/packages/cmdlime/package.py
@@ -0,0 +1,16 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Cmdlime(CMakePackage):
+    """cmdlime is a C++17 header-only library for command line parsing that
+    provides a concise, declarative interface without many details to remember."""
+
+    homepage = "https://github.com/kamchatka-volcano/cmdlime"
+    url = "https://github.com/kamchatka-volcano/cmdlime/archive/refs/tags/v2.5.0.tar.gz"
+
+    version("2.5.0", sha256="d5188d7f075142fcb546099a4ee2a967f8248109c0bee8c084e0e00f37603481")
diff --git a/var/spack/repos/builtin/packages/cnpy/package.py b/var/spack/repos/builtin/packages/cnpy/package.py
index 7727709a3a2814..fd01cdd3d94cdd 100644
--- a/var/spack/repos/builtin/packages/cnpy/package.py
+++ b/var/spack/repos/builtin/packages/cnpy/package.py
@@ -16,7 +16,7 @@ class Cnpy(CMakePackage):
 
     version("master", branch="master")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     def cmake_args(self):
         args = []
diff --git a/var/spack/repos/builtin/packages/cnvnator/package.py b/var/spack/repos/builtin/packages/cnvnator/package.py
index 69b284bf2994a6..01489a2d7539e4 100644
--- a/var/spack/repos/builtin/packages/cnvnator/package.py
+++ b/var/spack/repos/builtin/packages/cnvnator/package.py
@@ -21,7 +21,7 @@ class Cnvnator(MakefilePackage):
     depends_on("bzip2")
     depends_on("curl")
     depends_on("xz")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libdeflate")
     depends_on("openssl")
 
@@ -42,7 +42,7 @@ def edit(self, spec, prefix):
 
         # Link more libs
         ldflags = [
-            spec["zlib"].libs.ld_flags,
+            spec["zlib-api"].libs.ld_flags,
             spec["bzip2"].libs.ld_flags,
             spec["curl"].libs.ld_flags,
             spec["xz"].libs.ld_flags,
diff --git a/var/spack/repos/builtin/packages/coin3d/package.py b/var/spack/repos/builtin/packages/coin3d/package.py
index 2626248e342a5f..a7aaea76b98feb 100644
--- a/var/spack/repos/builtin/packages/coin3d/package.py
+++ b/var/spack/repos/builtin/packages/coin3d/package.py
@@ -59,6 +59,24 @@ def url_for_version(self, version):
             url = "https://github.com/coin3d/coin/archive/Coin-{0}.tar.gz"
         return url.format(version.dotted)
 
+    # 2 patches for fixing missing dlopen, ... on CentOS 8
+    patch(
+        "https://github.com/coin3d/coin/commit/962e613609ba93301999f83e283b7f489dfac503.patch?full_index=1",
+        sha256="33da720d6ff50916403320c47c72600737bdfb4113a4d527bc3fa9e8fe368f49",
+        when="@4.0.0",
+    )
+    patch(
+        "https://github.com/coin3d/coin/commit/9a36b15d66c5e340cd8695415ce5867ad07ee2bb.patch?full_index=1",
+        sha256="fcfe07c45e4981976ccf8df377f8cea24c48231b9e5d9538e5c7e4e038970597",
+        when="@4.0.0",
+    )
+    # fix missing X11 libraries on CentOS 8
+    patch(
+        "https://github.com/coin3d/coin/commit/58a1b4c3e968f96d3a2091fa5cb625f360ce6811.patch?full_index=1",
+        sha256="a4fe39d430068a7b107d4bca1fdb932e40cf92c3b5be0f846c70cbdc333e924d",
+        when="@4.0.0",
+    )
+
 
 class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder):
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/collier/package.py b/var/spack/repos/builtin/packages/collier/package.py
index c29704e354b0d2..98407d7c22076a 100644
--- a/var/spack/repos/builtin/packages/collier/package.py
+++ b/var/spack/repos/builtin/packages/collier/package.py
@@ -18,6 +18,7 @@ class Collier(CMakePackage):
 
     maintainers("vvolkl")
 
+    version("1.2.8", sha256="5cb24ce24ba1f62b7a96c655b31e9fddccc603eff31e60f9033b16354a6afd89")
     version("1.2.7", sha256="fde4b144a17c1bf5aa2ceaa86c71c79da10c9de8fec7bd33c8bffb4198acd5ca")
     version("1.2.6", sha256="b0d517868c71d2d1b8b6d3e0c370a43c9eb18ea8393a6e80070a5a2206f7de36")
     version("1.2.5", sha256="3ec58a975ff0c3b1ca870bc38973476c923ff78fd3dd5850e296037852b94a8b")
diff --git a/var/spack/repos/builtin/packages/comgr/package.py b/var/spack/repos/builtin/packages/comgr/package.py
index 597655e3ab738c..a17bcc7e9426a6 100644
--- a/var/spack/repos/builtin/packages/comgr/package.py
+++ b/var/spack/repos/builtin/packages/comgr/package.py
@@ -14,14 +14,17 @@ class Comgr(CMakePackage):
 
     homepage = "https://github.com/RadeonOpenCompute/ROCm-CompilerSupport"
     git = "https://github.com/RadeonOpenCompute/ROCm-CompilerSupport.git"
-    url = "https://github.com/RadeonOpenCompute/ROCm-CompilerSupport/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/ROCm-CompilerSupport/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath", "haampie")
     libraries = ["libamd_comgr"]
 
     version("master", branch="amd-stg-open")
-
+    version("5.6.1", sha256="0a85d84619f98be26ca7a32c71f94ed3c4e9866133789eabb451be64ce739300")
+    version("5.6.0", sha256="9396a7238b547ee68146c669b10b9d5de8f1d76527c649133c75d8076a185a72")
+    version("5.5.1", sha256="0fbb15fe5a95c2e141ccd360bc413e1feda283334781540a6e5095ab27fd8019")
+    version("5.5.0", sha256="97dfff03226ce0902b9d5d1c8c7bebb7a15978a81b6e9c750bf2d2473890bd42")
     version("5.4.3", sha256="8af18035550977fe0aa9cca8dfacbe65fe292e971de5a0e160710bafda05a81f")
     version("5.4.0", sha256="f4b83b27ff6195679d695c3f41fa25456e9c50bae6d978f46d3541b472aef757")
     version("5.3.3", sha256="6a4ef69e672a077b5909977248445f0eedf5e124af9812993a4d444be030c78b")
@@ -102,13 +105,6 @@ class Comgr(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     # Disable the hip compile tests.  Spack should not be using
     # /opt/rocm, and this breaks the build when /opt/rocm exists.
     patch("hip-tests.patch", when="@:4.2.0")
@@ -116,7 +112,7 @@ class Comgr(CMakePackage):
     depends_on("cmake@3.2.0:", type="build", when="@:3.8")
     depends_on("cmake@3.13.4:", type="build", when="@3.9.0:")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("z3", type="link")
     depends_on("ncurses", type="link")
 
@@ -146,6 +142,10 @@ class Comgr(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
     ]:
         # llvm libs are linked statically, so this *could* be a build dep
@@ -157,10 +157,15 @@ class Comgr(CMakePackage):
             "rocm-device-libs@" + ver, when="@{0} ^llvm-amdgpu ~rocm-device-libs".format(ver)
         )
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     root_cmakelists_dir = join_path("lib", "comgr")
 
     def cmake_args(self):
         args = [self.define("BUILD_TESTING", self.run_tests)]
+        if self.spec.satisfies("@5.4.3:"):
+            args.append("-DCMAKE_INSTALL_LIBDIR=lib")
         return args
 
     @classmethod
diff --git a/var/spack/repos/builtin/packages/composable-kernel/package.py b/var/spack/repos/builtin/packages/composable-kernel/package.py
new file mode 100644
index 00000000000000..85b383896a8af9
--- /dev/null
+++ b/var/spack/repos/builtin/packages/composable-kernel/package.py
@@ -0,0 +1,77 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class ComposableKernel(CMakePackage):
+    """Composable Kernel: Performance Portable Programming Model
+    for Machine Learning Tensor Operators."""
+
+    homepage = "https://github.com/ROCmSoftwarePlatform/composable_kernel"
+    git = "https://github.com/ROCmSoftwarePlatform/composable_kernel.git"
+
+    maintainers("srekolam", "afzpatel")
+
+    version("master", branch="develop")
+    version("5.6.1", commit="f5ec04f091fa5c48c67d7bacec36a414d0be06a5")
+    version("5.6.0", commit="f0fd02634c2f8f8c70f5a0ab2a8c84db5e36eeca")
+    version("5.5.1", commit="ac9e01e2cc3721be24619807adc444e1f59a9d25")
+    version("5.5.0", commit="8b76b832420a3d69708401de6607a033163edcce")
+    version("5.4.3", commit="bb3d9546f186e39cefedc3e7f01d88924ba20168")
+    version("5.4.0", commit="236bd148b98c7f1ec61ee850fcc0c5d433576305")
+
+    amdgpu_targets = ROCmPackage.amdgpu_targets
+    variant(
+        "amdgpu_target",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
+        description="set gpu targets",
+    )
+
+    depends_on("python", type="build")
+    depends_on("z3", type="build")
+    depends_on("zlib", type="build")
+    depends_on("ncurses+termlib", type="build")
+    depends_on("bzip2", type="build")
+    depends_on("sqlite", type="build")
+    depends_on("half", type="build")
+    depends_on("pkgconfig", type="build")
+    depends_on("cmake@3.16:", type="build")
+
+    for ver in ["master", "5.6.1", "5.6.0", "5.5.1", "5.5.0", "5.4.3", "5.4.0"]:
+        depends_on("hip@" + ver, when="@" + ver)
+        depends_on("llvm-amdgpu@" + ver, when="@" + ver)
+        depends_on("rocm-cmake@" + ver, when="@" + ver, type="build")
+
+    def setup_build_environment(self, env):
+        env.set("CXX", self.spec["hip"].hipcc)
+
+    def cmake_args(self):
+        spec = self.spec
+        args = [
+            self.define(
+                "CMAKE_CXX_COMPILER", "{0}/bin/clang++".format(spec["llvm-amdgpu"].prefix)
+            ),
+            self.define("CMAKE_C_COMPILER", "{0}/bin/clang".format(spec["llvm-amdgpu"].prefix)),
+            self.define("HIP_PATH", spec["hip"].prefix),
+            self.define("HIP_ROOT_DIR", "{0}".format(spec["hip"].prefix)),
+            self.define("CMAKE_CXX_FLAGS", "-O3"),
+            self.define("CMAKE_BUILD_TYPE", "Release"),
+        ]
+        if "auto" not in self.spec.variants["amdgpu_target"]:
+            args.append(self.define_from_variant("AMDGPU_TARGETS", "amdgpu_target"))
+        if self.spec.satisfies("@5.6.1:"):
+            args.append(self.define("INSTANCES_ONLY", "ON"))
+        return args
+
+    def build(self, spec, prefix):
+        with working_dir(self.build_directory):
+            # only instances is necessary to build and install
+            if self.spec.satisfies("@5.6.1:"):
+                make()
+            else:
+                make("instances")
diff --git a/var/spack/repos/builtin/packages/conduit/package.py b/var/spack/repos/builtin/packages/conduit/package.py
index cb6b5aefe16ed7..2762c5781d3db7 100644
--- a/var/spack/repos/builtin/packages/conduit/package.py
+++ b/var/spack/repos/builtin/packages/conduit/package.py
@@ -184,6 +184,8 @@ class Conduit(CMakePackage):
     depends_on("py-sphinx-rtd-theme", when="+python+doc", type="build")
     depends_on("doxygen", when="+doc+doxygen")
 
+    conflicts("+parmetis", when="~mpi", msg="Parmetis support requires MPI")
+
     # Tentative patch for fj compiler
     # Cmake will support fj compiler and this patch will be removed
     patch("fj_flags.patch", when="%fj")
@@ -530,9 +532,9 @@ def hostconfig(self):
 
         if "+hdf5" in spec:
             cfg.write(cmake_cache_entry("HDF5_DIR", spec["hdf5"].prefix))
-            if "zlib" in spec:
+            if "zlib-api" in spec:
                 # HDF5 depends on zlib
-                cfg.write(cmake_cache_entry("ZLIB_DIR", spec["zlib"].prefix))
+                cfg.write(cmake_cache_entry("ZLIB_DIR", spec["zlib-api"].prefix))
         else:
             cfg.write("# hdf5 not built by spack \n")
 
diff --git a/var/spack/repos/builtin/packages/connect-proxy/package.py b/var/spack/repos/builtin/packages/connect-proxy/package.py
index 2ddcffb8b22a9d..e1fbb1391b78e5 100644
--- a/var/spack/repos/builtin/packages/connect-proxy/package.py
+++ b/var/spack/repos/builtin/packages/connect-proxy/package.py
@@ -17,6 +17,9 @@ class ConnectProxy(MakefilePackage):
 
     version("1.105", sha256="07366026b1f81044ecd8da9b5b5b51321327ecdf6ba23576271a311bbd69d403")
 
+    def build(self, spec, prefix):
+        make("CC={0}".format(spack_cc))
+
     def install(self, spec, prefix):
         mkdir(prefix.bin)
         install("connect", prefix.bin)
diff --git a/var/spack/repos/builtin/packages/conquest/package.py b/var/spack/repos/builtin/packages/conquest/package.py
new file mode 100644
index 00000000000000..29e9fa5777bc3f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/conquest/package.py
@@ -0,0 +1,83 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Conquest(MakefilePackage):
+    """CONQUEST is a DFT code designed for large-scale calculations,
+    with excellent parallelisation."""
+
+    homepage = "http://www.order-n.org/"
+    url = "https://github.com/OrderN/CONQUEST-release/releases/download/v1.2/CONQUEST-release-1.2.tar.gz"
+    git = "https://github.com/OrderN/CONQUEST-release/"
+
+    # notify when the package is updated.
+    maintainers("davidbowler", "tkoskela", "ilectra")
+
+    version("1.2", sha256="74d974f20ec15ff31d97cd42aae6dbe95288eedfa785896d5872b9ff44ee7ae2")
+    version("1.1", sha256="772e058f073cccfee45521aa62bb13192ab07cb2979b6076ddbf21ba22f9ba5d")
+    version("master", branch="master")
+    version("develop", branch="develop")
+
+    depends_on("blas")
+    depends_on("lapack")
+    depends_on("scalapack")
+    depends_on("fftw-api")
+    depends_on("libxc@:5")
+    depends_on("mpi")
+
+    variant("openmp", default=False, description="Build with OpenMP support")
+    variant(
+        "mult_kern",
+        default="default",
+        values=[
+            "default",
+            "gemm",
+            "ompDoii",
+            "ompDoik",
+            "ompDoji",
+            "ompDojk",
+            "ompGemm",
+            "ompGemm_m",
+            "ompTsk",
+        ],
+        description="Matrix multiplication kernel type",
+    )
+
+    build_directory = "src"
+
+    def edit(self, spec, prefix):
+        fflags = "-O3 -fallow-argument-mismatch"
+        ldflags = ""
+
+        if "+openmp" in self.spec:
+            fflags += " " + self.compiler.openmp_flag
+            ldflags += " " + self.compiler.openmp_flag
+
+        libxc = self.spec["libxc:fortran"]
+        fflags += " " + libxc.headers.include_flags
+        ldflags += " " + self.spec["scalapack"].libs.ld_flags
+
+        lapack_ld = self.spec["lapack"].libs.ld_flags
+        blas_ld = self.spec["blas"].libs.ld_flags
+
+        defs_file = FileFilter("./src/system.make")
+
+        defs_file.filter("COMPFLAGS=.*", f"COMPFLAGS= {fflags}")
+        defs_file.filter("LINKFLAGS=.*", f"LINKFLAGS= {ldflags}")
+        defs_file.filter("# BLAS=.*", f"BLAS= {lapack_ld} -llapack {blas_ld} -lblas")
+
+        if "+openmp" in self.spec:
+            defs_file.filter("OMP_DUMMY = DUMMY", "OMP_DUMMY = ")
+
+        if self.spec.variants["mult_kern"].value != "default":
+            defs_file.filter(
+                "MULT_KERN =.*", f"MULT_KERN = {self.spec.variants['mult_kern'].value}"
+            )
+
+    def install(self, spec, prefix):
+        mkdirp(prefix.bin)
+        install("./bin/Conquest", prefix.bin)
diff --git a/var/spack/repos/builtin/packages/cool/package.py b/var/spack/repos/builtin/packages/cool/package.py
index 91772198e828e9..9418a16cbeb5d4 100644
--- a/var/spack/repos/builtin/packages/cool/package.py
+++ b/var/spack/repos/builtin/packages/cool/package.py
@@ -16,11 +16,11 @@ class Cool(CMakePackage):
     tags = ["hep"]
     maintainers("iarspider")
 
-    version("3.3.10", tag="COOL_3_3_10")
-    version("3.3.7", tag="COOL_3_3_7")
-    version("3.3.5", tag="COOL_3_3_5")
-    version("3.3.4", tag="COOL_3_3_4")
-    version("3.3.3", tag="COOL_3_3_3")
+    version("3.3.10", tag="COOL_3_3_10", commit="110b51c2b50af07cbe1f64a1c67ce9f737c4421d")
+    version("3.3.7", tag="COOL_3_3_7", commit="6f9a29d903e51ecbb26bdc8a694a67db9f28e234")
+    version("3.3.5", tag="COOL_3_3_5", commit="9af359de6a14350b9ab4cab572c638df73edfe84")
+    version("3.3.4", tag="COOL_3_3_4", commit="c3f9f780e0949fc78277c05d21d06fd7ddc6ea48")
+    version("3.3.3", tag="COOL_3_3_3", commit="42137f0ecd5028c41a46a99f0b95b56e105ef4e3")
 
     # Spack-specific patches:
     # * Create python/PyCool/_internal directory
diff --git a/var/spack/repos/builtin/packages/coral/package.py b/var/spack/repos/builtin/packages/coral/package.py
index 5708e9d598e974..8d1db303b06cb3 100644
--- a/var/spack/repos/builtin/packages/coral/package.py
+++ b/var/spack/repos/builtin/packages/coral/package.py
@@ -16,9 +16,9 @@ class Coral(CMakePackage):
 
     tags = ["hep"]
 
-    version("3.3.10", tag="CORAL_3_3_10")
-    version("3.3.3", tag="CORAL_3_3_3")
-    variant("binary_tag", default="auto")
+    version("3.3.10", tag="CORAL_3_3_10", commit="d79c4d94f74e8eaf518841e70c70c1d9b2f816fe")
+    version("3.3.3", tag="CORAL_3_3_3", commit="1c0393b2aa8d03748208ce564070d96f1bbd0c29")
+    variant("binary_tag", default="auto", description="Build type")
 
     depends_on("ninja")
     depends_on("ccache")
diff --git a/var/spack/repos/builtin/packages/cosma/package.py b/var/spack/repos/builtin/packages/cosma/package.py
index 2fccafe1872b0e..19db9a0531859c 100644
--- a/var/spack/repos/builtin/packages/cosma/package.py
+++ b/var/spack/repos/builtin/packages/cosma/package.py
@@ -48,6 +48,9 @@ class Cosma(CMakePackage):
     with when("+cuda"):
         variant("nccl", default=False, description="Use cuda nccl")
 
+    with when("+rocm"):
+        variant("rccl", default=False, description="Use rocm rccl")
+
     depends_on("cmake@3.22:", type="build")
     depends_on("mpi@3:")
     depends_on("blas", when="~cuda ~rocm")
@@ -114,6 +117,7 @@ def cmake_args(self):
             self.define_from_variant("COSMA_WITH_TESTS", "tests"),
             self.define_from_variant("COSMA_WITH_APPS", "apps"),
             self.define_from_variant("COSMA_WITH_NCCL", "nccl"),
+            self.define_from_variant("COSMA_WITH_RCCL", "rccl"),
             self.define_from_variant("COSMA_WITH_GPU_AWARE_MPI", "gpu_direct"),
             self.define_from_variant("COSMA_WITH_PROFILING", "profiling"),
             self.define("COSMA_WITH_BENCHMARKS", False),
diff --git a/var/spack/repos/builtin/packages/cp2k/backport_avoid_null_2022.x.patch b/var/spack/repos/builtin/packages/cp2k/backport_avoid_null_2022.x.patch
new file mode 100644
index 00000000000000..ecd3db979ca10f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/cp2k/backport_avoid_null_2022.x.patch
@@ -0,0 +1,21 @@
+diff -Naru a/src/qs_kpp1_env_methods.F b/src/qs_kpp1_env_methods.F
+--- a/src/qs_kpp1_env_methods.F	2022-10-03 01:14:25.720416300 +0530
++++ b/src/qs_kpp1_env_methods.F	2023-06-14 02:33:05.205287205 +0530
+@@ -214,7 +214,6 @@
+                                                             output_unit
+       LOGICAL                                            :: gapw, gapw_xc, lsd, my_calc_forces
+       REAL(KIND=dp)                                      :: alpha, energy_hartree, energy_hartree_1c
+-      REAL(KIND=dp), DIMENSION(:, :, :, :), POINTER      :: vxg
+       TYPE(atomic_kind_type), DIMENSION(:), POINTER      :: atomic_kind_set
+       TYPE(cp_logger_type), POINTER                      :: logger
+       TYPE(cp_para_env_type), POINTER                    :: para_env
+@@ -373,7 +372,8 @@
+ 
+       CALL xc_calc_2nd_deriv(v_xc, v_xc_tau, p_env%kpp1_env%deriv_set, p_env%kpp1_env%rho_set, &
+                              rho1_r_pw, rho1_g_pw, tau1_r_pw, auxbas_pw_pool, xc_section, .FALSE., &
+-                             NULL(vxg), lsd_singlets, do_excitations, do_triplet, do_tddft, &
++                             lsd_singlets=lsd_singlets, do_excitations=do_excitations, &
++                             do_triplet=do_triplet, do_tddft=do_tddft, &
+                              compute_virial=calc_virial, virial_xc=virial)
+ 
+       DO ispin = 1, nspins
diff --git a/var/spack/repos/builtin/packages/cp2k/backport_avoid_null_9.1.patch b/var/spack/repos/builtin/packages/cp2k/backport_avoid_null_9.1.patch
new file mode 100644
index 00000000000000..7f580a043c41ff
--- /dev/null
+++ b/var/spack/repos/builtin/packages/cp2k/backport_avoid_null_9.1.patch
@@ -0,0 +1,21 @@
+diff -Naru a/src/qs_kpp1_env_methods.F b/src/qs_kpp1_env_methods.F
+--- a/src/qs_kpp1_env_methods.F	2021-11-20 14:35:36.103103400 +0530
++++ b/src/qs_kpp1_env_methods.F	2023-06-14 12:00:52.350584708 +0530
+@@ -220,7 +220,6 @@
+                                                             output_unit
+       LOGICAL                                            :: gapw, gapw_xc, lsd, my_calc_forces
+       REAL(KIND=dp)                                      :: alpha, energy_hartree, energy_hartree_1c
+-      REAL(KIND=dp), DIMENSION(:, :, :, :), POINTER      :: vxg
+       TYPE(atomic_kind_type), DIMENSION(:), POINTER      :: atomic_kind_set
+       TYPE(cp_logger_type), POINTER                      :: logger
+       TYPE(cp_para_env_type), POINTER                    :: para_env
+@@ -361,7 +360,8 @@
+ 
+       CALL xc_calc_2nd_deriv(v_xc, p_env%kpp1_env%deriv_set, p_env%kpp1_env%rho_set, &
+                              rho1_r_pw, rho1_g_pw, auxbas_pw_pool, xc_section, .FALSE., &
+-                             NULL(vxg), lsd_singlets, do_excitations, do_triplet, do_tddft, &
++                             lsd_singlets=lsd_singlets, do_excitations=do_excitations, &
++                             do_triplet=do_triplet, do_tddft=do_tddft, &
+                              compute_virial=calc_virial, virial_xc=virial)
+ 
+       DO ispin = 1, nspins
diff --git a/var/spack/repos/builtin/packages/cp2k/cmake-fixes-2023.2.patch b/var/spack/repos/builtin/packages/cp2k/cmake-fixes-2023.2.patch
new file mode 100644
index 00000000000000..985edad3aa5a1c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/cp2k/cmake-fixes-2023.2.patch
@@ -0,0 +1,714 @@
+From b75eb217115820059aba26d1ff1a8657e3841e7d Mon Sep 17 00:00:00 2001
+From: Mathieu Taillefumier 
+Date: Mon, 23 Oct 2023 15:50:44 +0200
+Subject: [PATCH] cmake-fixes-2023.2
+
+---
+ CMakeLists.txt            |  63 +++++++-----
+ cmake/FindBlas.cmake      | 174 +++++++++++++++++-----------------
+ cmake/FindLapack.cmake    |  47 ++++-----
+ cmake/cp2k.pc.in          |  19 ----
+ cmake/cp2kConfig.cmake.in | 195 ++++++++++++++++++++------------------
+ cmake/libcp2k.pc.in       |  11 +++
+ src/CMakeLists.txt        |  18 ++--
+ 7 files changed, 276 insertions(+), 251 deletions(-)
+ delete mode 100644 cmake/cp2k.pc.in
+ create mode 100644 cmake/libcp2k.pc.in
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 3f81c7b52..f2d85d033 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -49,7 +49,8 @@ if(NOT DEFINED CMAKE_CUDA_STANDARD)
+ endif()
+ 
+ # set language and standard
+-set(CMAKE_CXX_STANDARD 11)
++set(CMAKE_CXX_STANDARD 14)
++set(CMAKE_C_STANDARD 11)
+ 
+ find_package(PkgConfig)
+ 
+@@ -108,6 +109,10 @@ option(CP2K_USE_LIBXSMM "Use libxsmm for small gemms (supports x86 platforms)"
+        OFF)
+ option(CP2K_BUILD_DBCSR "Duild dbcsr at the same time than cp2k." OFF)
+ option(BUILD_SHARED_LIBS "Build cp2k shared library" ON)
++option(
++  CP2K_USE_FFTW3_WITH_MKL
++  "If set to ON use the original implementation of fftw3 instead of the MKL implementation."
++  OFF)
+ 
+ cmake_dependent_option(CP2K_ENABLE_ELPA_OPENMP_SUPPORT
+                        "Enable elpa openmp support" ON "CP2K_USE_ELPA" OFF)
+@@ -115,8 +120,8 @@ cmake_dependent_option(CP2K_ENABLE_FFTW3_OPENMP_SUPPORT
+                        "Enable FFTW openmp support" ON "CP2K_USE_FFTW3" OFF)
+ cmake_dependent_option(CP2K_ENABLE_FFTW3_THREADS_SUPPORT
+                        "Enable FFTW THREADS support" OFF "CP2K_USE_FFTW3" OFF)
+-cmake_dependent_option(CP2K_ENABLE_F08_MPI "Enable MPI Fortran 2008 interface"
+-                       OFF "CP2K_USE_MPI" OFF)
++cmake_dependent_option(CP2K_USE_MPI_F08 "Enable MPI Fortran 2008 interface" OFF
++                       "CP2K_USE_MPI" OFF)
+ 
+ cmake_dependent_option(
+   DBCSR_USE_ACCEL
+@@ -527,7 +532,7 @@ if(CP2K_USE_ACCEL MATCHES "CUDA")
+   endif()
+ 
+   set(CP2K_USE_CUDA ON)
+-  message(STATUS ``"-- CUDA compiler and libraries found")
++  message(STATUS "-- CUDA compiler and libraries found")
+ elseif(CP2K_USE_ACCEL MATCHES "HIP")
+   enable_language(HIP)
+   # Find hip
+@@ -620,27 +625,36 @@ endif()
+ 
+ # FFTW3
+ 
++set(CP2K_USE_FFTW3_ OFF)
+ if(CP2K_USE_FFTW3)
+-  find_package(Fftw REQUIRED)
+-  if(CP2K_ENABLE_FFTW3_THREADS_SUPPORT AND CP2K_ENABLE_FFTW3_OPENMP_SUPPORT)
+-    message(
+-      FATAL_ERROR
+-        "Fftw3 threads and openmp supports can not be used at the same time")
+-  endif()
++  if(CP2K_USE_FFTW3_WITH_MKL OR NOT CP2K_BLAS_VENDOR MATCHES "MKL")
++    find_package(Fftw REQUIRED)
++    if(CP2K_ENABLE_FFTW3_THREADS_SUPPORT AND CP2K_ENABLE_FFTW3_OPENMP_SUPPORT)
++      message(
++        FATAL_ERROR
++          "Fftw3 threads and openmp supports can not be used at the same time")
++    endif()
+ 
+-  if((CP2K_ENABLE_FFTW3_THREADS_SUPPORT) AND (NOT TARGET
+-                                              CP2K::FFTW3::fftw3_threads))
+-    message(
+-      FATAL_ERROR
+-        "fftw3 was compiled without multithreading support (--enable-threads option in fftw build system)."
+-    )
+-  endif()
++    if((CP2K_ENABLE_FFTW3_THREADS_SUPPORT) AND (NOT TARGET
++                                                CP2K::FFTW3::fftw3_threads))
++      message(
++        FATAL_ERROR
++          "fftw3 was compiled without multithreading support (--enable-threads option in fftw build system)."
++      )
++    endif()
+ 
+-  if((CP2K_ENABLE_FFTW3_OPENMP_SUPPORT) AND (NOT TARGET CP2K::FFTW3::fftw3_omp))
+-    message(
+-      FATAL_ERROR
+-        "fftw3 was compiled without openmp support  (--enable-openmp option in fftw build system)."
+-    )
++    if((CP2K_ENABLE_FFTW3_OPENMP_SUPPORT) AND (NOT TARGET CP2K::FFTW3::fftw3_omp
++                                              ))
++      message(
++        FATAL_ERROR
++          "fftw3 was compiled without openmp support  (--enable-openmp option in fftw build system)."
++      )
++    endif()
++    # we use this variable later on to include the fftw target whenever mkl is
++    # found or not
++    set(CP2K_USE_FFTW3_ ON)
++  else()
++    message("-- Using the MKL implementation of FFTW3.")
+   endif()
+ endif()
+ 
+@@ -748,7 +762,7 @@ add_subdirectory(src)
+ include(GNUInstallDirs)
+ 
+ get_target_property(CP2K_LIBS cp2k_link_libs INTERFACE_LINK_LIBRARIES)
+-configure_file(cmake/cp2k.pc.in cp2k.pc @ONLY)
++configure_file(cmake/libcp2k.pc.in libcp2k.pc @ONLY)
+ 
+ message(
+   STATUS "--------------------------------------------------------------------")
+@@ -1039,6 +1053,9 @@ install(FILES "${PROJECT_BINARY_DIR}/cp2kConfig.cmake"
+               "${PROJECT_BINARY_DIR}/cp2kConfigVersion.cmake"
+         DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/cp2k")
+ 
++install(FILES "${PROJECT_BINARY_DIR}/libcp2k.pc"
++        DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
++
+ install(
+   DIRECTORY "${PROJECT_SOURCE_DIR}/cmake"
+   DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/cp2k"
+diff --git a/cmake/FindBlas.cmake b/cmake/FindBlas.cmake
+index 6e5fb7824..335cbd964 100644
+--- a/cmake/FindBlas.cmake
++++ b/cmake/FindBlas.cmake
+@@ -15,104 +15,108 @@ if(NOT
+     OR CMAKE_Fortran_COMPILER_LOADED))
+   message(FATAL_ERROR "FindBLAS requires Fortran, C, or C++ to be enabled.")
+ endif()
++if(NOT CP2K_CONFIG_PACKAGE)
++  set(CP2K_BLAS_VENDOR_LIST
++      "auto"
++      "MKL"
++      "OpenBLAS"
++      "SCI"
++      "GenericBLAS"
++      "Armpl"
++      "FlexiBLAS"
++      "Atlas"
++      "NVHPCBlas"
++      "CUSTOM")
++
++  set(__BLAS_VENDOR_LIST ${CP2K_BLAS_VENDOR_LIST})
++  list(REMOVE_ITEM __BLAS_VENDOR_LIST "auto")
++  list(REMOVE_ITEM __BLAS_VENDOR_LIST "CUSTOM")
++
++  # set(CP2K_BLAS_VENDOR "auto" CACHE STRING "Blas library for computations on
++  # host")
++  set_property(CACHE CP2K_BLAS_VENDOR PROPERTY STRINGS ${CP2K_BLAS_VENDOR_LIST})
++
++  if(NOT ${CP2K_BLAS_VENDOR} IN_LIST CP2K_BLAS_VENDOR_LIST)
++    message(FATAL_ERROR "Invalid Host BLAS backend")
++  endif()
+ 
+-set(CP2K_BLAS_VENDOR_LIST
+-    "auto"
+-    "MKL"
+-    "OpenBLAS"
+-    "SCI"
+-    "GenericBLAS"
+-    "Armpl"
+-    "FlexiBLAS"
+-    "Atlas"
+-    "NVHPCBlas"
+-    "CUSTOM")
+-
+-set(__BLAS_VENDOR_LIST ${CP2K_BLAS_VENDOR_LIST})
+-list(REMOVE_ITEM __BLAS_VENDOR_LIST "auto")
+-list(REMOVE_ITEM __BLAS_VENDOR_LIST "CUSTOM")
+-
+-# set(CP2K_BLAS_VENDOR "auto" CACHE STRING "Blas library for computations on
+-# host")
+-set_property(CACHE CP2K_BLAS_VENDOR PROPERTY STRINGS ${CP2K_BLAS_VENDOR_LIST})
+-
+-if(NOT ${CP2K_BLAS_VENDOR} IN_LIST CP2K_BLAS_VENDOR_LIST)
+-  message(FATAL_ERROR "Invalid Host BLAS backend")
+-endif()
+-
+-set(CP2K_BLAS_THREAD_LIST "sequential" "thread" "gnu-thread" "intel-thread"
+-                          "tbb-thread" "openmp")
+-
+-set(CP2K_BLAS_THREADING
+-    "sequential"
+-    CACHE STRING "threaded blas library")
+-set_property(CACHE CP2K_BLAS_THREADING PROPERTY STRINGS
+-                                                ${CP2K_BLAS_THREAD_LIST})
+-
+-if(NOT ${CP2K_BLAS_THREADING} IN_LIST CP2K_BLAS_THREAD_LIST)
+-  message(FATAL_ERROR "Invalid threaded BLAS backend")
+-endif()
++  set(CP2K_BLAS_THREAD_LIST "sequential" "thread" "gnu-thread" "intel-thread"
++                            "tbb-thread" "openmp")
+ 
+-set(CP2K_BLAS_INTERFACE_BITS_LIST "32bits" "64bits")
+-set(CP2K_BLAS_INTERFACE
+-    "32bits"
+-    CACHE STRING
+-          "32 bits integers are used for indices, matrices and vectors sizes")
+-set_property(CACHE CP2K_BLAS_INTERFACE
+-             PROPERTY STRINGS ${CP2K_BLAS_INTERFACE_BITS_LIST})
+-
+-if(NOT ${CP2K_BLAS_INTERFACE} IN_LIST CP2K_BLAS_INTERFACE_BITS_LIST)
+-  message(
+-    FATAL_ERROR
+-      "Invalid parameters. Blas and lapack can exist in two flavors 32 or 64 bits interfaces (relevant mostly for mkl)"
+-  )
+-endif()
++  set(CP2K_BLAS_THREADING
++      "sequential"
++      CACHE STRING "threaded blas library")
++  set_property(CACHE CP2K_BLAS_THREADING PROPERTY STRINGS
++                                                  ${CP2K_BLAS_THREAD_LIST})
+ 
+-set(CP2K_BLAS_FOUND FALSE)
++  if(NOT ${CP2K_BLAS_THREADING} IN_LIST CP2K_BLAS_THREAD_LIST)
++    message(FATAL_ERROR "Invalid threaded BLAS backend")
++  endif()
+ 
+-# first check for a specific implementation if requested
++  set(CP2K_BLAS_INTERFACE_BITS_LIST "32bits" "64bits")
++  set(CP2K_BLAS_INTERFACE
++      "32bits"
++      CACHE STRING
++            "32 bits integers are used for indices, matrices and vectors sizes")
++  set_property(CACHE CP2K_BLAS_INTERFACE
++               PROPERTY STRINGS ${CP2K_BLAS_INTERFACE_BITS_LIST})
+ 
+-if(NOT CP2K_BLAS_VENDOR MATCHES "auto|CUSTOM")
+-  find_package(${CP2K_BLAS_VENDOR} REQUIRED)
+-  if(TARGET CP2K::BLAS::${CP2K_BLAS_VENDOR}::blas)
+-    get_target_property(
+-      CP2K_BLAS_INCLUDE_DIRS CP2K::BLAS::${CP2K_BLAS_VENDOR}::blas
+-      INTERFACE_INCLUDE_DIRECTORIES)
+-    get_target_property(
+-      CP2K_BLAS_LINK_LIBRARIES CP2K::BLAS::${CP2K_BLAS_VENDOR}::blas
+-      INTERFACE_LINK_LIBRARIES)
+-    set(CP2K_BLAS_FOUND TRUE)
+-  endif()
+-else()
+-  if(CP2K_BLAS_VENDOR MATCHES "CUSTOM" AND NOT DEFINED CP2K_BLAS_LINK_LIBRARIES)
++  if(NOT ${CP2K_BLAS_INTERFACE} IN_LIST CP2K_BLAS_INTERFACE_BITS_LIST)
+     message(
+       FATAL_ERROR
+-        "Setting CP2K_BLAS_VENDOR=CUSTOM imply setting CP2K_BLAS_LINK_LIBRARIES\n and CP2K_LAPACK_LINK_LIBRARIES to the right libraries. See the README_cmake.md for more details"
++        "Invalid parameters. Blas and lapack can exist in two flavors 32 or 64 bits interfaces (relevant mostly for mkl)"
+     )
+   endif()
+ 
+-  if(DEFINED CP2K_BLAS_LINK_LIBRARIES)
+-    set(CP2K_BLAS_FOUND TRUE)
++  set(CP2K_BLAS_FOUND FALSE)
++
++  # first check for a specific implementation if requested
++
++  if(NOT CP2K_BLAS_VENDOR MATCHES "auto|CUSTOM")
++    find_package(${CP2K_BLAS_VENDOR} REQUIRED)
++    if(TARGET CP2K::BLAS::${CP2K_BLAS_VENDOR}::blas)
++      get_target_property(
++        CP2K_BLAS_INCLUDE_DIRS CP2K::BLAS::${CP2K_BLAS_VENDOR}::blas
++        INTERFACE_INCLUDE_DIRECTORIES)
++      get_target_property(
++        CP2K_BLAS_LINK_LIBRARIES CP2K::BLAS::${CP2K_BLAS_VENDOR}::blas
++        INTERFACE_LINK_LIBRARIES)
++      set(CP2K_BLAS_FOUND TRUE)
++    endif()
+   else()
+-    # search for any blas implementation and exit immediately if one is found.
+-    # we could also give a full list of found implementation and let the user
+-    # choose which implementation to use
+-    foreach(_libs ${__BLAS_VENDOR_LIST})
+-      # I exclude the first item of the list
+-      find_package(${_libs})
+-      if(TARGET CP2K::BLAS::${_libs}::blas)
+-        get_target_property(CP2K_BLAS_INCLUDE_DIRS CP2K::BLAS::${_libs}::blas
+-                            INTERFACE_INCLUDE_DIRECTORIES)
+-        get_target_property(CP2K_BLAS_LINK_LIBRARIES CP2K::BLAS::${_libs}::blas
+-                            INTERFACE_LINK_LIBRARIES)
+-        set(CP2K_BLAS_VENDOR "${_libs}")
+-        set(CP2K_BLAS_FOUND TRUE)
+-        break()
+-      endif()
+-    endforeach()
++    if(CP2K_BLAS_VENDOR MATCHES "CUSTOM" AND NOT DEFINED
++                                             CP2K_BLAS_LINK_LIBRARIES)
++      message(
++        FATAL_ERROR
++          "Setting CP2K_BLAS_VENDOR=CUSTOM imply setting CP2K_BLAS_LINK_LIBRARIES\n and CP2K_LAPACK_LINK_LIBRARIES to the right libraries. See the README_cmake.md for more details"
++      )
++    endif()
++
++    if(DEFINED CP2K_BLAS_LINK_LIBRARIES)
++      set(CP2K_BLAS_FOUND TRUE)
++    else()
++      # search for any blas implementation and exit immediately if one is found.
++      # we could also give a full list of found implementation and let the user
++      # choose which implementation to use
++      foreach(_libs ${__BLAS_VENDOR_LIST})
++        # I exclude the first item of the list
++        find_package(${_libs})
++        if(TARGET CP2K::BLAS::${_libs}::blas)
++          get_target_property(CP2K_BLAS_INCLUDE_DIRS CP2K::BLAS::${_libs}::blas
++                              INTERFACE_INCLUDE_DIRECTORIES)
++          get_target_property(
++            CP2K_BLAS_LINK_LIBRARIES CP2K::BLAS::${_libs}::blas
++            INTERFACE_LINK_LIBRARIES)
++          set(CP2K_BLAS_VENDOR "${_libs}")
++          set(CP2K_BLAS_FOUND TRUE)
++          break()
++        endif()
++      endforeach()
++    endif()
+   endif()
++else()
++  set(CP2K_BLAS_FOUND ON)
+ endif()
+-
+ # we exclude the CP2K_BLAS_INCLUDE_DIRS from the list of mandatory variables as
+ # having the fortran interface is usually enough. C, C++ and others languages
+ # might require this information though
+diff --git a/cmake/FindLapack.cmake b/cmake/FindLapack.cmake
+index 966e0d78d..77a1e0425 100644
+--- a/cmake/FindLapack.cmake
++++ b/cmake/FindLapack.cmake
+@@ -20,33 +20,34 @@ include(FindPackageHandleStandardArgs)
+ find_package(PkgConfig)
+ find_package(Blas REQUIRED)
+ 
+-if(CP2K_BLAS_FOUND)
+-  # LAPACK in the Intel MKL 10+ library?
+-  if(CP2K_BLAS_VENDOR MATCHES "MKL|OpenBLAS|Armpl|SCI|FlexiBLAS|NVHPC")
+-    # we just need to create the interface that's all
+-    set(CP2K_LAPACK_FOUND TRUE)
+-    get_target_property(CP2K_LAPACK_INCLUDE_DIRS CP2K::BLAS::blas
+-                        INTERFACE_INCLUDE_DIRECTORIES)
+-    get_target_property(CP2K_LAPACK_LINK_LIBRARIES CP2K::BLAS::blas
+-                        INTERFACE_LINK_LIBRARIES)
+-  else()
+-    # we might get lucky to find a pkgconfig package for lapack (fedora provides
+-    # one for instance)
+-    if(PKG_CONFIG_FOUND)
+-      pkg_check_modules(CP2K_LAPACK lapack)
+-    endif()
++if(NOT CP2K_CONFIG_PACKAGE)
++  if(CP2K_BLAS_FOUND)
++    # LAPACK in the Intel MKL 10+ library?
++    if(CP2K_BLAS_VENDOR MATCHES "MKL|OpenBLAS|Armpl|SCI|FlexiBLAS|NVHPC")
++      # we just need to create the interface that's all
++      set(CP2K_LAPACK_FOUND TRUE)
++      get_target_property(CP2K_LAPACK_INCLUDE_DIRS CP2K::BLAS::blas
++                          INTERFACE_INCLUDE_DIRECTORIES)
++      get_target_property(CP2K_LAPACK_LINK_LIBRARIES CP2K::BLAS::blas
++                          INTERFACE_LINK_LIBRARIES)
++    else()
++      # we might get lucky to find a pkgconfig package for lapack (fedora
++      # provides one for instance)
++      if(PKG_CONFIG_FOUND)
++        pkg_check_modules(CP2K_LAPACK lapack)
++      endif()
+ 
+-    if(NOT CP2K_LAPACK_FOUND)
+-      find_library(
+-        CP2K_LAPACK_LINK_LIBRARIES
+-        NAMES "lapack" "lapack64"
+-        PATH_SUFFIXES "openblas" "openblas64" "openblas-pthread"
+-                      "openblas-openmp" "lib" "lib64"
+-        NO_DEFAULT_PATH)
++      if(NOT CP2K_LAPACK_FOUND)
++        find_library(
++          CP2K_LAPACK_LINK_LIBRARIES
++          NAMES "lapack" "lapack64"
++          PATH_SUFFIXES "openblas" "openblas64" "openblas-pthread"
++                        "openblas-openmp" "lib" "lib64"
++          NO_DEFAULT_PATH)
++      endif()
+     endif()
+   endif()
+ endif()
+-
+ # check if found
+ find_package_handle_standard_args(Lapack
+                                   REQUIRED_VARS CP2K_LAPACK_LINK_LIBRARIES)
+diff --git a/cmake/cp2k.pc.in b/cmake/cp2k.pc.in
+deleted file mode 100644
+index 5b4a09566..000000000
+--- a/cmake/cp2k.pc.in
++++ /dev/null
+@@ -1,19 +0,0 @@
+-# this template is filled-in by CMake `configure_file(... @ONLY)`
+-# the `@....@` are filled in by CMake configure_file(), 
+-# from variables set in your CMakeLists.txt or by CMake itself
+-#
+-# Good tutoral for understanding .pc files: 
+-# https://people.freedesktop.org/~dbn/pkg-config-guide.html
+-
+-prefix="@CMAKE_INSTALL_PREFIX@"
+-exec_prefix="${prefix}"
+-libdir="${prefix}/lib"
+-includedir="${prefix}/include"
+-
+-Name: @PROJECT_NAME@
+-Description: @CMAKE_PROJECT_DESCRIPTION@
+-URL: @CMAKE_PROJECT_HOMEPAGE_URL@
+-Version: @PROJECT_VERSION@
+-Cflags: -I"${includedir}"
+-Libs: -L"${libdir}" -lcp2k -lcp2k_dbm -lcp2k_grid -lcp2k_offload
+-#Libs.private: -L"${libdir}" @CP2K_LIBS@
+\ No newline at end of file
+diff --git a/cmake/cp2kConfig.cmake.in b/cmake/cp2kConfig.cmake.in
+index a3acd4744..1c310e19b 100644
+--- a/cmake/cp2kConfig.cmake.in
++++ b/cmake/cp2kConfig.cmake.in
+@@ -5,112 +5,121 @@
+ #!   SPDX-License-Identifier: GPL-2.0-or-later                                                     !
+ #!-------------------------------------------------------------------------------------------------!
+ 
+-
+ cmake_minimum_required(VERSION 3.22)
++include(CMakeFindDependencyMacro)
++
++if(NOT TARGET cp2k::cp2k)
++  set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules"
++                        ${CMAKE_MODULE_PATH})
++
++  # store CXX compiler id. Used in MKL package.
++  set(CP2K_CXX_COMPILER_ID @CMAKE_CXX_COMPILER_ID@)
++  if(NOT ${CMAKE_CXX_COMPILER_ID})
++    set(CMAKE_CXX_COMPILER_ID ${CP2K_CXX_COMPILER_ID})
++  endif()
++
++  set(CP2K_BLAS_VENDOR @CP2K_BLAS_VENDOR@)
++  set(CP2K_SCALAPACK_VENDOR @CP2K_SCALAPACK_VENDOR@)
++  set(CP2K_BLAS_LINK_LIBRARIES @CP2K_BLAS_LINK_LIBRARIES@)
++  set(CP2K_LAPACK_LINK_LIBRARIES @CP2K_LAPACK_LINK_LIBRARIES@)
++  set(CP2K_SCALAPACK_LINK_LIBRARIES @CP2K_SCALAPACK_LINK_LIBRARIES@)
++
++  set(CP2K_CONFIG_PACKAGE ON)
++  find_dependency(Lapack REQUIRED)
++
++  # define lapack and blas TARGETS
++
++  if(@CP2K_USE_MPI@)
++    find_dependency(SCALAPACK REQUIRED)
++  endif()
++  unset(CP2K_CONFIG_PACKAGE)
++
++  set(cp2k_VERSION @cp2k_VERSION@)
++
++  find_dependency(DBCSR 2.5 REQUIRED)
++
++  if(@CP2K_USE_LIBXSMM@)
++    find_dependency(LibXSMM REQUIRED)
++  endif()
++
++  if(@CP2K_USE_HIP@)
++    # Find hip
++    find_dependency(hipfft REQUIRED IMPORTED CONFIG)
++    find_dependency(hipblas REQUIRED IMPORTED CONFIG)
++  endif()
++
++  if(@CP2K_USE_CUDA@)
++    find_dependency(CUDAToolkit REQUIRED)
++  endif()
++  if(@CP2K_USE_ELPA@)
++    find_dependency(Elpa REQUIRED)
++  endif()
++
++  if(@CP2K_USE_LIBXC@)
++    find_dependency(LibXC 6 REQUIRED EXACT)
++  endif()
++
++  if(@CP2K_USE_COSMA@)
++    find_dependency(cosma REQUIRED)
++  endif()
++
++  if(@CP2K_USE_MPI@)
++    find_dependency(MPI REQUIRED)
++  endif()
++
++  if(@CP2K_USE_FFTW3@ OR @CP2K_USE_FFTW3_WITH_MKL@)
++    find_dependency(Fftw REQUIRED)
++  endif()
++  
++  # QUIP
++  if(@CP2K_USE_QUIP@)
++    find_dependency(Quip REQUIRED)
++  endif()
+ 
+-# store CXX compiler id. Used in MKL package.
+-set(SIRIUS_CXX_COMPILER_ID @CMAKE_CXX_COMPILER_ID@)
+-if(NOT ${CMAKE_CXX_COMPILER_ID})
+-  set(CMAKE_CXX_COMPILER_ID ${SIRIUS_CXX_COMPILER_ID})
+-endif()
+-
+-set(CP2K_BLAS_VENDOR @CP2K_BLAS_VENDOR@)
+-set(CP2K_SCALAPACK_VENDOR @CP2K_SCALAPACK_VENDOR@)
+-
+-if (@CP2K_BLAS_VENDOR@ MATCHES "CUSTOM")
+-   set(CP2K_BLAS_LINK_LIBRARIES @CP2K_BLAS_LINK_LIBRARIES@)
+-   set(CP2K_LAPACK_LINK_LIBRARIES @CP2K_LAPACK_LINK_LIBRARIES@)
+-endif()
+-
+-if (@CP2K_SCALAPACK_VENDOR@ MATCHES "CUSTOM")
+-   set(CP2K_SCALAPACK_LINK_LIBRARIES @CP2K_SCALAPACK_LINK_LIBRARIES@)
+-endif()
+-
+-find_package(Lapack REQUIRED)
+-find_package(DBCSR 2.4 REQUIRED)
+-
+-if(@CP2K_USE_LIBXSMM@
+-  find_package(LibXSMM REQUIRED)
+-endif()
+-
+-if (@@CP2K_USE_HIP@)
+-  # Find hip
+-  find_package(hipfft REQUIRED IMPORTED CONFIG)
+-  find_package(hipblas REQUIRED IMPORTED CONFIG)
+-endif()
+-
+-if (@@CP2K_USE_CUDA@)
+-  find_package(CUDAToolkit REQUIRED)
+-endif()
+-if(@CP2K_USE_ELPA@)
+-  find_package(Elpa REQUIRED)
+-endif()
+-
+-if(@CP2K_USE_LIBXC@)
+-  find_package(LibXC 6 REQUIRED EXACT)
+-endif()
+-
+-if(@CP2K_USE_COSMA@)
+-  find_package(cosma REQUIRED)
+-endif()
++  # libint
+ 
+-if (@@CP2K_USE_MPI@)
+-  find_package(MPI REQUIRED)
+-  find_package(SCALAPACK REQUIRED)
+-endif()
++  if(@CP2K_USE_LIBINT2@)
++    find_dependency(Libint2 REQUIRED)
++  endif()
+ 
+-if(@CP2K_USE_FFTW3@)
+-  find_package(Fftw REQUIRED)
+-endif()
+-  # QUIP
+-if(@CP2K_USE_QUIP@)
+-  find_package(Quip REQUIRED)
+-endif()
++  # spglib
+ 
+-# libint
++  if(@CP2K_USE_SPGLIB@)
++    find_dependency(LibSPG REQUIRED)
++  endif()
+ 
+-if(@CP2K_USE_LIBINT2@)
+-  find_package(Libint2 REQUIRED)
+-endif()
++  if(@CP2K_USE_SPLA@)
++    find_dependency(SPLA REQUIRED)
++  endif()
+ 
+-# spglib
++  if(@CP2K_USE_SIRIUS@)
++    find_dependency(sirius REQUIRED)
++  endif()
+ 
+-if(@CP2K_USE_SPGLIB@)
+-  find_package(LibSPG REQUIRED)
+-endif()
++  if(@CP2K_USE_SUPERLU@)
++    find_dependency(SuperLU REQUIRED)
++  endif()
+ 
+-if(@CP2K_USE_SPLA@)
+-  find_package(SPLA REQUIRED)
+-endif()
++  if(@CP2K_USE_METIS@)
++    find_dependency(Metis)
++  endif()
+ 
+-if(@CP2K_USE_SIRIUS@)
+-  find_package(sirius REQUIRED)
+-endif()
+-
+-if(@CP2K_USE_SUPERLU@)
+-  find_package(SuperLU REQUIRED)
+-endif()
++  if(@CP2K_USE_PEXSI@)
++    # PEXSI 1.2 uses cmake as build system
++    find_dependency(PEXSI REQUIRED)
++  endif()
+ 
+-if(@CP2K_USE_PARMETIS@)
+-  find_package(Metis)
+-endif()
++  if(@CP2K_USE_PLUMED@)
++    find_dependency(Plumed REQUIRED)
++  endif()
+ 
+-if(@CP2K_USE_PTSCOTCH@)
+-  find_package(Ptscotch REQUIRED)
+-endif()
++  if(@CP2K_USE_LIBTORCH@)
++    find_dependency(Torch REQUIRED)
++  endif()
+ 
+-if(@CP2K_USE_PEXSI@)
+-  # PEXSI 1.2 uses cmake as build system
+-  find_package(PEXSI REQUIRED)
+-endif()
++  include("${CMAKE_CURRENT_LIST_DIR}/cp2kTargets.cmake")
+ 
+-if(@CP2K_USE_PLUMED@)
+-  find_package(Plumed REQUIRED)
+-endif()
++  # Clean-up module path.
++  list(REMOVE_ITEM CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules")
+ 
+-if(@CP2K_USE_LIBTORCH@)
+-  find_package(Torch REQUIRED)
+ endif()
+-
+-# Include SIRIUS target
+-include("${CMAKE_CURRENT_LIST_DIR}/cp2kTargets.cmake")
+diff --git a/cmake/libcp2k.pc.in b/cmake/libcp2k.pc.in
+new file mode 100644
+index 000000000..618af55e2
+--- /dev/null
++++ b/cmake/libcp2k.pc.in
+@@ -0,0 +1,11 @@
++prefix="@CMAKE_INSTALL_PREFIX@"
++exec_prefix="${prefix}"
++libdir="${prefix}/@CMAKE_INSTALL_LIBDIR@"
++includedir="${prefix}/@CMAKE_INSTALL_INCLUDEDIR@"
++
++Name: @PROJECT_NAME@
++Description: @CMAKE_PROJECT_DESCRIPTION@
++URL: @CMAKE_PROJECT_HOMEPAGE_URL@
++Version: @PROJECT_VERSION@
++Cflags: -I"${includedir}/cp2k" -I"${includedir}/cp2k/@CMAKE_Fortran_COMPILER_ID@-@CMAKE_Fortran_COMPILER_VERSION@"
++Libs: -L"${libdir}" -lcp2k
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index dbc955885..1178101ad 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -1536,9 +1536,9 @@ target_link_libraries(
+     $<$:CP2K::LIBSPG::libspg>
+     $<$:CP2K::Libxc::xc>
+     $<$:CP2K::ELPA::elpa>
+-    $<$:CP2K::FFTW3::fftw3>
+-    $<$:CP2K::FFTW3::fftw3_threads>
+-    $<$:CP2K::FFTW3::fftw3_omp>
++    $<$:CP2K::FFTW3::fftw3>
++    $<$,$>:CP2K::FFTW3::fftw3_threads>
++    $<$,$>:CP2K::FFTW3::fftw3_omp>
+     $<$:SPLA::spla>
+     $<$:CP2K::Libint2::int2>
+     $<$:${TORCH_LIBRARIES}>
+@@ -1555,7 +1555,7 @@ target_compile_definitions(
+   cp2k
+   PUBLIC $<$:__parallel>
+          $<$:__SCALAPACK>
+-         $<$:__MPI_08>
++         $<$:__MPI_F08>
+          __COMPILE_DATE=\"${CP2K_TIMESTAMP}\"
+          __COMPILE_HOST=\"${CP2K_HOST_NAME}\"
+          __COMPILE_REVISION=\"${CP2K_GIT_HASH}\"
+@@ -1577,7 +1577,7 @@ target_compile_definitions(
+          $<$:__OFFLOAD_GEMM>
+          $<$:__ELPA>
+          $<$:__LIBXC>
+-         $<$:__FFTW3>
++         $<$:__FFTW3>
+          $<$:__LIBINT>
+          $<$:__LIBPEXSI>
+          $<$:__LIBTORCH>
+@@ -1774,12 +1774,14 @@ install(
+   EXPORT cp2k_targets
+   FILE cp2kTargets.cmake
+   NAMESPACE cp2k::
+-  DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/cp2k")
++  DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
+ 
+-install(FILES start/libcp2k.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/cp2k")
++install(FILES start/libcp2k.h
++        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}")
+ 
+ install(
+   DIRECTORY "${PROJECT_BINARY_DIR}/src/mod_files"
+-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/cp2k"
++  DESTINATION
++    "${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/${CMAKE_Fortran_COMPILER_ID}-${CMAKE_Fortran_COMPILER_VERSION}"
+   FILES_MATCHING
+   PATTERN "*.mod")
+-- 
+2.41.0
+
diff --git a/var/spack/repos/builtin/packages/cp2k/package.py b/var/spack/repos/builtin/packages/cp2k/package.py
index 6f012bdf4c6247..2e765c7539fffb 100644
--- a/var/spack/repos/builtin/packages/cp2k/package.py
+++ b/var/spack/repos/builtin/packages/cp2k/package.py
@@ -20,11 +20,7 @@ class Cp2k(MakefilePackage, CudaPackage, CMakePackage, ROCmPackage):
     periodic, material, crystal, and biological systems
     """
 
-    build_system(
-        conditional("cmake", when="@master:"),
-        conditional("makefile", when="@:2023.1"),
-        default="makefile",
-    )
+    build_system(conditional("cmake", when="@2023.2:"), "makefile", default="makefile")
 
     homepage = "https://www.cp2k.org"
     url = "https://github.com/cp2k/cp2k/releases/download/v3.0.0/cp2k-3.0.tar.bz2"
@@ -33,6 +29,7 @@ class Cp2k(MakefilePackage, CudaPackage, CMakePackage, ROCmPackage):
 
     maintainers("dev-zero", "mtaillefumier")
 
+    version("2023.2", sha256="adbcc903c1a78cba98f49fe6905a62b49f12e3dfd7cedea00616d1a5f50550db")
     version("2023.1", sha256="dff343b4a80c3a79363b805429bdb3320d3e1db48e0ff7d20a3dfd1c946a51ce")
     version("2022.2", sha256="1a473dea512fe264bb45419f83de432d441f90404f829d89cbc3a03f723b8354")
     version("2022.1", sha256="2c34f1a7972973c62d471cd35856f444f11ab22f2ff930f6ead20f3454fd228b")
@@ -40,10 +37,26 @@ class Cp2k(MakefilePackage, CudaPackage, CMakePackage, ROCmPackage):
     version("8.2", sha256="2e24768720efed1a5a4a58e83e2aca502cd8b95544c21695eb0de71ed652f20a")
     version("8.1", sha256="7f37aead120730234a60b2989d0547ae5e5498d93b1e9b5eb548c041ee8e7772")
     version("7.1", sha256="ccd711a09a426145440e666310dd01cc5772ab103493c4ae6a3470898cd0addb")
-    version("6.1", sha256="af803558e0a6b9e9d9ce8a3ab955ba32bacd179922455424e061c82c9fefa34b")
-    version("5.1", sha256="e23613b593354fa82e0b8410e17d94c607a0b8c6d9b5d843528403ab09904412")
-    version("4.1", sha256="4a3e4a101d8a35ebd80a9e9ecb02697fb8256364f1eccdbe4e5a85d31fe21343")
-    version("3.0", sha256="1acfacef643141045b7cbade7006f9b7538476d861eeecd9658c9e468dc61151")
+    version(
+        "6.1",
+        sha256="af803558e0a6b9e9d9ce8a3ab955ba32bacd179922455424e061c82c9fefa34b",
+        deprecated=True,
+    )
+    version(
+        "5.1",
+        sha256="e23613b593354fa82e0b8410e17d94c607a0b8c6d9b5d843528403ab09904412",
+        deprecated=True,
+    )
+    version(
+        "4.1",
+        sha256="4a3e4a101d8a35ebd80a9e9ecb02697fb8256364f1eccdbe4e5a85d31fe21343",
+        deprecated=True,
+    )
+    version(
+        "3.0",
+        sha256="1acfacef643141045b7cbade7006f9b7538476d861eeecd9658c9e468dc61151",
+        deprecated=True,
+    )
     version("master", branch="master", submodules="True")
 
     variant("mpi", default=True, description="Enable MPI support")
@@ -70,6 +83,13 @@ class Cp2k(MakefilePackage, CudaPackage, CMakePackage, ROCmPackage):
         description="Enable optimised diagonalisation routines from ELPA",
         when="@6.1:",
     )
+    variant(
+        "dlaf",
+        default=False,
+        description="Enable DLA-Future eigensolver and Cholesky decomposition",
+        # TODO: Pin version when integrated in a release
+        when="@master build_system=cmake",
+    )
     variant(
         "sirius",
         default=False,
@@ -90,6 +110,7 @@ class Cp2k(MakefilePackage, CudaPackage, CMakePackage, ROCmPackage):
     )
     variant("pytorch", default=False, description="Enable libtorch support")
     variant("quip", default=False, description="Enable quip support")
+    variant("mpi_f08", default=False, description="Use MPI F08 module")
 
     variant(
         "enable_regtests",
@@ -180,17 +201,26 @@ class Cp2k(MakefilePackage, CudaPackage, CMakePackage, ROCmPackage):
         depends_on("libxc@5.1.3:5.1", when="@8.2:8")
         depends_on("libxc@5.1.7:5.1", when="@9:2022.2")
         depends_on("libxc@6.1:", when="@2023.1:")
+        depends_on("libxc@6.2:", when="@2023.2:")
+
+    with when("+spla"):
+        depends_on("spla+cuda+fortran", when="+cuda")
+        depends_on("spla+rocm+fortran", when="+rocm")
 
     with when("+mpi"):
         depends_on("mpi@2:")
         depends_on("mpi@3:", when="@2023.1:")
         depends_on("scalapack")
+        depends_on("mpich+fortran", when="^mpich")
+
+        conflicts("~mpi_f08", when="^mpich@4.1:")
 
     with when("+cosma"):
         depends_on("cosma+scalapack")
         depends_on("cosma@2.5.1:", when="@9:")
-        depends_on("cosma@2.6.3:", when="@master:")
+        depends_on("cosma@2.6.3:", when="@2023.2:")
         depends_on("cosma+cuda", when="+cuda")
+        depends_on("cosma+rocm", when="+rocm")
         conflicts("~mpi")
         # COSMA support was introduced in 8+
         conflicts("@:7")
@@ -201,6 +231,16 @@ class Cp2k(MakefilePackage, CudaPackage, CMakePackage, ROCmPackage):
         depends_on("elpa~openmp", when="~openmp")
         depends_on("elpa@2021.05:", when="@8.3:")
         depends_on("elpa@2021.11.001:", when="@9.1:")
+        depends_on("elpa@2023.05.001:", when="@2023.2:")
+
+    with when("+dlaf"):
+        conflicts(
+            "~mpi", msg="DLA-Future requires MPI. Only the distributed eigensolver is available."
+        )
+        depends_on("dla-future@0.2.1: +scalapack")
+        depends_on("dla-future ~cuda~rocm", when="~cuda~rocm")
+        depends_on("dla-future +cuda", when="+cuda")
+        depends_on("dla-future +rocm", when="+rocm")
 
     with when("+plumed"):
         depends_on("plumed+shared")
@@ -219,16 +259,17 @@ class Cp2k(MakefilePackage, CudaPackage, CMakePackage, ROCmPackage):
     # a consistent/compatible combination is pulled into the dependency graph.
     with when("+sirius"):
         depends_on("sirius+fortran+shared")
+        depends_on("sirius+cuda", when="+cuda")
+        depends_on("sirius+rocm", when="+rocm")
         depends_on("sirius+openmp", when="+openmp")
         depends_on("sirius~openmp", when="~openmp")
-        depends_on("sirius@:6", when="@:7")
         depends_on("sirius@7.0.0:7.0", when="@8:8.2")
         depends_on("sirius@7.2", when="@8.3:8.9")
         depends_on("sirius@7.3:", when="@9.1")
-        depends_on("sirius@7.4:", when="@master")
+        depends_on("sirius@7.4:", when="@2023.2")
         conflicts("~mpi", msg="SIRIUS requires MPI")
-        # sirius support was introduced in 7+
-        conflicts("@:6")
+        # sirius support was introduced in 7, but effectively usable starting from CP2K 9
+        conflicts("@:8")
 
     with when("+libvori"):
         depends_on("libvori@201219:", when="@8.1")
@@ -251,7 +292,7 @@ class Cp2k(MakefilePackage, CudaPackage, CMakePackage, ROCmPackage):
     depends_on("wannier90", when="@3.0+mpi")
 
     with when("build_system=cmake"):
-        depends_on("dbcsr")
+        depends_on("dbcsr@2.6:")
         depends_on("dbcsr+openmp", when="+openmp")
         depends_on("dbcsr+cuda", when="+cuda")
         depends_on("dbcsr+rocm", when="+rocm")
@@ -260,6 +301,10 @@ class Cp2k(MakefilePackage, CudaPackage, CMakePackage, ROCmPackage):
     conflicts("%apple-clang")
     conflicts("%clang")
     conflicts("%nag")
+    conflicts(
+        "%aocc@:3.2",
+        msg="Please use AOCC 4.0+ that better support modern Fortran features CP2K requires",
+    )
 
     conflicts("~openmp", when="@8:", msg="Building without OpenMP is not supported in CP2K 8+")
 
@@ -305,7 +350,33 @@ class Cp2k(MakefilePackage, CudaPackage, CMakePackage, ROCmPackage):
         sha256="3617abb877812c4b933f601438c70f95e21c6161bea177277b1d4125fd1c0bf9",
         when="@8.2",
     )
-    patch("posix_c_source.patch", when="%aocc")
+
+    # Patch for compilers with stricter C99 checks
+    patch("posix_c_source.patch", when="@7.1%aocc@4.0:")
+    patch("posix_c_source.patch", when="@7.1%gcc@13:")
+
+    # Fix missing variable in OpenMP private clause
+    patch(
+        "https://github.com/cp2k/cp2k/commit/be86bd7f6cd6af7d68f8957dcdb67e7c3d586741.patch?full_index=1",
+        sha256="1bb5a8e80603684a743e7821d24d41b31b60ccbb7d4257df1d2da53a3630e5bf",
+        when="@2022.1:2022.2",
+    )
+
+    # Avoid using NULL() as subroutine argument as doing so breaks some versions of AOCC compiler
+    # These patches backport 2023.x fixes to previous versions
+    patch("backport_avoid_null_2022.x.patch", when="@2022.1:2022.2 %aocc@:4.0")
+    patch("backport_avoid_null_9.1.patch", when="@9.1 %aocc@:4.0")
+    patch("cmake-fixes-2023.2.patch", when="@2023.2 build_system=cmake")
+
+    # Patch for an undefined constant due to incompatible changes in ELPA
+    @when("@9.1:2022.2 +elpa")
+    def patch(self):
+        if self.spec["elpa"].satisfies("@2022.05.001:"):
+            filter_file(
+                r"ELPA_2STAGE_REAL_INTEL_GPU",
+                "ELPA_2STAGE_REAL_INTEL_GPU_SYCL",
+                "src/fm/cp_fm_elpa.F",
+            )
 
     def url_for_version(self, version):
         url = "https://github.com/cp2k/cp2k/releases/download/v{0}/cp2k-{0}.tar.bz2"
@@ -350,7 +421,7 @@ def edit(self, spec, prefix):
             "nvhpc": ["-fast"],
             "cce": ["-O2"],
             "xl": ["-O3"],
-            "aocc": ["-O1"],
+            "aocc": ["-O2"],
         }
 
         dflags = ["-DNDEBUG"]
@@ -371,9 +442,13 @@ def edit(self, spec, prefix):
         ldflags = []
         libs = []
 
-        # CP2K Makefile doesn't set C standard, but the source code uses
-        # C99-style for-loops with inline definition of iterating variable.
-        cflags.append(self.compiler.c99_flag)
+        # CP2K Makefile doesn't set C standard
+        if spec.satisfies("@2023.2:"):
+            # Use of DBL_DECIMAL_DIG
+            cflags.append(self.compiler.c11_flag)
+        else:
+            # C99-style for-loops with inline definition of iterating variable.
+            cflags.append(self.compiler.c99_flag)
 
         if "%intel" in spec:
             cflags.append("-fp-model precise")
@@ -399,6 +474,8 @@ def edit(self, spec, prefix):
             fcflags += [
                 "-fallow-argument-mismatch"
             ]  # https://github.com/pmodels/mpich/issues/4300
+        if spec.satisfies("@7.1%gcc@13:"):
+            fcflags.append("-fallow-argument-mismatch")
 
         if "+openmp" in spec:
             cflags.append(self.compiler.openmp_flag)
@@ -487,6 +564,9 @@ def edit(self, spec, prefix):
             libs.extend(mpi)
             libs.extend(self.compiler.stdcxx_libs)
 
+            if "+mpi_f08" in spec:
+                cppflags.append("-D__MPI_F08")
+
             if "wannier90" in spec:
                 cppflags.append("-D__WANNIER90")
                 wannier = join_path(spec["wannier90"].libs.directories[0], "libwannier.a")
@@ -839,8 +919,7 @@ def check(self):
 
 
 class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder):
-    """Use the new cmake build system to build cp2k. It is the default when
-    building the master branch of cp2k."""
+    """Use the new CMake build system to build CP2K."""
 
     def cmake_args(self):
         spec = self.spec
@@ -864,20 +943,25 @@ def cmake_args(self):
                 raise InstallError("CP2K supports only one cuda_arch at a time.")
             else:
                 gpu_ver = gpu_map[spec.variants["cuda_arch"].value[0]]
-                args += ["-DCP2K_USE_ACCEL=CUDA"]
-                args += [self.define("CP2K_WITH_GPU", gpu_ver)]
+                args += [
+                    self.define("CP2K_USE_ACCEL", "CUDA"),
+                    self.define("CP2K_WITH_GPU", gpu_ver),
+                ]
 
         if "+rocm" in spec:
             if len(spec.variants["amdgpu_target"].value) > 1:
                 raise InstallError("CP2K supports only one amdgpu_target at a time.")
             else:
                 gpu_ver = gpu_map[spec.variants["amdgpu_target"].value[0]]
-                args += ["-DCP2K_USE_ACCEL=HIP"]
-                args += [self.define("CP2K_WITH_GPU", gpu_ver)]
+                args += [
+                    self.define("CP2K_USE_ACCEL", "HIP"),
+                    self.define("CP2K_WITH_GPU", gpu_ver),
+                ]
 
         args += [
             self.define_from_variant("CP2K_ENABLE_REGTESTS", "enable_regtests"),
             self.define_from_variant("CP2K_USE_ELPA", "elpa"),
+            self.define_from_variant("CP2K_USE_DLAF", "dlaf"),
             self.define_from_variant("CP2K_USE_LIBINT2", "libint"),
             self.define_from_variant("CP2K_USE_SIRIUS", "sirius"),
             self.define_from_variant("CP2K_USE_SPLA", "spla"),
@@ -891,6 +975,7 @@ def cmake_args(self):
             self.define_from_variant("CP2K_USE_VORI", "libvori"),
             self.define_from_variant("CP2K_USE_SPLA", "spla"),
             self.define_from_variant("CP2K_USE_QUIP", "quip"),
+            self.define_from_variant("CP2K_USE_MPI_F08", "mpi_f08"),
         ]
 
         # we force the use elpa openmp threading support. might need to be revisited though
diff --git a/var/spack/repos/builtin/packages/cpat/package.py b/var/spack/repos/builtin/packages/cpat/package.py
new file mode 100644
index 00000000000000..ef6415bfba84ea
--- /dev/null
+++ b/var/spack/repos/builtin/packages/cpat/package.py
@@ -0,0 +1,24 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Cpat(PythonPackage):
+    """CPAT is an alignment-free method to predict RNA coding potential using four sequence
+    features"""
+
+    homepage = "https://cpat.readthedocs.io/"
+    pypi = "CPAT/CPAT-3.0.4.tar.gz"
+
+    version("3.0.4", sha256="6d832f20729f8fc814384a27a4fcebcf81b11c0e6d80a404b4c4860d17e7d935")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-nose@0.10.4:", type="build")
+    depends_on("py-cython@0.17:", type="build")
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-pysam", type=("build", "run"))
+
+    depends_on("r", type="run")
diff --git a/var/spack/repos/builtin/packages/cpmd/package.py b/var/spack/repos/builtin/packages/cpmd/package.py
index 4206f7b4ab6573..1782d07b6b07f9 100644
--- a/var/spack/repos/builtin/packages/cpmd/package.py
+++ b/var/spack/repos/builtin/packages/cpmd/package.py
@@ -36,6 +36,7 @@ def edit(self, spec, prefix):
         else:
             fc = spack_fc
             cc = spack_cc
+            cp.filter(r"FFLAGS='([^']*)'", "FFLAGS='\\1 -fallow-argument-mismatch'")
 
         cp.filter("FC=.+", "FC='{0}'".format(fc))
         cp.filter("CC=.+", "CC='{0}'".format(cc))
@@ -71,7 +72,7 @@ def edit(self, spec, prefix):
     def install(self, spec, prefix):
         install_tree(".", prefix)
 
-    def test(self):
+    def test_cpmd(self):
         test_dir = self.test_suite.current_test_data_dir
         test_file = join_path(test_dir, "1-h2o-pbc-geoopt.inp")
         opts = []
@@ -83,9 +84,12 @@ def test(self):
             exe_name = "cpmd.x"
         opts.append(test_file)
         opts.append(test_dir)
+        cpmd = which(exe_name)
+        out = cpmd(*opts, output=str.split, error=str.split)
+
         expected = [
             "2       1        H        O              1.84444     0.97604",
             "3       1        H        O              1.84444     0.97604",
             "2   1   3         H     O     H              103.8663",
         ]
-        self.run_test(exe_name, options=opts, expected=expected)
+        check_outputs(expected, out)
diff --git a/var/spack/repos/builtin/packages/cpp-logger/package.py b/var/spack/repos/builtin/packages/cpp-logger/package.py
new file mode 100644
index 00000000000000..f325fa09dfc2ba
--- /dev/null
+++ b/var/spack/repos/builtin/packages/cpp-logger/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class CppLogger(CMakePackage):
+    """A simple C++ logger"""
+
+    homepage = "https://github.com/hariharan-devarajan/cpp-logger"
+    git = "https://github.com/hariharan-devarajan/cpp-logger.git"
+    maintainers("hariharan-devarajan")
+
+    version("develop", branch="develop")
+    version("master", branch="master")
+    version("0.0.1", tag="v0.0.1", commit="d48b38ab14477bb7c53f8189b8b4be2ea214c28a")
diff --git a/var/spack/repos/builtin/packages/cpr/package.py b/var/spack/repos/builtin/packages/cpr/package.py
new file mode 100644
index 00000000000000..71e32d9960d536
--- /dev/null
+++ b/var/spack/repos/builtin/packages/cpr/package.py
@@ -0,0 +1,30 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Cpr(CMakePackage):
+    """C++ Requests: Curl for People, a spiritual port of Python Requests."""
+
+    homepage = "https://docs.libcpr.org/"
+    url = "https://github.com/libcpr/cpr/archive/refs/tags/1.10.4.tar.gz"
+
+    maintainers("sethrj")
+
+    version("1.10.4", sha256="88462d059cd3df22c4d39ae04483ed50dfd2c808b3effddb65ac3b9aa60b542d")
+    version("1.9.2", sha256="3bfbffb22c51f322780d10d3ca8f79424190d7ac4b5ad6ad896de08dbd06bf31")
+
+    depends_on("curl")
+    depends_on("git", when="build")
+
+    def cmake_args(self):
+        _force = "_FORCE" if self.spec.satisfies("@:1.9") else ""
+
+        return [
+            self.define("CPR_USE_SYSTEM_GTEST", True),
+            self.define(f"CPR{_force}_USE_SYSTEM_CURL", True),
+            self.define("CPR_ENABLE_SSL", True),
+        ]
diff --git a/var/spack/repos/builtin/packages/cracklib/package.py b/var/spack/repos/builtin/packages/cracklib/package.py
index 1b5d2f090f8f08..82d8a411f31e21 100644
--- a/var/spack/repos/builtin/packages/cracklib/package.py
+++ b/var/spack/repos/builtin/packages/cracklib/package.py
@@ -22,7 +22,7 @@ class Cracklib(AutotoolsPackage):
     depends_on("python", type=("build", "run"))
     depends_on("gettext")
     depends_on("fmt")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     configure_directory = "src"
 
diff --git a/var/spack/repos/builtin/packages/creduce/package.py b/var/spack/repos/builtin/packages/creduce/package.py
index b7b01ada36269d..2756a80a21b43d 100644
--- a/var/spack/repos/builtin/packages/creduce/package.py
+++ b/var/spack/repos/builtin/packages/creduce/package.py
@@ -18,7 +18,7 @@ class Creduce(CMakePackage):
     maintainers("olupton")
 
     version("develop", branch="master")
-    version("2.10.0", tag="creduce-2.10.0")
+    version("2.10.0", tag="creduce-2.10.0", commit="fb91843c547794f165e5764a003166191e6c6643")
 
     depends_on("flex")
     depends_on("libxml2")
@@ -29,4 +29,4 @@ class Creduce(CMakePackage):
     depends_on("perl-getopt-tabular")
     depends_on("perl-regexp-common")
     depends_on("perl-termreadkey")
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/crosstool-ng/package.py b/var/spack/repos/builtin/packages/crosstool-ng/package.py
new file mode 100644
index 00000000000000..defc7095e48dfd
--- /dev/null
+++ b/var/spack/repos/builtin/packages/crosstool-ng/package.py
@@ -0,0 +1,46 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class CrosstoolNg(AutotoolsPackage):
+    """Crosstool-NG is a versatile (cross) toolchain generator.
+
+    It supports many architectures and components and has a simple yet powerful
+    menuconfig-style interface.
+    """
+
+    homepage = "https://crosstool-ng.github.io/"
+    url = "https://github.com/crosstool-ng/crosstool-ng/releases/download/crosstool-ng-1.25.0/crosstool-ng-1.25.0.tar.xz"
+
+    maintainers("alalazo")
+
+    version("1.26.0", sha256="e8ce69c5c8ca8d904e6923ccf86c53576761b9cf219e2e69235b139c8e1b74fc")
+    version("1.25.0", sha256="68162f342243cd4189ed7c1f4e3bb1302caa3f2cbbf8331879bd01fe06c60cd3")
+
+    depends_on("ncurses")
+
+    depends_on("bash", type=("build", "run"))
+    depends_on("binutils", type=("build", "run"))
+    depends_on("coreutils", type=("build", "run"))
+    depends_on("elfutils~exeprefix", type=("build", "run"))
+    depends_on("gawk", type=("build", "run"))
+    depends_on("gmake", type=("build", "run"))
+    depends_on("patch", type=("build", "run"))
+    depends_on("sed", type=("build", "run"))
+    depends_on("python@3.7:", type=("build", "run"))
+
+    depends_on("wget", type="run")
+    depends_on("curl", type="run")
+
+    depends_on("autoconf", type=("build", "run"))
+    depends_on("automake", type=("build", "run"))
+    depends_on("libtool", type=("build", "run"))
+
+    depends_on("bison", type="build")
+    depends_on("flex", type="build")
+    depends_on("pkgconfig", type="build")
+    depends_on("grep", type="build")
+    depends_on("help2man", type="build")
diff --git a/var/spack/repos/builtin/packages/crtm/package.py b/var/spack/repos/builtin/packages/crtm/package.py
index 3b49598716683e..df93eaafc5681b 100644
--- a/var/spack/repos/builtin/packages/crtm/package.py
+++ b/var/spack/repos/builtin/packages/crtm/package.py
@@ -52,20 +52,22 @@ class Crtm(CMakePackage):
     depends_on("ecbuild", type=("build"), when="@v2.4.1-jedi")
     depends_on("ecbuild", type=("build"), when="@v3.0.0-rc.1")
 
-    version("v2.4.1-jedi", sha256="fd8bf4db4f2a3b420b4186de84483ba2a36660519dffcb1e0ff14bfe8c6f6a14")
+    version(
+        "v2.4.1-jedi", sha256="fd8bf4db4f2a3b420b4186de84483ba2a36660519dffcb1e0ff14bfe8c6f6a14"
+    )
     # REL-2.4.0_emc (v2.4.0 ecbuild does not work)
-    version("2.4.0", commit="5ddd0d6")
+    version("2.4.0", commit="5ddd0d6b0138284764065feda73b5adf599082a2")
     # Uses the tip of REL-2.3.0_emc branch
-    version("2.3.0", commit="99760e6")
+    version("2.3.0", commit="99760e693ce3b90a3b3b0e97d80972b4dfb61196")
     # JEDI applications so far use these versions
     # Branch release/crtm_jedi
-    version("v2.3-jedi.4", commit="bfede42")
+    version("v2.3-jedi.4", commit="bfede42adc6149213f28f58bf4e02fa8f7cb0198")
     # Branch release/crtm_jedi_v2.4.0
-    version("v2.4-jedi.1", commit="8222341")
-    version("v2.4-jedi.2", commit="62831cb")
+    version("v2.4-jedi.1", commit="82223419fdb479d76c2f2109c2b704e1d9618f22")
+    version("v2.4-jedi.2", commit="62831cbb6c1ffcbb219eeec60e1b1c422526f597")
 
     def url_for_version(self, version):
-        if self.spec.satisfies("@v3") or version >=  Version("3.0.0"):
+        if self.spec.satisfies("@v3") or version >= Version("3.0.0"):
             return "https://github.com/JCSDA/crtmv3/archive/refs/tags/{}.tar.gz".format(version)
         else:
             return "https://github.com/JCSDA/crtm/archive/refs/tags/{}.tar.gz".format(version)
diff --git a/var/spack/repos/builtin/packages/ctffind/power9.patch b/var/spack/repos/builtin/packages/ctffind/no_sincos_asm.patch
similarity index 100%
rename from var/spack/repos/builtin/packages/ctffind/power9.patch
rename to var/spack/repos/builtin/packages/ctffind/no_sincos_asm.patch
diff --git a/var/spack/repos/builtin/packages/ctffind/package.py b/var/spack/repos/builtin/packages/ctffind/package.py
index a57833be160870..ac7bc960c358fb 100644
--- a/var/spack/repos/builtin/packages/ctffind/package.py
+++ b/var/spack/repos/builtin/packages/ctffind/package.py
@@ -33,13 +33,14 @@ def url_for_version(self, version):
     depends_on("jpeg")
 
     patch("configure.patch", when="@4.1.8")
-    patch("power9.patch", when="@4.1.14 target=power9le")
+    patch("no_sincos_asm.patch", when="@4.1.14 target=power9le")
+    patch("no_sincos_asm.patch", when="@4.1.14 target=aarch64:")
     patch("fix_return_types.patch", when="@4.1.13:4.1.14")
 
     def configure_args(self):
         config_args = []
 
-        if "^mkl" in self.spec:
+        if self.spec["fftw-api"].name in INTEL_MATH_LIBRARIES:
             config_args.extend(
                 [
                     "--enable-mkl",
diff --git a/var/spack/repos/builtin/packages/cube/package.py b/var/spack/repos/builtin/packages/cube/package.py
index 220d63f394e80d..f3d91693230842 100644
--- a/var/spack/repos/builtin/packages/cube/package.py
+++ b/var/spack/repos/builtin/packages/cube/package.py
@@ -47,7 +47,7 @@ class Cube(AutotoolsPackage):
 
     depends_on("pkgconfig", type="build")
     depends_on("dbus")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     depends_on("qt@5:", when="@4.3.0: +gui")
     depends_on("qt@4.8:", when="@4.2.0:4.2 +gui")
diff --git a/var/spack/repos/builtin/packages/cubelib/package.py b/var/spack/repos/builtin/packages/cubelib/package.py
index 8a37906d7b5cb7..919a001fedaa4f 100644
--- a/var/spack/repos/builtin/packages/cubelib/package.py
+++ b/var/spack/repos/builtin/packages/cubelib/package.py
@@ -11,7 +11,9 @@ class Cubelib(AutotoolsPackage):
 
     homepage = "https://www.scalasca.org/software/cube-4.x/download.html"
     url = "https://apps.fz-juelich.de/scalasca/releases/cube/4.4/dist/cubelib-4.4.tar.gz"
+    maintainers = ("swat-jsc", "wrwilliams")
 
+    version("4.8.2", sha256="d6fdef57b1bc9594f1450ba46cf08f431dd0d4ae595c47e2f3454e17e4ae74f4")
     version("4.8", sha256="171c93ac5afd6bc74c50a9a58efdaf8589ff5cc1e5bd773ebdfb2347b77e2f68")
     version("4.7.1", sha256="62cf33a51acd9a723fff9a4a5411cd74203e24e0c4ffc5b9e82e011778ed4f2f")
     version("4.7", sha256="e44352c80a25a49b0fa0748792ccc9f1be31300a96c32de982b92477a8740938")
@@ -43,7 +45,7 @@ class Cubelib(AutotoolsPackage):
     )
 
     depends_on("pkgconfig", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def url_for_version(self, version):
         url = "http://apps.fz-juelich.de/scalasca/releases/cube/{0}/dist/cubelib-{1}.tar.gz"
@@ -52,7 +54,7 @@ def url_for_version(self, version):
 
     def configure_args(self):
         configure_args = ["--enable-shared"]
-        configure_args.append("--with-frontend-zlib=%s" % self.spec["zlib"].prefix.lib)
+        configure_args.append("--with-frontend-zlib=%s" % self.spec["zlib-api"].prefix.lib)
         return configure_args
 
     def install(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/cubew/package.py b/var/spack/repos/builtin/packages/cubew/package.py
index 6b8aa968413cbd..bcab0920fd1833 100644
--- a/var/spack/repos/builtin/packages/cubew/package.py
+++ b/var/spack/repos/builtin/packages/cubew/package.py
@@ -11,7 +11,9 @@ class Cubew(AutotoolsPackage):
 
     homepage = "https://www.scalasca.org/software/cube-4.x/download.html"
     url = "https://apps.fz-juelich.de/scalasca/releases/cube/4.4/dist/cubew-4.4.tar.gz"
+    maintainers = ("swat-jsc", "wrwilliams")
 
+    version("4.8.2", sha256="4f3bcf0622c2429b8972b5eb3f14d79ec89b8161e3c1cc5862ceda417d7975d2")
     version("4.8", sha256="73c7f9e9681ee45d71943b66c01cfe675b426e4816e751ed2e0b670563ca4cf3")
     version("4.7.1", sha256="0d364a4930ca876aa887ec40d12399d61a225dbab69e57379b293516d7b6db8d")
     version("4.7", sha256="a7c7fca13e6cb252f08d4380223d7c56a8e86a67de147bcc0279ebb849c884a5")
@@ -43,7 +45,7 @@ class Cubew(AutotoolsPackage):
     )
 
     depends_on("pkgconfig", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def url_for_version(self, version):
         url = "http://apps.fz-juelich.de/scalasca/releases/cube/{0}/dist/cubew-{1}.tar.gz"
@@ -52,8 +54,8 @@ def url_for_version(self, version):
 
     def configure_args(self):
         configure_args = ["--enable-shared"]
-        configure_args.append("--with-frontend-zlib=%s" % self.spec["zlib"].prefix.lib)
-        configure_args.append("--with-backend-zlib=%s" % self.spec["zlib"].prefix.lib)
+        configure_args.append("--with-frontend-zlib=%s" % self.spec["zlib-api"].prefix.lib)
+        configure_args.append("--with-backend-zlib=%s" % self.spec["zlib-api"].prefix.lib)
 
         return configure_args
 
diff --git a/var/spack/repos/builtin/packages/cuda/package.py b/var/spack/repos/builtin/packages/cuda/package.py
index 93573d46987199..e624acbaa9db8d 100644
--- a/var/spack/repos/builtin/packages/cuda/package.py
+++ b/var/spack/repos/builtin/packages/cuda/package.py
@@ -25,6 +25,34 @@
 
 preferred_ver = "11.8.0"
 _versions = {
+    "12.2.1": {
+        "Linux-aarch64": (
+            "a9ae6bd02684c7acfb229484368bf2691d592767ce1aed10ae9aed92c81b9f09",
+            "https://developer.download.nvidia.com/compute/cuda/12.2.1/local_installers/cuda_12.2.1_535.86.10_linux_sbsa.run",
+        ),
+        "Linux-x86_64": (
+            "c51e814eaa756160a73289b1f35880cc38c18e1ddedc87f46f7ff0d1696bdb95",
+            "https://developer.download.nvidia.com/compute/cuda/12.2.1/local_installers/cuda_12.2.1_535.86.10_linux.run",
+        ),
+        "Linux-ppc64le": (
+            "8f1aa60cc1e14fd7853eeecd53a78b7dfb2b2da0512a312cc35e4f5b79454f94",
+            "https://developer.download.nvidia.com/compute/cuda/12.2.1/local_installers/cuda_12.2.1_535.86.10_linux_ppc64le.run",
+        ),
+    },
+    "12.2.0": {
+        "Linux-aarch64": (
+            "9c90d79bb63952cd30f1f57f9f8fe11e7a8967bba6f824df45d3464b5d37e5d8",
+            "https://developer.download.nvidia.com/compute/cuda/12.2.0/local_installers/cuda_12.2.0_535.54.03_linux_sbsa.run",
+        ),
+        "Linux-x86_64": (
+            "ecf3d2afadcbac029f0f4505785810d52d006e4b87ba79ff3f984336a2bbf518",
+            "https://developer.download.nvidia.com/compute/cuda/12.2.0/local_installers/cuda_12.2.0_535.54.03_linux.run",
+        ),
+        "Linux-ppc64le": (
+            "f7b5fde5f727b51c380c6199200d0525e8585475944ff8ad2791d06734f54230",
+            "https://developer.download.nvidia.com/compute/cuda/12.2.0/local_installers/cuda_12.2.0_535.54.03_linux_ppc64le.run",
+        ),
+    },
     "12.1.1": {
         "Linux-aarch64": (
             "45ea4cd860f0a26d3db8ce032530f2ee0b55abdd587545213d395a73623b4278",
@@ -494,6 +522,8 @@ class Cuda(Package):
     maintainers("ax3l", "Rombur")
     executables = ["^nvcc$"]
 
+    skip_version_audit = ["platform=darwin"]
+
     for ver, packages in _versions.items():
         key = "{0}-{1}".format(platform.system(), platform.machine())
         pkg = packages.get(key)
@@ -552,6 +582,8 @@ def setup_build_environment(self, env):
 
     def setup_dependent_build_environment(self, env, dependent_spec):
         env.set("CUDAHOSTCXX", dependent_spec.package.compiler.cxx)
+        env.set("CUDA_HOME", self.prefix)
+        env.set("NVHPC_CUDA_HOME", self.prefix)
 
     @property
     def cmake_prefix_paths(self):
@@ -563,6 +595,7 @@ def cmake_prefix_paths(self):
 
     def setup_run_environment(self, env):
         env.set("CUDA_HOME", self.prefix)
+        env.set("NVHPC_CUDA_HOME", self.prefix)
 
     def install(self, spec, prefix):
         if os.path.exists("/tmp/cuda-installer.log"):
diff --git a/var/spack/repos/builtin/packages/cudnn/package.py b/var/spack/repos/builtin/packages/cudnn/package.py
index df98518c8c0070..54d6708e40337a 100644
--- a/var/spack/repos/builtin/packages/cudnn/package.py
+++ b/var/spack/repos/builtin/packages/cudnn/package.py
@@ -260,6 +260,8 @@ class Cudnn(Package):
     # need to use modified URLs like in url_for_version.
     maintainers("adamjstewart", "bvanessen")
 
+    skip_version_audit = ["platform=darwin"]
+
     for ver, packages in _versions.items():
         key = "{0}-{1}".format(platform.system(), platform.machine())
         pkg = packages.get(key)
diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py
index 4aa89db0b9238c..f26247a44ad00d 100644
--- a/var/spack/repos/builtin/packages/curl/package.py
+++ b/var/spack/repos/builtin/packages/curl/package.py
@@ -26,11 +26,25 @@ class Curl(NMakePackage, AutotoolsPackage):
 
     maintainers("alecbcs")
 
-    version("8.1.2", sha256="b54974d32fd610acace92e3df1f643144015ac65847f0a041fdc17db6f43f243")
-    version("8.0.1", sha256="9b6b1e96b748d04b968786b6bdf407aa5c75ab53a3d37c1c8c81cdb736555ccf")
-    version("7.88.1", sha256="8224b45cce12abde039c12dc0711b7ea85b104b9ad534d6e4c5b4e188a61c907")
+    version("8.4.0", sha256="e5250581a9c032b1b6ed3cf2f9c114c811fc41881069e9892d115cc73f9e88c6")
 
     # Deprecated versions due to CVEs
+    # CVE-2023-38545
+    version(
+        "8.1.2",
+        sha256="b54974d32fd610acace92e3df1f643144015ac65847f0a041fdc17db6f43f243",
+        deprecated=True,
+    )
+    version(
+        "8.0.1",
+        sha256="9b6b1e96b748d04b968786b6bdf407aa5c75ab53a3d37c1c8c81cdb736555ccf",
+        deprecated=True,
+    )
+    version(
+        "7.88.1",
+        sha256="8224b45cce12abde039c12dc0711b7ea85b104b9ad534d6e4c5b4e188a61c907",
+        deprecated=True,
+    )
     # https://nvd.nist.gov/vuln/detail/CVE-2022-43551
     version(
         "7.87.0",
@@ -252,7 +266,7 @@ class Curl(NMakePackage, AutotoolsPackage):
         ),
         multi=True,
     )
-    variant("nghttp2", default=False, description="build nghttp2 library (requires C++11)")
+    variant("nghttp2", default=True, description="build nghttp2 library (requires C++11)")
     variant("libssh2", default=False, description="enable libssh2 support")
     variant("libssh", default=False, description="enable libssh support", when="@7.58:")
     variant("gssapi", default=False, description="enable Kerberos support")
@@ -286,11 +300,12 @@ class Curl(NMakePackage, AutotoolsPackage):
         depends_on("openssl@:1", when="@:7.76")
 
     depends_on("libidn2", when="+libidn2")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("nghttp2", when="+nghttp2")
     depends_on("libssh2", when="+libssh2")
     depends_on("libssh", when="+libssh")
     depends_on("krb5", when="+gssapi")
+    depends_on("rtmpdump", when="+librtmp")
 
     # https://github.com/curl/curl/pull/9054
     patch("easy-lock-sched-header.patch", when="@7.84.0")
@@ -342,7 +357,7 @@ def configure_args(self):
         spec = self.spec
 
         args = [
-            "--with-zlib=" + spec["zlib"].prefix,
+            "--with-zlib=" + spec["zlib-api"].prefix,
             # Prevent unintentional linking against system libraries: we could
             # add variants for these in the future
             "--without-brotli",
@@ -431,7 +446,7 @@ def nmake_args(self):
         mode = "dll" if "libs=dll" in self.spec else "static"
         args.append("mode=%s" % mode)
         args.append("WITH_ZLIB=%s" % mode)
-        args.append("ZLIB_PATH=%s" % self.spec["zlib"].prefix)
+        args.append("ZLIB_PATH=%s" % self.spec["zlib-api"].prefix)
         if "+libssh" in self.spec:
             args.append("WITH_SSH=%s" % mode)
         if "+libssh2" in self.spec:
diff --git a/var/spack/repos/builtin/packages/cusz/package.py b/var/spack/repos/builtin/packages/cusz/package.py
index abfec5ad343650..0482f34812d611 100644
--- a/var/spack/repos/builtin/packages/cusz/package.py
+++ b/var/spack/repos/builtin/packages/cusz/package.py
@@ -20,12 +20,13 @@ class Cusz(CMakePackage, CudaPackage):
     conflicts("cuda_arch=none", when="+cuda")
 
     version("develop", branch="develop")
+    version("0.3.1", commit="02be3cbd07db467decaf45ec9eb593ba6173c809")
     version("0.3", sha256="0feb4f7fd64879fe147624dd5ad164adf3983f79b2e0383d35724f8d185dcb11")
 
     # these version of Cuda provide the CUB headers, but not CUB cmake configuration that we use.
-    conflicts("cuda@11.0.2:11.2.2")
+    conflicts("^cuda@11.0.2:11.2.2")
 
-    depends_on("cub", when="^ cuda@:10.2.89")
+    depends_on("cub", when="^cuda@:10.2.89")
 
     def cmake_args(self):
         cuda_arch = self.spec.variants["cuda_arch"].value
diff --git a/var/spack/repos/builtin/packages/cutensor/package.py b/var/spack/repos/builtin/packages/cutensor/package.py
index 15bc9fc22ae0a5..9f42a27e9546f5 100644
--- a/var/spack/repos/builtin/packages/cutensor/package.py
+++ b/var/spack/repos/builtin/packages/cutensor/package.py
@@ -27,6 +27,8 @@ class Cutensor(Package):
     maintainers("bvanessen")
     url = "cutensor"
 
+    skip_version_audit = ["platform=darwin"]
+
     for ver, packages in _versions.items():
         key = "{0}-{1}".format(platform.system(), platform.machine())
         pkg = packages.get(key)
diff --git a/var/spack/repos/builtin/packages/cvise/package.py b/var/spack/repos/builtin/packages/cvise/package.py
index a5b7855c5214c1..097d9f68149567 100644
--- a/var/spack/repos/builtin/packages/cvise/package.py
+++ b/var/spack/repos/builtin/packages/cvise/package.py
@@ -15,7 +15,7 @@ class Cvise(CMakePackage):
     git = "https://github.com/marxin/cvise.git"
 
     version("master", branch="master")
-    version("2.7.0", tag="v2.7.0")
+    version("2.7.0", tag="v2.7.0", commit="d9e4a50514d9931b2a1293755a7e96e0f9520032")
 
     variant("pytest", default=False, description="Add py-pytest as dependency")
     variant("colordiff", default=False, description="Add colordiff support")
diff --git a/var/spack/repos/builtin/packages/cxxopts/package.py b/var/spack/repos/builtin/packages/cxxopts/package.py
index 19cafc40345126..1f7e90393184a0 100644
--- a/var/spack/repos/builtin/packages/cxxopts/package.py
+++ b/var/spack/repos/builtin/packages/cxxopts/package.py
@@ -13,6 +13,7 @@ class Cxxopts(CMakePackage):
     url = "https://github.com/jarro2783/cxxopts/archive/v2.2.0.tar.gz"
     maintainers("haampie")
 
+    version("3.1.1", sha256="523175f792eb0ff04f9e653c90746c12655f10cb70f1d5e6d6d9491420298a08")
     version("3.0.0", sha256="36f41fa2a46b3c1466613b63f3fa73dc24d912bc90d667147f1e43215a8c6d00")
     version("2.2.1", sha256="984aa3c8917d649b14d7f6277104ce38dd142ce378a9198ec926f03302399681")
     version("2.2.0", sha256="447dbfc2361fce9742c5d1c9cfb25731c977b405f9085a738fbd608626da8a4d")
diff --git a/var/spack/repos/builtin/packages/dakota/package.py b/var/spack/repos/builtin/packages/dakota/package.py
index 4d302d05c4fce6..e0374ad927880e 100644
--- a/var/spack/repos/builtin/packages/dakota/package.py
+++ b/var/spack/repos/builtin/packages/dakota/package.py
@@ -7,6 +7,16 @@
 from spack.pkg.builtin.boost import Boost
 
 
+def submodules(package):
+    submodules = []
+    submodules.append("dakota-examples")
+    submodules.append("packages/external")
+    submodules.append("packages/pecos")
+    submodules.append("packages/surfpack")
+
+    return submodules
+
+
 class Dakota(CMakePackage):
     """The Dakota toolkit provides a flexible, extensible interface between
     analysis codes and iterative systems analysis methods. Dakota
@@ -27,8 +37,15 @@ class Dakota(CMakePackage):
     """
 
     homepage = "https://dakota.sandia.gov/"
+    git = "https://github.com/snl-dakota/dakota.git"
     url = "https://dakota.sandia.gov/sites/default/files/distributions/public/dakota-6.12-release-public.src.tar.gz"
 
+    version(
+        "6.18",
+        tag="v6.18.0",
+        commit="f6cb33b517bb304795e1e14d3673fe289df2ec9b",
+        submodules=submodules,
+    )
     version("6.12", sha256="4d69f9cbb0c7319384ab9df27643ff6767eb410823930b8fbd56cc9de0885bc9")
     version("6.9", sha256="989b689278964b96496e3058b8ef5c2724d74bcd232f898fe450c51eba7fe0c2")
     version("6.3", sha256="0fbc310105860d77bb5c96de0e8813d75441fca1a5e6dfaf732aa095c4488d52")
@@ -46,12 +63,14 @@ class Dakota(CMakePackage):
     depends_on("python")
     depends_on("perl-data-dumper", type="build", when="@6.12:")
     depends_on("boost@:1.68.0", when="@:6.12")
+    depends_on("boost@1.69.0:", when="@6.18:")
 
     # TODO: replace this with an explicit list of components of Boost,
     # for instance depends_on('boost +filesystem')
     # See https://github.com/spack/spack/pull/22303 for reference
     depends_on(Boost.with_default_variants, when="@:6.12")
     depends_on("cmake@2.8.9:", type="build")
+    depends_on("cmake@3.17:", type="build", when="@6.18:")
 
     def cmake_args(self):
         spec = self.spec
diff --git a/var/spack/repos/builtin/packages/dalton/package.py b/var/spack/repos/builtin/packages/dalton/package.py
index 2cf7fe14491aaf..c221a9aa2ad429 100644
--- a/var/spack/repos/builtin/packages/dalton/package.py
+++ b/var/spack/repos/builtin/packages/dalton/package.py
@@ -18,8 +18,12 @@ class Dalton(CMakePackage):
     maintainers("foeroyingur")
 
     version("master", branch="master", submodules=True)
-    version("2020.0", tag="2020.0", submodules=True)
-    version("2018.2", tag="2018.2", submodules=True)
+    version(
+        "2020.0", tag="2020.0", commit="66052b3af5ea7225e31178bf9a8b031913c72190", submodules=True
+    )
+    version(
+        "2018.2", tag="2018.2", commit="4aa945ecd235fbf67ed0c1609617c553ef40be89", submodules=True
+    )
 
     variant(
         "build_type",
diff --git a/var/spack/repos/builtin/packages/damaris/package.py b/var/spack/repos/builtin/packages/damaris/package.py
index 413e3a4453168e..3f8b6f156bead8 100644
--- a/var/spack/repos/builtin/packages/damaris/package.py
+++ b/var/spack/repos/builtin/packages/damaris/package.py
@@ -16,11 +16,29 @@ class Damaris(CMakePackage):
     maintainers("jcbowden")
 
     version("master", branch="master")
-    version("1.6.0", tag="v1.6.0")
-    version("1.5.0", tag="v1.5.0")
-    version("1.3.3", tag="v1.3.3")
-    version("1.3.2", tag="v1.3.2")
-    version("1.3.1", tag="v1.3.1")
+    version("1.9.2", tag="v1.9.2", commit="22c146b4b4ca047d4d36fd904d248e0280b3c0ea")
+    version("1.9.1", tag="v1.9.1", commit="2fe83f587837b7ad0b5c187b8ff453f7d3ad2c18")
+    version("1.9.0", tag="v1.9.0", commit="23cac3a8ade9f9c20499081a8ed10b3e51801428")
+    version("1.8.2", tag="v1.8.2", commit="bd447e677cdf81389f93bea3139af0fa54554a01")
+    version("1.8.1", tag="v1.8.1", commit="18513edb1e11974a4296263ff8499d2802e17891")
+    version("1.8.0", tag="v1.8.0", commit="56701eee59d464cc73d248fbd5e7a8a70e7a3933")
+    version("1.7.1", tag="v1.7.1", commit="09dfbe7828ee295b4433c9e01c6523fa6b4adab5")
+    version("1.7.0", tag="v1.7.0", commit="9ab3ea4c568de16f5d43b8b5ad71feb4864a5584")
+    version(
+        "1.6.0", tag="v1.6.0", commit="1fe4c61cce03babd24315b8e6156f226baac97a2", deprecated=True
+    )
+    version(
+        "1.5.0", tag="v1.5.0", commit="68206a696ad430aa8426ca370501aa71914fbc87", deprecated=True
+    )
+    version(
+        "1.3.3", tag="v1.3.3", commit="f1c473507c080738f7092f6a7d72deb938ade786", deprecated=True
+    )
+    version(
+        "1.3.2", tag="v1.3.2", commit="38b50664523e56900809a19f0cf52fc0ab5dca53", deprecated=True
+    )
+    version(
+        "1.3.1", tag="v1.3.1", commit="6cee3690fa7d387acc8f5f650a7b019e13b90284", deprecated=True
+    )
 
     variant("fortran", default=True, description="Enables Fortran support")
     variant("hdf5", default=False, description="Enables the HDF5 storage plugin")
@@ -36,24 +54,20 @@ class Damaris(CMakePackage):
     variant(
         "python",
         default=False,
-        description="Enables building of Python enabled Damaris library - "
-        "boost::python boost::numpy needed",
+        description="Enables building of Python enabled Damaris library using Boost::python",
     )
+    extends("python", when="+python")
 
-    depends_on("mpi")
-    depends_on("cmake@3.18.0:", type=("build"))
-    depends_on(
-        "boost"
-        "+exception+locale+system+serialization+chrono+atomic"
-        "+container+regex+thread+log+filesystem+date_time"
-        "@1.67:"
-    )
     depends_on("xsd")
     depends_on("xerces-c")
+    depends_on("mpi")
+    depends_on("cmake@3.18.0:", type=("build"))
+    depends_on("boost@1.67:+thread+log+filesystem+date_time+system")
+    depends_on("boost+python", when="+python")
+    depends_on("py-mpi4py", when="+python", type=("build", "run"))
     depends_on("hdf5@1.8.20:", when="+hdf5")
-    depends_on("paraview+python", when="+catalyst")
+    depends_on("paraview+python+mpi+development_files", when="+catalyst")
     depends_on("visit+mpi", when="+visit")
-    depends_on("boost+thread+log+filesystem+date_time+python+numpy @1.67:", when="+python")
 
     def cmake_args(self):
         args = []
@@ -77,6 +91,7 @@ def cmake_args(self):
         if self.spec.variants["catalyst"].value:
             args.extend(["-DENABLE_CATALYST:BOOL=ON"])
             args.extend(["-DParaView_ROOT:PATH=%s" % self.spec["catalyst"].prefix])
+
         if self.spec.variants["examples"].value:
             args.extend(["-DENABLE_EXAMPLES:BOOL=ON"])
 
@@ -85,6 +100,8 @@ def cmake_args(self):
 
         if self.spec.variants["python"].value:
             args.extend(["-DENABLE_PYTHON:BOOL=ON"])
+            args.extend(["-DENABLE_PYTHONMOD:BOOL=ON"])
+            args.append(self.define("PYTHON_MODULE_INSTALL_PATH", python_platlib))
 
         if self.spec.variants["visit"].value:
             args.extend(["-DENABLE_VISIT:BOOL=ON"])
diff --git a/var/spack/repos/builtin/packages/daos/package.py b/var/spack/repos/builtin/packages/daos/package.py
index 838f009092798d..3b62ecd8052fdd 100644
--- a/var/spack/repos/builtin/packages/daos/package.py
+++ b/var/spack/repos/builtin/packages/daos/package.py
@@ -16,7 +16,9 @@ class Daos(SConsPackage):
     maintainers("hyoklee")
 
     version("master", branch="master", submodules=True)
-    version("2.2.0", tag="v2.2.0", submodules=True)
+    version(
+        "2.2.0", tag="v2.2.0", commit="d2a1f2790c946659c9398926254e6203fd957b7c", submodules=True
+    )
     variant(
         "debug", default=False, description="Enable debugging info and strict compile warnings"
     )
diff --git a/var/spack/repos/builtin/packages/darshan-runtime/package.py b/var/spack/repos/builtin/packages/darshan-runtime/package.py
index 5fd2340014b596..1b4e06b5919a04 100644
--- a/var/spack/repos/builtin/packages/darshan-runtime/package.py
+++ b/var/spack/repos/builtin/packages/darshan-runtime/package.py
@@ -25,6 +25,7 @@ class DarshanRuntime(AutotoolsPackage):
     test_requires_compiler = True
 
     version("main", branch="main", submodules=True)
+    version("3.4.4", sha256="d9c9df5aca94dc5ca3d56fd763bec2f74771d35126d61cb897373d2166ccd867")
     version("3.4.3", sha256="dca5f9f9b0ead55a8724b218071ecbb5c4f2ef6027eaade3a6477256930ccc2c")
     version("3.4.2", sha256="b095c3b7c059a8eba4beb03ec092b60708780a3cae3fc830424f6f9ada811c6b")
     version("3.4.1", sha256="77c0a4675d94a0f9df5710e5b8658cc9ef0f0981a6dafb114d0389b1af64774c")
@@ -49,7 +50,7 @@ class DarshanRuntime(AutotoolsPackage):
     version("3.0.0", sha256="95232710f5631bbf665964c0650df729c48104494e887442596128d189da43e0")
 
     depends_on("mpi", when="+mpi")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("hdf5", when="+hdf5")
     depends_on("parallel-netcdf", when="+parallel-netcdf")
     depends_on("papi", when="+apxc")
@@ -121,7 +122,7 @@ def configure_args(self):
         extra_args.append("--with-mem-align=8")
         extra_args.append("--with-log-path-by-env=DARSHAN_LOG_DIR_PATH")
         extra_args.append("--with-jobid-env=%s" % job_id)
-        extra_args.append("--with-zlib=%s" % spec["zlib"].prefix)
+        extra_args.append("--with-zlib=%s" % spec["zlib-api"].prefix)
 
         if "+mpi" in spec:
             extra_args.append("CC=%s" % self.spec["mpi"].mpicc)
diff --git a/var/spack/repos/builtin/packages/darshan-util/package.py b/var/spack/repos/builtin/packages/darshan-util/package.py
index 9dcc5e219ef209..56f85a9f006ec9 100644
--- a/var/spack/repos/builtin/packages/darshan-util/package.py
+++ b/var/spack/repos/builtin/packages/darshan-util/package.py
@@ -21,6 +21,7 @@ class DarshanUtil(AutotoolsPackage):
     tags = ["e4s"]
 
     version("main", branch="main", submodules="True")
+    version("3.4.4", sha256="d9c9df5aca94dc5ca3d56fd763bec2f74771d35126d61cb897373d2166ccd867")
     version("3.4.3", sha256="dca5f9f9b0ead55a8724b218071ecbb5c4f2ef6027eaade3a6477256930ccc2c")
     version("3.4.2", sha256="b095c3b7c059a8eba4beb03ec092b60708780a3cae3fc830424f6f9ada811c6b")
     version("3.4.1", sha256="77c0a4675d94a0f9df5710e5b8658cc9ef0f0981a6dafb114d0389b1af64774c")
@@ -55,7 +56,7 @@ class DarshanUtil(AutotoolsPackage):
         "apxc", default=False, description="Compile with AutoPerf XC module support", when="@3.3:"
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2", when="+bzip2", type=("build", "link", "run"))
     depends_on("autoconf", type="build", when="@main")
     depends_on("automake", type="build", when="@main")
@@ -77,7 +78,7 @@ def configure_args(self):
         extra_args = []
 
         extra_args.append("CC=%s" % self.compiler.cc)
-        extra_args.append("--with-zlib=%s" % spec["zlib"].prefix)
+        extra_args.append("--with-zlib=%s" % spec["zlib-api"].prefix)
         if "+apmpi" in spec:
             if self.version < Version("3.3.2"):
                 extra_args.append("--enable-autoperf-apmpi")
diff --git a/var/spack/repos/builtin/packages/datatransferkit/package.py b/var/spack/repos/builtin/packages/datatransferkit/package.py
index 1829e2414df090..7e0d050069d0cd 100644
--- a/var/spack/repos/builtin/packages/datatransferkit/package.py
+++ b/var/spack/repos/builtin/packages/datatransferkit/package.py
@@ -11,7 +11,7 @@ class Datatransferkit(CMakePackage):
     parallel solution transfer services for multiphysics simulations"""
 
     homepage = "https://datatransferkit.readthedoc.io"
-    url = "https://github.com/ORNL-CEES/DataTransferKit/archive/3.1-rc3.tar.gz"
+    url = "https://github.com/ORNL-CEES/DataTransferKit/archive/3.1.1.tar.gz"
     git = "https://github.com/ORNL-CEES/DataTransferKit.git"
 
     tags = ["e4s"]
@@ -19,6 +19,8 @@ class Datatransferkit(CMakePackage):
     maintainers("Rombur")
 
     version("master", branch="master", submodules=True)
+    version("3.1.1", commit="bfb7673cc233c26a6a541cbf096f37f26df1e5fb", submodules=True)
+    version("3.1.0", commit="60a4cbd0a55505e0450f1ac979e1eef8966dc03f", submodules=True)
     version("3.1-rc3", commit="691d5a1540f7cd42141a3b3d2a7c8370cbc3560a", submodules=True)
     version("3.1-rc2", commit="1abc1a43b33dffc7a16d7497b4185d09d865e36a", submodules=True)
 
@@ -37,7 +39,8 @@ class Datatransferkit(CMakePackage):
     depends_on("trilinos+intrepid2+shards~dtk")
     depends_on("trilinos+openmp", when="+openmp")
     depends_on("trilinos+stratimikos+belos", when="@master")
-    depends_on("trilinos@13:", when="@3.1-rc2:")
+    depends_on("trilinos@13:13.4.1", when="@3.1-rc2:3.1-rc3")
+    depends_on("trilinos@14:", when="@3.1.0:")
 
     def cmake_args(self):
         spec = self.spec
diff --git a/var/spack/repos/builtin/packages/davix/package.py b/var/spack/repos/builtin/packages/davix/package.py
index 3f351c65b00e88..ea0eb7eb35dbc9 100644
--- a/var/spack/repos/builtin/packages/davix/package.py
+++ b/var/spack/repos/builtin/packages/davix/package.py
@@ -35,7 +35,7 @@ class Davix(CMakePackage):
     depends_on("uuid")
     depends_on("openssl")
 
-    variant("thirdparty", default=False)
+    variant("thirdparty", default=False, description="Build vendored libraries")
     depends_on("gsoap", when="+thirdparty")
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/dbcsr/package.py b/var/spack/repos/builtin/packages/dbcsr/package.py
index df0a6035e9ab32..2a3251304f44c2 100644
--- a/var/spack/repos/builtin/packages/dbcsr/package.py
+++ b/var/spack/repos/builtin/packages/dbcsr/package.py
@@ -14,9 +14,10 @@ class Dbcsr(CMakePackage, CudaPackage, ROCmPackage):
     url = "https://github.com/cp2k/dbcsr/releases/download/v2.2.0/dbcsr-2.2.0.tar.gz"
     list_url = "https://github.com/cp2k/dbcsr/releases"
 
-    maintainers("dev-zero")
+    maintainers("dev-zero", "mtaillefumier")
 
     version("develop", branch="develop")
+    version("2.6.0", sha256="c67b02ff9abc7c1f529af446a9f01f3ef9e5b0574f220259128da8d5ca7e9dc6")
     version("2.5.0", sha256="91fda9b2502e5d0a2a6cdd5a73ef096253cc7e75bd01ba5189a4726ad86aef08")
     version("2.4.1", sha256="b3d5ae62ca582b72707a2c932e8074a4f2f61d61085d97bd374213c70b8dbdcf")
     version("2.4.0", sha256="cf2b774328c9a30677501f49b79955841bd08915a7ca53c8533bfdf14a8f9bd4")
@@ -45,15 +46,20 @@ class Dbcsr(CMakePackage, CudaPackage, ROCmPackage):
     )
 
     variant("opencl", default=False, description="Enable OpenCL backend")
+    variant("mpi_f08", default=False, when="@2.6:", description="Use mpi F08 module")
 
     depends_on("blas")
     depends_on("lapack")
     depends_on("mpi", when="+mpi")
     depends_on("libxsmm@1.11:~header-only", when="smm=libxsmm")
 
-    depends_on("cmake@3.17:", type="build", when="@2.1:")
     depends_on("cmake@3.10:", type="build")
+    depends_on("cmake@3.12:", type="build", when="@2.1:")
+    depends_on("cmake@3.17:", type="build", when="@2.2:")
+    depends_on("cmake@3.22:", type="build", when="@2.3:")
+
     depends_on("py-fypp", type="build")
+    depends_on("py-fypp@3.1:", type="build", when="@2.6:")
     depends_on("pkgconfig", type="build")
     depends_on("python@3.6:", type="build", when="+cuda")
 
@@ -65,7 +71,7 @@ class Dbcsr(CMakePackage, CudaPackage, ROCmPackage):
     # for optimal kernels. Note that we don't override the parent class arch
     # properties, since the parent class defines constraints for different archs
     # Instead just mark all unsupported cuda archs as conflicting.
-    dbcsr_cuda_archs = ("35", "37", "60", "70")
+    dbcsr_cuda_archs = ("35", "37", "60", "70", "80")
     cuda_msg = "dbcsr only supports cuda_arch {0}".format(dbcsr_cuda_archs)
 
     for arch in CudaPackage.cuda_arch_values:
@@ -74,7 +80,7 @@ class Dbcsr(CMakePackage, CudaPackage, ROCmPackage):
 
     conflicts("+cuda", when="cuda_arch=none", msg=cuda_msg)
 
-    dbcsr_amdgpu_targets = "gfx906"
+    dbcsr_amdgpu_targets = {"gfx906", "gfx910", "gfx90a", "gfx90a:xnack-", "gfx90a:xnack+"}
     amd_msg = "DBCSR only supports amdgpu_target {0}".format(dbcsr_amdgpu_targets)
 
     for arch in ROCmPackage.amdgpu_targets:
@@ -92,6 +98,14 @@ class Dbcsr(CMakePackage, CudaPackage, ROCmPackage):
 
     conflicts("smm=blas", when="+opencl")
 
+    with when("+mpi"):
+        # When using mpich 4.1 or higher, mpi_f08 has to be used, otherwise:
+        # Error: Type mismatch in argument 'baseptr' at (1); passed TYPE(c_ptr)
+        # to INTEGER(8)
+        conflicts("^mpich@4.1:", when="@:2.5")
+        conflicts("~mpi_f08", when="^mpich@4.1:")
+        depends_on("mpich+fortran", when="^mpich")
+
     generator("ninja")
     depends_on("ninja@1.10:", type="build")
 
@@ -124,13 +138,9 @@ def cmake_args(self):
         if self.spec.satisfies("+cuda"):
             cuda_arch = self.spec.variants["cuda_arch"].value[0]
 
-            gpu_map = {"35": "K40", "37": "K80", "60": "P100", "70": "V100"}
-
-            if "@2.3:" in spec:
-                gpu_map["80"] = "A100"
+            gpu_map = {"35": "K40", "37": "K80", "60": "P100", "70": "V100", "80": "A100"}
 
             gpuver = gpu_map[cuda_arch]
-
             if cuda_arch == "35" and self.spec.satisfies("+cuda_arch_35_k20x"):
                 gpuver = "K20X"
 
@@ -138,14 +148,22 @@ def cmake_args(self):
 
         if self.spec.satisfies("+rocm"):
             amd_arch = self.spec.variants["amdgpu_target"].value[0]
-
-            gpuver = {"gfx906": "Mi50"}[amd_arch]
+            gpuver = {
+                "gfx906": "Mi50",
+                "gfx908": "Mi100",
+                "gfx90a": "Mi250",
+                "gfx90a:xnack-": "Mi250",
+                "gfx90a:xnack+": "Mi250",
+            }[amd_arch]
 
             args += ["-DWITH_GPU={0}".format(gpuver), "-DUSE_ACCEL=hip"]
 
         if self.spec.satisfies("+opencl"):
             args += ["-DUSE_ACCEL=opencl"]
 
+        if self.spec.satisfies("+mpi_f08"):
+            args += ["-DUSE_MPI_F08=ON"]
+
         return args
 
     def check(self):
diff --git a/var/spack/repos/builtin/packages/dbus/package.py b/var/spack/repos/builtin/packages/dbus/package.py
index 69cda7b4771ddd..37a1b8a694b907 100644
--- a/var/spack/repos/builtin/packages/dbus/package.py
+++ b/var/spack/repos/builtin/packages/dbus/package.py
@@ -29,6 +29,7 @@ class Dbus(AutotoolsPackage):
     version("1.8.2", sha256="5689f7411165adc953f37974e276a3028db94447c76e8dd92efe910c6d3bae08")
 
     variant("xml_docs", default=False, description="Build XML documentation")
+    variant("system-socket", default="default", description="Location for the DBus system socket")
 
     depends_on("pkgconfig", type="build")
     depends_on("docbook-xml", type="build")
@@ -41,6 +42,9 @@ class Dbus(AutotoolsPackage):
     def configure_args(self):
         args = ["--disable-systemd", "--disable-launchd"]
         args += self.enable_or_disable("xml-docs", variant="xml_docs")
+        socket = self.spec.variants["system-socket"].value
+        if socket != "default":
+            args += ["--with-system-socket={0}".format(socket)]
         return args
 
     @run_after("install")
diff --git a/var/spack/repos/builtin/packages/dcmtk/package.py b/var/spack/repos/builtin/packages/dcmtk/package.py
index 64efc612c96e40..9a9c1ce98d116a 100644
--- a/var/spack/repos/builtin/packages/dcmtk/package.py
+++ b/var/spack/repos/builtin/packages/dcmtk/package.py
@@ -13,6 +13,7 @@ class Dcmtk(CMakePackage):
     homepage = "https://dicom.offis.de"
     url = "https://github.com/DCMTK/dcmtk/archive/DCMTK-3.6.3.tar.gz"
 
+    version("3.6.7", sha256="17705dcdb2047d1266bb4e92dbf4aa6d4967819e8e3e94f39b7df697661b4860")
     version("3.6.6", sha256="117097da6d50ddbad0e48bb1e6cdc61468e82ba1d32001dd8e2366b445133a8c")
     version("3.6.5", sha256="37dad355d5513b4de4a86b5b7b0c3e9ec059860d88781b80916bba2a04e6d5b8")
     version("3.6.4", sha256="e4b1de804a3fef38fe8cb9edd00262c3cbbd114b305511c14479dd888a9337d2")
@@ -27,7 +28,7 @@ class Dcmtk(CMakePackage):
         default=True,
         description="Support 'Deflated Explicit VR Little Endian' Transfer Syntax",
     )
-    depends_on("zlib", type=("build", "link"), when="+zlib")
+    depends_on("zlib-api", type=("build", "link"), when="+zlib")
 
     variant("tiff", default=True, description="Support for TIFF output")
     depends_on("libtiff", type=("build", "link"), when="+tiff")
@@ -41,9 +42,12 @@ class Dcmtk(CMakePackage):
     variant("iconv", default=True, description="Charset conversion support (iconv)")
     depends_on("iconv", type=("build", "link"))
 
+    variant("pic", default=False, description="Produce position-independent code")
     variant("cxx11", default=False, description="Enable c++11 features")
     variant("stl", default=True, description="Use native STL implementation")
 
+    conflicts("platform=darwin target=aarch64:", when="@:3.6.6")
+
     def patch(self):
         # Backport 3.6.4
         if self.spec.satisfies("@:3.6.3 %fj"):
@@ -55,12 +59,15 @@ def patch(self):
             )
 
     def cmake_args(self):
-        args = ["-DDCMTK_WITH_OPENSSL={0}".format("ON" if "+ssl" in self.spec else "OFF")]
-        args += ["-DDCMTK_WITH_ZLIB={0}".format("ON" if "+zlib" in self.spec else "OFF")]
-        args += ["-DDCMTK_WITH_TIFF={0}".format("ON" if "+tiff" in self.spec else "OFF")]
-        args += ["-DDCMTK_WITH_PNG={0}".format("ON" if "+png" in self.spec else "OFF")]
-        args += ["-DDCMTK_WITH_XML={0}".format("ON" if "+xml" in self.spec else "OFF")]
-        args += ["-DDCMTK_WITH_ICONV={0}".format("ON" if "+iconv" in self.spec else "OFF")]
-        args += ["-DDCMTK_ENABLE_CXX11={0}".format("ON" if "+cxx11" in self.spec else "OFF")]
-        args += ["-DDCMTK_ENABLE_STL={0}".format("ON" if "+stl" in self.spec else "OFF")]
+        args = [
+            self.define_from_variant("DCMTK_WITH_OPENSSL", "ssl"),
+            self.define_from_variant("DCMTK_WITH_ZLIB", "zlib"),
+            self.define_from_variant("DCMTK_WITH_TIFF", "tiff"),
+            self.define_from_variant("DCMTK_WITH_PNG", "png"),
+            self.define_from_variant("DCMTK_WITH_XML", "xml"),
+            self.define_from_variant("DCMTK_WITH_ICONV", "iconv"),
+            self.define_from_variant("DCMTK_ENABLE_CXX11", "cxx11"),
+            self.define_from_variant("DCMTK_ENABLE_STL", "stl"),
+            self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
+        ]
         return args
diff --git a/var/spack/repos/builtin/packages/dd4hep/package.py b/var/spack/repos/builtin/packages/dd4hep/package.py
index 7589feabf97b2d..77c3934bdff7fd 100644
--- a/var/spack/repos/builtin/packages/dd4hep/package.py
+++ b/var/spack/repos/builtin/packages/dd4hep/package.py
@@ -24,6 +24,8 @@ class Dd4hep(CMakePackage):
     tags = ["hep"]
 
     version("master", branch="master")
+    version("1.27", sha256="51fbd0f91f2511261d9b01e4b3528c658bea1ea1b5d67b25b6812615e782a902")
+    version("1.26", sha256="de2cc8d8e99217e23fdf0a55b879d3fd3a864690d6660e7808f1ff99eb47f384")
     version("1.25.1", sha256="6267e76c74fbb346aa881bc44de84434ebe788573f2997a189996252fc5b271b")
     version("1.25", sha256="102a049166a95c2f24fc1c03395a819fc4501c175bf7915d69ccc660468d094d")
     version("1.24", sha256="361a932b9af2479458c0759281fef0161439d8bd119da426ce462a0467adc679")
@@ -131,6 +133,12 @@ class Dd4hep(CMakePackage):
     # variants for other build options
     variant("xercesc", default=False, description="Enable 'Detector Builders' based on XercesC")
     variant("hepmc3", default=False, description="Enable build with hepmc3")
+    variant(
+        "hepmc3-gz",
+        default=False,
+        description="Enable build with compressed hepmc3",
+        when="@1.26: +hepmc3",
+    )
     variant("lcio", default=False, description="Enable build with lcio")
     variant("edm4hep", default=True, description="Enable build with edm4hep")
     variant("geant4units", default=False, description="Use geant4 units throughout")
@@ -143,6 +151,7 @@ class Dd4hep(CMakePackage):
     )
 
     depends_on("cmake @3.12:", type="build")
+    depends_on("cmake @3.14:", type="build", when="@1.26:")
     depends_on("boost @1.49:")
     depends_on("boost +iostreams", when="+ddg4")
     depends_on("boost +system +filesystem", when="%gcc@:7")
@@ -158,6 +167,10 @@ class Dd4hep(CMakePackage):
     depends_on("geant4@10.2.2:", when="+ddg4")
     depends_on("assimp@5.0.2:", when="+ddcad")
     depends_on("hepmc3", when="+hepmc3")
+    depends_on("hepmc3@3.2.6:", when="+hepmc3-gz")
+    depends_on("bzip2", when="+hepmc3-gz")
+    depends_on("xz", when="+hepmc3-gz")
+    depends_on("zlib-api", when="+hepmc3-gz")
     depends_on("tbb", when="+tbb")
     depends_on("intel-tbb@:2020.3", when="+tbb @:1.23")
     depends_on("lcio", when="+lcio")
@@ -165,6 +178,7 @@ class Dd4hep(CMakePackage):
     depends_on("podio", when="+edm4hep")
     depends_on("podio@:0.16.03", when="@:1.23 +edm4hep")
     depends_on("podio@0.16:", when="@1.24: +edm4hep")
+    depends_on("podio@0.16.3:", when="@1.26: +edm4hep")
     depends_on("py-pytest", type=("build", "test"))
 
     # See https://github.com/AIDASoft/DD4hep/pull/771 and https://github.com/AIDASoft/DD4hep/pull/876
@@ -175,6 +189,13 @@ class Dd4hep(CMakePackage):
     )
     conflicts("~ddrec+dddetectors", msg="Need to enable +ddrec to build +dddetectors.")
 
+    @property
+    def libs(self):
+        # We need to override libs here, because we don't build a libdd4hep so
+        # the default discovery fails. All libraries that are built by DD4hep
+        # start with libDD
+        return find_libraries("libDD*", root=self.prefix, shared=True, recursive=True)
+
     def cmake_args(self):
         spec = self.spec
         cxxstd = spec["root"].variants["cxxstd"].value
@@ -190,6 +211,10 @@ def cmake_args(self):
             self.define_from_variant("DD4HEP_USE_HEPMC3", "hepmc3"),
             self.define_from_variant("DD4HEP_USE_GEANT4_UNITS", "geant4units"),
             self.define_from_variant("DD4HEP_BUILD_DEBUG", "debug"),
+            # DD4hep@1.26: with hepmc3@3.2.6: allows compressed hepmc3 files
+            self.define(
+                "DD4HEP_HEPMC3_COMPRESSION_SUPPORT", self.spec.satisfies("@1.26: ^hepmc3@3.2.6:")
+            ),
             # Downloads assimp from github and builds it on the fly.
             # However, with spack it is preferrable to have a proper external
             # dependency, so we disable it.
@@ -229,8 +254,7 @@ def setup_run_environment(self, env):
         env.set("DD4HEP", self.prefix.examples)
         env.set("DD4hep_DIR", self.prefix)
         env.set("DD4hep_ROOT", self.prefix)
-        env.set("LD_LIBRARY_PATH", self.prefix.lib)
-        env.set("LD_LIBRARY_PATH", self.prefix.lib64)
+        env.prepend_path("LD_LIBRARY_PATH", self.libs.directories[0])
 
     def url_for_version(self, version):
         # dd4hep releases are dashes and padded with a leading zero
diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py
index 93723cd29f3d0e..df6f514134a799 100644
--- a/var/spack/repos/builtin/packages/dealii/package.py
+++ b/var/spack/repos/builtin/packages/dealii/package.py
@@ -152,7 +152,7 @@ class Dealii(CMakePackage, CudaPackage):
     depends_on(Boost.with_default_variants)
     depends_on("lapack")
     depends_on("suite-sparse")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # Optional dependencies: Configuration
     depends_on("cuda@8:", when="+cuda")
@@ -361,9 +361,7 @@ class Dealii(CMakePackage, CudaPackage):
         )
 
     # Optional dependencies:
-    conflicts(
-        "+adol-c", when="^netcdf", msg="Symbol clash between the ADOL-C library and " "Netcdf."
-    )
+    conflicts("+adol-c", when="+netcdf", msg="Symbol clash between the ADOL-C library and Netcdf.")
     conflicts(
         "+adol-c",
         when="^trilinos+chaco",
@@ -414,7 +412,7 @@ def cmake_args(self):
                 self.define("LAPACK_INCLUDE_DIRS", ";".join(lapack_blas_headers.directories)),
                 self.define("LAPACK_LIBRARIES", lapack_blas_libs.joined(";")),
                 self.define("UMFPACK_DIR", spec["suite-sparse"].prefix),
-                self.define("ZLIB_DIR", spec["zlib"].prefix),
+                self.define("ZLIB_DIR", spec["zlib-api"].prefix),
                 self.define("DEAL_II_ALLOW_BUNDLED", False),
             ]
         )
diff --git a/var/spack/repos/builtin/packages/delly2/package.py b/var/spack/repos/builtin/packages/delly2/package.py
index 8c9b3cd7247f3f..1005d5eddc437d 100644
--- a/var/spack/repos/builtin/packages/delly2/package.py
+++ b/var/spack/repos/builtin/packages/delly2/package.py
@@ -18,7 +18,7 @@ class Delly2(MakefilePackage):
     maintainers("snehring")
 
     version("1.1.6", sha256="08961e9c81431eb486476fa71eea94941ad24ec1970b71e5a7720623a39bfd2a")
-    version("0.9.1", tag="v0.9.1")
+    version("0.9.1", tag="v0.9.1", commit="ef1cd626a85cfd1c1b7acfca2b5fd5957f2a05f1")
     version("2017-08-03", commit="e32a9cd55c7e3df5a6ae4a91f31a0deb354529fc", deprecated=True)
 
     variant("openmp", default=False, description="Build with openmp support")
diff --git a/var/spack/repos/builtin/packages/denovogear/package.py b/var/spack/repos/builtin/packages/denovogear/package.py
index 57c6d435f0e790..63860f42205c32 100644
--- a/var/spack/repos/builtin/packages/denovogear/package.py
+++ b/var/spack/repos/builtin/packages/denovogear/package.py
@@ -26,7 +26,7 @@ class Denovogear(CMakePackage):
     )
     depends_on("htslib@1.2:", type=("build"))
     depends_on("eigen", type=("build"))
-    depends_on("zlib", type=("link"))
+    depends_on("zlib-api", type=("link"))
 
     patch("stream-open.patch", when="@:1.1.1")
     # fix: ordered comparison between pointer and zero.
diff --git a/var/spack/repos/builtin/packages/diamond/package.py b/var/spack/repos/builtin/packages/diamond/package.py
index f0fd7aa29141b7..a97a341eb17600 100644
--- a/var/spack/repos/builtin/packages/diamond/package.py
+++ b/var/spack/repos/builtin/packages/diamond/package.py
@@ -32,7 +32,7 @@ class Diamond(CMakePackage):
     version("0.8.38", sha256="582a7932f3aa73b0eac2275dd773818665f0b067b32a79ff5a13b0e3ca375f60")
     version("0.8.26", sha256="00d2be32dad76511a767ab8e917962c0ecc572bc808080be60dec028df45439f")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     conflicts("target=aarch64:", when="@:0.9.25")
 
diff --git a/var/spack/repos/builtin/packages/digitrounding/package.py b/var/spack/repos/builtin/packages/digitrounding/package.py
index 5360a31489f6a8..63c9632f0b2eeb 100644
--- a/var/spack/repos/builtin/packages/digitrounding/package.py
+++ b/var/spack/repos/builtin/packages/digitrounding/package.py
@@ -17,7 +17,7 @@ class Digitrounding(CMakePackage):
     version("master", branch="master")
     version("2020-02-27", commit="7b18679aded7a85e6f221f7f5cd4f080f322bc33")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     variant("shared", default=True, description="build shared libraries")
 
diff --git a/var/spack/repos/builtin/packages/dire/package.py b/var/spack/repos/builtin/packages/dire/package.py
index 152df2776bb473..f2a76e88460ce8 100644
--- a/var/spack/repos/builtin/packages/dire/package.py
+++ b/var/spack/repos/builtin/packages/dire/package.py
@@ -23,7 +23,7 @@ class Dire(Package):
 
     version("2.004", sha256="8cc1213b58fec744fdaa50834560a14b141de99efb2c3e3d3d47f3d6d84b179f")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # TODO: replace this with an explicit list of components of Boost,
     # for instance depends_on('boost +filesystem')
@@ -33,7 +33,7 @@ class Dire(Package):
     depends_on("hepmc")
     depends_on("pythia8@8.226:")
 
-    conflicts("pythia8@8.301:", msg="Dire is included in Pythia8 since version 8.301")
+    conflicts("^pythia8@8.301:", msg="Dire is included in Pythia8 since version 8.301")
 
     def install(self, spec, prefix):
         configure_args = ["--prefix={0}".format(prefix)]
diff --git a/var/spack/repos/builtin/packages/discotec/package.py b/var/spack/repos/builtin/packages/discotec/package.py
new file mode 100644
index 00000000000000..7693f97c83c805
--- /dev/null
+++ b/var/spack/repos/builtin/packages/discotec/package.py
@@ -0,0 +1,53 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Discotec(CMakePackage):
+    """This project contains DisCoTec, a code for the distributed sparse
+    grid combination technique with MPI parallelization."""
+
+    homepage = "https://github.com/SGpp/DisCoTec"
+    url = "https://github.com/SGpp/DisCoTec"
+    git = "https://github.com/SGpp/DisCoTec"
+
+    maintainers("freifrauvonbleifrei", "pfluegdk")
+
+    version("main", branch="main")
+
+    variant("ft", default=False, description="DisCoTec with algorithm-based fault tolerance")
+    variant("gene", default=False, description="Build for GENE (as task library)")
+    variant("hdf5", default=True, description="Interpolation output with HDF5")
+    variant("lto", default=True, description="Build with link-time optimization")
+    variant("openmp", default=True, description="Parallelize with OpenMP")
+    variant("timing", default=True, description="With high-res timers")
+    variant("selalib", default=False, description="Build selalib example")
+    variant("vtk", default=False, description="Build with VTK support")
+
+    depends_on("boost +test +serialization +filesystem +system +program_options +date_time")
+    depends_on("cmake@3.24.2:", type="build")
+    depends_on("glpk")
+    depends_on("highfive+mpi+boost+ipo", when="+hdf5")
+    depends_on("mpi")
+    depends_on("selalib", when="+selalib")
+    depends_on("vtk", when="+vtk")
+
+    def cmake_args(self):
+        args = [
+            self.define("DISCOTEC_BUILD_MISSING_DEPS", False),
+            self.define_from_variant("DISCOTEC_ENABLEFT", "ft"),
+            self.define_from_variant("DISCOTEC_GENE", "gene"),
+            self.define_from_variant("DISCOTEC_OPENMP", "openmp"),
+            self.define_from_variant("DISCOTEC_TIMING", "timing"),
+            self.define_from_variant("DISCOTEC_USE_HIGHFIVE", "hdf5"),
+            self.define_from_variant("DISCOTEC_USE_LTO", "lto"),
+            self.define_from_variant("DISCOTEC_USE_VTK", "vtk"),
+            self.define_from_variant("DISCOTEC_WITH_SELALIB", "selalib"),
+        ]
+        if "+selalib" in self.spec:
+            args.append(self.define("SELALIB_DIR", self.spec["selalib"].prefix.cmake))
+
+        return args
diff --git a/var/spack/repos/builtin/packages/dla-future/package.py b/var/spack/repos/builtin/packages/dla-future/package.py
index c08736e85fe59f..8c0590d9f7b949 100644
--- a/var/spack/repos/builtin/packages/dla-future/package.py
+++ b/var/spack/repos/builtin/packages/dla-future/package.py
@@ -14,27 +14,51 @@ class DlaFuture(CMakePackage, CudaPackage, ROCmPackage):
     git = "https://github.com/eth-cscs/DLA-Future.git"
     maintainers = ["rasolca", "albestro", "msimberg", "aurianer"]
 
+    version("0.2.1", sha256="4c2669d58f041304bd618a9d69d9879a42e6366612c2fc932df3894d0326b7fe")
+    version("0.2.0", sha256="da73cbd1b88287c86d84b1045a05406b742be924e65c52588bbff200abd81a10")
     version("0.1.0", sha256="f7ffcde22edabb3dc24a624e2888f98829ee526da384cd752b2b271c731ca9b1")
     version("master", branch="master")
 
     variant("shared", default=True, description="Build shared libraries.")
 
+    variant(
+        "hdf5",
+        default=False,
+        when="@0.2.0:",
+        description="HDF5 support for dealing with matrices on disk.",
+    )
+
     variant("doc", default=False, description="Build documentation.")
 
     variant("miniapps", default=False, description="Build miniapps.")
 
+    variant(
+        "scalapack",
+        default=False,
+        when="@0.2.0:",
+        description="Build C API compatible with ScaLAPACK",
+    )
+
     depends_on("cmake@3.22:", type="build")
     depends_on("doxygen", type="build", when="+doc")
     depends_on("mpi")
     depends_on("blaspp@2022.05.00:")
     depends_on("lapackpp@2022.05.00:")
 
+    depends_on("blas")
+    depends_on("lapack")
+    depends_on("scalapack", when="+scalapack")
+
     depends_on("umpire~examples")
+    depends_on("umpire~cuda", when="~cuda")
+    depends_on("umpire~rocm", when="~rocm")
     depends_on("umpire+cuda~shared", when="+cuda")
     depends_on("umpire+rocm~shared", when="+rocm")
     depends_on("umpire@4.1.0:")
 
-    depends_on("pika@0.15.1:")
+    depends_on("pika@0.15.1:", when="@0.1")
+    depends_on("pika@0.16:", when="@0.2.0")
+    depends_on("pika@0.17:", when="@0.2.1:")
     depends_on("pika-algorithms@0.1:")
     depends_on("pika +mpi")
     depends_on("pika +cuda", when="+cuda")
@@ -50,6 +74,8 @@ class DlaFuture(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("rocsolver", when="+rocm")
     depends_on("rocthrust", when="+rocm")
 
+    depends_on("hdf5 +cxx+mpi+threadsafe+shared", when="+hdf5")
+
     conflicts("+cuda", when="+rocm")
 
     with when("+rocm"):
@@ -84,22 +110,35 @@ def cmake_args(self):
         args.append(self.define_from_variant("BUILD_SHARED_LIBS", "shared"))
 
         # BLAS/LAPACK
-        if "^mkl" in spec:
+        if self.spec["lapack"].name in INTEL_MATH_LIBRARIES:
             vmap = {
                 "none": "seq",
                 "openmp": "omp",
                 "tbb": "tbb",
             }  # Map MKL variants to LAPACK target name
+            mkl_threads = vmap[spec["intel-mkl"].variants["threads"].value]
             # TODO: Generalise for intel-oneapi-mkl
             args += [
                 self.define("DLAF_WITH_MKL", True),
-                self.define(
-                    "MKL_LAPACK_TARGET",
-                    "mkl::mkl_intel_32bit_{0}_dyn".format(
-                        vmap[spec["intel-mkl"].variants["threads"].value]
-                    ),
-                ),
+                self.define("MKL_LAPACK_TARGET", f"mkl::mkl_intel_32bit_{mkl_threads}_dyn"),
             ]
+            if "+scalapack" in spec:
+                if (
+                    "^mpich" in spec
+                    or "^cray-mpich" in spec
+                    or "^intel-mpi" in spec
+                    or "^mvapich" in spec
+                    or "^mvapich2" in spec
+                ):
+                    mkl_mpi = "mpich"
+                elif "^openmpi" in spec:
+                    mkl_mpi = "ompi"
+                args.append(
+                    self.define(
+                        "MKL_SCALAPACK_TARGET",
+                        f"mkl::scalapack_{mkl_mpi}_intel_32bit_{mkl_threads}_dyn",
+                    )
+                )
         else:
             args.append(self.define("DLAF_WITH_MKL", False))
             args.append(
@@ -108,6 +147,10 @@ def cmake_args(self):
                     " ".join([spec[dep].libs.ld_flags for dep in ["blas", "lapack"]]),
                 )
             )
+            if "+scalapack" in spec:
+                args.append(self.define("SCALAPACK_LIBRARY", spec["scalapack"].libs.ld_flags))
+
+        args.append(self.define_from_variant("DLAF_WITH_SCALAPACK", "scalapack"))
 
         # CUDA/HIP
         args.append(self.define_from_variant("DLAF_WITH_CUDA", "cuda"))
@@ -123,6 +166,9 @@ def cmake_args(self):
                 arch_str = ";".join(archs)
                 args.append(self.define("CMAKE_CUDA_ARCHITECTURES", arch_str))
 
+        # HDF5 support
+        args.append(self.define_from_variant("DLAF_WITH_HDF5", "hdf5"))
+
         # DOC
         args.append(self.define_from_variant("DLAF_BUILD_DOC", "doc"))
 
diff --git a/var/spack/repos/builtin/packages/dlib/package.py b/var/spack/repos/builtin/packages/dlib/package.py
index 7c8f1a634b9bed..0100a2379fc756 100644
--- a/var/spack/repos/builtin/packages/dlib/package.py
+++ b/var/spack/repos/builtin/packages/dlib/package.py
@@ -23,7 +23,7 @@ class Dlib(CMakePackage):
 
     variant("shared", default=True, description="build the shared libraries")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libpng")
     depends_on("libjpeg")
     depends_on("blas")
diff --git a/var/spack/repos/builtin/packages/duckdb/package.py b/var/spack/repos/builtin/packages/duckdb/package.py
index a7585b9c12d652..aae9bca08b7e01 100644
--- a/var/spack/repos/builtin/packages/duckdb/package.py
+++ b/var/spack/repos/builtin/packages/duckdb/package.py
@@ -11,8 +11,9 @@ class Duckdb(CMakePackage):
     """DuckDB is an in-process SQL OLAP Database Management System."""
 
     homepage = "https://duckdb.org"
-    url = "https://github.com/duckdb/duckdb/archive/refs/tags/v0.7.1.tar.gz"
+    url = "https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz"
     git = "https://github.com/duckdb/duckdb.git"
 
     version("master", branch="master")
+    version("0.8.1", sha256="a0674f7e320dc7ebcf51990d7fc1c0e7f7b2c335c08f5953702b5285e6c30694")
     version("0.7.1", sha256="67f840f861e5ffbe137d65a8543642d016f900b89dd035492d562ad11acf0e1e")
diff --git a/var/spack/repos/builtin/packages/dxt-explorer/package.py b/var/spack/repos/builtin/packages/dxt-explorer/package.py
index 4f7df14c186af9..90ef64818346c4 100644
--- a/var/spack/repos/builtin/packages/dxt-explorer/package.py
+++ b/var/spack/repos/builtin/packages/dxt-explorer/package.py
@@ -26,5 +26,5 @@ class DxtExplorer(PythonPackage):
 
     depends_on("darshan-util", type=("run"))
 
-    depends_on("python@3.6:", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
     depends_on("py-pandas", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/dyninst/package.py b/var/spack/repos/builtin/packages/dyninst/package.py
index 4bf1c354009ec0..3055f090d53361 100644
--- a/var/spack/repos/builtin/packages/dyninst/package.py
+++ b/var/spack/repos/builtin/packages/dyninst/package.py
@@ -32,11 +32,21 @@ class Dyninst(CMakePackage):
     version("10.2.0", sha256="4212b93bef4563c7de7dce4258e899bcde52315a571087e87fde9f8040123b43")
     version("10.1.0", sha256="4a121d70c1bb020408a7a697d74602e18250c3c85800f230566fcccd593c0129")
     version("10.0.0", sha256="542fccf5c57c4fe784b1a9a9e3db01d40b16ad04e7174dc6f7eb23440485ba06")
-    version("9.3.2", tag="v9.3.2", deprecated=True)
-    version("9.3.0", tag="v9.3.0", deprecated=True)
-    version("9.2.0", tag="v9.2.0", deprecated=True)
-    version("9.1.0", tag="v9.1.0", deprecated=True)
-    version("8.2.1", tag="v8.2.1", deprecated=True)
+    version(
+        "9.3.2", tag="v9.3.2", commit="5d2ddacb273682daa014ae22f17f3575e05b411e", deprecated=True
+    )
+    version(
+        "9.3.0", tag="v9.3.0", commit="9b8e9c1f16d4616b827d2d36955604a8e3fb915c", deprecated=True
+    )
+    version(
+        "9.2.0", tag="v9.2.0", commit="3a6ad66df7294417cf61618acdcfcc0fecccb045", deprecated=True
+    )
+    version(
+        "9.1.0", tag="v9.1.0", commit="df6d090061bae7ff2ba5a6bd57bb2ecbf538ef7a", deprecated=True
+    )
+    version(
+        "8.2.1", tag="v8.2.1", commit="939afcbad1a8273636a3686a31b51dae4f1f0c11", deprecated=True
+    )
 
     variant(
         "openmp",
@@ -73,9 +83,9 @@ class Dyninst(CMakePackage):
 
     # findtbb.cmake in the dynist repo does not work with recent tbb
     # package layout. Need to use tbb provided config instead.
-    conflicts("intel-tbb@2021.1:")
-    conflicts("intel-oneapi-tbb@2021.1:")
-    conflicts("intel-parallel-studio", when="@12.0.0:")
+    conflicts("^intel-tbb@2021.1:")
+    conflicts("^intel-oneapi-tbb@2021.1:")
+    conflicts("^intel-parallel-studio", when="@12.0.0:")
     depends_on("tbb@2018.6.0:", when="@10.0.0:")
 
     depends_on("cmake@3.4.0:", type="build", when="@10.1.0:")
diff --git a/var/spack/repos/builtin/packages/e3sm-scorpio/package.py b/var/spack/repos/builtin/packages/e3sm-scorpio/package.py
index 32375cfd4a093e..c9958c35bd1a5b 100644
--- a/var/spack/repos/builtin/packages/e3sm-scorpio/package.py
+++ b/var/spack/repos/builtin/packages/e3sm-scorpio/package.py
@@ -16,8 +16,8 @@ class E3smScorpio(CMakePackage):
 
     version("1.4.1", sha256="7cb4589410080d7e547ef17ddabe68f749e6af019c1d0e6ee9f11554f3ff6b1a")
 
-    variant("timing", default="False", description="Enable timing")
-    variant("mpi", default="True", description="Enable MPI")
+    variant("timing", default=False, description="Enable timing")
+    variant("mpi", default=True, description="Enable MPI")
 
     depends_on("gptl", when="+timing")
     depends_on("mpi", when="+mpi")
diff --git a/var/spack/repos/builtin/packages/ea-utils/package.py b/var/spack/repos/builtin/packages/ea-utils/package.py
index b18e016fee7b62..afd6611dd20232 100644
--- a/var/spack/repos/builtin/packages/ea-utils/package.py
+++ b/var/spack/repos/builtin/packages/ea-utils/package.py
@@ -26,7 +26,7 @@ class EaUtils(MakefilePackage):
     )
 
     depends_on("sparsehash")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("gsl")
     depends_on("bamtools")
     depends_on("perl", type=["build", "run"])
diff --git a/var/spack/repos/builtin/packages/eagle/package.py b/var/spack/repos/builtin/packages/eagle/package.py
index 9353da9042b741..3070c64439bf6d 100644
--- a/var/spack/repos/builtin/packages/eagle/package.py
+++ b/var/spack/repos/builtin/packages/eagle/package.py
@@ -17,7 +17,7 @@ class Eagle(MakefilePackage):
     version("1.1.2", sha256="afe967560d1f8fdbd0caf4b93b5f2a86830e9e4d399fee4a526140431343045e")
 
     depends_on("curl")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("lzma")
     depends_on("htslib")
 
diff --git a/var/spack/repos/builtin/packages/easi/package.py b/var/spack/repos/builtin/packages/easi/package.py
index 37d60dc448a457..a6d8b3ae32152a 100644
--- a/var/spack/repos/builtin/packages/easi/package.py
+++ b/var/spack/repos/builtin/packages/easi/package.py
@@ -17,8 +17,8 @@ class Easi(CMakePackage):
     maintainers("ravil-mobile", "Thomas-Ulrich", "krenzland", "ThrudPrimrose")
 
     version("develop", branch="master")
-    version("1.2.0", tag="v1.2.0")
-    version("1.1.2", tag="v1.1.2")
+    version("1.2.0", tag="v1.2.0", commit="305a119338116a0ceac6b68b36841a50250d05b1")
+    version("1.1.2", tag="v1.1.2", commit="4c87ef3b3dca9415d116ef102cb8de750ef7e1a0")
 
     variant("asagi", default=True, description="build with ASAGI support")
     variant(
diff --git a/var/spack/repos/builtin/packages/eccodes/package.py b/var/spack/repos/builtin/packages/eccodes/package.py
index aa6d1b97067824..36ab4019f077dd 100644
--- a/var/spack/repos/builtin/packages/eccodes/package.py
+++ b/var/spack/repos/builtin/packages/eccodes/package.py
@@ -82,17 +82,9 @@ class Eccodes(CMakePackage):
     variant("shared", default=True, description="Build shared versions of the libraries")
 
     variant(
-        "definitions",
-        values=disjoint_sets(("auto",), ("default",) + tuple(_definitions.keys())).with_default(
-            "auto"
-        ),
-        description="List of definitions to install",
-    )
-
-    variant(
-        "samples",
-        values=disjoint_sets(("auto",), ("default",)).with_default("auto"),
-        description="List of samples to install",
+        "extra_definitions",
+        values=any_combination_of(*_definitions.keys()),
+        description="List of extra definitions to install",
     )
 
     depends_on("netcdf-c", when="+netcdf")
@@ -133,7 +125,7 @@ class Eccodes(CMakePackage):
     for center, definitions in _definitions.items():
         kwargs = definitions.get("conflicts", None)
         if kwargs:
-            conflicts("definitions={0}".format(center), **kwargs)
+            conflicts("extra_definitions={0}".format(center), **kwargs)
         for kwargs in definitions.get("resources", []):
             resource(
                 name=center,
@@ -333,48 +325,34 @@ def cmake_args(self):
         ]
 
         if "+netcdf" in self.spec:
-            args.extend(
-                [
-                    # Prevent possible overriding by environment variables
-                    # NETCDF_ROOT, NETCDF_DIR, and NETCDF_PATH:
-                    self.define("NETCDF_PATH", self.spec["netcdf-c"].prefix),
-                    # Prevent overriding by environment variable HDF5_ROOT:
-                    self.define("HDF5_ROOT", self.spec["hdf5"].prefix),
-                ]
-            )
+            # Prevent possible overriding by environment variables NETCDF_ROOT, NETCDF_DIR, and
+            # NETCDF_PATH:
+            args.append(self.define("NETCDF_PATH", self.spec["netcdf-c"].prefix))
+            # Prevent overriding by environment variable HDF5_ROOT (starting version 2.14.0,
+            # ecCodes is shipped with ecBuild 3.1.0+, which does not seem to rely on the HDF5_ROOT
+            # variable):
+            if self.spec.satisfies("@:2.13"):
+                args.append(self.define("HDF5_ROOT", self.spec["hdf5"].prefix))
 
         if jp2k == "openjpeg":
             args.append(self.define("OPENJPEG_PATH", self.spec["openjpeg"].prefix))
 
         if "+png" in self.spec:
-            args.append(self.define("ZLIB_ROOT", self.spec["zlib"].prefix))
+            args.append(self.define("ZLIB_ROOT", self.spec["zlib-api"].prefix))
 
         if "+aec" in self.spec:
             # Prevent overriding by environment variables AEC_DIR and AEC_PATH:
             args.append(self.define("AEC_DIR", self.spec["libaec"].prefix))
 
-        if "^python" in self.spec:
+        if "+memfs" in self.spec:
             args.append(self.define("PYTHON_EXECUTABLE", python.path))
 
-        definitions = self.spec.variants["definitions"].value
-
-        if "auto" not in definitions:
-            args.append(
-                self.define("ENABLE_INSTALL_ECCODES_DEFINITIONS", "default" in definitions)
-            )
-
-        samples = self.spec.variants["samples"].value
-
-        if "auto" not in samples:
-            args.append(self.define("ENABLE_INSTALL_ECCODES_SAMPLES", "default" in samples))
-
         return args
 
     @run_after("install")
     def install_extra_definitions(self):
-        noop = set(["auto", "none", "default"])
-        for center in self.spec.variants["definitions"].value:
-            if center not in noop:
+        for center in self.spec.variants["extra_definitions"].value:
+            if center != "none":
                 center_dir = "definitions.{0}".format(center)
                 install_tree(
                     join_path(self.stage.source_path, "spack-definitions", center_dir),
diff --git a/var/spack/repos/builtin/packages/ecflow/package.py b/var/spack/repos/builtin/packages/ecflow/package.py
index ea20062825d8a0..7fa77e386a0e2d 100644
--- a/var/spack/repos/builtin/packages/ecflow/package.py
+++ b/var/spack/repos/builtin/packages/ecflow/package.py
@@ -21,7 +21,7 @@ class Ecflow(CMakePackage):
     homepage = "https://confluence.ecmwf.int/display/ECFLOW/"
     url = "https://confluence.ecmwf.int/download/attachments/8650755/ecFlow-4.11.1-Source.tar.gz"
 
-    maintainers("climbfuji")
+    maintainers("climbfuji", "AlexanderRichert-NOAA")
 
     version("5.11.4", sha256="4836a876277c9a65a47a3dc87cae116c3009699f8a25bab4e3afabf160bcf212")
     version("5.8.4", sha256="bc628556f8458c269a309e4c3b8d5a807fae7dfd415e27416fe9a3f544f88951")
@@ -63,6 +63,7 @@ class Ecflow(CMakePackage):
     )
 
     depends_on("openssl@1:", when="@5:")
+    depends_on("pkgconfig", type="build", when="+ssl ^openssl ~shared")
     depends_on("qt@5:", when="+ui")
     # Requirement to use the Python3_EXECUTABLE variable
     depends_on("cmake@3.16:", type="build")
@@ -96,9 +97,8 @@ def cmake_args(self):
         ]
 
         if spec.satisfies("+ssl ^openssl ~shared"):
-            ssl_libs = [os.path.join(spec["openssl"].prefix.lib, "libcrypto.a")]
-            ssl_libs.extend(spec["zlib"].libs)
-            args.append(self.define("OPENSSL_CRYPTO_LIBRARY", ";".join(ssl_libs)))
+            ssllibs = ";".join(spec["openssl"].libs + spec["zlib"].libs)
+            args.append(self.define("OPENSSL_CRYPTO_LIBRARY", ssllibs))
 
         if self.spec.satisfies("@5.8.3:"):
             args.append("-DCMAKE_CXX_FLAGS=-DBOOST_NO_CXX98_FUNCTION_BASE")
@@ -115,5 +115,12 @@ def remove_recursive_symlink_in_source_code(self):
 
     @when("+ssl ^openssl~shared")
     def patch(self):
+        pkgconf = which("pkg-config")
+        liblist_l = pkgconf("--libs-only-l", "--static", "openssl", output=str).split()
+        liblist = " ".join([ll.replace("-l", "") for ll in liblist_l])
         for sdir in ["Client", "Server"]:
-            filter_file("(target_link_libraries.*pthread)", r"\1 ssl crypto z", os.path.join(sdir, "CMakeLists.txt"))
+            filter_file(
+                "(target_link_libraries.*pthread)",
+                f"\\1 {liblist}",
+                os.path.join(sdir, "CMakeLists.txt"),
+            )
diff --git a/var/spack/repos/builtin/packages/ecmwf-atlas/package.py b/var/spack/repos/builtin/packages/ecmwf-atlas/package.py
index d720ed1ff06277..44cdd7adc2ff62 100644
--- a/var/spack/repos/builtin/packages/ecmwf-atlas/package.py
+++ b/var/spack/repos/builtin/packages/ecmwf-atlas/package.py
@@ -52,9 +52,8 @@ class EcmwfAtlas(CMakePackage):
     variant("openmp", default=True, description="Use OpenMP")
     depends_on("llvm-openmp", when="+openmp %apple-clang", type=("build", "run"))
     variant("shared", default=True, description="Build shared libraries")
-
-    variant("ectrans", default=False, description="Enable ectrans", when="@0.31.0:")
-    depends_on("ectrans@1.1.0:", when="+ectrans")
+    variant("trans", default=False, description="Enable trans")
+    depends_on("ectrans@1.1.0:", when="@0.31.0: +trans")
     variant("eigen", default=True, description="Enable eigen")
     depends_on("eigen", when="+eigen")
     variant("fftw", default=True, description="Enable fftw")
@@ -73,9 +72,9 @@ def cmake_args(self):
             "-DPYTHON_EXECUTABLE:FILEPATH=" + self.spec["python"].command.path,
         ]
         if self.spec.satisfies("@0.31:0.34"):
-            args.append(self.define_from_variant("ENABLE_TRANS", "ectrans"))
+            args.append(self.define_from_variant("ENABLE_TRANS", "trans"))
         if self.spec.satisfies("@0.35:"):
-            args.append(self.define_from_variant("ENABLE_ECTRANS", "ectrans"))
+            args.append(self.define_from_variant("ENABLE_ECTRANS", "trans"))
             args.append(self.define_from_variant("ENABLE_TESSELATION", "tesselation"))
         if "~shared" in self.spec:
             args.append("-DBUILD_SHARED_LIBS=OFF")
diff --git a/var/spack/repos/builtin/packages/ecp-data-vis-sdk/package.py b/var/spack/repos/builtin/packages/ecp-data-vis-sdk/package.py
index fff25128219b99..f23a736569f24a 100644
--- a/var/spack/repos/builtin/packages/ecp-data-vis-sdk/package.py
+++ b/var/spack/repos/builtin/packages/ecp-data-vis-sdk/package.py
@@ -102,7 +102,7 @@ class EcpDataVisSdk(BundlePackage, CudaPackage, ROCmPackage):
     amdgpu_target_variants = ["amdgpu_target={0}".format(x) for x in ROCmPackage.amdgpu_targets]
 
     dav_sdk_depends_on(
-        "adios2+shared+mpi+python+blosc+sst+ssc+dataman",
+        "adios2+shared+mpi+python+sst+dataman",
         when="+adios2",
         propagate=["cuda", "hdf5", "sz", "zfp", "fortran"] + cuda_arch_variants,
     )
@@ -141,7 +141,7 @@ class EcpDataVisSdk(BundlePackage, CudaPackage, ROCmPackage):
     dav_sdk_depends_on("parallel-netcdf+shared", when="+pnetcdf", propagate=["fortran"])
 
     dav_sdk_depends_on("unifyfs", when="+unifyfs ")
-    conflicts("unifyfs@develop")
+    conflicts("^unifyfs@develop")
 
     dav_sdk_depends_on("veloc", when="+veloc")
 
@@ -161,23 +161,22 @@ class EcpDataVisSdk(BundlePackage, CudaPackage, ROCmPackage):
 
     # Need to explicitly turn off conduit hdf5_compat in order to build
     # hdf5@1.12 which is required for SDK
-    depends_on("ascent ^conduit ~hdf5_compat", when="+ascent +hdf5")
+    depends_on("conduit ~hdf5_compat", when="+ascent +hdf5")
     # Disable configuring with @develop. This should be removed after ascent
     # releases 0.8 and ascent can build with conduit@0.8: and vtk-m@1.7:
-    conflicts("ascent@develop", when="+ascent")
+    conflicts("^ascent@develop", when="+ascent")
 
     depends_on("py-cinemasci", when="+cinema")
 
     # ParaView needs @5.11: in order to use CUDA/ROCM, therefore it is the minimum
     # required version since GPU capability is desired for ECP
     dav_sdk_depends_on(
-        "paraview@5.11:+mpi+openpmd+python+kits+shared+catalyst+libcatalyst+raytracing"
-        " use_vtkm=on",
+        "paraview@5.11:+mpi+openpmd+python+kits+shared+catalyst+libcatalyst" " use_vtkm=on",
         when="+paraview",
         propagate=["adios2", "cuda", "hdf5", "rocm"] + amdgpu_target_variants + cuda_arch_variants,
     )
     dav_sdk_depends_on("libcatalyst@2:+mpi", when="+paraview")
-    conflicts("paraview@master", when="+paraview")
+    conflicts("^paraview@master", when="+paraview")
 
     dav_sdk_depends_on("visit+mpi+python+silo", when="+visit", propagate=["hdf5", "adios2"])
 
diff --git a/var/spack/repos/builtin/packages/ecp-proxy-apps/package.py b/var/spack/repos/builtin/packages/ecp-proxy-apps/package.py
index 20572b951adbb0..1427e7a72f9f2f 100644
--- a/var/spack/repos/builtin/packages/ecp-proxy-apps/package.py
+++ b/var/spack/repos/builtin/packages/ecp-proxy-apps/package.py
@@ -43,7 +43,7 @@ class EcpProxyApps(BundlePackage):
     depends_on("laghos@3.0", when="@3.0:")
 
     # Added with release 2.1
-    depends_on("amg@1.2", when="@2.1:")
+    depends_on("amg2013@1.2", when="@2.1:")
     depends_on("miniamr@1.4.3", when="@2.1")
 
     # Added with release 2.0
@@ -60,7 +60,7 @@ class EcpProxyApps(BundlePackage):
     depends_on("xsbench@18", when="@2.0:2.1")
 
     # Dependencies for version 2.0
-    depends_on("amg@1.1", when="@2.0")
+    depends_on("amg2013@1.1", when="@2.0")
     depends_on("miniamr@1.4.1", when="@2.0:2.1")
 
     # Added with release 1.1
@@ -70,7 +70,7 @@ class EcpProxyApps(BundlePackage):
     depends_on("swfft@1.0", when="@1.0:")
 
     # Dependencies for versions 1.0:1.1
-    depends_on("amg@1.0", when="@1.0:1.1")
+    depends_on("amg2013@1.0", when="@1.0:1.1")
     depends_on("candle-benchmarks@0.0", when="+candle @1.0:1.1")
     depends_on("laghos@1.0", when="@1.0:1.1")
     depends_on("macsio@1.0", when="@1.0:1.1")
diff --git a/var/spack/repos/builtin/packages/ectrans/package.py b/var/spack/repos/builtin/packages/ectrans/package.py
index 3b8f7417d5dca0..900aaaf1d1a498 100644
--- a/var/spack/repos/builtin/packages/ectrans/package.py
+++ b/var/spack/repos/builtin/packages/ectrans/package.py
@@ -47,7 +47,7 @@ class Ectrans(CMakePackage):
     depends_on("blas")
     depends_on("lapack")
     # ectrans distinguishes between mkl and fftw
-    depends_on("fftw", when="+fftw")
+    depends_on("fftw-api", when="+fftw")
     depends_on("mkl", when="+mkl")
     conflicts("+mkl", when="+fftw")
 
diff --git a/var/spack/repos/builtin/packages/edm4hep/package.py b/var/spack/repos/builtin/packages/edm4hep/package.py
index af6ea16231997e..b8d4238ce70522 100644
--- a/var/spack/repos/builtin/packages/edm4hep/package.py
+++ b/var/spack/repos/builtin/packages/edm4hep/package.py
@@ -14,18 +14,48 @@ class Edm4hep(CMakePackage):
     url = "https://github.com/key4hep/EDM4hep/archive/v00-01.tar.gz"
     git = "https://github.com/key4hep/EDM4hep.git"
 
-    maintainers("vvolkl", "jmcarcell")
+    maintainers("vvolkl", "jmcarcell", "tmadlener")
 
     tags = ["hep", "key4hep"]
 
     version("master", branch="master")
-    version("0.9", sha256="170ef84822761c4b02da9047f2b4d0dd0f48ed1c027b10171d4207b1542fbd5c")
-    version("0.8", sha256="102d57167885eba3bea79f6b6647e5303ad8732c5784590abdcdd816b2411c79")
-    version("0.7.2", sha256="e289280d5de2c0a3b542bf9dfe04b9f6471b0a0fcf33f5c8101ea7252e2a7643")
-    version("0.7.1", sha256="82e215a532f548a73a6f6094eaa8b436c553994e135f6d63a674543dc89a9f1b")
-    version("0.7", sha256="0cef3f06d86c13e87e3343ac9d5db0b3087c421e8bda4bd2623858acb1af60c9")
-    version("0.6", sha256="625a5a939cb8d7a0a6ab5874a3e076d7dd5338446be3921b0cbc09de4d96b315")
-    version("0.5", sha256="aae4f001412d57585751d858999fe78e004755aa0303a503d503a325ef97d7e0")
+    version("0.10.1", sha256="28a3bd4df899309b14ec0d441f8b6ed0065206a08a0018113bb490e9d008caed")
+    version("0.10", sha256="a95c917c19793cfad6b0959854a653c5ce698c965598cabd649d544da07712c0")
+    version(
+        "0.9",
+        sha256="170ef84822761c4b02da9047f2b4d0dd0f48ed1c027b10171d4207b1542fbd5c",
+        deprecated=True,
+    )
+    version(
+        "0.8",
+        sha256="102d57167885eba3bea79f6b6647e5303ad8732c5784590abdcdd816b2411c79",
+        deprecated=True,
+    )
+    version(
+        "0.7.2",
+        sha256="e289280d5de2c0a3b542bf9dfe04b9f6471b0a0fcf33f5c8101ea7252e2a7643",
+        deprecated=True,
+    )
+    version(
+        "0.7.1",
+        sha256="82e215a532f548a73a6f6094eaa8b436c553994e135f6d63a674543dc89a9f1b",
+        deprecated=True,
+    )
+    version(
+        "0.7",
+        sha256="0cef3f06d86c13e87e3343ac9d5db0b3087c421e8bda4bd2623858acb1af60c9",
+        deprecated=True,
+    )
+    version(
+        "0.6",
+        sha256="625a5a939cb8d7a0a6ab5874a3e076d7dd5338446be3921b0cbc09de4d96b315",
+        deprecated=True,
+    )
+    version(
+        "0.5",
+        sha256="aae4f001412d57585751d858999fe78e004755aa0303a503d503a325ef97d7e0",
+        deprecated=True,
+    )
     version(
         "0.4.2",
         sha256="5f2ff3a14729cbd4da370c7c768c2a09eb9f68f814d61690b1cc99c4248994f4",
@@ -95,6 +125,10 @@ def cmake_args(self):
         args.append(self.define("BUILD_TESTING", self.run_tests))
         return args
 
+    def setup_run_environment(self, env):
+        env.prepend_path("LD_LIBRARY_PATH", self.spec["edm4hep"].libs.directories[0])
+        env.prepend_path("PYTHONPATH", self.prefix.python)
+
     def url_for_version(self, version):
         """Translate version numbers to ilcsoft conventions.
         in spack, the convention is: 0.1 (or 0.1.0) 0.1.1, 0.2, 0.2.1 ...
diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py
index e78d563f78836d..889ca904f259bf 100644
--- a/var/spack/repos/builtin/packages/eigen/package.py
+++ b/var/spack/repos/builtin/packages/eigen/package.py
@@ -43,9 +43,9 @@ class Eigen(CMakePackage):
 
     # there is a bug in 3.3.8 that provokes a compile error in dependent packages, see https://gitlab.com/libeigen/eigen/-/issues/2011
     patch(
-        "https://gitlab.com/libeigen/eigen/-/commit/6d822a1052fc665f06dc51b4729f6a38e0da0546.diff",
+        "https://gitlab.com/libeigen/eigen/-/commit/ef3cc72cb65e2d500459c178c63e349bacfa834f.diff",
         when="@3.3.8",
-        sha256="62590e9b33a8f72b608a72b87147a306e7cb20766ea53c6b8e0a183fa6cb7635",
+        sha256="b8877a84c4338f08ab8a6bb8b274c768e93d36ac05b733b078745198919a74bf",
     )
 
     # there is a bug in 3.3.4 that provokes a compile error with the xl compiler
diff --git a/var/spack/repos/builtin/packages/elbencho/package.py b/var/spack/repos/builtin/packages/elbencho/package.py
new file mode 100644
index 00000000000000..a9417a4dfc135a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/elbencho/package.py
@@ -0,0 +1,71 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+
+
+class Elbencho(MakefilePackage):
+
+    """
+    Elbencho storage benchmark
+    """
+
+    homepage = "https://github.com/breuner/elbencho"
+    url = "https://github.com/breuner/elbencho/archive/refs/tags/v3.0-1.tar.gz"
+
+    maintainers("ethanjjjjjjj")
+
+    version("3.0-1", sha256="19dad85e1fc74419dcdf740f11a47d3f6d566770a06e40976755a3404566c11d")
+    version("2.2-5", sha256="4b598639452665a8b79c4c9d8a22ae63fb9b04057635a45e686aa3939ee255b4")
+    version("2.2-3", sha256="0ae2d495d2863b84f21f55b7c526674fab1be723d0697087017946647f79d0e6")
+    version("2.1-5", sha256="5d2293dcdb766e9194bed964486a10b4c8c308cc1ba8c0044c6e5a2aadd4f199")
+    version("2.1-3", sha256="9d08aa0e83753666cb16a78320dfa5710350879f9f4f1e281dacd69f53249d01")
+    version("2.1-1", sha256="18be49f521df2fab4f16a1a9f00dd6104a25e5ea335ce8801bf07268ed9271a9")
+    version("2.0-9", sha256="fe0f67fbb7dd7c743f8b3e0a92358f7393f2950da456474d4adb38690fab1878")
+    version("2.0-7", sha256="a2e49cb2cf1ae99e46e9fa95b42ece250cb58fbadb4c393f9776b40204e8b2c0")
+
+    variant("s3", default=False, description="Enable support for s3 api")
+    variant("cuda", default=True, description="Enable CUDA support", when="+cufile")
+    variant("cuda", default=False, description="Enable CUDA support")
+    variant("cufile", default=False, description="GPU Direct Storage")
+
+    depends_on("cuda", when="+cuda")
+    depends_on(
+        """
+           boost
+           +filesystem+program_options
+           +thread
+           +system+date_time
+           +regex
+           +serialization
+           +iostreams
+        """
+    )
+    depends_on("ncurses")
+    depends_on("numactl")
+    depends_on("libaio")
+    depends_on("curl", when="+s3")
+    depends_on("libarchive", when="+s3")
+    depends_on("openssl", when="+s3")
+    depends_on("libuuid", when="+s3")
+    depends_on("zlib", when="+s3")
+    depends_on("cmake", when="+s3")
+
+    conflicts("+cufile", when="~cuda")
+
+    def edit(self, spec, prefix):
+        os.mkdir(prefix.bin)
+        os.environ["INST_PATH"] = prefix.bin
+        if "+s3" in spec:
+            os.environ["S3_SUPPORT"] = "1"
+        if "+cuda" in spec:
+            os.environ["CUDA_SUPPORT"] = "1"
+        if "+cufile" in spec:
+            os.environ["CUFILE_SUPPORT"] = "1"
+        makefile = FileFilter("Makefile")
+        makefile.filter(r"\s+/etc/bash_completion.d/", f" {prefix}/etc/bash_completion.d/")
+        makefile.filter(r"-lncurses", "-ltinfo -lncurses")
diff --git a/var/spack/repos/builtin/packages/elfutils/package.py b/var/spack/repos/builtin/packages/elfutils/package.py
index 0d168df75d6199..15f0b11130e7bf 100644
--- a/var/spack/repos/builtin/packages/elfutils/package.py
+++ b/var/spack/repos/builtin/packages/elfutils/package.py
@@ -48,6 +48,7 @@ class Elfutils(AutotoolsPackage, SourcewarePackage):
 
     # Native language support from libintl.
     variant("nls", default=True, description="Enable Native Language Support.")
+    variant("exeprefix", default=True, description="Add a prefix to generated executables.")
 
     # libdebuginfod support
     # NB: For 0.181 and newer, this enables _both_ the client and server
@@ -65,7 +66,7 @@ class Elfutils(AutotoolsPackage, SourcewarePackage):
 
     depends_on("bzip2", type="link")
     depends_on("xz", type="link")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("zstd", type="link", when="@0.182:")
 
     depends_on("gettext", when="+nls")
@@ -84,17 +85,14 @@ class Elfutils(AutotoolsPackage, SourcewarePackage):
 
     provides("elf@1")
 
-    # libarchive with iconv doesn't configure.
+    # libarchive with iconv doesn't configure (still broken as of libarchive@3.7.1)
     # see https://github.com/spack/spack/issues/36710
     # and https://github.com/libarchive/libarchive/issues/1819
-    conflicts("^libarchive@3.6.2 +iconv", when="+debuginfod")
+    conflicts("^libarchive +iconv", when="+debuginfod")
 
-    # Elfutils uses nested functions in C code, which is implemented
-    # in gcc, but not in clang. C code compiled with gcc is
-    # binary-compatible with clang, so it should be possible to build
-    # elfutils with gcc, and then link it to clang-built libraries.
+    # https://sourceware.org/bugzilla/show_bug.cgi?id=24964
     conflicts("%apple-clang")
-    conflicts("%clang")
+    conflicts("%clang", when="@:0.185")
     conflicts("%cce")
 
     # Elfutils uses -Wall and we don't want to fail the build over a
@@ -115,9 +113,14 @@ def configure_args(self):
         args = [
             "--with-bzlib=%s" % spec["bzip2"].prefix,
             "--with-lzma=%s" % spec["xz"].prefix,
-            "--with-zlib=%s" % spec["zlib"].prefix,
+            "--with-zlib=%s" % spec["zlib-api"].prefix,
         ]
 
+        if spec.satisfies("+exeprefix"):
+            args.append("--program-prefix='eu-'")
+        else:
+            args.append("--program-prefix=''")
+
         if "@0.182:" in spec:
             args.append("--with-zstd=%s" % spec["zstd"].prefix)
 
diff --git a/var/spack/repos/builtin/packages/elpa/package.py b/var/spack/repos/builtin/packages/elpa/package.py
index 12f68226d4ca16..30061eb9eff857 100644
--- a/var/spack/repos/builtin/packages/elpa/package.py
+++ b/var/spack/repos/builtin/packages/elpa/package.py
@@ -22,6 +22,9 @@ class Elpa(AutotoolsPackage, CudaPackage, ROCmPackage):
 
     version("master", branch="master")
 
+    version(
+        "2023.05.001", sha256="ec64be5d6522810d601a3b8e6a31720e3c3eb4af33a434d8a64570d76e6462b6"
+    )
     version(
         "2022.11.001", sha256="75db3ac146f9a6a1598e3418ddcab2be2f40a30ef9ec4c00a3b5d3808c99c430"
     )
@@ -182,6 +185,8 @@ def configure_args(self):
 
         if "+autotune" in self.spec:
             options.append("--enable-autotune-redistribute-matrix")
+            # --enable-autotune-redistribute-matrix requires --enable-scalapack-tests as well
+            options.append("--enable-scalapack-tests")
 
         options.append("--disable-silent-rules")
         options.append("--without-threading-support-check-during-build")
diff --git a/var/spack/repos/builtin/packages/emacs/package.py b/var/spack/repos/builtin/packages/emacs/package.py
index be45473e8bcd8e..8bf6ee8c476f96 100644
--- a/var/spack/repos/builtin/packages/emacs/package.py
+++ b/var/spack/repos/builtin/packages/emacs/package.py
@@ -19,6 +19,7 @@ class Emacs(AutotoolsPackage, GNUMirrorPackage):
     maintainers("alecbcs")
 
     version("master", branch="master")
+    version("29.1", sha256="5b80e0475b0e619d2ad395ef5bc481b7cb9f13894ed23c301210572040e4b5b1")
     version("28.2", sha256="a6912b14ef4abb1edab7f88191bfd61c3edd7085e084de960a4f86485cb7cad8")
     version("28.1", sha256="1439bf7f24e5769f35601dbf332e74dfc07634da6b1e9500af67188a92340a28")
     version("27.2", sha256="80ff6118fb730a6d8c704dccd6915a6c0e0a166ab1daeef9fe68afa9073ddb73")
@@ -48,7 +49,7 @@ class Emacs(AutotoolsPackage, GNUMirrorPackage):
 
     depends_on("ncurses")
     depends_on("pcre")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libxml2")
     depends_on("libtiff", when="+X")
     depends_on("libpng", when="+X")
diff --git a/var/spack/repos/builtin/packages/embree/package.py b/var/spack/repos/builtin/packages/embree/package.py
index b7c7fc0495606a..6a6e91115eb4f3 100644
--- a/var/spack/repos/builtin/packages/embree/package.py
+++ b/var/spack/repos/builtin/packages/embree/package.py
@@ -13,6 +13,15 @@ class Embree(CMakePackage):
     url = "https://github.com/embree/embree/archive/v3.7.0.tar.gz"
     maintainers("aumuell")
 
+    version("4.3.0", sha256="baf0a57a45837fc055ba828a139467bce0bc0c6a9a5f2dccb05163d012c12308")
+    version("4.2.0", sha256="b0479ce688045d17aa63ce6223c84b1cdb5edbf00d7eda71c06b7e64e21f53a0")
+    version("4.1.0", sha256="117efd87d6dddbf7b164edd94b0bc057da69d6422a25366283cded57ed94738b")
+    version("4.0.1", sha256="1fa3982fa3531f1b6e81f19e6028ae8a62b466597f150b853440fe35ef7c6c06")
+    version("4.0.0", sha256="bb967241f9516712a9f8e399ed7f756d7baeec3c85c223c0005ede8b95c9fa61")
+    version("3.13.5", sha256="b8c22d275d9128741265537c559d0ea73074adbf2f2b66b0a766ca52c52d665b")
+    version("3.13.4", sha256="e6a8d1d4742f60ae4d936702dd377bc4577a3b034e2909adb2197d0648b1cb35")
+    version("3.13.3", sha256="74ec785afb8f14d28ea5e0773544572c8df2e899caccdfc88509f1bfff58716f")
+    version("3.13.2", sha256="dcda827e5b7a606c29d00c1339f1ef00f7fa6867346bc46a2318e8f0a601c6f9")
     version("3.13.1", sha256="00dbd852f19ae2b95f5106dd055ca4b304486436ced0ccf842aec4e38a4df425")
     version("3.13.0", sha256="4d86a69508a7e2eb8710d571096ad024b5174834b84454a8020d3a910af46f4f")
     version("3.12.2", sha256="22a527622497e07970e733f753cc9c10b2bd82c3b17964e4f71a5fd2cdfca210")
@@ -29,6 +38,17 @@ class Embree(CMakePackage):
 
     depends_on("tbb")
 
+    # official aarch64 support on macOS starting with 3.13.0, on Linux since 4.0.0
+    # upstream patch for Linux/aarch64 applies cleanly to 3.13.5, and 3.13.3 works by chance
+    conflicts("@:3.12", when="target=aarch64:")
+    conflicts("@:3.13.2", when="target=aarch64: platform=linux")
+    conflicts("@3.13.4", when="target=aarch64: platform=linux")
+    patch(
+        "https://github.com/embree/embree/commit/82ca6b5ccb7abe0403a658a0e079926478f04cb1.patch?full_index=1",
+        sha256="3af5a65e8875549b4c930d4b0f2840660beba4a7f295d8c89068250a1df376f2",
+        when="@3.13.5",
+    )
+
     def cmake_args(self):
         spec = self.spec
 
@@ -37,23 +57,33 @@ def cmake_args(self):
             "-DEMBREE_TUTORIALS=OFF",
             "-DEMBREE_IGNORE_CMAKE_CXX_FLAGS=ON",
             self.define_from_variant("EMBREE_ISPC_SUPPORT", "ispc"),
+            self.define("EMBREE_TBB_ROOT", spec["tbb"].prefix),
+        ]
+
+        if spec.satisfies("target=x86_64:") or spec.satisfies("target=x86:"):
             # code selection and defines controlling namespace names are based on
             # defines controlled by compiler flags, so disable ISAs below compiler
             # flags chosen by spack
-            self.define("EMBREE_ISA_SSE2", "sse4_2" not in spec.target),
-            self.define("EMBREE_ISA_SSE42", "avx" not in spec.target),
-            self.define("EMBREE_ISA_AVX", "avx2" not in spec.target),
-            self.define("EMBREE_ISA_AVX2", "avx512" not in spec.target),
-            self.define("EMBREE_ISA_AVX512SKX", True),
-        ]
+            args.append(self.define("EMBREE_ISA_SSE2", "sse4_2" not in spec.target))
+            args.append(self.define("EMBREE_ISA_SSE42", "avx" not in spec.target))
+            args.append(self.define("EMBREE_ISA_AVX", "avx2" not in spec.target))
+            args.append(self.define("EMBREE_ISA_AVX2", "avx512" not in spec.target))
 
-        if spec.satisfies("%gcc@:7"):
-            # remove unsupported -mprefer-vector-width=256, otherwise copied
-            # from common/cmake/gnu.cmake
-            args.append(
-                "-DFLAGS_AVX512SKX=-mavx512f -mavx512dq -mavx512cd"
-                " -mavx512bw -mavx512vl -mf16c -mavx2 -mfma -mlzcnt"
-                " -mbmi -mbmi2"
-            )
+            # during the 3.12 cycle AVX512SKX was renamed to AVX512,
+            # but for compatibility, the old name is still supported
+            avx512_suffix = ""
+            if spec.satisfies("@:3.12"):
+                avx512_suffix = "SKX"
+            args.append(self.define("EMBREE_ISA_AVX512" + avx512_suffix, True)),
+            if spec.satisfies("%gcc@:7"):
+                # remove unsupported -mprefer-vector-width=256, otherwise copied
+                # from common/cmake/gnu.cmake
+                args.append(
+                    self.define(
+                        "FLAGS_AVX512" + avx512_suffix,
+                        "-mavx512f -mavx512dq -mavx512cd -mavx512bw -mavx512vl"
+                        " -mf16c -mavx2 -mfma -mlzcnt -mbmi -mbmi2",
+                    )
+                )
 
         return args
diff --git a/var/spack/repos/builtin/packages/environment-modules/package.py b/var/spack/repos/builtin/packages/environment-modules/package.py
index 50c2d11a015a0d..38bcd9b3bac08b 100644
--- a/var/spack/repos/builtin/packages/environment-modules/package.py
+++ b/var/spack/repos/builtin/packages/environment-modules/package.py
@@ -59,13 +59,15 @@ class EnvironmentModules(Package):
 
     variant("X", default=True, description="Build with X functionality")
 
-    depends_on("autoconf", type="build", when="@main")
-    depends_on("automake", type="build", when="@main")
-    depends_on("libtool", type="build", when="@main")
-    depends_on("m4", type="build", when="@main")
-    depends_on("python", type="build", when="@main")
-    depends_on("py-sphinx@1.0:", type="build", when="@main")
-    depends_on("gzip", type="build", when="@main")
+    depends_on("less", type=("build", "run"), when="@4.1:")
+    with when("@main"):
+        depends_on("autoconf", type="build")
+        depends_on("automake", type="build")
+        depends_on("libtool", type="build")
+        depends_on("m4", type="build")
+        depends_on("python", type="build")
+        depends_on("py-sphinx@1.0:", type="build")
+        depends_on("gzip", type="build")
 
     # Dependencies:
     depends_on("tcl", type=("build", "link", "run"))
@@ -135,6 +137,9 @@ def install(self, spec, prefix):
                 ]
             )
 
+        if self.spec.satisfies("@4.1:"):
+            config_args.append(f"--with-pager={str(self.spec['less'].prefix.bin.less)}")
+
         configure(*config_args)
         make()
         make("install")
diff --git a/var/spack/repos/builtin/packages/enzyme/package.py b/var/spack/repos/builtin/packages/enzyme/package.py
index 37a5bfff185431..2d4edd17ffcaab 100644
--- a/var/spack/repos/builtin/packages/enzyme/package.py
+++ b/var/spack/repos/builtin/packages/enzyme/package.py
@@ -24,6 +24,7 @@ class Enzyme(CMakePackage):
     root_cmakelists_dir = "enzyme"
 
     version("main", branch="main")
+    version("0.0.81", sha256="4c17d0c28f0572a3ab97a60f1e56bbc045ed5dd64c2daac53ae34371ca5e8b34")
     version("0.0.69", sha256="144d964187551700fdf0a4807961ceab1480d4e4cd0bb0fc7bbfab48fe053aa2")
     version("0.0.48", sha256="f5af62448dd2a8a316e59342ff445003581bc154f06b9b4d7a5a2c7259cf5769")
     version("0.0.32", sha256="9d42e42f7d0faf9beed61b2b1d27c82d1b369aeb9629539d5b7eafbe95379292")
@@ -34,7 +35,8 @@ class Enzyme(CMakePackage):
     depends_on("llvm@7:12", when="@0.0.13:0.0.15")
     depends_on("llvm@7:14", when="@0.0.32:0.0.47")
     depends_on("llvm@7:14", when="@0.0.48:0.0.68")
-    depends_on("llvm@9:16", when="@0.0.69:")
+    depends_on("llvm@9:16", when="@0.0.69:0.0.79")
+    depends_on("llvm@11:16", when="@0.0.80:")
     depends_on("cmake@3.13:", type="build")
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/eospac/package.py b/var/spack/repos/builtin/packages/eospac/package.py
index 0339931eb3c30b..8587a4b583e1b4 100644
--- a/var/spack/repos/builtin/packages/eospac/package.py
+++ b/var/spack/repos/builtin/packages/eospac/package.py
@@ -23,8 +23,13 @@ class Eospac(Package):
     # - alpha and beta versions are marked with 'deprecated=True' to help
     #   spack's version comparison.
     version(
-        "6.5.6",
+        "6.5.7",
         preferred=True,
+        sha256="e59bd449bf97ce977309c6fc8a54fa30f4db9b2ca3e21f996095d78e23799e42",
+        url="https://laws.lanl.gov/projects/data/eos/get_file.php?package=eospac&filename=eospac_v6.5.7_9a867a15ae4137d22e1b52199d6a46b486fc4376.tgz",
+    )
+    version(
+        "6.5.6",
         sha256="49bae52a0c1dc249ca75d47fc1ff9d6367221aa5a5a9c4664efaea6f0292eaff",
         url="https://laws.lanl.gov/projects/data/eos/get_file.php?package=eospac&filename=eospac_v6.5.6_a523d0d98e0323fc0e3aa1c6ad540ebc741c3982.tgz",
     )
diff --git a/var/spack/repos/builtin/packages/epics-base/package.py b/var/spack/repos/builtin/packages/epics-base/package.py
index d885ea7d205bb4..9aae46a46bea54 100644
--- a/var/spack/repos/builtin/packages/epics-base/package.py
+++ b/var/spack/repos/builtin/packages/epics-base/package.py
@@ -23,6 +23,10 @@ class EpicsBase(MakefilePackage):
     depends_on("readline")
     depends_on("perl", type=("build", "run"))
 
+    def patch(self):
+        filter_file(r"^\s*CC\s*=.*", "CC = " + spack_cc, "configure/CONFIG.gnuCommon")
+        filter_file(r"^\s*CCC\s*=.*", "CCC = " + spack_cxx, "configure/CONFIG.gnuCommon")
+
     @property
     def install_targets(self):
         return ["INSTALL_LOCATION={0}".format(self.prefix), "install"]
diff --git a/var/spack/repos/builtin/packages/er/package.py b/var/spack/repos/builtin/packages/er/package.py
index 88eeb63504ec82..19d7faae8bc0be 100644
--- a/var/spack/repos/builtin/packages/er/package.py
+++ b/var/spack/repos/builtin/packages/er/package.py
@@ -29,7 +29,7 @@ class Er(CMakePackage):
     depends_on("rankstr", when="@0.0.4:")
     depends_on("redset")
     depends_on("shuffile")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     depends_on("kvtree@:1.2.0", when="@:0.1.0")
     depends_on("kvtree@1.3.0", when="@0.2.0:0.3.0")
diff --git a/var/spack/repos/builtin/packages/esmf/package.py b/var/spack/repos/builtin/packages/esmf/package.py
index 77cdb222ca4ed4..16fa6c9001eb2f 100644
--- a/var/spack/repos/builtin/packages/esmf/package.py
+++ b/var/spack/repos/builtin/packages/esmf/package.py
@@ -43,7 +43,7 @@ class Esmf(MakefilePackage):
         sha256="0ff43ede83d1ac6beabd3d5e2a646f7574174b28a48d1b9f2c318a054ba268fd",
         deprecated=True,
     )
-    version("8.3.0b09", commit="5b7e546c4b", deprecated=True)
+    version("8.3.0b09", commit="5b7e546c4ba350bff9c9ebd00e5fa1c6315d17da", deprecated=True)
     version("8.2.0", sha256="27866c31fdb63c58e78211de970470ca02d274f5d4d6d97e94284d63b1c1d9e4")
     version("8.1.1", sha256="629690c7a488e84ac7252470349458d7aaa98b54c260f8b3911a2e2f3e713dd0")
     version(
@@ -77,7 +77,7 @@ class Esmf(MakefilePackage):
     variant("debug", default=False, description="Build with debugging symbols and options enabled")
     variant("shared", default=True, description="Build shared library")
 
-    # 'esmf_comm' and 'esmf_os' variants allow override values for their corresponding
+    # 'esmf_os', 'esmf_comm', 'esmf_pio' variants allow override values for their corresponding
     # build environment variables. Documentation, including valid values, can be found at
     # https://earthsystemmodeling.org/docs/release/latest/ESMF_usrdoc/node10.html#SECTION000105000000000000000
     variant("esmf_os", default="auto", description="Override for ESMF_OS variable")
@@ -98,26 +98,13 @@ class Esmf(MakefilePackage):
     depends_on("netcdf-fortran@3.6:", when="+netcdf")
     depends_on("parallel-netcdf@1.2.0:", when="+pnetcdf")
     depends_on("xerces-c@3.1.0:", when="+xerces")
-    depends_on(
-        "parallelio@2.5.7: +mpi+pnetcdf", when="@8.3.0:8.3.99+external-parallelio+mpi+pnetcdf"
-    )
-    depends_on("parallelio@2.5.7: +mpi", when="@8.3.0:8.3.99+external-parallelio+mpi~pnetcdf")
-    depends_on(
-        "parallelio@2.5.7: ~mpi+pnetcdf", when="@8.3.0:8.3.99+external-parallelio~mpi+pnetcdf"
-    )
-    depends_on("parallelio@2.5.7: ~mpi", when="@8.3.0:8.3.99+external-parallelio~mpi~pnetcdf")
-    depends_on(
-        "parallelio@2.5.9: +mpi+pnetcdf", when="@8.4.0:8.4.99+external-parallelio+mpi+pnetcdf"
-    )
-    depends_on("parallelio@2.5.9: +mpi", when="@8.4.0:8.4.99+external-parallelio+mpi~pnetcdf")
-    depends_on(
-        "parallelio@2.5.9: ~mpi+pnetcdf", when="@8.4.0:8.4.99+external-parallelio~mpi+pnetcdf"
-    )
-    depends_on("parallelio@2.5.9: ~mpi", when="@8.4.0:8.4.99+external-parallelio~mpi~pnetcdf")
-    depends_on("parallelio@2.5.10: +mpi+pnetcdf", when="@8.5.0:+external-parallelio+mpi+pnetcdf")
-    depends_on("parallelio@2.5.10: +mpi", when="@8.5.0:+external-parallelio+mpi~pnetcdf")
-    depends_on("parallelio@2.5.10: ~mpi+pnetcdf", when="@8.5.0:+external-parallelio~mpi+pnetcdf")
-    depends_on("parallelio@2.5.10: ~mpi", when="@8.5.0:+external-parallelio~mpi~pnetcdf")
+    depends_on("parallelio@2.5.7: +mpi", when="@8.3+external-parallelio+mpi")
+    depends_on("parallelio@2.5.7: ~mpi", when="@8.3+external-parallelio~mpi")
+    depends_on("parallelio@2.5.9: +mpi", when="@8.4+external-parallelio+mpi")
+    depends_on("parallelio@2.5.9: ~mpi", when="@8.4+external-parallelio~mpi")
+    depends_on("parallelio@2.5.10: +mpi", when="@8.5:+external-parallelio+mpi")
+    depends_on("parallelio@2.5.10: ~mpi", when="@8.5:+external-parallelio~mpi")
+    depends_on("cmake@3.5.2:", type="build", when="~external-parallelio")
 
     # Testing dependencies
     depends_on("perl", type="test")
@@ -356,11 +343,7 @@ def setup_build_environment(self, env):
             netcdfc = spec["netcdf-c"]
             if netcdfc.satisfies("~shared"):
                 nc_config = which(os.path.join(netcdfc.prefix.bin, "nc-config"))
-                # DH* 20230710
-                # original spack code
-                # nc_flags = nc_config("--libs", output=str).strip()
                 nc_flags = nc_config("--static", "--libs", output=str).strip()
-                # *DH 20230710
                 env.set("ESMF_NETCDF_LIBS", nc_flags)
 
         ###################
diff --git a/var/spack/repos/builtin/packages/essl/package.py b/var/spack/repos/builtin/packages/essl/package.py
index a99c44b29e26ad..4b8572834c43aa 100644
--- a/var/spack/repos/builtin/packages/essl/package.py
+++ b/var/spack/repos/builtin/packages/essl/package.py
@@ -22,13 +22,15 @@ class Essl(BundlePackage):
         multi=False,
     )
     variant("cuda", default=False, description="CUDA acceleration")
+    variant("lapackforessl", default=False, description="Provides lapackforessl lapack library")
 
     provides("blas")
+    provides("lapack", when="+lapackforessl")
 
     conflicts(
         "+cuda",
         when="+ilp64",
-        msg="ESSL+cuda+ilp64 cannot combine CUDA acceleration" " 64 bit integers",
+        msg="ESSL+cuda+ilp64 cannot combine CUDA acceleration 64 bit integers",
     )
 
     conflicts(
@@ -63,3 +65,10 @@ def blas_libs(self):
         essl_libs = find_libraries(essl_lib, root=essl_root, shared=True)
 
         return essl_libs
+
+    @property
+    def lapack_libs(self):
+        essl_libs = find_libraries(
+            ["liblapackforessl", "liblapackforessl_"], root=self.prefix.lib64, shared=True
+        )
+        return essl_libs
diff --git a/var/spack/repos/builtin/packages/ethminer/package.py b/var/spack/repos/builtin/packages/ethminer/package.py
index 9c8871e9ea8609..91b42ef93a5499 100644
--- a/var/spack/repos/builtin/packages/ethminer/package.py
+++ b/var/spack/repos/builtin/packages/ethminer/package.py
@@ -27,7 +27,7 @@ class Ethminer(CMakePackage):
     depends_on(Boost.with_default_variants)
     depends_on("json-c")
     depends_on("curl")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("cuda", when="+cuda")
     depends_on("mesa", when="+opencl")
 
diff --git a/var/spack/repos/builtin/packages/exaca/package.py b/var/spack/repos/builtin/packages/exaca/package.py
index 18a3bf473fbb88..d5d6b8785a2601 100644
--- a/var/spack/repos/builtin/packages/exaca/package.py
+++ b/var/spack/repos/builtin/packages/exaca/package.py
@@ -12,13 +12,14 @@ class Exaca(CMakePackage):
 
     homepage = "https://github.com/LLNL/ExaCA"
     git = "https://github.com/LLNL/ExaCA.git"
-    url = "https://github.com/LLNL/ExaCA/archive/1.0.0.tar.gz"
+    url = "https://github.com/LLNL/ExaCA/archive/1.2.0.tar.gz"
 
     maintainers("streeve", "MattRolchigo")
 
     tags = ["ecp"]
 
     version("master", branch="master")
+    version("1.2.0", sha256="5038d63de96c6142ddea956998e1f4ebffbc4a5723caa4da0e73eb185e6623e4")
     version("1.1.0", sha256="10106fb1836964a19bc5bab3f374baa24188ba786c768e554442ab896b31ff24")
     version("1.0.0", sha256="48556233360a5e15e1fc20849e57dd60739c1991c7dfc7e6b2956af06688b96a")
 
@@ -34,8 +35,9 @@ class Exaca(CMakePackage):
     depends_on("cmake@3.12:", type="build", when="@master")
     depends_on("googletest@1.10:", type="test", when="@1.1:+testing")
     depends_on("kokkos@3.0:", when="@:1.1")
-    depends_on("kokkos@3.2:", when="@master")
+    depends_on("kokkos@3.2:", when="@1.2:")
     depends_on("mpi")
+    depends_on("nlohmann-json", when="@1.2:")
 
     def cmake_args(self):
         options = [self.define_from_variant("BUILD_SHARED_LIBS", "shared")]
diff --git a/var/spack/repos/builtin/packages/exago/package.py b/var/spack/repos/builtin/packages/exago/package.py
index 4534afecf6b3e5..fe7b67cc11d949 100644
--- a/var/spack/repos/builtin/packages/exago/package.py
+++ b/var/spack/repos/builtin/packages/exago/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import os
+
 from spack.package import *
 
 
@@ -11,35 +13,63 @@ class Exago(CMakePackage, CudaPackage, ROCmPackage):
     problems on parallel and distributed architectures, particularly targeted
     for exascale machines."""
 
-    homepage = "https://gitlab.pnnl.gov/exasgd/frameworks/exago"
-    git = "https://gitlab.pnnl.gov/exasgd/frameworks/exago.git"
-    maintainers("ryandanehy", "CameronRutherford", "pelesh")
-
-    version("1.5.1", commit="7abe482c8da0e247f9de4896f5982c4cacbecd78", submodules=True)
-    version("1.5.0", commit="227f49573a28bdd234be5500b3733be78a958f15", submodules=True)
-    version("1.4.1", commit="ea607c685444b5f345bfdc9a59c345f0f30adde2", submodules=True)
-    version("1.4.0", commit="4f4c3fdb40b52ace2d6ba000e7f24b340ec8e886", submodules=True)
-    version("1.3.0", commit="58b039d746a6eac8e84b0afc01354cd58caec485", submodules=True)
-    version("1.2.0", commit="255a214e", submodules=True)
-    version("1.1.2", commit="db3bb16e", submodules=True)
-    version("1.1.1", commit="0e0a3f27", submodules=True)
-    version("1.1.0", commit="dc8dd855", submodules=True)
-    version("1.0.0", commit="230d7df2")
-    version("0.99.2", commit="56961641")
-    version("0.99.1", commit="0ae426c7")
+    homepage = "https://github.com/pnnl/ExaGO"
+    git = "https://github.com/pnnl/ExaGO.git"
+    maintainers("ryandanehy", "cameronrutherford", "pelesh")
+
+    version(
+        "1.6.0", tag="v1.6.0", commit="159cd173572280ac0f6f094a71dcc3ebeeb34076", submodules=True
+    )
+    version(
+        "1.5.1", tag="v1.5.1", commit="84e9faf9d9dad8d851075eba26038338d90e6d3a", submodules=True
+    )
+    version(
+        "1.5.0", tag="v1.5.0", commit="227f49573a28bdd234be5500b3733be78a958f15", submodules=True
+    )
+    version(
+        "1.4.1", tag="v1.4.1", commit="ea607c685444b5f345bfdc9a59c345f0f30adde2", submodules=True
+    )
+    version(
+        "1.4.0", tag="v1.4.0", commit="4f4c3fdb40b52ace2d6ba000e7f24b340ec8e886", submodules=True
+    )
+    version(
+        "1.3.0", tag="v1.3.0", commit="58b039d746a6eac8e84b0afc01354cd58caec485", submodules=True
+    )
+    version(
+        "1.1.2", tag="v1.1.2", commit="db3bb16e19c09e01402071623258dae4d13e5133", submodules=True
+    )
+    version(
+        "1.1.1", tag="v1.1.1", commit="0e0a3f27604876749d47c06ec71daaca4b270df9", submodules=True
+    )
+    version(
+        "1.1.0", tag="v1.1.0", commit="dc8dd85544ff1b55a64a3cbbbdf12b8a0c6fdaf6", submodules=True
+    )
+    version("1.0.0", tag="v1.0.0", commit="230d7df2f384f68b952a1ea03aad41431eaad283")
+    version("0.99.2", tag="v0.99.2", commit="56961641f50827b3aa4c14524f2f978dc48b9ce5")
+    version("0.99.1", tag="v0.99.1", commit="0ae426c76651ba5a9dbcaeb95f18d1b8ba961690")
     version("main", branch="main", submodules=True)
     version("develop", branch="develop", submodules=True)
-    version("5-18-2022-snapshot", tag="5-18-2022-snapshot", submodules=True)
+    version(
+        "snapshot.5-18-2022",
+        tag="5-18-2022-snapshot",
+        commit="3eb58335db71bb72341153a7867eb607402067ca",
+        submodules=True,
+    )
+    version("kpp2", tag="kpp2", commit="1da764d80a2db793f4c43ca50e50981f7ed3880a", submodules=True)
 
     # Progrmming model options
     variant("mpi", default=True, description="Enable/Disable MPI")
     variant("raja", default=False, description="Enable/Disable RAJA")
     variant("python", default=True, when="@1.4:", description="Enable/Disable Python bindings")
     variant("logging", default=True, description="Enable/Disable spdlog based logging")
+
     conflicts(
         "+python", when="+ipopt+rocm", msg="Python bindings require -fPIC with Ipopt for rocm."
     )
 
+    # Adds ExaGO's python wrapper to PYTHONPATH
+    extends("python", when="+python")
+
     # Solver options
     variant("hiop", default=False, description="Enable/Disable HiOp")
     variant("ipopt", default=False, description="Enable/Disable IPOPT")
@@ -53,9 +83,14 @@ class Exago(CMakePackage, CudaPackage, ROCmPackage):
         "~hiop~ipopt+python @:1.5.0",
         msg="ExaGO Python wrapper requires at least one solver enabled.",
     )
-
+    conflicts(
+        "+hiop~mpi ^hiop@1.0.0:~mpi",
+        when="@1.5.1:1.6.1",
+        msg="#18 - builds with hiop and without MPI cause compile time errors",
+    )
+    conflicts("+python~mpi", msg="#16 - Python wrapper requires MPI enabled")
     # Dependencies
-    depends_on("python@3.6:", when="@1.3.0:+python")
+    depends_on("python@3.6:3.10", when="@1.3.0:1.5+python")
     depends_on("py-pytest", type=("build", "run"), when="@1.5.0:+python")
     depends_on("py-mpi4py", when="@1.3.0:+mpi+python")
     depends_on("pkgconfig", type="build")
@@ -65,7 +100,6 @@ class Exago(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("cuda", when="+cuda")
     depends_on("raja", when="+raja")
     depends_on("umpire", when="+raja")
-
     depends_on("cmake@3.18:", type="build")
 
     # Profiling
@@ -106,7 +140,7 @@ class Exago(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("hiop@0.3.99:", when="@0.99:+hiop")
     depends_on("hiop@0.5.1:", when="@1.1.0:+hiop")
     depends_on("hiop@0.5.3:", when="@1.3.0:+hiop")
-    depends_on("hiop@0.7.0:", when="@1.5.0:+hiop")
+    depends_on("hiop@0.7.0:1.0.0", when="@1.5.0:+hiop")
 
     depends_on("hiop~mpi", when="+hiop~mpi")
     depends_on("hiop+mpi", when="+hiop+mpi")
@@ -123,9 +157,10 @@ class Exago(CMakePackage, CudaPackage, ROCmPackage):
     # This is no longer a requirement in RAJA > 0.14
     depends_on("umpire+cuda~shared", when="+raja+cuda ^raja@:0.14")
 
-    depends_on("petsc@3.13:3.14", when="@:1.2.99")
-    depends_on("petsc@3.16.0:3.16", when="@1.3.0:1.4")
-    depends_on("petsc@3.18.0:3.19", when="@1.5.0:")
+    depends_on("petsc@3.13:3.14", when="@:1.2")
+    depends_on("petsc@3.16", when="@1.3:1.4")
+    depends_on("petsc@3.18:3.19", when="@1.5")
+    depends_on("petsc@3.20:", when="@1.6:")
 
     depends_on("petsc~mpi", when="~mpi")
 
@@ -169,17 +204,17 @@ def cmake_args(self):
         args.extend(
             [
                 self.define("EXAGO_ENABLE_GPU", "+cuda" in spec or "+rocm" in spec),
+                self.define("PETSC_DIR", spec["petsc"].prefix),
+                self.define("EXAGO_RUN_TESTS", self.run_tests),
+                self.define("LAPACK_LIBRARIES", spec["lapack"].libs + spec["blas"].libs),
                 self.define_from_variant("EXAGO_ENABLE_CUDA", "cuda"),
                 self.define_from_variant("EXAGO_ENABLE_HIP", "rocm"),
-                self.define("PETSC_DIR", spec["petsc"].prefix),
-                self.define("EXAGO_RUN_TESTS", True),
+                self.define_from_variant("EXAGO_ENABLE_LOGGING", "logging"),
                 self.define_from_variant("EXAGO_ENABLE_MPI", "mpi"),
                 self.define_from_variant("EXAGO_ENABLE_RAJA", "raja"),
                 self.define_from_variant("EXAGO_ENABLE_HIOP", "hiop"),
                 self.define_from_variant("EXAGO_ENABLE_IPOPT", "ipopt"),
                 self.define_from_variant("EXAGO_ENABLE_PYTHON", "python"),
-                self.define_from_variant("EXAGO_ENABLE_LOGGING", "logging"),
-                self.define("LAPACK_LIBRARIES", spec["lapack"].libs + spec["blas"].libs),
             ]
         )
 
diff --git a/var/spack/repos/builtin/packages/exciting/package.py b/var/spack/repos/builtin/packages/exciting/package.py
index 215b6de5f52a3b..c74da1013c1ebf 100644
--- a/var/spack/repos/builtin/packages/exciting/package.py
+++ b/var/spack/repos/builtin/packages/exciting/package.py
@@ -39,23 +39,9 @@ class Exciting(MakefilePackage):
     depends_on("scalapack", when="+scalapack")
     # conflicts('%gcc@10:', msg='exciting cannot be built with GCC 10')
 
-    for __compiler in spack.compilers.supported_compilers():
-        if __compiler != "intel":
-            conflicts(
-                "%{0}".format(__compiler),
-                when="^mkl",
-                msg="Intel MKL only works with the Intel compiler",
-            )
-            conflicts(
-                "%{0}".format(__compiler),
-                when="^intel-mkl",
-                msg="Intel MKL only works with the Intel compiler",
-            )
-            conflicts(
-                "%{0}".format(__compiler),
-                when="^intel-mpi",
-                msg="Intel MPI only works with the Intel compiler",
-            )
+    requires("%intel", when="^mkl", msg="Intel MKL only works with the Intel compiler")
+    requires("%intel", when="^intel-mkl", msg="Intel MKL only works with the Intel compiler")
+    requires("%intel", when="^intel-mpi", msg="Intel MPI only works with the Intel compiler")
 
     def patch(self):
         """Fix bad logic in m_makespectrum.f90 for the Oxygen release"""
diff --git a/var/spack/repos/builtin/packages/exempi/package.py b/var/spack/repos/builtin/packages/exempi/package.py
index c193c4f03f42b8..3a3f6be668b5c0 100644
--- a/var/spack/repos/builtin/packages/exempi/package.py
+++ b/var/spack/repos/builtin/packages/exempi/package.py
@@ -20,7 +20,7 @@ class Exempi(AutotoolsPackage):
     version("2.6.1", sha256="072451ac1e0dc97ed69a2e5bfc235fd94fe093d837f65584d0e3581af5db18cd")
     version("2.5.2", sha256="52f54314aefd45945d47a6ecf4bd21f362e6467fa5d0538b0d45a06bc6eaaed5")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("iconv")
     # needs +test variant to prevent following error:
     # 118    checking for the Boost unit_test_framework library... no
diff --git a/var/spack/repos/builtin/packages/exiv2/package.py b/var/spack/repos/builtin/packages/exiv2/package.py
index e19d578a0a65f4..b1509359da51d2 100644
--- a/var/spack/repos/builtin/packages/exiv2/package.py
+++ b/var/spack/repos/builtin/packages/exiv2/package.py
@@ -20,5 +20,5 @@ class Exiv2(CMakePackage):
     version("0.27.3", sha256="6398bc743c32b85b2cb2a604273b8c90aa4eb0fd7c1700bf66cbb2712b4f00c1")
     version("0.27.2", sha256="3dbcaf01fbc5b98d42f091d1ff0d4b6cd9750dc724de3d9c0d113948570b2934")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("expat@2.2.6:", type="link")
diff --git a/var/spack/repos/builtin/packages/exodusii/package.py b/var/spack/repos/builtin/packages/exodusii/package.py
index d057a84bb82e86..53b472d908be96 100644
--- a/var/spack/repos/builtin/packages/exodusii/package.py
+++ b/var/spack/repos/builtin/packages/exodusii/package.py
@@ -52,7 +52,7 @@ class Exodusii(CMakePackage):
     version(
         "2019-10-14", sha256="f143d90e8a7516d25979d1416e580dea638332db723f26ae94a712dfe4052e8f"
     )
-    version("2016-08-09", commit="2ffeb1b")
+    version("2016-08-09", commit="2ffeb1bd39454ad5aa230e12969ce976f3d1c92b")
     version("master", branch="master")
 
     variant("mpi", default=True, description="Enables MPI parallelism.")
diff --git a/var/spack/repos/builtin/packages/express/package.py b/var/spack/repos/builtin/packages/express/package.py
index 4a3b1464d3ecb8..4dcd6d72d505d7 100644
--- a/var/spack/repos/builtin/packages/express/package.py
+++ b/var/spack/repos/builtin/packages/express/package.py
@@ -28,7 +28,7 @@ class Express(CMakePackage):
         "+atomic+container+math+thread+program_options"
     )
     depends_on("bamtools")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # patch from the debian package repo:
     # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=811859
diff --git a/var/spack/repos/builtin/packages/extrae/package.py b/var/spack/repos/builtin/packages/extrae/package.py
index f091439afd3286..25b763700f4f8a 100644
--- a/var/spack/repos/builtin/packages/extrae/package.py
+++ b/var/spack/repos/builtin/packages/extrae/package.py
@@ -40,6 +40,8 @@ class Extrae(AutotoolsPackage):
     homepage = "https://tools.bsc.es/extrae"
     url = "https://ftp.tools.bsc.es/extrae/extrae-3.4.1-src.tar.bz2"
 
+    version("4.0.6", sha256="b5060336cac57f1345faa09009b1940edf1e6991aae05cc10d0b714d31360a92")
+    version("4.0.5", sha256="8f5eefa95f2e94a3b5f9b7f7cbaaed523862f190575ee797113b1e97deff1586")
     version("4.0.4", sha256="003bede870de6d88b705c1a13eabe63b6beb928d8f389f5dd70ca5db8450a1f9")
     version("4.0.3", sha256="b5139a07dbb1f4aa9758c1d62d54e42c01125bcfa9aa0cb9ee4f863afae93db1")
     version("3.8.3", sha256="c3bf27fb6f18e66200e40a0b4c35bc257766e5c1a525dc5725f561879e88bf32")
diff --git a/var/spack/repos/builtin/packages/eztrace/package.py b/var/spack/repos/builtin/packages/eztrace/package.py
index 8c7f4b67d29964..5fa16fe9c44da0 100644
--- a/var/spack/repos/builtin/packages/eztrace/package.py
+++ b/var/spack/repos/builtin/packages/eztrace/package.py
@@ -15,6 +15,7 @@ class Eztrace(Package):
     git = "https://gitlab.com/eztrace/eztrace.git"
 
     version("master", branch="master")
+    version("2.1", sha256="ab5076086eced78e4c6cf7736e7765ca1337dec95a881c9270a42b3251aeea19")
     version("2.0", sha256="67bd296f059cdfab303c62f674af3e1e858213d6945bd79cb8ede4a035c0c2d6")
     version("1.1-13", sha256="6144d04fb62b3ccad41af0268cd921161f168d0cca3f6c210c448bb0b07be7e0")
     version("1.1-10", sha256="63d1af2db38b04efa817614574f381e7536e12db06a2c75375d1795adda3d1d8")
@@ -35,6 +36,15 @@ class Eztrace(Package):
     depends_on("cmake@3.1:", type="build", when="@2.0:")
     depends_on("otf2", when="@2.0:")
 
+    variant("starpu", default=False, description="Enable StarPU support", when="@2.1:")
+    depends_on("starpu", when="@2.1:+starpu")
+    variant("cuda", default=False, description="Enable CUDA support", when="@2.1:")
+    depends_on("cuda", when="@2.1:+cuda")
+    variant("netcdf", default=False, description="Enable NetCDF support", when="@2.1:")
+    depends_on("netcdf-c", when="@2.1:+netcdf")
+    variant("pnetcdf", default=False, description="Enable PNetCDF support", when="@2.1:")
+    depends_on("parallel-netcdf", when="@2.1:+pnetcdf")
+
     def url_for_version(self, version):
         url = "https://gitlab.com/eztrace/eztrace/-/archive/{0}/eztrace-{1}.tar.gz"
         return url.format(version, version)
@@ -54,8 +64,17 @@ def install(self, spec, prefix):
             "-DOTF2_LIBRARY_PATH=%s" % spec["otf2"].libs,
         ]
 
-        if spec.satisfies("%llvm-openmp-ompt"):
-            args.append("-DEZTRACE_ENABLE_OMPT=ON")
+        if spec.satisfies("@2.1:"):
+            if spec.satisfies("%llvm-openmp-ompt"):
+                args.extend(["-DEZTRACE_ENABLE_OMPT=ON"])
+            if "+starpu" in spec:
+                args.extend(["-DEZTRACE_ENABLE_STARPU=ON"])
+            if "+cuda" in spec:
+                args.extend(["-DEZTRACE_ENABLE_CUDA=ON"])
+            if "+netcdf" in spec:
+                args.extend(["-DEZTRACE_ENABLE_NETCDF=ON"])
+            if "+pnetcdf" in spec:
+                args.extend(["-DEZTRACE_ENABLE_PNETCDF=ON"])
 
         args.extend(std_cmake_args)
 
diff --git a/var/spack/repos/builtin/packages/falco/package.py b/var/spack/repos/builtin/packages/falco/package.py
new file mode 100644
index 00000000000000..5acecbf5c0f3de
--- /dev/null
+++ b/var/spack/repos/builtin/packages/falco/package.py
@@ -0,0 +1,26 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Falco(AutotoolsPackage):
+    """A C++ drop-in replacement of FastQC to assess the quality of sequence read data"""
+
+    homepage = "https://github.com/smithlabcode/falco"
+    url = "https://github.com/smithlabcode/falco/releases/download/v1.2.1/falco-1.2.1.tar.gz"
+
+    version("1.2.1", sha256="33de8aafac45c7aea055ed7ab837d0a39d12dcf782816cea8a6c648acb911057")
+
+    variant("htslib", default=False, description="Add support for BAM files")
+
+    depends_on("gmake", type="build")
+    depends_on("zlib-ng")
+    depends_on("htslib", when="+htslib")
+
+    def configure_args(self):
+        if self.spec.satisfies("+htslib"):
+            return ["--enable-htslib"]
+        return []
diff --git a/var/spack/repos/builtin/packages/faodel/package.py b/var/spack/repos/builtin/packages/faodel/package.py
index e93dc45f01bae7..263ff3521df487 100644
--- a/var/spack/repos/builtin/packages/faodel/package.py
+++ b/var/spack/repos/builtin/packages/faodel/package.py
@@ -9,9 +9,9 @@
 class Faodel(CMakePackage):
     """Flexible, Asynchronous, Object Data-Exchange Libraries"""
 
-    homepage = "https://github.com/faodel/faodel"
-    url = "https://github.com/faodel/faodel/archive/v1.2108.1.tar.gz"
-    git = "https://github.com/faodel/faodel.git"
+    homepage = "https://github.com/sandialabs/faodel"
+    url = "https://github.com/sandialabs/faodel/archive/v1.2108.1.tar.gz"
+    git = "https://github.com/sandialabs/faodel.git"
 
     maintainers("tkordenbrock", "craigulmer")
 
diff --git a/var/spack/repos/builtin/packages/fasta/package.py b/var/spack/repos/builtin/packages/fasta/package.py
index be625a7384badb..763e6ec9dbce0c 100644
--- a/var/spack/repos/builtin/packages/fasta/package.py
+++ b/var/spack/repos/builtin/packages/fasta/package.py
@@ -26,7 +26,7 @@ class Fasta(MakefilePackage):
         url="https://github.com/wrpearson/fasta36/archive/refs/tags/v36.3.8h_04-May-2020.tar.gz",
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # The src tree includes a plethora of variant Makefiles and the
     # builder is expected to choose one that's appropriate.  This'll
diff --git a/var/spack/repos/builtin/packages/fastani/package.py b/var/spack/repos/builtin/packages/fastani/package.py
index 1eacdf840ae252..d5c9444005f789 100644
--- a/var/spack/repos/builtin/packages/fastani/package.py
+++ b/var/spack/repos/builtin/packages/fastani/package.py
@@ -6,18 +6,23 @@
 from spack.package import *
 
 
-class Fastani(AutotoolsPackage):
+class Fastani(CMakePackage, AutotoolsPackage):
     """Fast, alignment-free computation of whole-genome Average Nucleotide
     Identity (ANI)"""
 
     homepage = "https://github.com/ParBLiSS/FastANI"
     url = "https://github.com/ParBLiSS/FastANI/archive/v1.33.tar.gz"
+    maintainers("snehring")
 
+    version("1.34", sha256="dc185cf29b9fa40cdcc2c83bb48150db46835e49b9b64a3dbff8bc4d0f631cb1")
     version("1.33", sha256="0b18b3074094722fb1b2247c1a1c4eb96295fff369b837f422e05072740e0013")
 
-    depends_on("autoconf", type="build")
+    build_system(conditional("cmake", when="@1.34:"), "autotools", default="cmake")
+
+    depends_on("autoconf", type="build", when="build_system=autotools")
+    depends_on("automake", type="build", when="build_system=autotools")
+    depends_on("cmake@3.20:", type="build", when="build_system=cmake")
+    depends_on("libtool", type="build", when="build_system=autotools")
+    depends_on("m4", type="build", when="build_system=autotools")
     depends_on("gsl", type=("build", "link"))
-    depends_on("zlib", type=("build", "link"))
-    depends_on("automake", type="build")
-    depends_on("libtool", type="build")
-    depends_on("m4", type="build")
+    depends_on("zlib-api", type=("build", "link"))
diff --git a/var/spack/repos/builtin/packages/fastjar/package.py b/var/spack/repos/builtin/packages/fastjar/package.py
index e11d68bb9ab8ef..e27320d50eacbc 100644
--- a/var/spack/repos/builtin/packages/fastjar/package.py
+++ b/var/spack/repos/builtin/packages/fastjar/package.py
@@ -14,4 +14,4 @@ class Fastjar(AutotoolsPackage):
 
     version("0.98", sha256="f156abc5de8658f22ee8f08d7a72c88f9409ebd8c7933e9466b0842afeb2f145")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/fastqvalidator/package.py b/var/spack/repos/builtin/packages/fastqvalidator/package.py
index e7b312f55a121f..46792698087e00 100644
--- a/var/spack/repos/builtin/packages/fastqvalidator/package.py
+++ b/var/spack/repos/builtin/packages/fastqvalidator/package.py
@@ -20,7 +20,7 @@ class Fastqvalidator(MakefilePackage):
         commit="9db9c23e176a6ce6f421a3c21ccadedca892ac0c",
     )
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     patch("chromosome.patch", when="@2017-01-10", working_dir="libStatGen")
 
     @property
diff --git a/var/spack/repos/builtin/packages/fdb/package.py b/var/spack/repos/builtin/packages/fdb/package.py
index c6a2d6aa66b298..7dc2f75e76b031 100644
--- a/var/spack/repos/builtin/packages/fdb/package.py
+++ b/var/spack/repos/builtin/packages/fdb/package.py
@@ -18,6 +18,8 @@ class Fdb(CMakePackage):
 
     # master version of fdb is subject to frequent changes and is to be used experimentally.
     version("master", branch="master")
+    version("5.11.23", sha256="09b1d93f2b71d70c7b69472dfbd45a7da0257211f5505b5fcaf55bfc28ca6c65")
+    version("5.11.17", sha256="375c6893c7c60f6fdd666d2abaccb2558667bd450100817c0e1072708ad5591e")
     version("5.10.8", sha256="6a0db8f98e13c035098dd6ea2d7559f883664cbf9cba8143749539122ac46099")
     version("5.7.8", sha256="6adac23c0d1de54aafb3c663d077b85d0f804724596623b381ff15ea4a835f60")
 
@@ -39,6 +41,7 @@ class Fdb(CMakePackage):
 
     depends_on("cmake@3.12:", type="build")
     depends_on("ecbuild@3.4:", type="build")
+    depends_on("ecbuild@3.7:", type="build", when="@5.11.6:")
 
     depends_on("eckit@1.16:")
     depends_on("eckit+admin", when="+tools")
diff --git a/var/spack/repos/builtin/packages/fenics-basix/package.py b/var/spack/repos/builtin/packages/fenics-basix/package.py
index e4a3833bc05b9a..0de31ae59ff624 100644
--- a/var/spack/repos/builtin/packages/fenics-basix/package.py
+++ b/var/spack/repos/builtin/packages/fenics-basix/package.py
@@ -15,6 +15,7 @@ class FenicsBasix(CMakePackage):
     maintainers("mscroggs", "chrisrichardson", "garth-wells", "jhale")
 
     version("main", branch="main")
+    version("0.7.0", sha256="9bee81b396ee452eec8d9735f278cb44cb6994c6bc30aec8ed9bb4b12d83fa7f")
     version("0.6.0", sha256="687ae53153c98facac4080dcdc7081701db1dcea8c5e7ae3feb72aec17f83304")
     version("0.5.1", sha256="69133476ac35f0bd0deccb480676030378c341d7dfb2adaca22cd16b7e1dc1cb")
     version("0.4.2", sha256="a54f5e442b7cbf3dbb6319c682f9161272557bd7f42e2b8b8ccef88bc1b7a22f")
diff --git a/var/spack/repos/builtin/packages/fenics/package.py b/var/spack/repos/builtin/packages/fenics/package.py
index 736812e242621e..e8e5f89cce324e 100644
--- a/var/spack/repos/builtin/packages/fenics/package.py
+++ b/var/spack/repos/builtin/packages/fenics/package.py
@@ -106,7 +106,7 @@ class Fenics(CMakePackage):
     depends_on("python@3.5:", type=("build", "run"), when="+python")
     depends_on("eigen@3.2.0:")
     depends_on("pkgconfig", type="build")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
 
     depends_on("boost+filesystem+program_options+system+iostreams+timer+regex+chrono")
     depends_on(
diff --git a/var/spack/repos/builtin/packages/feq-parse/package.py b/var/spack/repos/builtin/packages/feq-parse/package.py
index 9c79422702eb3f..687ae6f66fd297 100644
--- a/var/spack/repos/builtin/packages/feq-parse/package.py
+++ b/var/spack/repos/builtin/packages/feq-parse/package.py
@@ -12,11 +12,15 @@ class FeqParse(CMakePackage):
     provided as strings."""
 
     homepage = "https://github.com/FluidNumerics/feq-parse"
-    url = "https://github.com/FluidNumerics/feq-parse/archive/v1.0.0.tar.gz"
+    url = "https://github.com/FluidNumerics/feq-parse/archive/v2.0.1.tar.gz"
 
     maintainers("fluidnumerics-joe")
 
+    version("2.0.3", sha256="a1c42507801adc55a63a9a904807058079d54e002e10f2b29a916b06fc815f80")
+    version("2.0.1", sha256="08dd08bd100a0a2eb672a5b2792ad56a337df575c634aac0d7a300d7e484b21c")
     version("1.1.0", sha256="d33a4fd6904939bb70780e8f25f37c1291c4f24fd207feb4ffc0f8d89637d1e3")
     version("1.0.2", sha256="1cd1db7562908ea16fc65dc5268b654405d0b3d9dcfe11f409949c431b48a3e8")
 
     depends_on("cmake@3.0.2:", type="build")
+
+    parallel = False
diff --git a/var/spack/repos/builtin/packages/fermi/package.py b/var/spack/repos/builtin/packages/fermi/package.py
index 23e71310ebea5c..df03daa2899dd0 100644
--- a/var/spack/repos/builtin/packages/fermi/package.py
+++ b/var/spack/repos/builtin/packages/fermi/package.py
@@ -14,7 +14,7 @@ class Fermi(MakefilePackage):
 
     version("1.1", sha256="f1351b52a4ff40e5d708899e90ecf747e7af8d4eac795f6968e5b58c2ba11a67")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("perl", type="run")
     depends_on("sse2neon", when="target=aarch64:")
 
diff --git a/var/spack/repos/builtin/packages/fermikit/package.py b/var/spack/repos/builtin/packages/fermikit/package.py
index f8bb48f1d6ffca..a17f9e8f3a5b3c 100644
--- a/var/spack/repos/builtin/packages/fermikit/package.py
+++ b/var/spack/repos/builtin/packages/fermikit/package.py
@@ -15,7 +15,7 @@ class Fermikit(MakefilePackage):
 
     version("2017-11-7", commit="bf9c7112221577ba110665bddca8f1987250bdc7", submodules=True)
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("sse2neon", when="target=aarch64:")
 
     patch("ksw_for_aarch64.patch", when="target=aarch64:")
diff --git a/var/spack/repos/builtin/packages/ferret/package.py b/var/spack/repos/builtin/packages/ferret/package.py
index 09f08d6f81a8d3..0d7dc892ee3aec 100644
--- a/var/spack/repos/builtin/packages/ferret/package.py
+++ b/var/spack/repos/builtin/packages/ferret/package.py
@@ -32,7 +32,7 @@ class Ferret(Package):
     depends_on("netcdf-c")
     depends_on("netcdf-fortran")
     depends_on("readline")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libx11")
     depends_on("curl")
 
@@ -66,7 +66,7 @@ def patch(self):
         hdf5_prefix = spec["hdf5"].prefix
         netcdff_prefix = spec["netcdf-fortran"].prefix
         readline_prefix = spec["readline"].prefix
-        libz_prefix = spec["zlib"].prefix
+        libz_prefix = spec["zlib-api"].prefix
 
         work_dir = "FERRET" if "@:7.2" in spec else "."
         with working_dir(work_dir, create=False):
diff --git a/var/spack/repos/builtin/packages/ffmpeg/package.py b/var/spack/repos/builtin/packages/ffmpeg/package.py
index fbbfb746133eec..ff597653ff987b 100644
--- a/var/spack/repos/builtin/packages/ffmpeg/package.py
+++ b/var/spack/repos/builtin/packages/ffmpeg/package.py
@@ -77,7 +77,7 @@ class Ffmpeg(AutotoolsPackage):
     depends_on("alsa-lib", when="platform=linux")
     depends_on("iconv")
     depends_on("yasm@1.2.0:")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     depends_on("aom", when="+libaom")
     depends_on("bzip2", when="+bzlib")
@@ -111,6 +111,13 @@ class Ffmpeg(AutotoolsPackage):
         when="@5.1.2",
     )
 
+    # Patch fixing a build failure with binutils 2.41.0
+    patch(
+        "https://git.ffmpeg.org/gitweb/ffmpeg.git/commitdiff_plain/effadce6c756247ea8bae32dc13bb3e6f464f0eb",
+        sha256="d1ea47c29968507fee772234bc734d29958b62ab92400801ef28559b538a9168",
+        when="@6.0",
+    )
+
     @property
     def libs(self):
         return find_libraries("*", self.prefix, recursive=True)
@@ -121,6 +128,10 @@ def headers(self):
         headers.directories = [self.prefix.include]
         return headers
 
+    @when("@:6.0 %apple-clang@15:")
+    def setup_build_environment(self, env):
+        env.append_flags("LDFLAGS", "-Wl,-ld_classic")
+
     def enable_or_disable_meta(self, variant, options):
         switch = "enable" if "+{0}".format(variant) in self.spec else "disable"
         return ["--{0}-{1}".format(switch, option) for option in options]
diff --git a/var/spack/repos/builtin/packages/fftw/package.py b/var/spack/repos/builtin/packages/fftw/package.py
index 4ce6d24dd72cef..976d8183f893fa 100644
--- a/var/spack/repos/builtin/packages/fftw/package.py
+++ b/var/spack/repos/builtin/packages/fftw/package.py
@@ -64,6 +64,13 @@ def libs(self):
 
         return find_libraries(libraries, root=self.prefix, recursive=True)
 
+    def flag_handler(self, name, flags):
+        if name == "cflags":
+            if self.spec.satisfies("%clang@15:"):
+                flags.append("-Wno-error=int-conversion")
+
+        return flags, None, None
+
     def patch(self):
         # If fftw/config.h exists in the source tree, it will take precedence
         # over the copy in build dir.  As only the latter has proper config
diff --git a/var/spack/repos/builtin/packages/fftx/package.py b/var/spack/repos/builtin/packages/fftx/package.py
index 423cf5e8b2f9aa..b9229216ff1510 100644
--- a/var/spack/repos/builtin/packages/fftx/package.py
+++ b/var/spack/repos/builtin/packages/fftx/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import os.path
+
 from spack.package import *
 
 
@@ -11,20 +13,23 @@ class Fftx(CMakePackage, CudaPackage, ROCmPackage):
     package for executing the Fast Fourier Transform as well as higher-level
     operations composed of linear operations combined with DFT transforms."""
 
-    homepage = "https://spiral.net"
-    url = "https://github.com/spiral-software/fftx/archive/refs/tags/1.0.3.tar.gz"
+    homepage = "https://spiralgen.com"
+    url = "https://github.com/spiral-software/fftx/archive/refs/tags/1.1.3.tar.gz"
     git = "https://github.com/spiral-software/fftx.git"
 
     maintainers("spiralgen")
 
     version("develop", branch="develop")
     version("main", branch="main")
+    version("1.1.3", sha256="17ed0baf9c2dcf30c789fdae530e006ae3ff2d2c9006989b1e6348e4ae50cef9")
+    version("1.1.2", sha256="b2c4a7791305481af9e1bd358c1215efa4506c91c943cddca3780a1ccbc27810")
+    version("1.1.1", sha256="5cbca66ef09eca02ee8f336f58eb45cfac69cfb29cd6eb945852ad74085d8a60")
+    version("1.1.0", sha256="a6f95605abc11460bbf51839727a456a31488e27e12a970fc29a1b8c42f4e3b5")
     version("1.0.3", sha256="b5ff275facce4a2fbabd0aecc65dd55b744794f2e07cd8cfa91363001c664896")
 
-    depends_on("spiral-software")
-    depends_on("spiral-package-fftx")
-    depends_on("spiral-package-simt")
-    #  depends_on('spiral-package-mpi')
+    depends_on("spiral-software+fftx+simt+jit+mpi")
+    # depend only on spiral-software, but spiral-software must be installed with variants:
+    # +fftx +simt +mpi +jit
 
     conflicts("+rocm", when="+cuda", msg="FFTX only supports one GPU backend at a time")
 
@@ -39,41 +44,37 @@ def create_lib_source_code(self):
             backend = "HIP"
         self.build_config = "-D_codegen=%s" % backend
 
-        #  From directory examples/library run the build-lib-code.sh script
-        with working_dir(join_path(self.stage.source_path, "src", "library")):
+        #  From the root directory run the config-fftx-libs.sh script
+        with working_dir(self.stage.source_path):
             bash = which("bash")
-            bash("./build-lib-code.sh", backend)
+            bash("./config-fftx-libs.sh", backend)
 
     def cmake_args(self):
         spec = self.spec
         args = ["-DSPIRAL_HOME:STRING={0}".format(spec["spiral-software"].prefix)]
-        args.append("-DCMAKE_INSTALL_PREFIX:PATH={0}".format(self.stage.source_path))
+        args.append("-DCMAKE_INSTALL_PREFIX:PATH={0}".format(self.prefix))
+        if "+rocm" in spec:
+            args.append("-DCMAKE_CXX_COMPILER={0}".format(self.spec["hip"].hipcc))
         args.append(self.build_config)
+
         print("Args = " + str(args))
         return args
 
     @property
     def build_targets(self):
-        return ["-j1", "install"]
+        return ["install"]
 
     def install(self, spec, prefix):
-        mkdirp(prefix.bin)
-        mkdirp(prefix.CMakeIncludes)
-        mkdirp(prefix.examples)
-        mkdirp(prefix.include)
-        mkdirp(prefix.lib)
-
         with working_dir(self.stage.source_path):
-            files = ("License.txt", "README.md", "ReleaseNotes.md")
+            files = ("License.txt", "README.md", "ReleaseNotes.md", "supercomputer-README.md")
             for fil in files:
                 install(fil, prefix)
 
+        mkdirp(prefix.cache_jit_files)
         with working_dir(self.stage.source_path):
-            install_tree("bin", prefix.bin)
-            install_tree("CMakeIncludes", prefix.CMakeIncludes)
-            install_tree("examples", prefix.examples)
-            install_tree("include", prefix.include)
-            install_tree("lib", prefix.lib)
+            dir = join_path(self.stage.source_path, "cache_jit_files")
+            if os.path.isdir(dir):
+                install_tree("cache_jit_files", prefix.cache_jit_files)
 
     def setup_dependent_build_environment(self, env, dependent_spec):
         env.set("FFTX_HOME", self.prefix)
diff --git a/var/spack/repos/builtin/packages/figcone/package.py b/var/spack/repos/builtin/packages/figcone/package.py
new file mode 100644
index 00000000000000..43b0f1a1cf0db4
--- /dev/null
+++ b/var/spack/repos/builtin/packages/figcone/package.py
@@ -0,0 +1,16 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Figcone(CMakePackage):
+    """figcone - is a C++17 library, providing a convenient declarative interface for configuration
+    parsers and built-in support for reading JSON, YAML, TOML, XML, INI and shoal config files."""
+
+    homepage = "https://github.com/kamchatka-volcano/figcone"
+    url = "https://github.com/kamchatka-volcano/figcone/archive/refs/tags/v2.4.9.tar.gz"
+
+    version("2.4.9", sha256="735399e849621a4923e71a50d5e2ba928d5dfa3b01e54d56e0bac8e5102b7697")
diff --git a/var/spack/repos/builtin/packages/file/package.py b/var/spack/repos/builtin/packages/file/package.py
index 69609c0036ed5d..23394ab2ab8af9 100644
--- a/var/spack/repos/builtin/packages/file/package.py
+++ b/var/spack/repos/builtin/packages/file/package.py
@@ -32,7 +32,7 @@ class File(AutotoolsPackage):
 
     depends_on("bzip2")
     depends_on("xz", when="@5.38:")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("zstd", when="@5.44:")
 
     @classmethod
diff --git a/var/spack/repos/builtin/packages/filtlong/gcc13.patch b/var/spack/repos/builtin/packages/filtlong/gcc13.patch
new file mode 100644
index 00000000000000..6ff541ab88bc17
--- /dev/null
+++ b/var/spack/repos/builtin/packages/filtlong/gcc13.patch
@@ -0,0 +1,10 @@
+--- src/kmers.h	2018-01-04 03:52:20.000000000 +0000
++++ src/kmers.h.patched	2023-09-04 13:19:09.206573971 +0100
+@@ -17,6 +17,7 @@
+ #define KMERS_H
+ 
+ 
++#include 
+ #include 
+ #include 
+ #include 
diff --git a/var/spack/repos/builtin/packages/filtlong/package.py b/var/spack/repos/builtin/packages/filtlong/package.py
index e9fc522b36be62..fa32ca6633607e 100644
--- a/var/spack/repos/builtin/packages/filtlong/package.py
+++ b/var/spack/repos/builtin/packages/filtlong/package.py
@@ -13,10 +13,14 @@ class Filtlong(MakefilePackage):
     homepage = "https://github.com/rrwick/Filtlong"
     url = "https://github.com/rrwick/Filtlong/archive/v0.2.0.tar.gz"
 
+    version("0.2.1", sha256="e6f47675e87f98cf2481a60bef5cad38396f1e4db653a5c1673139f37770273a")
     version("0.2.0", sha256="a4afb925d7ced8d083be12ca58911bb16d5348754e7c2f6431127138338ee02a")
     version("0.1.1", sha256="ddae7a5850efeb64424965a443540b1ced34286fbefad9230ab71f4af314081b")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
+
+    # %gcc@13: requires std libraries be manually added - add an include for `cstdint`
+    patch("gcc13.patch", level=0, when="%gcc@13:")
 
     def install(self, spec, prefix):
         mkdir(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/fish/package.py b/var/spack/repos/builtin/packages/fish/package.py
index 16e4d2ff20b7dd..14c9057ba8018a 100644
--- a/var/spack/repos/builtin/packages/fish/package.py
+++ b/var/spack/repos/builtin/packages/fish/package.py
@@ -13,13 +13,15 @@ class Fish(CMakePackage):
     """
 
     homepage = "https://fishshell.com/"
-    url = "https://github.com/fish-shell/fish-shell/releases/download/3.5.1/fish-3.5.1.tar.xz"
+    url = "https://github.com/fish-shell/fish-shell/releases/download/3.6.1/fish-3.6.1.tar.xz"
     git = "https://github.com/fish-shell/fish-shell.git"
     list_url = homepage
 
     maintainers("funnell", "adamjstewart")
 
     version("master", branch="master")
+    version("3.6.1", sha256="55402bb47ca6739d8aba25e41780905b5ce1bce0a5e0dd17dca908b5bc0b49b2")
+    version("3.6.0", sha256="97044d57773ee7ca15634f693d917ed1c3dc0fa7fde1017f1626d60b83ea6181")
     version("3.5.1", sha256="a6d45b3dc5a45dd31772e7f8dfdfecabc063986e8f67d60bd7ca60cc81db6928")
     version("3.4.1", sha256="b6f23b3843b04db6b0a90fea1f6f0d0e40cc027b4a732098200863f2864a94ea")
     version("3.3.1", sha256="b5b4ee1a5269762cbbe993a4bd6507e675e4100ce9bbe84214a5eeb2b19fae89")
@@ -30,6 +32,7 @@ class Fish(CMakePackage):
     variant("docs", default=False, description="Build documentation")
 
     # https://github.com/fish-shell/fish-shell#dependencies-1
+    depends_on("cmake@3.5:", when="@3.4:", type="build")
     depends_on("cmake@3.2:", type="build")
     depends_on("ncurses")
     depends_on("pcre2@10.21:")
diff --git a/var/spack/repos/builtin/packages/flash/package.py b/var/spack/repos/builtin/packages/flash/package.py
index 9e72cdf528828d..78451f9ddf99e4 100644
--- a/var/spack/repos/builtin/packages/flash/package.py
+++ b/var/spack/repos/builtin/packages/flash/package.py
@@ -16,7 +16,7 @@ class Flash(MakefilePackage):
 
     version("1.2.11", sha256="685ca6f7fedda07434d8ee03c536f4763385671c4509c5bb48beb3055fd236ac")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/flcl/package.py b/var/spack/repos/builtin/packages/flcl/package.py
index 8460f2b9ac888f..31ba3ed59b38ee 100644
--- a/var/spack/repos/builtin/packages/flcl/package.py
+++ b/var/spack/repos/builtin/packages/flcl/package.py
@@ -25,7 +25,9 @@ class Flcl(CMakePackage):
     depends_on("cmake@3.17:", type="build", when="@:0.4.0")
     depends_on("cmake@3.19:", type="build", when="@0.5.0:")
 
-    conflicts("kokkos@3.3.00:", when="@:0.4.99", msg="Requires FLCL >= 0.5.0 to use Kokkos >= 3.3")
+    conflicts(
+        "^kokkos@3.3.00:", when="@:0.4.99", msg="Requires FLCL >= 0.5.0 to use Kokkos >= 3.3"
+    )
 
     def cmake_args(self):
         args = [self.define("BUILD_TESTING", self.run_tests)]
diff --git a/var/spack/repos/builtin/packages/flecsi/package.py b/var/spack/repos/builtin/packages/flecsi/package.py
index 6ec0e50151b1e8..1ae350aac74b48 100644
--- a/var/spack/repos/builtin/packages/flecsi/package.py
+++ b/var/spack/repos/builtin/packages/flecsi/package.py
@@ -12,22 +12,25 @@ class Flecsi(CMakePackage, CudaPackage, ROCmPackage):
     provide a very general set of infrastructure design patterns that can
     be specialized and extended to suit the needs of a broad variety of
     solver and data requirements. Current support includes multi-dimensional
-    mesh topology, mesh geometry, and mesh adjacency information,
-    n-dimensional hashed-tree data structures, graph partitioning
-    interfaces,and dependency closures.
+    mesh topology, mesh geometry, and mesh adjacency information.
     """
 
     homepage = "http://flecsi.org/"
     git = "https://github.com/flecsi/flecsi.git"
-    maintainers("rspavel", "ktsai7", "rbberger")
+    maintainers("ktsai7", "rbberger")
 
     tags = ["e4s"]
 
     version("develop", branch="develop")
-    version("2.2.0", tag="v2.2.0", preferred=True)
-    version("2.1.0", tag="v2.1.0")
-    version("2.0.0", tag="v2.0.0")
-    version("1.4.1", tag="v1.4.1", submodules=True)
+    version(
+        "2.2.1", tag="v2.2.1", commit="84b5b232aebab40610f57387778db80f6c8c84c5", preferred=True
+    )
+    version("2.2.0", tag="v2.2.0", commit="dd531ac16c5df124d76e385c6ebe9b9589c2d3ad")
+    version("2.1.0", tag="v2.1.0", commit="533df139c267e2a93c268dfe68f9aec55de11cf0")
+    version("2.0.0", tag="v2.0.0", commit="5ceebadf75d1c98999ea9e9446926722d061ec22")
+    version(
+        "1.4.1", tag="v1.4.1", commit="ab974c3164056e6c406917c8ca771ffd43c5a031", submodules=True
+    )
     version(
         "1.4.develop",
         git="https://github.com/laristra/flecsi.git",
@@ -58,7 +61,7 @@ class Flecsi(CMakePackage, CudaPackage, ROCmPackage):
         multi=False,
     )
     variant("shared", default=True, description="Build shared libraries")
-    variant("flog", default=False, description="Enable flog testing")
+    variant("flog", default=False, description="Enable logging support")
     variant("graphviz", default=False, description="Enable GraphViz Support")
     variant("doc", default=False, description="Enable documentation")
     variant("hdf5", default=True, description="Enable HDF5 Support")
@@ -88,14 +91,14 @@ class Flecsi(CMakePackage, CudaPackage, ROCmPackage):
     for level in ("low", "medium", "high"):
         depends_on("caliper@2.0.1~adiak~libdw", when="@:1 caliper_detail=%s" % level)
         depends_on("caliper", when="@2.0: caliper_detail=%s" % level)
-        conflicts("caliper@2.6", when="@2.0: caliper_detail=%s" % level)
-        conflicts("caliper@2.7", when="@2.0: caliper_detail=%s" % level)
+        conflicts("^caliper@2.6", when="@2.0: caliper_detail=%s" % level)
+        conflicts("^caliper@2.7", when="@2.0: caliper_detail=%s" % level)
 
     depends_on("graphviz", when="+graphviz")
     depends_on("hdf5+hl+mpi", when="+hdf5")
     depends_on("metis@5.1.0:")
     depends_on("parmetis@4.0.3:")
-    depends_on("boost@1.70.0: cxxstd=17 +program_options")
+    depends_on("boost@1.70.0: cxxstd=17 +program_options +stacktrace")
     depends_on("legion network=gasnet", when="backend=legion")
 
     # FleCSI@1.x
@@ -122,19 +125,19 @@ class Flecsi(CMakePackage, CudaPackage, ROCmPackage):
     # FleCSI@2.x
     depends_on("cmake@3.15:", when="@2.0:")
     depends_on("cmake@3.19:", when="@2.2:")
-    depends_on("boost +atomic +filesystem +regex +system", when="@2.0:")
-    depends_on(
-        "boost@1.79.0: cxxstd=17 +program_options +atomic +filesystem +regex +system", when="@2.2:"
-    )
+    depends_on("boost +atomic +filesystem +regex +system", when="@2.0:2.2.1")
+    depends_on("boost@1.79.0:", when="@2.2:")
     depends_on("kokkos@3.2.00:", when="+kokkos @2.0:")
     depends_on("kokkos +cuda +cuda_constexpr +cuda_lambda", when="+kokkos +cuda @2.0:")
     depends_on("kokkos +rocm", when="+kokkos +rocm @2.0:")
     depends_on("legion@cr", when="backend=legion @2.0:")
     depends_on("legion+shared", when="backend=legion +shared @2.0:")
     depends_on("legion+hdf5", when="backend=legion +hdf5 @2.0:")
-    depends_on("legion +kokkos +cuda", when="backend=legion +kokkos +cuda @2.0:")
+    depends_on("legion+kokkos", when="backend=legion +kokkos @2.0:")
+    depends_on("legion+cuda", when="backend=legion +cuda @2.0:")
+    depends_on("legion+rocm", when="backend=legion +rocm @2.0:")
     depends_on("hdf5@1.10.7:", when="backend=legion +hdf5 @2.0:")
-    depends_on("hpx@1.3.0 cxxstd=17 malloc=system", when="backend=hpx @2.0:")
+    depends_on("hpx@1.9.1: cxxstd=17 malloc=system", when="backend=hpx @2.0:")
     depends_on("mpi", when="@2.0:")
     depends_on("mpich@3.4.1:", when="@2.0: ^mpich")
     depends_on("openmpi@4.1.0:", when="@2.0: ^openmpi")
@@ -144,10 +147,10 @@ class Flecsi(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("py-sphinx-rtd-theme", when="@2.2: +doc")
     depends_on("py-recommonmark", when="@2.2: +doc")
     depends_on("doxygen", when="@2.2: +doc")
+    depends_on("graphviz", when="@2.2: +doc")
 
     # Propagate cuda_arch requirement to dependencies
-    cuda_arch_list = ("60", "70", "75", "80")
-    for _flag in cuda_arch_list:
+    for _flag in CudaPackage.cuda_arch_values:
         depends_on("kokkos cuda_arch=" + _flag, when="+cuda+kokkos cuda_arch=" + _flag + " @2.0:")
         depends_on(
             "legion cuda_arch=" + _flag, when="backend=legion +cuda cuda_arch=" + _flag + " @2.0:"
@@ -156,8 +159,12 @@ class Flecsi(CMakePackage, CudaPackage, ROCmPackage):
     # Propagate amdgpu_target requirement to dependencies
     for _flag in ROCmPackage.amdgpu_targets:
         depends_on("kokkos amdgpu_target=" + _flag, when="+kokkos +rocm amdgpu_target=" + _flag)
+        depends_on(
+            "legion amdgpu_target=" + _flag,
+            when="backend=legion +rocm amdgpu_target=" + _flag + " @2.0:",
+        )
 
-    conflicts("%gcc@:8", when="@2.1:")
+    requires("%gcc@9:", when="@2: %gcc", msg="Version 9 or newer of GNU compilers required!")
 
     conflicts("+tutorial", when="backend=hpx")
     # FleCSI@2: no longer supports serial or charmpp backends
@@ -198,6 +205,8 @@ def cmake_args(self):
             if "+rocm" in self.spec:
                 options.append(self.define("CMAKE_CXX_COMPILER", self.spec["hip"].hipcc))
                 options.append(self.define("CMAKE_C_COMPILER", self.spec["hip"].hipcc))
+            elif "+kokkos" in self.spec:
+                options.append(self.define("CMAKE_CXX_COMPILER", self.spec["kokkos"].kokkos_cxx))
         else:
             # kept for supporing version prior to 2.2
             options = [
diff --git a/var/spack/repos/builtin/packages/flecsph/package.py b/var/spack/repos/builtin/packages/flecsph/package.py
index 08d064e5f32733..9d7985df08fb3b 100644
--- a/var/spack/repos/builtin/packages/flecsph/package.py
+++ b/var/spack/repos/builtin/packages/flecsph/package.py
@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 from spack.package import *
-from spack.pkg.builtin.boost import Boost
 
 
 class Flecsph(CMakePackage):
@@ -15,37 +14,32 @@ class Flecsph(CMakePackage):
     includes support for gravitational forces using the fast multipole method
     (FMM)."""
 
-    homepage = "http://flecsi.lanl.com"
+    homepage = "http://flecsi.org"
     git = "https://github.com/laristra/flecsph.git"
 
+    maintainers("JulienLoiseau")
     version("master", branch="master", submodules=True, preferred=True)
 
-    variant("test", default=True, description="Adding tests")
+    variant("debug_tree", default=False, description="Enable debug for Ntree")
 
     depends_on("cmake@3.15:", type="build")
-
-    # TODO: replace this with an explicit list of components of Boost,
-    # for instance depends_on('boost +filesystem')
-    # See https://github.com/spack/spack/pull/22303 for reference
-    depends_on(Boost.with_default_variants)
-    depends_on("boost@1.70.0: cxxstd=17 +program_options")
-
+    depends_on("boost@1.70.0: +atomic +filesystem +regex +system")
     depends_on("mpi")
     depends_on("hdf5+hl@1.8:")
-    depends_on("flecsi@1.4.2 +external_cinch backend=mpi")
+    depends_on("flecsi@2.2 +flog backend=mpi")
     depends_on("gsl")
-    depends_on("googletest", when="+test")
+    depends_on("googletest", type="test")
     depends_on("pkgconfig", type="build")
 
     def setup_run_environment(self, env):
         env.set("HDF5_ROOT", self.spec["hdf5"].prefix)
 
     def cmake_args(self):
-        options = ["-DCMAKE_BUILD_TYPE=debug"]
-        options.append("-DENABLE_UNIT_TESTS=ON")
-        options.append("-DENABLE_DEBUG=OFF")
-        options.append("-DLOG_STRIP_LEVEL=1")
-        options.append("-DENABLE_UNIT_TESTS=ON")
-        options.append("-DENABLE_DEBUG_TREE=OFF")
-        # add option to build the tests
+        options = [
+            self.define("LOG_STRIP_LEVEL", True),
+            self.define("ENABLE_UNIT_TESTS", self.run_tests),
+            self.define_from_variant("ENABLE_DEBUG_TREE", "debug_tree"),
+            self.define_from_variant("ENABLE_DEBUG", "debug_tree"),
+        ]
+
         return options
diff --git a/var/spack/repos/builtin/packages/fleur/package.py b/var/spack/repos/builtin/packages/fleur/package.py
index c854f0f46fd352..a4a2b60cb2d826 100644
--- a/var/spack/repos/builtin/packages/fleur/package.py
+++ b/var/spack/repos/builtin/packages/fleur/package.py
@@ -15,10 +15,10 @@ class Fleur(Package):
     git = "https://iffgit.fz-juelich.de/fleur/fleur.git"
 
     version("develop", branch="develop")
-    version("5.1", tag="MaX-R5.1")
-    version("5.0", tag="MaX-R5")
-    version("4.0", tag="MaX-R4")
-    version("3.1", tag="MaX-R3.1")
+    version("5.1", tag="MaX-R5.1", commit="a482abd9511b16412c2222e2ac1b1a303acd454b")
+    version("5.0", tag="MaX-R5", commit="f2df362c3dad6ef39938807ea14e4ec4cb677723")
+    version("4.0", tag="MaX-R4", commit="ea0db7877451e6240124e960c5546318c9ab3953")
+    version("3.1", tag="MaX-R3.1", commit="f6288a0699604ad9e11efbfcde824b96db429404")
 
     variant("mpi", default=True, description="Enable MPI support")
     variant("hdf5", default=False, description="Enable HDF5 support")
diff --git a/var/spack/repos/builtin/packages/flexi/package.py b/var/spack/repos/builtin/packages/flexi/package.py
index 9a976d4d9980ac..ef0aa67961d027 100644
--- a/var/spack/repos/builtin/packages/flexi/package.py
+++ b/var/spack/repos/builtin/packages/flexi/package.py
@@ -14,7 +14,7 @@ class Flexi(CMakePackage):
     git = "https://github.com/flexi-framework/flexi.git"
 
     version("master", preferred=True)
-    version("21.03.0", tag="v21.03.0")
+    version("21.03.0", tag="v21.03.0", commit="d061978e5d96cfc96c06edc1bae9d92cbe540c18")
 
     patch("for_aarch64.patch", when="target=aarch64:")
 
@@ -97,7 +97,7 @@ class Flexi(CMakePackage):
     depends_on("hdf5+fortran+mpi", when="+mpi")
     depends_on("hdf5+fortran~mpi", when="~mpi")
     depends_on("lapack")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("fftw", when="+channel_fft")
 
     def flag_handler(self, name, flags):
diff --git a/var/spack/repos/builtin/packages/flux-core/package.py b/var/spack/repos/builtin/packages/flux-core/package.py
index 96e879cea023ea..bb150b154dc9b8 100644
--- a/var/spack/repos/builtin/packages/flux-core/package.py
+++ b/var/spack/repos/builtin/packages/flux-core/package.py
@@ -20,6 +20,9 @@ class FluxCore(AutotoolsPackage):
     maintainers("grondo")
 
     version("master", branch="master")
+    version("0.55.0", sha256="2925b8a084e9d1069a96de7689b515ad6f2051ecfb9fbbe4d2643507de7ccd30")
+    version("0.54.0", sha256="721fc3fff64b3b167ae55d0e29379ff3211729248ef97e3b9855816219063b42")
+    version("0.53.0", sha256="2f14d032a2d54f34e066c8a15c79917089e9f7f8558baa03dbfe63dbf56918b7")
     version("0.52.0", sha256="dca434238405e4cae4686c8143f2cc79919bfd9e26b09c980e1e5f69ffd0c448")
     version("0.51.0", sha256="e57b71b708482f20d2a2195a000c0c3b9176faa6aaadfad4d2117f8671ca67ce")
     version("0.50.0", sha256="77414299a7ca081199aa0f57bcaea3e05860e2095df73c0f6b7672b88fadf683")
@@ -123,7 +126,7 @@ class FluxCore(AutotoolsPackage):
     conflicts("platform=darwin", msg="flux-core does not support MacOS based platforms.")
     conflicts("platform=windows", msg="flux-core does not support Windows based platforms.")
 
-    depends_on("libarchive", when="@0.38.0:")
+    depends_on("libarchive+iconv", when="@0.38.0:")
     depends_on("ncurses@6.2:", when="@0.32.0:")
     depends_on("libzmq@4.0.4:")
     depends_on("czmq@3.0.1:")
@@ -139,6 +142,8 @@ class FluxCore(AutotoolsPackage):
     # `link` dependency on python due to Flux's `pymod` module
     depends_on("python@3.6:", when="@0.17:", type=("build", "link", "run"))
     depends_on("python@2.7:", type=("build", "link", "run"))
+    # Use of distutils in configure script dropped in v0.55
+    depends_on("python@:3.11", when="@:0.54", type=("build", "link", "run"))
     depends_on("py-cffi@1.1:", type=("build", "run"))
     depends_on("py-six@1.9:", when="@:0.24", type=("build", "run"))
     depends_on("py-pyyaml@3.10:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/flux-pmix/package.py b/var/spack/repos/builtin/packages/flux-pmix/package.py
index 5969da47b2f50f..db6d580a61c5c3 100644
--- a/var/spack/repos/builtin/packages/flux-pmix/package.py
+++ b/var/spack/repos/builtin/packages/flux-pmix/package.py
@@ -18,10 +18,11 @@ class FluxPmix(AutotoolsPackage):
     maintainers("grondo")
 
     version("main", branch="main")
+    version("0.4.0", sha256="f7f58891fc9d9a97a0399b3ab186f2cae30a75806ba0b4d4c1307f07b3f6d1bc")
     version("0.3.0", sha256="88edb2afaeb6058b56ff915105a36972acc0d83204cff7f4a4d2f65a5dee9d34")
     version("0.2.0", sha256="d09f1fe6ffe54f83be4677e1e727640521d8110090515d94013eba0f58216934")
 
-    depends_on("flux-core@0.49.0:", when="@0.3.0:")
+    depends_on("flux-core@0.49:", when="@0.3:")
     depends_on("flux-core@0.30.0:")
     depends_on("pmix@v4.1.0:")
     depends_on("openmpi")
diff --git a/var/spack/repos/builtin/packages/flux-sched/package.py b/var/spack/repos/builtin/packages/flux-sched/package.py
index 122cb40dd55912..df38f938296043 100644
--- a/var/spack/repos/builtin/packages/flux-sched/package.py
+++ b/var/spack/repos/builtin/packages/flux-sched/package.py
@@ -6,11 +6,13 @@
 import os
 
 import spack.util.executable
+from spack.build_systems.autotools import AutotoolsBuilder
+from spack.build_systems.cmake import CMakeBuilder
 from spack.package import *
 
 
-class FluxSched(AutotoolsPackage):
-    """A scheduler for flux-core (pre-alpha)"""
+class FluxSched(CMakePackage, AutotoolsPackage):
+    """A scheduler for flux-core"""
 
     homepage = "https://github.com/flux-framework/flux-sched"
     url = "https://github.com/flux-framework/flux-sched/releases/download/v0.5.0/flux-sched-0.5.0.tar.gz"
@@ -20,6 +22,8 @@ class FluxSched(AutotoolsPackage):
     maintainers("grondo")
 
     version("master", branch="master")
+    version("0.29.0", sha256="b93b18788e677535aa8ef945cdbeeced6d1408a4d16cb4a816ead53f31dd78d2")
+    version("0.28.0", sha256="9431c671bed5d76fd95b4a4a7f36224d4bf76f416a2a1a5c4908f3ca790d434d")
     version("0.27.0", sha256="1e131924440c904fa0c925b7aa14c47b97f4e67b43af7efd2ebc0ef7ce90eb7c")
     version("0.26.0", sha256="184faec800cf45952ef79bda113f710bf91a05be584034d36a3234627d4a54c7")
     version("0.25.0", sha256="a984b238d8b6968ef51f1948a550bf57887bf3da8002dcd1734ce26afc4bff07")
@@ -80,6 +84,18 @@ class FluxSched(AutotoolsPackage):
     depends_on("automake", type="build", when="@master")
     depends_on("libtool", type="build", when="@master")
 
+    # Set default to cmake so master (and branches) use it
+    build_system(
+        conditional("cmake", when="@0.29.0:"),
+        conditional("autotools", when="@:0.28.0"),
+        default="cmake",
+    )
+
+    # Required dependencies
+    with when("build_system=cmake"):
+        generator("ninja")
+        depends_on("cmake@3.18:", type="build")
+
     # Disable t5000-valgrind.t by default due to false positives not yet
     # in the suppressions file. (This patch will be in v0.21.0)
     patch("no-valgrind.patch", when="@:0.20.0")
@@ -135,19 +151,6 @@ def patch(self):
         filter_file("NULL", "nullptr", "resource/schema/sched_data.hpp")
         filter_file("size_t", "std::size_t", "resource/planner/planner.h")
 
-    def configure_args(self):
-        args = []
-        if self.spec.satisfies("@0.9.0:"):
-            args.append("CXXFLAGS=-Wno-uninitialized")
-        if self.spec.satisfies("%clang@12:"):
-            args.append("CXXFLAGS=-Wno-defaulted-function-deleted")
-        if self.spec.satisfies("%oneapi"):
-            args.append("CXXFLAGS=-Wno-tautological-constant-compare")
-        # flux-sched's ax_boost is sometimes weird about non-system locations
-        # explicitly setting the path guarantees success
-        args.append("--with-boost={0}".format(self.spec["boost"].prefix))
-        return args
-
     @property
     def lua_version(self):
         return self.spec["lua"].version.up_to(2)
@@ -172,3 +175,23 @@ def setup_run_environment(self, env):
         env.prepend_path("FLUX_MODULE_PATH", self.prefix.lib.flux.modules.sched)
         env.prepend_path("FLUX_EXEC_PATH", self.prefix.libexec.flux.cmd)
         env.prepend_path("FLUX_RC_EXTRA", self.prefix.etc.flux)
+
+
+class CMakeBuilder(CMakeBuilder):
+    def cmake_args(self):
+        return []
+
+
+class AutotoolsBuilder(AutotoolsBuilder):
+    def configure_args(self):
+        args = []
+        if self.spec.satisfies("@0.9.0:"):
+            args.append("CXXFLAGS=-Wno-uninitialized")
+        if self.spec.satisfies("%clang@12:"):
+            args.append("CXXFLAGS=-Wno-defaulted-function-deleted")
+        if self.spec.satisfies("%oneapi"):
+            args.append("CXXFLAGS=-Wno-tautological-constant-compare")
+        # flux-sched's ax_boost is sometimes weird about non-system locations
+        # explicitly setting the path guarantees success
+        args.append("--with-boost={0}".format(self.spec["boost"].prefix))
+        return args
diff --git a/var/spack/repos/builtin/packages/flux-security/package.py b/var/spack/repos/builtin/packages/flux-security/package.py
index f8c5c45933399d..71f0081f7125be 100644
--- a/var/spack/repos/builtin/packages/flux-security/package.py
+++ b/var/spack/repos/builtin/packages/flux-security/package.py
@@ -20,6 +20,7 @@ class FluxSecurity(AutotoolsPackage):
     maintainers("grondo")
 
     version("master", branch="master")
+    version("0.10.0", sha256="b0f39c5e32322f901454469ffd6154019b6dffafc064b55b3e593f70db6a6f68")
     version("0.9.0", sha256="2258120c6f32ca0b5b13b166bae56d9bd82a44c6eeaa6bc6187e4a4419bdbcc0")
     version("0.8.0", sha256="9963628063b4abdff6bece03208444c8f23fbfda33c20544c48b21e9f4819ce2")
 
diff --git a/var/spack/repos/builtin/packages/fms/package.py b/var/spack/repos/builtin/packages/fms/package.py
index f7bc88af28a146..8e351b1bba8d3f 100644
--- a/var/spack/repos/builtin/packages/fms/package.py
+++ b/var/spack/repos/builtin/packages/fms/package.py
@@ -23,6 +23,9 @@ class Fms(CMakePackage):
         "2023.02.01", sha256="1597f7a485d02e401ce76444b2401060d74bd032cbb060cef917f001b4ff14bc"
     )
     version("2023.02", sha256="dc029ffadfd82c334f104268bedd8635c77976485f202f0966ae4cf06d2374be")
+    version(
+        "2023.01.01", sha256="f83e2814a1e3ba439ab847ec8bb251f3889d5ca14fb20849507590adbbe8e899"
+    )
     version("2023.01", sha256="6079ea885e9365513b453c77aadfc7c305bf413b840656bb333db1eabba0f18e")
     version("2022.04", sha256="f741479128afc2b93ca8291a4c5bcdb024a8cbeda1a26bf77a236c0f629e1b03")
     version("2022.03", sha256="42d2ac53d3c889a8177a6d7a132583364c0f6e5d5cbde0d980443b6797ad4838")
@@ -98,7 +101,10 @@ class Fms(CMakePackage):
         "pic", default=False, description="Build with position independent code", when="@2022.02:"
     )
     variant(
-        "use_fmsio", default=False, description="Enable deprecated fms_io API", when="@2023.02:"
+        "deprecated_io",
+        default=False,
+        description="Compiles with support for deprecated io modules fms_io and mpp_io",
+        when="@2023.02:",
     )
 
     depends_on("netcdf-c")
@@ -122,9 +128,8 @@ def cmake_args(self):
             self.define("32BIT", "precision=32" in self.spec),
             self.define("64BIT", "precision=64" in self.spec),
             self.define_from_variant("FPIC", "pic"),
+            self.define_from_variant("USE_DEPRECATED_IO", "deprecated_io"),
         ]
-        with when("@2023.02:"):
-            args.append(self.define_from_variant("USE_DEPRECATED_IO", "use_fmsio"))
 
         args.append(self.define("CMAKE_C_COMPILER", self.spec["mpi"].mpicc))
         args.append(self.define("CMAKE_CXX_COMPILER", self.spec["mpi"].mpicxx))
diff --git a/var/spack/repos/builtin/packages/fmt/package.py b/var/spack/repos/builtin/packages/fmt/package.py
index 878cb2e7b1b7b1..ea7abc909284ec 100644
--- a/var/spack/repos/builtin/packages/fmt/package.py
+++ b/var/spack/repos/builtin/packages/fmt/package.py
@@ -15,6 +15,8 @@ class Fmt(CMakePackage):
     url = "https://github.com/fmtlib/fmt/releases/download/7.1.3/fmt-7.1.3.zip"
     maintainers("msimberg")
 
+    version("10.1.1", sha256="b84e58a310c9b50196cda48d5678d5fa0849bca19e5fdba6b684f0ee93ed9d1b")
+    version("10.1.0", sha256="d725fa83a8b57a3cedf238828fa6b167f963041e8f9f7327649bddc68ae316f4")
     version("10.0.0", sha256="4943cb165f3f587f26da834d3056ee8733c397e024145ca7d2a8a96bb71ac281")
     version("9.1.0", sha256="cceb4cb9366e18a5742128cb3524ce5f50e88b476f1e54737a47ffdf4df4c996")
     version("9.0.0", sha256="fc96dd2d2fdf2bded630787adba892c23cb9e35c6fd3273c136b0c57d4651ad6")
@@ -39,7 +41,7 @@ class Fmt(CMakePackage):
     variant(
         "cxxstd",
         default="11",
-        values=("98", "11", "14", "17", "20"),
+        values=("98", "11", "14", "17", "20", conditional("23", when="^cmake@3.20.3:")),
         multi=False,
         description="Use the specified C++ standard when building",
     )
diff --git a/var/spack/repos/builtin/packages/foam-extend/package.py b/var/spack/repos/builtin/packages/foam-extend/package.py
index ac1c7be88eeaec..ffd431f18a460c 100644
--- a/var/spack/repos/builtin/packages/foam-extend/package.py
+++ b/var/spack/repos/builtin/packages/foam-extend/package.py
@@ -76,7 +76,7 @@ class FoamExtend(Package):
 
     depends_on("mpi")
     depends_on("python")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("flex", type="build")
     depends_on("cmake", type="build")
 
@@ -261,7 +261,7 @@ def configure(self, spec, prefix):
             },
             "flex": {"FLEX_SYSTEM": 1, "FLEX_DIR": spec["flex"].prefix},
             "bison": {"BISON_SYSTEM": 1, "BISON_DIR": spec["flex"].prefix},
-            "zlib": {"ZLIB_SYSTEM": 1, "ZLIB_DIR": spec["zlib"].prefix},
+            "zlib": {"ZLIB_SYSTEM": 1, "ZLIB_DIR": spec["zlib-api"].prefix},
         }
         # Adjust configuration via prefs - sort second
         self.etc_prefs["001"].update(self.foam_arch.foam_dict())
diff --git a/var/spack/repos/builtin/packages/fontconfig/package.py b/var/spack/repos/builtin/packages/fontconfig/package.py
index c87c631df0e321..d45b261d133aa4 100644
--- a/var/spack/repos/builtin/packages/fontconfig/package.py
+++ b/var/spack/repos/builtin/packages/fontconfig/package.py
@@ -62,7 +62,6 @@ def configure_args(self):
         args.extend(self.with_or_without("pic"))
 
         return args
-        
 
     @run_after("install")
     def system_fonts(self):
diff --git a/var/spack/repos/builtin/packages/foonathan-memory/package.py b/var/spack/repos/builtin/packages/foonathan-memory/package.py
index 2941cc0a0a4f03..6ff55036c1a53a 100644
--- a/var/spack/repos/builtin/packages/foonathan-memory/package.py
+++ b/var/spack/repos/builtin/packages/foonathan-memory/package.py
@@ -13,4 +13,8 @@ class FoonathanMemory(CMakePackage):
     homepage = "https://memory.foonathan.net/"
     url = "https://github.com/foonathan/memory/archive/v0.7.tar.gz"
 
+    version("0.7-3", sha256="4203d15db22a94a3978eeb1afb59a37d35c57c0f148733f0f1a53a6281cb74dd")
     version("0.7", sha256="01a7cc5a5ebddbd71bec69c89562a4a2ecd7c29334c0a29d38d83e7f7f66eb53")
+
+    depends_on("cmake@3.14:", when="@0.7-3:", type="build")
+    depends_on("cmake@3.1:", type="build")
diff --git a/var/spack/repos/builtin/packages/form/package.py b/var/spack/repos/builtin/packages/form/package.py
index cffb758552c046..13c303ca508802 100644
--- a/var/spack/repos/builtin/packages/form/package.py
+++ b/var/spack/repos/builtin/packages/form/package.py
@@ -23,7 +23,7 @@ class Form(AutotoolsPackage):
     )
 
     depends_on("gmp", type="link", when="+gmp")
-    depends_on("zlib", type="link", when="+zlib")
+    depends_on("zlib-api", type="link", when="+zlib")
     depends_on("mpi", type="link", when="+parform")
 
     variant("gmp", default=True, description="Use GMP for long integer arithmetic")
@@ -35,7 +35,10 @@ class Form(AutotoolsPackage):
     def configure_args(self):
         args = []
         args += self.with_or_without("gmp", "prefix")
-        args += self.with_or_without("zlib", "prefix")
+        if "+zlib" in self.spec:
+            args.append("--with-zlib=%s" % self.spec["zlib-api"].prefix)
+        else:
+            args.append("--without-zlib")
         args += self.enable_or_disable("scalar")
         args += self.enable_or_disable("threaded")
         args += self.enable_or_disable("parform")
diff --git a/var/spack/repos/builtin/packages/fortrilinos/package.py b/var/spack/repos/builtin/packages/fortrilinos/package.py
index 0ed163705bd862..f1a51a8909ceb3 100644
--- a/var/spack/repos/builtin/packages/fortrilinos/package.py
+++ b/var/spack/repos/builtin/packages/fortrilinos/package.py
@@ -27,6 +27,7 @@ class Fortrilinos(CMakePackage):
     tags = ["e4s"]
     test_requires_compiler = True
 
+    version("2.3.0", sha256="7be5efecaea61ad773d3fe182aa28735ebc3e7af821e1805ad284e4ed4e31a49")
     version("2.2.0", sha256="9e73fc71066bfaf7cde040e1467baf7a1ec797ff2874add49f9741e93f9fffb5")
     version("2.1.0", sha256="2c62bb6106ae86a804497d549080cb6877c5d860b6bf2e72ec5cbcbbe63e3b5b")
     version("2.0.1", sha256="291a62c885cd4ffd76cbebafa02789649bd4fa73f1005cf8da51fd153acb9e1a")
@@ -50,9 +51,10 @@ class Fortrilinos(CMakePackage):
     variant("shared", default=True, description="Build shared libraries")
 
     # Trilinos version dependencies
-    depends_on("trilinos@13.4.0:13.4", when="@2.2.0:2.2")
-    depends_on("trilinos@13.2.0:13.2", when="@2.1.0:2.1")
-    depends_on("trilinos@13.0.0:13.2", when="@2.0.0:2.0")
+    depends_on("trilinos@14.0", when="@2.3")
+    depends_on("trilinos@13.4", when="@2.2")
+    depends_on("trilinos@13.2", when="@2.1.0:2.1")
+    depends_on("trilinos@13:13.2", when="@2.0")
     depends_on("trilinos@12.18.1", when="@2.0.dev3")
     depends_on("trilinos@12.18.1", when="@2.0.dev2")
 
diff --git a/var/spack/repos/builtin/packages/fox/package.py b/var/spack/repos/builtin/packages/fox/package.py
index b50623bcb1cec1..2d0166e3fcf496 100644
--- a/var/spack/repos/builtin/packages/fox/package.py
+++ b/var/spack/repos/builtin/packages/fox/package.py
@@ -32,7 +32,7 @@ class Fox(AutotoolsPackage):
     depends_on("jpeg")
     depends_on("libpng")
     depends_on("libtiff")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libx11")
     depends_on("libsm")
     depends_on("libxft")
diff --git a/var/spack/repos/builtin/packages/fp16/package.py b/var/spack/repos/builtin/packages/fp16/package.py
index f3d535efbdccb1..5e56aec0113a74 100644
--- a/var/spack/repos/builtin/packages/fp16/package.py
+++ b/var/spack/repos/builtin/packages/fp16/package.py
@@ -14,7 +14,7 @@ class Fp16(CMakePackage):
     git = "https://github.com/Maratyszcza/FP16.git"
 
     version("master", branch="master")
-    version("2020-05-14", commit="4dfe081cf6bcd15db339cf2680b9281b8451eeb3")  # py-torch@1.5:1.9
+    version("2020-05-14", commit="4dfe081cf6bcd15db339cf2680b9281b8451eeb3")  # py-torch@1.5:
     version("2018-11-28", commit="febbb1c163726b5db24bed55cc9dc42529068997")  # py-torch@1.1:1.4
     version("2018-10-10", commit="34d4bf01bbf7376f2baa71b8fa148b18524d45cf")  # py-torch@1.0
     version("2018-02-25", commit="43d6d17df48ebf622587e7ed9472ea76573799b9")  # py-torch@:0.4
@@ -29,31 +29,11 @@ class Fp16(CMakePackage):
         destination="deps",
         placement="psimd",
     )
-    resource(
-        name="googletest",
-        url="https://github.com/google/googletest/archive/release-1.8.0.zip",
-        sha256="f3ed3b58511efd272eb074a3a6d6fb79d7c2e6a0e374323d1e6bcbcc1ef141bf",
-        destination="deps",
-        placement="googletest",
-    )
-    resource(
-        name="googlebenchmark",
-        url="https://github.com/google/benchmark/archive/v1.2.0.zip",
-        sha256="cc463b28cb3701a35c0855fbcefb75b29068443f1952b64dd5f4f669272e95ea",
-        destination="deps",
-        placement="googlebenchmark",
-    )
 
     def cmake_args(self):
         return [
             self.define("PSIMD_SOURCE_DIR", join_path(self.stage.source_path, "deps", "psimd")),
-            self.define(
-                "GOOGLETEST_SOURCE_DIR", join_path(self.stage.source_path, "deps", "googletest")
-            ),
-            self.define(
-                "GOOGLEBENCHMARK_SOURCE_DIR",
-                join_path(self.stage.source_path, "deps", "googlebenchmark"),
-            ),
-            self.define("FP16_BUILD_TESTS", self.run_tests),
-            self.define("FP16_BUILD_BENCHMARKS", self.run_tests),
+            self.define("FP16_BUILD_TESTS", False),
+            # https://github.com/Maratyszcza/FP16/issues/21
+            self.define("FP16_BUILD_BENCHMARKS", False),
         ]
diff --git a/var/spack/repos/builtin/packages/fpchecker/package.py b/var/spack/repos/builtin/packages/fpchecker/package.py
index bd3eb9ed3db730..ca832e9d3f030b 100644
--- a/var/spack/repos/builtin/packages/fpchecker/package.py
+++ b/var/spack/repos/builtin/packages/fpchecker/package.py
@@ -19,6 +19,7 @@ class Fpchecker(CMakePackage):
     maintainers("ilagunap")
 
     version("master", branch="master")
+    version("0.4.0", sha256="59d18a3fb5211645bb1cfde1502ba78a9cca4234604dfd879534d7f13ae5d5b9")
     version("0.3.5", sha256="ed7277318af8e0a22b05c5655c9acc99e1d3036af41095ec2f1b1ada4d6e90f6")
     version("0.3.4", sha256="ecea778dcddc8347da86b02069e12d574a3ba27a4f7c6224bf492fbff6cd162a")
 
diff --git a/var/spack/repos/builtin/packages/fplo/package.py b/var/spack/repos/builtin/packages/fplo/package.py
index 2c025218ac8ac2..a9e8cbc2c2fb11 100644
--- a/var/spack/repos/builtin/packages/fplo/package.py
+++ b/var/spack/repos/builtin/packages/fplo/package.py
@@ -83,7 +83,7 @@ def edit(self, spec, prefix):
         filter_file(r"^\s*F90\s*=.*", "F90=" + spack_fc, *files)
 
         # patch for 64 bit integers
-        if "^mkl+ilp64" in spec:
+        if spec["mkl"].satisfies("+ilp64"):
             setuphelper = FileFilter(join_path(self.build_directory, "PYTHON", "setuphelper.py"))
             setuphelper.filter("mkl 64bit integer 32bit", "mkl 64bit integer 64bit")
 
diff --git a/var/spack/repos/builtin/packages/fpm/package.py b/var/spack/repos/builtin/packages/fpm/package.py
index 9069d6652e28cf..6c8d1811bb79f6 100644
--- a/var/spack/repos/builtin/packages/fpm/package.py
+++ b/var/spack/repos/builtin/packages/fpm/package.py
@@ -23,6 +23,10 @@ class Fpm(Package):
 
     maintainers("awvwgk")
 
+    version("0.9.0", sha256="484debabd7d22186ac41f865ddf63475c279a61a51aaff5636ed615860b5b8d7")
+    version("0.8.2", sha256="67fd8f4f78d19662c61855f531465e347ab0bc913ba59bd419f75f4022d2cd70")
+    version("0.8.1", sha256="0bd978bb1d3f2a3297d82a0d6ac009746a466cfa9a59ba3b6513b74e5ce4b7bf")
+    version("0.8.0", sha256="d63162a2ab013c19cefc938e52717c30f78e04de94384d4589c55a48be2724f1")
     version("0.7.0", sha256="536dec7d4502221734683b15e6ff64a6ab3f9910df122d18f851c9a68711f91f")
     version("0.6.0", sha256="365516f66b116a112746af043e8eccb3d854d6feb1fad0507c570433dacbf7be")
     version("0.5.0", sha256="e4a06956d2300f9aa1d06bd3323670480e946549617582e32684ded6921a921e")
diff --git a/var/spack/repos/builtin/packages/fpocket/package.py b/var/spack/repos/builtin/packages/fpocket/package.py
index 831283a4ef66dd..bf8d64aa9e8e12 100644
--- a/var/spack/repos/builtin/packages/fpocket/package.py
+++ b/var/spack/repos/builtin/packages/fpocket/package.py
@@ -3,23 +3,27 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+
 from spack.package import *
 
 
 class Fpocket(MakefilePackage):
-    """fpocket is a very fast open source protein pocket detection algorithm
-    based on Voronoi tessellation."""
+    """The fpocket suite of programs is a very fast open source
+    protein pocket detection algorithm based on Voronoi tessellation."""
 
     homepage = "https://github.com/Discngine/fpocket"
-    version("master", branch="master", git="https://github.com/Discngine/fpocket.git")
+    url = "https://github.com/Discngine/fpocket/archive/refs/tags/4.1.tar.gz"
+
+    version("4.1", "1a2af2d3f2df42de67301996db3b93c7eaff0375f866443c0468dcf4b1750688")
 
     depends_on("netcdf-c")
+    depends_on("netcdf-cxx")
 
     def setup_build_environment(self, env):
         if self.compiler.name == "gcc":
             env.set("CXX", "g++")
 
-    def edit(self):
+    def edit(self, spec, prefix):
         makefile = FileFilter("makefile")
-        makefile.filter("BINDIR .*", "BINDIR = %s/bin" % self.prefix)
-        makefile.filter("MANDIR .*", "MANDIR = %s/man/man8" % self.prefix)
+        makefile.filter("BINDIR .*", f"BINDIR = {prefix}/bin")
+        makefile.filter("MANDIR .*", f"MANDIR = {prefix}/man/man8")
diff --git a/var/spack/repos/builtin/packages/fq/package.py b/var/spack/repos/builtin/packages/fq/package.py
new file mode 100644
index 00000000000000..e0b73c3906bba4
--- /dev/null
+++ b/var/spack/repos/builtin/packages/fq/package.py
@@ -0,0 +1,24 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Fq(Package):
+    """fq is a library to generate and validate FASTQ file pairs.
+
+    fq provides subcommands for filtering, generating, subsampling, and validating FASTQ files."""
+
+    homepage = "https://github.com/stjude-rust-labs/fq"
+    url = "https://github.com/stjude-rust-labs/fq/archive/refs/tags/v0.10.0.tar.gz"
+    maintainers("pabloaledo")
+
+    version("0.10.0", sha256="34007ab71a873e1b066d910e90c5bdac3dcc4299ae6c9891ac6d8233cffeabb8")
+
+    depends_on("rust")
+
+    def install(self, spec, prefix):
+        cargo = which("cargo")
+        cargo("install", "--root", prefix, "--path", ".")
diff --git a/var/spack/repos/builtin/packages/freebayes/package.py b/var/spack/repos/builtin/packages/freebayes/package.py
index 89a3f7eb4d040a..f54c7955211c23 100644
--- a/var/spack/repos/builtin/packages/freebayes/package.py
+++ b/var/spack/repos/builtin/packages/freebayes/package.py
@@ -24,13 +24,13 @@ class Freebayes(MesonPackage):
     )
 
     depends_on("cmake", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # Deps for @1.3.5 and beyond
     depends_on("ninja", type="build", when="@1.3.5:")
     depends_on("pkgconfig", type="build", when="@1.3.5:")
     depends_on("htslib", when="@1.3.5:")
-    depends_on("zlib", when="@1.3.5:")
+    depends_on("zlib-api", when="@1.3.5:")
     depends_on("xz", when="@1.3.5:")
     depends_on("parallel", when="@1.3.5:")
     depends_on("vcftools", when="@1.3.5:")
diff --git a/var/spack/repos/builtin/packages/freeglut/package.py b/var/spack/repos/builtin/packages/freeglut/package.py
index 60faf18cf9a58f..9114f6e1e2a943 100644
--- a/var/spack/repos/builtin/packages/freeglut/package.py
+++ b/var/spack/repos/builtin/packages/freeglut/package.py
@@ -23,7 +23,7 @@ class Freeglut(CMakePackage, SourceforgePackage):
     depends_on("glu")
 
     # FreeGLUT does not support OSMesa
-    conflicts("osmesa")
+    conflicts("^osmesa")
 
     # FreeGLUT only works with GLX on linux (cray is also linux)
     with when("platform=linux"):
@@ -71,3 +71,7 @@ def cmake_args(self):
         ]
 
         return args
+
+    @property
+    def libs(self):
+        return find_libraries(["libglut"], root=self.prefix, recursive=True)
diff --git a/var/spack/repos/builtin/packages/freesasa/libcpp.patch b/var/spack/repos/builtin/packages/freesasa/libcpp.patch
new file mode 100644
index 00000000000000..e62fbdc10383a3
--- /dev/null
+++ b/var/spack/repos/builtin/packages/freesasa/libcpp.patch
@@ -0,0 +1,21 @@
+diff --git a/src/Makefile.am b/src/Makefile.am
+index 140259f..4e2ed3f 100644
+--- a/src/Makefile.am
++++ b/src/Makefile.am
+@@ -1,6 +1,5 @@
+ AM_LDFLAGS =
+ AM_CFLAGS =
+-freesasa_LDADD = -lc++
+ AM_CPPFLAGS = -I$(top_srcdir)/third-party/gemmi/include/
+ 
+ if COND_GCOV
+@@ -29,7 +28,7 @@ libfreesasa_a_SOURCES = classifier.c classifier.h \
+        selection.h selection.c $(lp_output)
+ freesasa_SOURCES = main.cc cif.cc cif.hh
+ example_SOURCES = example.c
+-freesasa_LDADD += libfreesasa.a
++freesasa_LDADD = libfreesasa.a
+ example_LDADD = libfreesasa.a
+ 
+ lp_output = lexer.c lexer.h parser.c parser.h
+
diff --git a/var/spack/repos/builtin/packages/freesasa/package.py b/var/spack/repos/builtin/packages/freesasa/package.py
index 135b65363d28f9..201fbd596527de 100644
--- a/var/spack/repos/builtin/packages/freesasa/package.py
+++ b/var/spack/repos/builtin/packages/freesasa/package.py
@@ -16,9 +16,9 @@ class Freesasa(AutotoolsPackage):
 
     version("2.1.2", sha256="a031c4eb8cd59e802d715a37ef72930ec2d90ec53dfcf1bea0b0255980490fd5")
 
-    variant("json", default=True)
-    variant("xml", default=True)
-    variant("threads", default=True)
+    variant("json", default=True, description="Build with support for JSON output")
+    variant("xml", default=True, description="Build with support for XML output")
+    variant("threads", default=True, description="Build with support for multiple threads")
 
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
@@ -30,6 +30,10 @@ class Freesasa(AutotoolsPackage):
     depends_on("json-c", when="+json")
     depends_on("libxml2", when="+xml")
 
+    # Remove hard-coded -lc++ flag from Makefile, preventing successful
+    # compilation with GCC 11 (see #36566 for details)
+    patch("libcpp.patch", when="@2.1.2")
+
     def autoreconf(self, spec, prefix):
         autoreconf("--install", "--verbose", "--force")
 
diff --git a/var/spack/repos/builtin/packages/freesurfer/package.py b/var/spack/repos/builtin/packages/freesurfer/package.py
index 4bf4a4a2f5ef71..0e7188db06c51a 100644
--- a/var/spack/repos/builtin/packages/freesurfer/package.py
+++ b/var/spack/repos/builtin/packages/freesurfer/package.py
@@ -3,7 +3,11 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import glob
+import os
+
 from spack.package import *
+from spack.util.environment import EnvironmentModifications
 
 
 class Freesurfer(Package):
@@ -14,11 +18,11 @@ class Freesurfer(Package):
 
     # A license is required, but is free to obtain.
     license_required = True
-    license_files = ["./license.txt"]
+    license_files = [".license"]
 
     maintainers("robgics")
 
-    version("7.4.1", sha256="eb6545d1ffdee17a90abd2e7dc444aa1091a6138e257f6f956a7ff214635b092")
+    version("7.4.1", sha256="313a96caeb246c5985f483633b5cf43f86ed8f7ccc6d6acfac8eedb638443010")
     version("7.4.0", sha256="6b65c2edf3b88973ced0324269a88966c541f221b799337c6570c38c2f884431")
     version("7.3.2", sha256="58518d3ee5abd2e05109208aed2eef145c4e3b994164df8c4e0033c1343b9e56")
     version("7.2.0", sha256="4cca78602f898bf633428b9d82cbb9b07e3ab97a86c620122050803779c86d62")
@@ -27,6 +31,9 @@ class Freesurfer(Package):
 
     depends_on("mesa-glu")
     depends_on("qt")
+    depends_on("tcsh")
+    depends_on("bc")
+    depends_on("perl")
 
     def url_for_version(self, version):
         return "https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/{0}/freesurfer-linux-centos7_x86_64-{1}.tar.gz".format(
@@ -34,10 +41,24 @@ def url_for_version(self, version):
         )
 
     def setup_run_environment(self, env):
+        source_file = join_path(self.prefix, "SetUpFreeSurfer.sh")
         env.prepend_path("PATH", self.prefix.bin)
         env.set("FREESURFER_HOME", self.prefix)
         env.set("SUBJECTS_DIR", join_path(self.prefix, "subjects"))
         env.set("FUNCTIONALS_DIR", join_path(self.prefix, "sessions"))
+        env.append_path("PERL5LIB", join_path(self.prefix, "mni/share/perl5"))
+        env.append_path("PATH", join_path(self.prefix, "mni/bin"))
+        env.extend(EnvironmentModifications.from_sourcing_file(source_file))
 
     def install(self, spec, prefix):
+        scripts = ["sources.csh", "SetUpFreeSurfer.csh"]
+        scripts.extend(glob.glob("bin/*"))
+        scripts.extend(glob.glob("subjects/**/*", recursive=True))
+        scripts.extend(glob.glob("fsfast/bin/*", recursive=True))
+        scripts.extend(glob.glob("mni/bin/*", recursive=True))
+        for s in scripts:
+            if os.path.isfile(s):
+                filter_file(r"(\/usr)?(\/local?)\/bin\/tcsh", "/usr/bin/env -S tcsh", s)
+                filter_file(r"(\/usr)?(\/local?)\/bin\/csh", "/usr/bin/env -S csh", s)
+                filter_file(r"(\/usr)?(\/local)?\/bin\/perl", "/usr/bin/env -S perl", s)
         install_tree(".", prefix)
diff --git a/var/spack/repos/builtin/packages/frontier-client/package.py b/var/spack/repos/builtin/packages/frontier-client/package.py
index 616d1f0c8e14ff..a25c0c854f6d2b 100644
--- a/var/spack/repos/builtin/packages/frontier-client/package.py
+++ b/var/spack/repos/builtin/packages/frontier-client/package.py
@@ -23,7 +23,7 @@ class FrontierClient(MakefilePackage):
     depends_on("pacparser")
     depends_on("expat")
     depends_on("openssl")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     patch("frontier-client.patch", level=0)
 
@@ -71,7 +71,7 @@ def build(self, spec, prefix):
                 "PACPARSER_DIR=" + self.spec["pacparser"].prefix,
                 "EXPAT_DIR=" + self.spec["expat"].prefix,
                 "OPENSSL_DIR=" + self.spec["openssl"].prefix,
-                "ZLIB_DIR=" + self.spec["zlib"].prefix,
+                "ZLIB_DIR=" + self.spec["zlib-api"].prefix,
             )
 
     def install(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/frontistr/package.py b/var/spack/repos/builtin/packages/frontistr/package.py
index 54380b40be53cc..c86304e00e93a6 100644
--- a/var/spack/repos/builtin/packages/frontistr/package.py
+++ b/var/spack/repos/builtin/packages/frontistr/package.py
@@ -50,9 +50,9 @@ class Frontistr(FrontistrBase):
 
     maintainers("hiroshi.okuda", "kgoto", "morita", "inagaki", "michioga")
 
-    version("5.3", tag="v5.3")
-    version("5.2", tag="v5.2")
-    version("5.1.1", tag="v5.1.1")
-    version("5.1", tag="v5.1")
-    version("5.0", tag="v5.0")
+    version("5.3", tag="v5.3", commit="5db1d80452b951905658da828285c2fd0537603c")
+    version("5.2", tag="v5.2", commit="c66bdc397de319ca59a0565b3f3b1a3b33f0c50c")
+    version("5.1.1", tag="v5.1.1", commit="57e9bbd529a6062f55e03c884b59af22f920eef1")
+    version("5.1", tag="v5.1", commit="f3fe347a8fd83cd45983476521d43061c8528da0")
+    version("5.0", tag="v5.0", commit="39b83f057a2639af4b5083fb911e0726f0972b75")
     version("master", tag="master")
diff --git a/var/spack/repos/builtin/packages/ftk/package.py b/var/spack/repos/builtin/packages/ftk/package.py
index 1f6a75793d1c4d..f3e4e0f8a8c795 100644
--- a/var/spack/repos/builtin/packages/ftk/package.py
+++ b/var/spack/repos/builtin/packages/ftk/package.py
@@ -28,14 +28,14 @@ class Ftk(CMakePackage):
     version("0.0.4", sha256="1674904da8d88dbd4c7d2b6a2629883f0444e70aefc99b48d285735d394897fa")
 
     # variants
-    variant("adios2", default=False)
-    variant("cuda", default=False)
-    variant("gmp", default=False)
-    variant("hdf5", default=False)
-    variant("metis", default=False)
-    variant("mpi", default=False)
-    variant("netcdf", default=False)
-    variant("vtk", default=False)
+    variant("adios2", default=False, description="Use ADIOS2")
+    variant("cuda", default=False, description="Use CUDA")
+    variant("gmp", default=False, description="Use GMP")
+    variant("hdf5", default=False, description="Use HDF5")
+    variant("metis", default=False, description="Use METIS")
+    variant("mpi", default=False, description="Use MPI")
+    variant("netcdf", default=False, description="Use NetCDF")
+    variant("vtk", default=False, description="Use VTK")
 
     # optional dependencies
     depends_on("adios2", when="+adios2")
diff --git a/var/spack/repos/builtin/packages/funwave/package.py b/var/spack/repos/builtin/packages/funwave/package.py
index 6c047ba17df196..76b8e6376a95b8 100644
--- a/var/spack/repos/builtin/packages/funwave/package.py
+++ b/var/spack/repos/builtin/packages/funwave/package.py
@@ -24,9 +24,9 @@ class Funwave(MakefilePackage):
 
     maintainers("stevenrbrandt", "fengyanshi")
 
-    version("3.2", tag="v3.2")
-    version("3.1", tag="v3.1")
-    version("3.0", tag="v3.0")
+    version("3.2", tag="v3.2", commit="6e57952d1cef1bb6c0930f07387ed41694bd604b")
+    version("3.1", tag="v3.1", commit="d99502f9288380e0c823f7cc619c6c7c1897f2b9")
+    version("3.0", tag="v3.0", commit="b569dfe08ab379d7ec375c71304ff95bd6f5ddb6")
 
     depends_on("mpi")
 
diff --git a/var/spack/repos/builtin/packages/fuse-overlayfs/package.py b/var/spack/repos/builtin/packages/fuse-overlayfs/package.py
index dbb65b730900ae..a6968e68a58f6f 100644
--- a/var/spack/repos/builtin/packages/fuse-overlayfs/package.py
+++ b/var/spack/repos/builtin/packages/fuse-overlayfs/package.py
@@ -29,4 +29,5 @@ class FuseOverlayfs(AutotoolsPackage):
     depends_on("automake", type="build")
     depends_on("libtool", type="build")
     depends_on("m4", type="build")
+    depends_on("pkgconfig", type="build")
     depends_on("fuse")
diff --git a/var/spack/repos/builtin/packages/fzf/package.py b/var/spack/repos/builtin/packages/fzf/package.py
index d9e37802097c76..95362138f1ec0c 100644
--- a/var/spack/repos/builtin/packages/fzf/package.py
+++ b/var/spack/repos/builtin/packages/fzf/package.py
@@ -17,6 +17,7 @@ class Fzf(MakefilePackage):
 
     executables = ["^fzf$"]
 
+    version("0.42.0", sha256="743c1bfc7851b0796ab73c6da7db09d915c2b54c0dd3e8611308985af8ed3df2")
     version("0.41.1", sha256="982682eaac377c8a55ae8d7491fcd0e888d6c13915d01da9ebb6b7c434d7f4b5")
     version("0.40.0", sha256="9597f297a6811d300f619fff5aadab8003adbcc1566199a43886d2ea09109a65")
 
diff --git a/var/spack/repos/builtin/packages/g2/package.py b/var/spack/repos/builtin/packages/g2/package.py
index 28dfcc9d54cb9a..de4dcd4bcbc175 100644
--- a/var/spack/repos/builtin/packages/g2/package.py
+++ b/var/spack/repos/builtin/packages/g2/package.py
@@ -15,26 +15,56 @@ class G2(CMakePackage):
 
     homepage = "https://noaa-emc.github.io/NCEPLIBS-g2"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-g2/archive/refs/tags/v3.4.3.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-g2"
 
-    maintainers("t-brown", "AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
+    maintainers("AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
 
+    version("develop", branch="develop")
+    version("3.4.8", sha256="071a6f799c4c4fdfd5d0478152a0cbb9d668d12d71c78d5bda71845fc5580a7f")
+    version("3.4.7", sha256="d6530611e3a515122f11ed4aeede7641f6f8932ef9ee0d4828786572767304dc")
+    version("3.4.6", sha256="c4b03946365ce0bacf1e10e8412a5debd72d8671d1696aa4fb3f3adb119175fe")
     version("3.4.5", sha256="c18e991c56964953d778632e2d74da13c4e78da35e8d04cb742a2ca4f52737b6")
     version("3.4.3", sha256="679ea99b225f08b168cbf10f4b29f529b5b011232f298a5442ce037ea84de17c")
 
-    variant("pic", default=True, description="Build with position-independent-code")
+    variant("pic", default=True, description="Build with position-independent code (PIC)")
+    variant(
+        "precision",
+        default=("4", "d"),
+        values=("4", "d"),
+        multi=True,
+        description="Set precision (_4/_d library versions)",
+        when="@3.4.6:",
+    )
+    variant("w3emc", default=True, description="Enable GRIB1 through w3emc", when="@3.4.6:")
 
-    depends_on("jasper@:2.0.32")
+    depends_on("jasper@:2.0.32", when="@:3.4.7")
+    depends_on("jasper")
     depends_on("libpng")
+    depends_on("bacio", when="@3.4.6:")
+    with when("+w3emc"):
+        depends_on("w3emc")
+        depends_on("w3emc precision=4", when="precision=4")
+        depends_on("w3emc precision=d", when="precision=d")
 
     def cmake_args(self):
         args = [
-            self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic")
+            self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
+            self.define_from_variant("BUILD_WITH_W3EMC", "w3emc"),
+            self.define("BUILD_4", self.spec.satisfies("precision=4")),
+            self.define("BUILD_D", self.spec.satisfies("precision=d")),
         ]
 
         return args
 
     def setup_run_environment(self, env):
-        for suffix in ("4", "d"):
+        precisions = (
+            self.spec.variants["precision"].value if self.spec.satisfies("@3.4.6:") else ("4", "d")
+        )
+        for suffix in precisions:
             lib = find_libraries("libg2_" + suffix, root=self.prefix, shared=False, recursive=True)
             env.set("G2_LIB" + suffix, lib[0])
             env.set("G2_INC" + suffix, join_path(self.prefix, "include_" + suffix))
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/g2c/package.py b/var/spack/repos/builtin/packages/g2c/package.py
index ad2f87aed8688a..9f3d93c71ed38b 100644
--- a/var/spack/repos/builtin/packages/g2c/package.py
+++ b/var/spack/repos/builtin/packages/g2c/package.py
@@ -13,28 +13,82 @@ class G2c(CMakePackage):
 
     homepage = "https://github.com/NOAA-EMC/NCEPLIBS-g2c"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-g2c/archive/refs/tags/v1.6.4.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-g2c"
 
     maintainers("AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
 
+    version("develop", branch="develop")
+    version("1.8.0", sha256="4ce9f5a7cb0950699fe08ebc5a463ab4d09ef550c050391a319308a2494f971f")
     version("1.7.0", sha256="73afba9da382fed73ed8692d77fa037bb313280879cd4012a5e5697dccf55175")
     version("1.6.4", sha256="5129a772572a358296b05fbe846bd390c6a501254588c6a223623649aefacb9d")
     version("1.6.2", sha256="b5384b48e108293d7f764cdad458ac8ce436f26be330b02c69c2a75bb7eb9a2c")
 
-    variant("png", default=True)
-    variant("jasper", default=True)
-    variant("openjpeg", default=False)
+    variant("aec", default=True, description="Use AEC library")
+    variant("png", default=True, description="Use PNG library")
+    variant("jasper", default=True, description="Use Jasper library")
+    variant("openjpeg", default=False, description="Use OpenJPEG library")
     variant("pic", default=True, description="Build with position-independent-code")
+    variant(
+        "libs",
+        default=("shared", "static"),
+        values=("shared", "static"),
+        multi=True,
+        description="Build shared libs, static libs or both",
+        when="@1.7:",
+    )
+    variant(
+        "pthreads",
+        default=False,
+        description="Turn on thread-safety with pthreads",
+        when="@develop",
+    )
+    variant(
+        "utils",
+        default=True,
+        description="Build and install some utility programs",
+        when="@develop",
+    )
+    variant(
+        "build_g2c",
+        default=False,
+        description="Build new g2c API, experimental until 2.0.0 release",
+        when="@develop",
+    )
 
+    depends_on("libaec", when="+aec")
     depends_on("libpng", when="+png")
     depends_on("jasper", when="+jasper")
     depends_on("openjpeg", when="+openjpeg")
+    depends_on("libxml2@2.9:", when="+build_g2c")
+
+    conflicts("+jasper +openjpeg", msg="Either Jasper or OpenJPEG should be used, not both")
 
     def cmake_args(self):
-        args = [self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic")]
+        args = [
+            self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
+            self.define("BUILD_SHARED_LIBS", self.spec.satisfies("libs=shared")),
+            self.define("BUILD_STATIC_LIBS", self.spec.satisfies("libs=static")),
+            self.define_from_variant("USE_AEC", "aec"),
+            self.define_from_variant("USE_PNG", "png"),
+            self.define_from_variant("USE_Jasper", "jasper"),
+            self.define_from_variant("USE_OpenJPEG", "openjpeg"),
+            self.define_from_variant("PTHREADS", "pthreads"),
+            self.define_from_variant("UTILS", "utils"),
+            self.define_from_variant("BUILD_G2C", "build_g2c"),
+            self.define("BUILD_TESTING", self.run_tests),
+        ]
 
         return args
 
     def setup_run_environment(self, env):
-        lib = find_libraries("libg2c", root=self.prefix, shared=False, recursive=True)
+        if self.spec.satisfies("@:1.6"):
+            shared = False
+        else:
+            shared = self.spec.satisfies("libs=shared")
+        lib = find_libraries("libg2c", root=self.prefix, shared=shared, recursive=True)
         env.set("G2C_LIB", lib[0])
         env.set("G2C_INC", join_path(self.prefix, "include"))
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/g2tmpl/package.py b/var/spack/repos/builtin/packages/g2tmpl/package.py
index 55318f8a051898..e5a94b4e451877 100644
--- a/var/spack/repos/builtin/packages/g2tmpl/package.py
+++ b/var/spack/repos/builtin/packages/g2tmpl/package.py
@@ -13,9 +13,21 @@ class G2tmpl(CMakePackage):
 
     homepage = "https://github.com/NOAA-EMC/NCEPLIBS-g2tmpl"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-g2tmpl/archive/refs/tags/v1.10.0.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-g2tmpl"
 
-    maintainers("t-brown", "edwardhartnett", "AlexanderRichert-NOAA", "Hang-Lei-NOAA")
+    maintainers("edwardhartnett", "AlexanderRichert-NOAA", "Hang-Lei-NOAA")
 
+    version("develop", branch="develop")
     version("1.10.2", sha256="4063361369f3691f75288c801fa9d1a2414908b7d6c07bbf69d4165802e2a7fc")
     version("1.10.1", sha256="0be425e5128fabb89915a92261aa75c27a46a3e115e00c686fc311321e5d1e2a")
     version("1.10.0", sha256="dcc0e40b8952f91d518c59df7af64e099131c17d85d910075bfa474c8822649d")
+
+    variant("shared", default=False, description="Build shared library")
+
+    def cmake_args(self):
+        args = [self.define_from_variant("BUILD_SHARED_LIBS", "shared")]
+        return args
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/gadap/cxx-updates.patch b/var/spack/repos/builtin/packages/gadap/cxx-updates.patch
new file mode 100644
index 00000000000000..d9a4be125f6ed8
--- /dev/null
+++ b/var/spack/repos/builtin/packages/gadap/cxx-updates.patch
@@ -0,0 +1,62 @@
+--- a/src/gaBaseTypes.cc	2008-07-25 14:43:25.000000000 -0600
++++ b/src/gaBaseTypes.cc	2023-09-13 14:59:50.000000000 -0600
+@@ -31,6 +31,7 @@
+ #include "gaBaseTypes.h"
+ #include "gaTypeFactory.h"
+ 
++
+ /** class declarations for gaStr and gaUrl are in gaBaseTypes.h so we
+  * can't declare them again, we have to define each function
+  * individually
+--- a/src/gaBaseTypes.h	2008-07-25 14:43:25.000000000 -0600
++++ b/src/gaBaseTypes.h	2023-09-13 14:44:39.000000000 -0600
+@@ -31,6 +31,7 @@
+  * 
+  */
+ 
++using namespace libdap;
+ 
+ class gaStr: public Str {
+ public:
+--- a/src/gaReports.cc	2008-07-25 14:43:25.000000000 -0600
++++ b/src/gaReports.cc	2023-09-13 14:50:28.000000000 -0600
+@@ -12,6 +12,7 @@
+  */
+ 
+ #include 
++#include 
+ #include "gaReports.h"
+ #include "gaUtils.h"
+ #include "Grid.h"
+--- a/src/gaTypeFactory.h	2008-07-25 14:43:25.000000000 -0600
++++ b/src/gaTypeFactory.h	2023-09-13 14:44:43.000000000 -0600
+@@ -28,6 +28,8 @@
+ 
+ #include 
+ 
++using namespace libdap;
++
+ // Class declarations; Make sure to include the corresponding headers in the
+ // implementation file.
+ 
+--- a/src/gaUtils.h	2008-07-25 14:43:25.000000000 -0600
++++ b/src/gaUtils.h	2023-09-13 14:46:27.000000000 -0600
+@@ -23,6 +23,8 @@
+ #include "Sequence.h"
+ #include "gadap.h"
+ 
++using namespace libdap;
++
+ typedef struct varinfo {
+   string name;
+   string longname;
+--- a/test/test.cc	2008-07-30 06:35:43.000000000 -0600
++++ b/test/test.cc	2023-09-13 15:12:48.000000000 -0600
+@@ -9,6 +9,7 @@
+ #include "gadap.h"
+ #include 
+ #include 
++#include 
+ 
+ using namespace std;
+ 
diff --git a/var/spack/repos/builtin/packages/gadap/package.py b/var/spack/repos/builtin/packages/gadap/package.py
new file mode 100644
index 00000000000000..90cb596ec61c97
--- /dev/null
+++ b/var/spack/repos/builtin/packages/gadap/package.py
@@ -0,0 +1,28 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Gadap(AutotoolsPackage):
+    """Enables OPeNDAP access of in situ data."""
+
+    homepage = "http://cola.gmu.edu/grads/gadoc/supplibs.html"
+    url = "http://cola.gmu.edu/grads/Supplibs/2.1/src/gadap-2.0.tar.gz"
+
+    maintainers("vanderwb")
+
+    version("2.0", sha256="ae9a989ca00ec29fb40616383d170883f07c022456db338399982a8a94ec0100")
+
+    depends_on("curl@7.18.0:")
+    depends_on("libdap4")
+    depends_on("libxml2")
+
+    # libdap uses namespacing in recent versions, so we need to patch this source
+    patch("cxx-updates.patch")
+
+    def setup_build_environment(self, env):
+        env.set("CFLAGS", "-fPIC")
+        env.set("CXXFLAGS", "-fPIC")
diff --git a/var/spack/repos/builtin/packages/garfieldpp/package.py b/var/spack/repos/builtin/packages/garfieldpp/package.py
index f24d1dc5b20165..0bbdda3e3d9d8a 100644
--- a/var/spack/repos/builtin/packages/garfieldpp/package.py
+++ b/var/spack/repos/builtin/packages/garfieldpp/package.py
@@ -17,6 +17,11 @@ class Garfieldpp(CMakePackage):
     tags = ["hep"]
 
     maintainers("mirguest")
+    patch(
+        "https://gitlab.cern.ch/garfield/garfieldpp/-/commit/882c3023cfa89b45ca7a0c95ab1518454536e8e1.diff",
+        sha256="ea3b91d67011abe41e72c7b55578d14b77bd2ef5e7f344077091934b24f38f0d",
+        when="@4.0",
+    )
 
     variant("examples", default=False, description="Build garfield examples")
 
@@ -29,5 +34,24 @@ class Garfieldpp(CMakePackage):
     depends_on("geant4", when="+examples")
 
     def cmake_args(self):
-        args = [self.define_from_variant("WITH_EXAMPLES", "examples")]
+        args = [
+            "-DCMAKE_INSTALL_LIBDIR=lib",
+            self.define_from_variant("WITH_EXAMPLES", "examples"),
+        ]
         return args
+
+    def setup_run_environment(self, env):
+        env.set("GARFIELD_INSTALL", self.prefix)
+        env.set("HEED_DATABASE", self.prefix.share.Heed.database)
+
+        # In order to get Garfield work in python, need to setup both ROOT and GSL
+        pyver = self.spec["python"].version.up_to(2)
+        site_packages = "python{}/site-packages".format(pyver)
+        pypath = join_path(self.prefix.lib, site_packages)
+        env.prepend_path("PYTHONPATH", pypath)
+        env.prepend_path("LD_LIBRARY_PATH", self.spec["root"].prefix.lib.root)
+        env.prepend_path("LD_LIBRARY_PATH", self.spec["gsl"].prefix.lib.root)
+
+    def setup_dependent_build_environment(self, env, dependent_spec):
+        env.set("GARFIELD_INSTALL", self.prefix)
+        env.set("HEED_DATABASE", self.prefix.share.Heed.database)
diff --git a/var/spack/repos/builtin/packages/gasnet/package.py b/var/spack/repos/builtin/packages/gasnet/package.py
index 42a9fa3c583a91..516accb30db6a5 100644
--- a/var/spack/repos/builtin/packages/gasnet/package.py
+++ b/var/spack/repos/builtin/packages/gasnet/package.py
@@ -83,7 +83,7 @@ class Gasnet(Package, CudaPackage, ROCmPackage):
     depends_on("autoconf@2.69", type="build", when="@master:")
     depends_on("automake@1.16:", type="build", when="@master:")
 
-    conflicts("hip@:4.4.0", when="+rocm")
+    conflicts("^hip@:4.4.0", when="+rocm")
 
     def install(self, spec, prefix):
         if spec.satisfies("@master:"):
diff --git a/var/spack/repos/builtin/packages/gatb-core/package.py b/var/spack/repos/builtin/packages/gatb-core/package.py
index 24b7e14a14c6f5..9da5b143c53104 100644
--- a/var/spack/repos/builtin/packages/gatb-core/package.py
+++ b/var/spack/repos/builtin/packages/gatb-core/package.py
@@ -14,7 +14,7 @@ class GatbCore(CMakePackage):
 
     depends_on("cmake@3.1.0:", type="build")
 
-    version("1.4.2", tag="v1.4.2")
-    version("1.4.1", tag="v1.4.1")
+    version("1.4.2", tag="v1.4.2", commit="99f573a465beb30acc22ab20be458d2ea0277684")
+    version("1.4.1", tag="v1.4.1", commit="b45a6c213597b23f8f5221902e2b86b4009c11d9")
 
     root_cmakelists_dir = "gatb-core"
diff --git a/var/spack/repos/builtin/packages/gaudi/package.py b/var/spack/repos/builtin/packages/gaudi/package.py
index d51984244a9a2a..12a095c301989a 100644
--- a/var/spack/repos/builtin/packages/gaudi/package.py
+++ b/var/spack/repos/builtin/packages/gaudi/package.py
@@ -17,6 +17,8 @@ class Gaudi(CMakePackage):
     tags = ["hep"]
 
     version("master", branch="master")
+    version("37.1", sha256="1d7038fd5dfb5f2517ce57623cf8090549ffe2ea8f0171d534e5c1ca20bd009a")
+    version("37.0", sha256="823f3821a4f498ddd2dd123fbb8a3787b361ddfd818f4ab13572076fc9afdfe4")
     version("36.14", sha256="b11e0afcb797d61a305856dfe8079d48d74c6b6867ceccc0a83aab5978c9ba5f")
     version("36.13", sha256="41e711c83428663996c825044b268ce515bef85dad74b4a9453f2207b4b1be7b")
     version("36.12", sha256="dfce9156cedfa0a7234f880a3c395e592a5f3dc79070d5d196fdb94b83ae203e")
@@ -72,7 +74,8 @@ class Gaudi(CMakePackage):
     depends_on("cppgsl")
     depends_on("fmt", when="@33.2:")
     depends_on("fmt@:8", when="@:36.9")
-    depends_on("intel-tbb@:2020.3")
+    depends_on("intel-tbb@:2020.3", when="@:37.0")
+    depends_on("tbb", when="@37.1:")
     depends_on("uuid")
     depends_on("nlohmann-json", when="@35.0:")
     depends_on("python", type=("build", "run"))
@@ -82,7 +85,7 @@ class Gaudi(CMakePackage):
     depends_on("py-xenv@1:", when="@:34.9", type=("build", "run"))
     depends_on("range-v3")
     depends_on("root +python +root7 +ssl +tbb +threads")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # Testing dependencies
     # Note: gaudi only builds examples when testing enabled
diff --git a/var/spack/repos/builtin/packages/gaussian-src/package.py b/var/spack/repos/builtin/packages/gaussian-src/package.py
index d31ee97800d022..5cd0cf9e27e661 100644
--- a/var/spack/repos/builtin/packages/gaussian-src/package.py
+++ b/var/spack/repos/builtin/packages/gaussian-src/package.py
@@ -28,11 +28,7 @@ class GaussianSrc(Package):
     depends_on("tcsh", type="build")
 
     # All compilers except for pgi are in conflict:
-    for __compiler in spack.compilers.supported_compilers():
-        if __compiler != "pgi":
-            conflicts(
-                "%{0}".format(__compiler), msg="Gaussian can only be built with the PGI compiler"
-            )
+    requires("%pgi", msg="Gaussian can only be built with the PGI compiler")
 
     patch("16-C.01-replace-deprecated-pgf77-with-pgfortran.patch", when="@16-C.01")
     patch("16-C.01-fix-building-c-code-with-pgcc.patch", when="@16-C.01")
diff --git a/var/spack/repos/builtin/packages/gawk/package.py b/var/spack/repos/builtin/packages/gawk/package.py
index 56455d67f7db15..46f5034ad4486a 100644
--- a/var/spack/repos/builtin/packages/gawk/package.py
+++ b/var/spack/repos/builtin/packages/gawk/package.py
@@ -28,6 +28,7 @@ class Gawk(AutotoolsPackage, GNUMirrorPackage):
 
     tags = ["build-tools", "core-packages"]
 
+    version("5.2.2", sha256="3c1fce1446b4cbee1cd273bd7ec64bc87d89f61537471cd3e05e33a965a250e9")
     version("5.2.1", sha256="673553b91f9e18cc5792ed51075df8d510c9040f550a6f74e09c9add243a7e4f")
     version("5.1.1", sha256="d87629386e894bbea11a5e00515fc909dc9b7249529dad9e6a3a2c77085f7ea2")
     version("5.1.0", sha256="cf5fea4ac5665fd5171af4716baab2effc76306a9572988d5ba1078f196382bd")
diff --git a/var/spack/repos/builtin/packages/gcc/detection_test.yaml b/var/spack/repos/builtin/packages/gcc/detection_test.yaml
new file mode 100644
index 00000000000000..0930f82d936568
--- /dev/null
+++ b/var/spack/repos/builtin/packages/gcc/detection_test.yaml
@@ -0,0 +1,38 @@
+paths:
+  # Ubuntu 18.04, system compilers without Fortran
+  - layout:
+      - executables:
+        - "bin/gcc"
+        - "bin/g++"
+        script: "echo 7.5.0"
+    results:
+      - spec: "gcc@7.5.0 languages=c,c++"
+  # Mock a version < 7 of GCC that requires -dumpversion and
+  # errors with -dumpfullversion
+  - layout:
+      - executables:
+        - "bin/gcc-5"
+        - "bin/g++-5"
+        - "bin/gfortran-5"
+        script: |
+          if [[ "$1" == "-dumpversion" ]] ; then
+            echo "5.5.0"
+          else
+            echo "gcc-5: fatal error: no input files"
+            echo "compilation terminated."
+            exit 1
+          fi
+    results:
+      - spec: "gcc@5.5.0 languages=c,c++,fortran"
+  # Multiple compilers present at the same time
+  - layout:
+      - executables:
+        - "bin/x86_64-linux-gnu-gcc-6"
+        script: 'echo 6.5.0'
+      - executables:
+        - "bin/x86_64-linux-gnu-gcc-10"
+        - "bin/x86_64-linux-gnu-g++-10"
+        script: "echo 10.1.0"
+    results:
+      - spec: "gcc@6.5.0 languages=c"
+      - spec: "gcc@10.1.0 languages=c,c++"
diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py
index 5cb2a13c75ae0d..116371cdb272bc 100644
--- a/var/spack/repos/builtin/packages/gcc/package.py
+++ b/var/spack/repos/builtin/packages/gcc/package.py
@@ -35,6 +35,7 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
 
     version("master", branch="master")
 
+    version("13.2.0", sha256="e275e76442a6067341a27f04c5c6b83d8613144004c0413528863dc6b5c743da")
     version("13.1.0", sha256="61d684f0aa5e76ac6585ad8898a2427aade8979ed5e7f85492286c4dfc13ee86")
 
     version("12.3.0", sha256="949a5d4f99e786421a93b532b22ffab5578de7321369975b91aec97adfda8c3b")
@@ -181,7 +182,7 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
         depends_on("isl@0.15:0.20", when="@9:9.9")
         depends_on("isl@0.15:", when="@10:")
 
-    depends_on("zlib", when="@6:")
+    depends_on("zlib-api", when="@6:")
     depends_on("zstd", when="@10:")
     depends_on("diffutils", type="build")
     depends_on("iconv", when="platform=darwin")
@@ -336,11 +337,11 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
     #   on XCode 12.5
     conflicts("+bootstrap", when="@:11.1 %apple-clang@12.0.5")
 
-    # aarch64/M1 is supported in GCC 11.3-12.2
-    conflicts(
-        "@:11.2,12.3:",
+    # aarch64/M1 is supported in GCC 11.3-12.2 and 13
+    requires(
+        "@11.3,12.2,13.1:",
         when="target=aarch64: platform=darwin",
-        msg="Only GCC 11.3-12.2 support macOS M1 (aarch64)",
+        msg="Only GCC 11.3-12.2, 13.1+ support macOS M1 (aarch64)",
     )
 
     # Newer binutils than RHEL's is required to run `as` on some instructions
@@ -412,7 +413,17 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
             sha256="a7843b5c6bf1401e40c20c72af69c8f6fc9754ae980bb4a5f0540220b3dcb62d",
             when="@12.2.0 target=aarch64:",
         )
-        conflicts("+bootstrap", when="@11.3.0 target=aarch64:")
+        patch(
+            "https://raw.githubusercontent.com/Homebrew/formula-patches/5c206c47/gcc/gcc-13.1.0.diff",
+            sha256="cb4e8a89387f748a744da0273025d0dc2e3c76780cc390b18ada704676afea11",
+            when="@13.1.0 target=aarch64:",
+        )
+        patch(
+            "https://raw.githubusercontent.com/Homebrew/formula-patches/3c5cbc8e9cf444a1967786af48e430588e1eb481/gcc/gcc-13.2.0.diff",
+            sha256="2df7ef067871a30b2531a2013b3db661ec9e61037341977bfc451e30bf2c1035",
+            when="@13.2.0 target=aarch64:",
+        )
+        conflicts("+bootstrap", when="@11.3.0,13.1: target=aarch64:")
 
         # Use -headerpad_max_install_names in the build,
         # otherwise updated load commands won't fit in the Mach-O header.
@@ -651,9 +662,11 @@ def patch(self):
 
         # Use installed libz
         if self.version >= Version("6"):
-            filter_file("@zlibdir@", "-L{0}".format(spec["zlib"].prefix.lib), "gcc/Makefile.in")
             filter_file(
-                "@zlibinc@", "-I{0}".format(spec["zlib"].prefix.include), "gcc/Makefile.in"
+                "@zlibdir@", "-L{0}".format(spec["zlib-api"].prefix.lib), "gcc/Makefile.in"
+            )
+            filter_file(
+                "@zlibinc@", "-I{0}".format(spec["zlib-api"].prefix.include), "gcc/Makefile.in"
             )
 
         if spec.satisfies("+nvptx"):
@@ -770,6 +783,11 @@ def configure_args(self):
                     "--with-as=" + binutils.join("as"),
                 ]
             )
+        elif spec.satisfies("%apple-clang@15:"):
+            # https://github.com/iains/gcc-darwin-arm64/issues/117
+            # https://github.com/iains/gcc-12-branch/issues/22
+            # https://github.com/iains/gcc-13-branch/issues/8
+            options.append("--with-ld=/Library/Developer/CommandLineTools/usr/bin/ld-classic")
 
         # enable_bootstrap
         if spec.satisfies("+bootstrap"):
@@ -1017,11 +1035,11 @@ def detect_gdc(self):
         """
         # Detect GCC package in the directory of the GCC compiler
         # or in the $PATH if self.compiler.cc is not an absolute path:
-        from spack.detection import by_executable
+        from spack.detection import by_path
 
         compiler_dir = os.path.dirname(self.compiler.cc)
-        detected_packages = by_executable(
-            [self.__class__], path_hints=([compiler_dir] if os.path.isdir(compiler_dir) else None)
+        detected_packages = by_path(
+            [self.name], path_hints=([compiler_dir] if os.path.isdir(compiler_dir) else None)
         )
 
         # We consider only packages that satisfy the following constraint:
diff --git a/var/spack/repos/builtin/packages/gcta/package.py b/var/spack/repos/builtin/packages/gcta/package.py
index f8c084d37afdeb..538e26fe7d0b11 100644
--- a/var/spack/repos/builtin/packages/gcta/package.py
+++ b/var/spack/repos/builtin/packages/gcta/package.py
@@ -29,7 +29,7 @@ class Gcta(CMakePackage):
     depends_on("eigen@3.3.1", when="@1.91.2")
     depends_on("eigen@3.3.7:", when="@1.94.0beta:")
     depends_on("boost@1.79:", when="@1.94.0beta:")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("sqlite@3.3.1:", when="@1.94.0beta:")
     depends_on("zstd@1.4.4:", when="@1.94.0beta:")
     depends_on("spectra", when="@1.94.0beta:")
diff --git a/var/spack/repos/builtin/packages/gdal/package.py b/var/spack/repos/builtin/packages/gdal/package.py
index 28d3e386040b15..313a46f28bd969 100644
--- a/var/spack/repos/builtin/packages/gdal/package.py
+++ b/var/spack/repos/builtin/packages/gdal/package.py
@@ -30,6 +30,9 @@ class Gdal(CMakePackage, AutotoolsPackage, PythonExtension):
 
     maintainers("adamjstewart")
 
+    version("3.7.3", sha256="e0a6f0c453ea7eb7c09967f50ac49426808fcd8f259dbc9888140eb69d7ffee6")
+    version("3.7.2", sha256="40c0068591d2c711c699bbb734319398485ab169116ac28005d8302f80b923ad")
+    version("3.7.1", sha256="9297948f0a8ba9e6369cd50e87c7e2442eda95336b94d2b92ef1829d260b9a06")
     version("3.7.0", sha256="af4b26a6b6b3509ae9ccf1fcc5104f7fe015ef2110f5ba13220816398365adce")
     version("3.6.4", sha256="889894cfff348c04ac65b462f629d03efc53ea56cf04de7662fbe81a364e3df1")
     version("3.6.3", sha256="3cccbed883b1fb99b913966aa3a650ad930e7c3afc714f5823f9754176ee49ea")
@@ -252,7 +255,7 @@ class Gdal(CMakePackage, AutotoolsPackage, PythonExtension):
     depends_on("proj@:6", when="@2.5:2")
     depends_on("proj@:5", when="@2.4")
     depends_on("proj@:4", when="@:2.3")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libtiff@4:", when="@3:")
     depends_on("libtiff@3.6.0:")  # 3.9.0+ needed to pass testsuite
     depends_on("libgeotiff@1.5:", when="@3:")
@@ -291,7 +294,8 @@ class Gdal(CMakePackage, AutotoolsPackage, PythonExtension):
     depends_on("libheif@1.1:", when="+heif")
     depends_on("hdf", when="+hdf4")
     depends_on("hdf5+cxx", when="+hdf5")
-    depends_on("hdf5@:1.12", when="@:3.4.1 +hdf5")
+    depends_on("hdf5@:1.13", when="@:3.5 +hdf5")
+    depends_on("hdf5@:1.12", when="@:3.4 +hdf5")
     depends_on("hadoop", when="+hdfs")
     depends_on("iconv", when="+iconv")
     # depends_on('idb', when='+idb')
@@ -369,6 +373,8 @@ class Gdal(CMakePackage, AutotoolsPackage, PythonExtension):
     depends_on("python@3.6:", type=("build", "link", "run"), when="@3.3:+python")
     depends_on("python@2.0:", type=("build", "link", "run"), when="@3.2:+python")
     depends_on("python", type=("build", "link", "run"), when="+python")
+    # Uses distutils
+    depends_on("python@:3.11", type=("build", "link", "run"), when="@:3.4+python")
     # swig/python/setup.py
     depends_on("py-setuptools@:57", type="build", when="@:3.2+python")  # needs 2to3
     depends_on("py-setuptools", type="build", when="+python")
@@ -482,6 +488,10 @@ def cmake_args(self):
             self.define("GDAL_USE_JSONC", True),
             self.define("GDAL_USE_TIFF", True),
             self.define("GDAL_USE_ZLIB", True),
+            # zlib-ng + deflate64 doesn't compile (heavily relies on zlib)
+            # but since zlib-ng is faster than zlib, it deflate shouldn't
+            # be necessary.
+            self.define("ENABLE_DEFLATE64", "zlib-ng" not in self.spec),
             # Optional dependencies
             self.define_from_variant("GDAL_USE_ARMADILLO", "armadillo"),
             self.define_from_variant("GDAL_USE_ARROW", "arrow"),
@@ -607,7 +617,7 @@ def configure_args(self):
             "--with-geotiff={}".format(self.spec["libgeotiff"].prefix),
             "--with-libjson-c={}".format(self.spec["json-c"].prefix),
             "--with-libtiff={}".format(self.spec["libtiff"].prefix),
-            "--with-libz={}".format(self.spec["zlib"].prefix),
+            "--with-libz={}".format(self.spec["zlib-api"].prefix),
             # Optional dependencies
             self.with_or_without("armadillo", package="armadillo"),
             self.with_or_without("blosc", package="c-blosc"),
diff --git a/var/spack/repos/builtin/packages/gdb/package.py b/var/spack/repos/builtin/packages/gdb/package.py
index 22f26731063a82..f4c4a54396a725 100644
--- a/var/spack/repos/builtin/packages/gdb/package.py
+++ b/var/spack/repos/builtin/packages/gdb/package.py
@@ -79,7 +79,7 @@ class Gdb(AutotoolsPackage, GNUMirrorPackage):
     # https://bugzilla.redhat.com/show_bug.cgi?id=1829702
     depends_on("python@:3.8", when="@:9.2+python", type=("build", "link", "run"))
     depends_on("xz", when="+xz")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("zstd", when="@13.1:")
     depends_on("source-highlight", when="+source-highlight")
     depends_on("ncurses", when="+tui")
diff --git a/var/spack/repos/builtin/packages/gdk-pixbuf/package.py b/var/spack/repos/builtin/packages/gdk-pixbuf/package.py
index efb3d9d1372487..aac83b70013bd8 100644
--- a/var/spack/repos/builtin/packages/gdk-pixbuf/package.py
+++ b/var/spack/repos/builtin/packages/gdk-pixbuf/package.py
@@ -63,7 +63,7 @@ class GdkPixbuf(Package):
     depends_on("glib@2.38.0:")
     depends_on("jpeg")
     depends_on("libpng")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libtiff", when="+tiff")
     depends_on("gobject-introspection")
     depends_on("libx11", when="+x11")
diff --git a/var/spack/repos/builtin/packages/gdrcopy/package.py b/var/spack/repos/builtin/packages/gdrcopy/package.py
index f3e356a770cafb..441349bdbb2a78 100644
--- a/var/spack/repos/builtin/packages/gdrcopy/package.py
+++ b/var/spack/repos/builtin/packages/gdrcopy/package.py
@@ -6,13 +6,14 @@
 from spack.package import *
 
 
-class Gdrcopy(MakefilePackage):
+class Gdrcopy(MakefilePackage, CudaPackage):
     """A fast GPU memory copy library based on NVIDIA GPUDirect
     RDMA technology."""
 
     homepage = "https://github.com/NVIDIA/gdrcopy"
     url = "https://github.com/NVIDIA/gdrcopy/archive/v2.1.tar.gz"
     git = "https://github.com/NVIDIA/gdrcopy"
+    maintainers("scothalverson")
 
     version("master", branch="master")
     version("2.3", sha256="b85d15901889aa42de6c4a9233792af40dd94543e82abe0439e544c87fd79475")
@@ -24,13 +25,19 @@ class Gdrcopy(MakefilePackage):
     # Don't call ldconfig: https://github.com/NVIDIA/gdrcopy/pull/229
     patch("ldconfig.patch", when="@2.0:")
 
+    depends_on("check")
+    requires("+cuda")
+
     def build(self, spec, prefix):
         make("lib")
+        make("exes")
 
     def install(self, spec, prefix):
         mkdir(prefix.include)
         mkdir(prefix.lib64)
         if spec.satisfies("@2.2:"):
             make("lib_install", "prefix={0}".format(self.prefix))
+            make("exes_install", "prefix={0}".format(self.prefix))
         else:
             make("lib_install", "PREFIX={0}".format(self.prefix))
+            make("exes_install", "PREFIX={0}".format(self.prefix))
diff --git a/var/spack/repos/builtin/packages/geant4/package-cache.patch b/var/spack/repos/builtin/packages/geant4/package-cache.patch
new file mode 100644
index 00000000000000..835a4c34098d0e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/geant4/package-cache.patch
@@ -0,0 +1,48 @@
+diff --git a/cmake/Modules/G4CMakeUtilities.cmake b/cmake/Modules/G4CMakeUtilities.cmake
+index 16f7b3c8c0..84acfcd5e7 100644
+--- a/cmake/Modules/G4CMakeUtilities.cmake
++++ b/cmake/Modules/G4CMakeUtilities.cmake
+@@ -221,6 +221,21 @@ function(geant4_export_package_variables _file)
+       get_property(__var_value CACHE ${__var} PROPERTY VALUE)
+       get_property(__var_type CACHE ${__var} PROPERTY TYPE)
+       get_property(__var_help CACHE ${__var} PROPERTY HELPSTRING)
++      # Variable may not be in cache, only local (canonical case being EXPAT_LIBRARY since CMake 3.27)
++      # We still need to account for these because they may be required to be in the CACHE at least set in
++      # earlier versions.
++      # 1. Variable may not be in cache, only local (canonical case being EXPAT_LIBRARY since CMake 3.27)
++      #    We still need to account for these because they may be required to be in the CACHE at least set in
++      #    earlier versions.
++      # 2. Depending on CMake version, variable may be in cache but unitialized, here we want the local value
++      if(((NOT __var_value) AND (NOT __var_type) AND (NOT __var_help)) OR (__var_type STREQUAL "UNINITIALIZED"))
++        set(__var_value ${${__var}})
++        # TODO: set type based on whether it looks like a bool or path, but PATH almost invariably what we save
++        # Only important in cmake GUI and if value needs to be changed, which we don't if package cache is used
++        set(__var_type PATH)
++        set(__var_help "no documentation, not a cache value")
++      endif()
++
+       list(APPEND __local_build_setting "geant4_set_and_check_package_variable(${__var} \"${__var_value}\" ${__var_type} \"${__var_help}\")")
+     endforeach()
+ 
+diff --git a/cmake/Modules/G4OptionalComponents.cmake b/cmake/Modules/G4OptionalComponents.cmake
+index 7b3a1f9836..f503a2994a 100644
+--- a/cmake/Modules/G4OptionalComponents.cmake
++++ b/cmake/Modules/G4OptionalComponents.cmake
+@@ -78,6 +78,8 @@ else()
+       unset(EXPAT_FOUND)
+       unset(EXPAT_INCLUDE_DIR CACHE)
+       unset(EXPAT_LIBRARY CACHE)
++      unset(EXPAT_LIBRARY_RELEASE CACHE)
++      unset(EXPAT_LIBRARY_DEBUG CACHE)
+       message(FATAL_ERROR
+ "Detected system expat header and library:
+ EXPAT_INCLUDE_DIR = ${__badexpat_include_dir}
+@@ -88,7 +90,7 @@ Set the above CMake variables to point to an expat install of the required versi
+ 
+     # Backward compatibility for sources.cmake using the variable
+     set(EXPAT_LIBRARIES EXPAT::EXPAT)
+-    geant4_save_package_variables(EXPAT EXPAT_INCLUDE_DIR EXPAT_LIBRARY)
++    geant4_save_package_variables(EXPAT EXPAT_INCLUDE_DIR EXPAT_LIBRARY EXPAT_LIBRARY_RELEASE EXPAT_LIBRARY_DEBUG)
+   else()
+     set(EXPAT_FOUND TRUE)
+     set(GEANT4_USE_BUILTIN_EXPAT TRUE)
\ No newline at end of file
diff --git a/var/spack/repos/builtin/packages/geant4/package.py b/var/spack/repos/builtin/packages/geant4/package.py
index 76b6d1bef60284..afc4464b098bd8 100644
--- a/var/spack/repos/builtin/packages/geant4/package.py
+++ b/var/spack/repos/builtin/packages/geant4/package.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 from spack.package import *
+from spack.variant import _ConditionalVariantValues
 
 
 class Geant4(CMakePackage):
@@ -43,16 +44,18 @@ class Geant4(CMakePackage):
     version("10.4.0", sha256="e919b9b0a88476e00c0b18ab65d40e6a714b55ee4778f66bac32a5396c22aa74")
     version("10.3.3", sha256="bcd36a453da44de9368d1d61b0144031a58e4b43a6d2d875e19085f2700a89d8")
 
-    _cxxstd_values = ("11", "14", "17")
+    _cxxstd_values = (
+        conditional("11", "14", when="@:10"),
+        conditional("17", when="@10.4.1:"),
+        conditional("20", when="@10.7.0:"),
+    )
     variant(
         "cxxstd",
-        default=_cxxstd_values[0],
+        default="11",
         values=_cxxstd_values,
         multi=False,
         description="Use the specified C++ standard when building.",
     )
-    conflicts("cxxstd=11", when="@11:", msg="geant4@11: only supports cxxstd=17")
-    conflicts("cxxstd=14", when="@11:", msg="geant4@11: only supports cxxstd=17")
 
     variant("threads", default=True, description="Build with multithreading")
     variant("vecgeom", default=False, description="Enable vecgeom support")
@@ -88,7 +91,7 @@ class Geant4(CMakePackage):
         depends_on("geant4-data@" + _vers, type="run", when="@" + _vers)
 
     depends_on("expat")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     depends_on("tbb", when="+tbb")
     depends_on("vtk@8.2:", when="+vtk")
@@ -97,30 +100,39 @@ class Geant4(CMakePackage):
     depends_on("python@3:", when="+python")
     extends("python", when="+python")
 
-    for std in _cxxstd_values:
-        # CLHEP version requirements to be reviewed
-        depends_on("clhep@2.4.6.0: cxxstd=" + std, when="@11.1: cxxstd=" + std)
-
-        depends_on("clhep@2.4.5.1: cxxstd=" + std, when="@11.0.0: cxxstd=" + std)
-
-        depends_on("clhep@2.4.4.0: cxxstd=" + std, when="@10.7.0: cxxstd=" + std)
+    # CLHEP version requirements to be reviewed
+    depends_on("clhep@2.4.6.0:", when="@11.1:")
+    depends_on("clhep@2.4.5.1:", when="@11.0.0:")
+    depends_on("clhep@2.4.4.0:", when="@10.7.0:")
+    depends_on("clhep@2.3.3.0:", when="@10.3.3:10.6")
+
+    # Vecgeom specific versions for each Geant4 version
+    with when("+vecgeom"):
+        depends_on("vecgeom@1.2.0:", when="@11.1:")
+        depends_on("vecgeom@1.1.18:1.1", when="@11.0.0:11.0")
+        depends_on("vecgeom@1.1.8:1.1", when="@10.7.0:10.7")
+        depends_on("vecgeom@1.1.5", when="@10.6.0:10.6")
+        depends_on("vecgeom@1.1.0", when="@10.5.0:10.5")
+        depends_on("vecgeom@0.5.2", when="@10.4.0:10.4")
+        depends_on("vecgeom@0.3rc", when="@10.3.0:10.3")
+
+    def std_when(values):
+        for v in values:
+            if isinstance(v, _ConditionalVariantValues):
+                for c in v:
+                    yield (c.value, c.when)
+            else:
+                yield (v, "")
 
-        depends_on("clhep@2.3.3.0: cxxstd=" + std, when="@10.3.3:10.6 cxxstd=" + std)
+    for _std, _when in std_when(_cxxstd_values):
+        depends_on(f"clhep cxxstd={_std}", when=f"{_when} cxxstd={_std}")
+        depends_on(f"vecgeom cxxstd={_std}", when=f"{_when} +vecgeom cxxstd={_std}")
 
         # Spack only supports Xerces-c 3 and above, so no version req
-        depends_on("xerces-c netaccessor=curl cxxstd=" + std, when="cxxstd=" + std)
-
-        # Vecgeom specific versions for each Geant4 version
-        depends_on("vecgeom@1.2.0: cxxstd=" + std, when="@11.1: +vecgeom cxxstd=" + std)
-        depends_on("vecgeom@1.1.18:1.1 cxxstd=" + std, when="@11.0.0:11.0 +vecgeom cxxstd=" + std)
-        depends_on("vecgeom@1.1.8:1.1 cxxstd=" + std, when="@10.7.0:10.7 +vecgeom cxxstd=" + std)
-        depends_on("vecgeom@1.1.5 cxxstd=" + std, when="@10.6.0:10.6 +vecgeom cxxstd=" + std)
-        depends_on("vecgeom@1.1.0 cxxstd=" + std, when="@10.5.0:10.5 +vecgeom cxxstd=" + std)
-        depends_on("vecgeom@0.5.2 cxxstd=" + std, when="@10.4.0:10.4 +vecgeom cxxstd=" + std)
-        depends_on("vecgeom@0.3rc cxxstd=" + std, when="@10.3.0:10.3 +vecgeom cxxstd=" + std)
+        depends_on(f"xerces-c netaccessor=curl cxxstd={_std}", when=f"{_when} cxxstd={_std}")
 
         # Boost.python, conflict handled earlier
-        depends_on("boost@1.70: +python cxxstd=" + std, when="+python cxxstd=" + std)
+        depends_on(f"boost@1.70: +python cxxstd={_std}", when=f"{_when} +python cxxstd={_std}")
 
     # Visualization driver dependencies
     depends_on("gl", when="+opengl")
@@ -139,6 +151,9 @@ class Geant4(CMakePackage):
     patch("cxx17_geant4_10_0.patch", level=1, when="@10.4.0 cxxstd=17")
     patch("geant4-10.4.3-cxx17-removed-features.patch", level=1, when="@10.4.3 cxxstd=17")
 
+    # See https://bugzilla-geant4.kek.jp/show_bug.cgi?id=2556
+    patch("package-cache.patch", level=1, when="@10.7.0:11.2.0^cmake@3.17:")
+
     # NVHPC: "thread-local declaration follows non-thread-local declaration"
     conflicts("%nvhpc", when="+threads")
 
diff --git a/var/spack/repos/builtin/packages/gemma/package.py b/var/spack/repos/builtin/packages/gemma/package.py
index a5596d32adf0f5..625b30dfa204c6 100644
--- a/var/spack/repos/builtin/packages/gemma/package.py
+++ b/var/spack/repos/builtin/packages/gemma/package.py
@@ -18,7 +18,7 @@ class Gemma(MakefilePackage):
 
     version("0.98.5", sha256="3ed336deee29e370f96ec8f1a240f7b62550e57dcd1694245ce7ec8f42241677")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     # openblas is the default
     # other lapack implementors can be made to work
     # but must provide cblas, blas, and lapack libs
diff --git a/var/spack/repos/builtin/packages/generax/package.py b/var/spack/repos/builtin/packages/generax/package.py
index 149f1f6c9c4e04..16692d72c50aca 100644
--- a/var/spack/repos/builtin/packages/generax/package.py
+++ b/var/spack/repos/builtin/packages/generax/package.py
@@ -27,7 +27,7 @@ class Generax(CMakePackage):
     depends_on("bison")
     depends_on("flex")
 
-    variant("mpi", default=False)
+    variant("mpi", default=False, description="Build with MPI support")
 
     build_directory = "build"
 
diff --git a/var/spack/repos/builtin/packages/gengetopt/package.py b/var/spack/repos/builtin/packages/gengetopt/package.py
index ceff92c528c2c0..255f3308462da6 100644
--- a/var/spack/repos/builtin/packages/gengetopt/package.py
+++ b/var/spack/repos/builtin/packages/gengetopt/package.py
@@ -36,3 +36,6 @@ def url_for_version(self, version):
         else:
             suffix = "gz"
         return url.format(version, suffix)
+
+    def setup_dependent_package(self, module, dependent_spec):
+        setattr(module, "gengetopt", Executable(join_path(self.prefix.bin, "gengetopt")))
diff --git a/var/spack/repos/builtin/packages/genomeworks/package.py b/var/spack/repos/builtin/packages/genomeworks/package.py
index ba19c1c1c495be..9ea6b6c072f3f6 100644
--- a/var/spack/repos/builtin/packages/genomeworks/package.py
+++ b/var/spack/repos/builtin/packages/genomeworks/package.py
@@ -13,15 +13,33 @@ class Genomeworks(CMakePackage, CudaPackage):
     url = "https://github.com/clara-parabricks/GenomeWorks/archive/v0.5.3.tar.gz"
     git = "https://github.com/clara-parabricks/GenomeWorks.git"
 
-    version("0.5.3", tag="v0.5.3", submodules=True)
-    version("0.5.2", tag="v0.5.2", submodules=True)
-    version("0.5.1", tag="v0.5.1", submodules=True)
-    version("0.5.0", tag="v0.5.0", submodules=True)
-    version("0.4.4", tag="v0.4.4", submodules=True)
-    version("0.4.3", tag="v0.4.3", submodules=True)
-    version("0.4.0", tag="v0.4.0", submodules=True)
-    version("0.3.0", tag="v0.3.0", submodules=True)
-    version("0.2.0", tag="v0.2.0", submodules=True)
+    version(
+        "0.5.3", tag="v0.5.3", commit="b4b8bf76ea2ce44452d3a1107e66d47968414adb", submodules=True
+    )
+    version(
+        "0.5.2", tag="v0.5.2", commit="d94b6d55a7f9cca8056912ebe9281c77dfc89997", submodules=True
+    )
+    version(
+        "0.5.1", tag="v0.5.1", commit="8cade237403f5ece5b133772232766875f046f20", submodules=True
+    )
+    version(
+        "0.5.0", tag="v0.5.0", commit="3f3837c1a6f8cb6ee4c3d9d177ea38f7c325bf5a", submodules=True
+    )
+    version(
+        "0.4.4", tag="v0.4.4", commit="0cb889061cb4a8c134d96590cc73721601dec283", submodules=True
+    )
+    version(
+        "0.4.3", tag="v0.4.3", commit="97b0b704eee85304602495284343c2135a2ecc22", submodules=True
+    )
+    version(
+        "0.4.0", tag="v0.4.0", commit="fbf7a6a84c8a5681150c864d5180729226bf48d8", submodules=True
+    )
+    version(
+        "0.3.0", tag="v0.3.0", commit="957d4497f8867f1368382c096e2cf7523dd847fb", submodules=True
+    )
+    version(
+        "0.2.0", tag="v0.2.0", commit="416af9f1817a4a70745b3f7cdb7418125159f75c", submodules=True
+    )
 
     depends_on("cmake@3.10.2:", type=("build"))
     depends_on("cuda@11:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/genrich/package.py b/var/spack/repos/builtin/packages/genrich/package.py
index 74134b48007e42..93253354dd3cdb 100644
--- a/var/spack/repos/builtin/packages/genrich/package.py
+++ b/var/spack/repos/builtin/packages/genrich/package.py
@@ -15,7 +15,7 @@ class Genrich(MakefilePackage):
     version("0.6.1", sha256="2c70239e1caf33519b9e99142470bb4dd2f4c69e71f68cee33d6d6a1032d0e33")
     version("0.6", sha256="4c87aca8b7789f28b0c5c2c0ccea75668f19fa6a4cb38cd3c06d80ffd98d396f")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/gettext/package.py b/var/spack/repos/builtin/packages/gettext/package.py
index 6067afd307782e..ee502c07853974 100644
--- a/var/spack/repos/builtin/packages/gettext/package.py
+++ b/var/spack/repos/builtin/packages/gettext/package.py
@@ -19,6 +19,7 @@ class Gettext(AutotoolsPackage, GNUMirrorPackage):
 
     executables = [r"^gettext$"]
 
+    version("0.22.3", sha256="b838228b3f8823a6c1eddf07297197c4db13f7e1b173b9ef93f3f945a63080b6")
     version("0.21.1", sha256="50dbc8f39797950aa2c98e939947c527e5ac9ebd2c1b99dd7b06ba33a6767ae6")
     version("0.21", sha256="d20fcbb537e02dcf1383197ba05bd0734ef7bf5db06bdb241eb69b7d16b73192")
     version("0.20.2", sha256="b22b818e644c37f6e3d1643a1943c32c3a9bff726d601e53047d2682019ceaba")
@@ -56,6 +57,8 @@ class Gettext(AutotoolsPackage, GNUMirrorPackage):
     depends_on("libunistring", when="+libunistring")
     # depends_on('cvs')
 
+    conflicts("+shared~pic")
+
     patch("test-verify-parallel-make-check.patch", when="@:0.19.8.1")
     patch("nvhpc-builtin.patch", when="@:0.21.0 %nvhpc")
     patch("nvhpc-export-symbols.patch", when="%nvhpc")
@@ -119,17 +122,18 @@ def configure_args(self):
         else:
             config_args.append("--with-included-libunistring")
 
-        if spec.satisfies("+pic"):
-            config_args.append("CFLAGS=-fPIC")
+        config_args.extend(self.with_or_without("pic"))
 
         return config_args
 
     @property
     def libs(self):
+        # Do not fail if the installed gettext did not yet have the shared variant:
+        shared_variant = self.spec.variants.get("shared")
         libs = find_libraries(
             ["libasprintf", "libgettextlib", "libgettextpo", "libgettextsrc", "libintl"],
             root=self.prefix,
             recursive=True,
-            shared=self.spec.variants["shared"].value,
+            shared=True if not shared_variant else shared_variant.value,
         )
         return libs
diff --git a/var/spack/repos/builtin/packages/gffcompare/package.py b/var/spack/repos/builtin/packages/gffcompare/package.py
new file mode 100644
index 00000000000000..7f1dda5318d140
--- /dev/null
+++ b/var/spack/repos/builtin/packages/gffcompare/package.py
@@ -0,0 +1,26 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Gffcompare(MakefilePackage):
+    """gffcompare: classify, merge, tracking and annotation of GFF files by comparing to a
+    reference annotation GFF"""
+
+    homepage = "https://ccb.jhu.edu/software/stringtie/gffcompare.shtml"
+    url = (
+        "https://github.com/gpertea/gffcompare/releases/download/v0.12.6/gffcompare-0.12.6.tar.gz"
+    )
+
+    version("0.12.6", sha256="0e713bc9177d874c935802d11669776da5e9377a8c4d031153b48a783d3391d0")
+
+    def build(self, spec, prefix):
+        make("release")
+
+    def install(self, spec, prefix):
+        mkdir(prefix.bin)
+        install("gffcompare", prefix.bin)
+        install("trmap", prefix.bin)
diff --git a/var/spack/repos/builtin/packages/gffread/package.py b/var/spack/repos/builtin/packages/gffread/package.py
new file mode 100644
index 00000000000000..3757aa52c1e098
--- /dev/null
+++ b/var/spack/repos/builtin/packages/gffread/package.py
@@ -0,0 +1,23 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Gffread(MakefilePackage):
+    """gffread: GFF/GTF utility providing format conversions, region filtering,
+    FASTA sequence extraction and more"""
+
+    homepage = "http://ccb.jhu.edu/software/stringtie/gff.shtml#gffread"
+    url = "https://github.com/gpertea/gffread/releases/download/v0.12.7/gffread-0.12.7.tar.gz"
+
+    version("0.12.7", sha256="bfde1c857495e578f5b3af3c007a9aa40593e69450eafcc6a42c3e8ef08ed1f5")
+
+    def build(self, spec, prefix):
+        make("release")
+
+    def install(self, spec, prefix):
+        mkdir(prefix.bin)
+        install("gffread", prefix.bin)
diff --git a/var/spack/repos/builtin/packages/gfsio/package.py b/var/spack/repos/builtin/packages/gfsio/package.py
index 5d284e07e36e21..2c03bb0fdba35a 100644
--- a/var/spack/repos/builtin/packages/gfsio/package.py
+++ b/var/spack/repos/builtin/packages/gfsio/package.py
@@ -14,9 +14,11 @@ class Gfsio(CMakePackage):
 
     homepage = "https://github.com/NOAA-EMC/NCEPLIBS-gfsio"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-gfsio/archive/refs/tags/v1.4.1.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-gfsio"
 
-    maintainers("t-brown", "AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
+    maintainers("AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
 
+    version("develop", branch="develop")
     version("1.4.1", sha256="eab106302f520600decc4f9665d7c6a55e7b4901fab6d9ef40f29702b89b69b1")
 
     def setup_run_environment(self, env):
diff --git a/var/spack/repos/builtin/packages/gh/package.py b/var/spack/repos/builtin/packages/gh/package.py
index c50fb7cff9b8f5..c9012abb89298b 100644
--- a/var/spack/repos/builtin/packages/gh/package.py
+++ b/var/spack/repos/builtin/packages/gh/package.py
@@ -14,6 +14,7 @@ class Gh(Package):
 
     maintainers("lcnzg")
 
+    version("2.32.1", sha256="1d569dc82eb6520e6a8959568c2db84fea3bbaab2604c8dd5901849d320e1eae")
     version("2.28.0", sha256="cf3c0fb7f601d717d8b5177707a197c49fd426f5dc3c9aa52a932e96ba7166af")
     version("2.25.1", sha256="d3b28da03f49600697d2e80c2393425bd382e340040c34641bf3569593c7fbe8")
     version("2.25.0", sha256="b445dbb863643d30cc7991b134c694ea14492e7fac363a9e2648f245f67184f7")
diff --git a/var/spack/repos/builtin/packages/ghostscript/package.py b/var/spack/repos/builtin/packages/ghostscript/package.py
index 7a81c5e584a1fe..9e812e638f1d8e 100644
--- a/var/spack/repos/builtin/packages/ghostscript/package.py
+++ b/var/spack/repos/builtin/packages/ghostscript/package.py
@@ -38,7 +38,7 @@ class Ghostscript(AutotoolsPackage):
     depends_on("lcms")
     depends_on("libpng")
     depends_on("libtiff")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libxext")
     depends_on("gtkplus")
 
@@ -81,7 +81,7 @@ def patch(self):
 
         filter_file(
             "ZLIBDIR=src",
-            "ZLIBDIR={0}".format(self.spec["zlib"].prefix.include),
+            "ZLIBDIR={0}".format(self.spec["zlib-api"].prefix.include),
             "configure.ac",
             "configure",
             string=True,
diff --git a/var/spack/repos/builtin/packages/gimp/package.py b/var/spack/repos/builtin/packages/gimp/package.py
index 3f79356daedc02..4d0c424150568b 100644
--- a/var/spack/repos/builtin/packages/gimp/package.py
+++ b/var/spack/repos/builtin/packages/gimp/package.py
@@ -81,7 +81,7 @@ class Gimp(AutotoolsPackage):
     depends_on("pango@1.29.4:")
     depends_on("poppler+glib")
     depends_on("poppler-data@0.4.7:")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def url_for_version(self, version):
         # ref: https://download.gimp.org/gimp/v2.10/gimp-2.10.32.tar.bz2"
diff --git a/var/spack/repos/builtin/packages/ginkgo/package.py b/var/spack/repos/builtin/packages/ginkgo/package.py
index 97bd7b064099e0..7bdfdb390370d5 100644
--- a/var/spack/repos/builtin/packages/ginkgo/package.py
+++ b/var/spack/repos/builtin/packages/ginkgo/package.py
@@ -50,6 +50,8 @@ class Ginkgo(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("hipsparse", when="+rocm")
     depends_on("hipblas", when="+rocm")
     depends_on("rocrand", when="+rocm")
+    depends_on("hiprand", when="+rocm")
+    depends_on("hipfft", when="+rocm")
     # ROCPRIM is not a direct dependency, but until we have reviewed our CMake
     # setup for rocthrust, this needs to also be added here.
     depends_on("rocprim", when="+rocm")
diff --git a/var/spack/repos/builtin/packages/git-annex/package.py b/var/spack/repos/builtin/packages/git-annex/package.py
index 4cc0490ecec193..ece00cd376225d 100644
--- a/var/spack/repos/builtin/packages/git-annex/package.py
+++ b/var/spack/repos/builtin/packages/git-annex/package.py
@@ -47,6 +47,8 @@ class GitAnnex(Package):
     # - $ git annex whereis git-annex/linux/current/git-annex-standalone-arm64.tar.gz
     #     -> gives web url
 
+    skip_version_audit = ["platform=darwin"]
+
     if platform.system() == "Linux" and platform.machine() == "aarch64":
         # git-annex-standalone-arm64.tar.gz
         version(
diff --git a/var/spack/repos/builtin/packages/git/package.py b/var/spack/repos/builtin/packages/git/package.py
index d6cc876eb8adf0..417600cc05d3f2 100644
--- a/var/spack/repos/builtin/packages/git/package.py
+++ b/var/spack/repos/builtin/packages/git/package.py
@@ -27,93 +27,94 @@ class Git(AutotoolsPackage):
     # Every new git release comes with a corresponding manpage resource:
     # https://www.kernel.org/pub/software/scm/git/git-manpages-{version}.tar.gz
     # https://mirrors.edge.kernel.org/pub/software/scm/git/sha256sums.asc
-    version("2.40.0", sha256="ab37c343c0ad097282fd311ab9ca521ab3da836e5c4ed2093994f1b7f8575b09")
-    version("2.39.2", sha256="fb6807d1eb4094bb2349ab97d203fe1e6c3eb28af73ea391decfbd3a03c02e85")
-    version("2.39.1", sha256="ae8d3427e4ccd677abc931f16183c0ec953e3bfcd866493601351e04a2b97398")
-    version("2.38.3", sha256="ba8f1c56763cfde0433657b045629a4c55047c243eb3091d39dea6f281c8d8e1")
-    version("2.37.5", sha256="5c11f90652afee6c77ef7ddfc672facd4bc6f2596d9627df2f1780664b058b9a")
-    version("2.36.3", sha256="0c831b88b0534f08051d1287505dfe45c367108ee043de6f1c0502711a7aa3a6")
-    version("2.35.6", sha256="6bd51e0487028543ba40fe3d5b33bd124526a7f7109824aa7f022e79edf93bd1")
-    version("2.34.6", sha256="01c0ae4161a07ffeb89cfb8bda564eb2dcb83b45b678cf2930cdbdd8e81784d0")
-    version("2.33.6", sha256="76f6a64a198bec38e83044f97fb5a2dfa8404091df5a905404615d2a4c5ebfb7")
-    version("2.32.5", sha256="9982e17209cf4a385ce4a6167863cdd29f68e425d4249aac186434dc3536fe5f")
-    version("2.31.6", sha256="73971208dccdd6d87639abe50ee3405421ec4ba05dec9f8aa90b4e7f1985e15c")
-    version("2.30.7", sha256="c98bf38a296f23ad5619a097df928044b31859df8f89b3ae5a8ea109d3ebd88e")
+    version("2.42.0", sha256="34aedd54210d7216a55d642bbb4cfb22695b7610719a106bf0ddef4c82a8beed")
+    version("2.41.0", sha256="c4a6a3dd1827895a80cbd824e14d94811796ae54037549e0da93f7b84cb45b9f")
+    version("2.40.1", sha256="55511f10f3b1cdf5db4e0e3dea61819dfb67661b0507a5a2b061c70e4f87e14c")
+    version("2.39.3", sha256="2f9aa93c548941cc5aff641cedc24add15b912ad8c9b36ff5a41b1a9dcad783e")
+    version("2.38.5", sha256="09392caf6ff296341022595a175d8b075bc98b6a82f6227d3bd21e36a2a812c3")
+    version("2.37.7", sha256="2108fa57b74add4300b8960e9404e0ed3e5f0efda7470450c67c67e8ab7616d5")
+    version("2.36.6", sha256="a8c09f46d5d16a8d8f19e8089aeb408d95d8097af03de297061e83a2c74890dd")
+    version("2.35.8", sha256="3a675e0128a7153e1492bbe14d08195d44b5916e6b8879addf94b1f4add77dca")
+    version("2.34.8", sha256="10a6c233471d7d4439cd4004961a3f4ff7e6de308645a1074ec3522b8ea52c83")
+    version("2.33.8", sha256="eafd10da9fdf86be0a79beb67c3537eead114f91836c685d5b9c969c961516ae")
+    version("2.32.7", sha256="f09904d13a9bfca5fcb228c3caba1d4c17426dec0000bf67672af257b8a73db4")
+    version("2.31.8", sha256="d2443e368b1394858a1040bd74dacfba46bce2cf3410ef3bc5089a703fc91e9a")
+    version("2.30.9", sha256="b14b5f4ce1fe23ed78839664c7ba888fb5cedba3dd98d9f5a499a36fa3a4a2d8")
 
     # Deprecated versions
     version(
-        "2.38.1",
-        sha256="620ed3df572a34e782a2be4c7d958d443469b2665eac4ae33f27da554d88b270",
+        "2.40.0",
+        sha256="ab37c343c0ad097282fd311ab9ca521ab3da836e5c4ed2093994f1b7f8575b09",
         deprecated=True,
     )
     version(
-        "2.37.0",
-        sha256="fc3ffe6c65c1f7c681a1ce6bb91703866e432c762731d4b57c566d696f6d62c3",
+        "2.39.2",
+        sha256="fb6807d1eb4094bb2349ab97d203fe1e6c3eb28af73ea391decfbd3a03c02e85",
         deprecated=True,
     )
     version(
-        "2.37.4",
-        sha256="a638c9bf9e45e8d48592076266adaa9b7aa272a99ee2aee2e166a649a9ba8a03",
+        "2.39.1",
+        sha256="ae8d3427e4ccd677abc931f16183c0ec953e3bfcd866493601351e04a2b97398",
         deprecated=True,
     )
     version(
-        "2.36.4",
-        sha256="8b99bd166d904103eccf90b7a9681247a1fb7eba82a72aba70625d9eb95c3db4",
+        "2.38.3",
+        sha256="ba8f1c56763cfde0433657b045629a4c55047c243eb3091d39dea6f281c8d8e1",
         deprecated=True,
     )
     version(
-        "2.36.1",
-        sha256="37d936fd17c81aa9ddd3dba4e56e88a45fa534ad0ba946454e8ce818760c6a2c",
+        "2.38.1",
+        sha256="620ed3df572a34e782a2be4c7d958d443469b2665eac4ae33f27da554d88b270",
         deprecated=True,
     )
     version(
-        "2.35.5",
-        sha256="2cca63fe7bebb5b4bf8efea7b46b12bb89c16ff9711b6b6d845928501d00d0a3",
+        "2.37.5",
+        sha256="5c11f90652afee6c77ef7ddfc672facd4bc6f2596d9627df2f1780664b058b9a",
         deprecated=True,
     )
     version(
-        "2.35.2",
-        sha256="0decc02a47e792f522df3183c38a61ad8fbb38927502ca6781467a6599a888cb",
+        "2.37.4",
+        sha256="a638c9bf9e45e8d48592076266adaa9b7aa272a99ee2aee2e166a649a9ba8a03",
         deprecated=True,
     )
     version(
-        "2.35.1",
-        sha256="9845a37dd01f9faaa7d8aa2078399d3aea91b43819a5efea6e2877b0af09bd43",
+        "2.36.3",
+        sha256="0c831b88b0534f08051d1287505dfe45c367108ee043de6f1c0502711a7aa3a6",
         deprecated=True,
     )
     version(
-        "2.35.0",
-        sha256="c1d0adc777a457a3d9b2759024f173b34e61be96f7480ac5bc44216617834412",
+        "2.35.6",
+        sha256="6bd51e0487028543ba40fe3d5b33bd124526a7f7109824aa7f022e79edf93bd1",
         deprecated=True,
     )
     version(
-        "2.34.5",
-        sha256="26831c5e48a8c2bf6a4fede1b38e1e51ffd6dad85952cf69ac520ebd81a5ae82",
+        "2.35.5",
+        sha256="2cca63fe7bebb5b4bf8efea7b46b12bb89c16ff9711b6b6d845928501d00d0a3",
         deprecated=True,
     )
     version(
-        "2.34.1",
-        sha256="fc4eb5ecb9299db91cdd156c06cdeb41833f53adc5631ddf8c0cb13eaa2911c1",
+        "2.34.6",
+        sha256="01c0ae4161a07ffeb89cfb8bda564eb2dcb83b45b678cf2930cdbdd8e81784d0",
         deprecated=True,
     )
     version(
-        "2.34.0",
-        sha256="0ce6222bfd31938b29360150286b51c77c643fa97740b1d35b6d1ceef8b0ecd7",
+        "2.34.5",
+        sha256="26831c5e48a8c2bf6a4fede1b38e1e51ffd6dad85952cf69ac520ebd81a5ae82",
         deprecated=True,
     )
     version(
-        "2.33.5",
-        sha256="d061ed97f890befaef18b4aad80a37b40db90bcf24113c42765fee157a69c7de",
+        "2.33.6",
+        sha256="76f6a64a198bec38e83044f97fb5a2dfa8404091df5a905404615d2a4c5ebfb7",
         deprecated=True,
     )
     version(
-        "2.33.1",
-        sha256="02047f8dc8934d57ff5e02aadd8a2fe8e0bcf94a7158da375e48086cc46fce1d",
+        "2.33.5",
+        sha256="d061ed97f890befaef18b4aad80a37b40db90bcf24113c42765fee157a69c7de",
         deprecated=True,
     )
     version(
-        "2.33.0",
-        sha256="02d909d0bba560d3a1008bd00dd577621ffb57401b09175fab2bf6da0e9704ae",
+        "2.32.5",
+        sha256="9982e17209cf4a385ce4a6167863cdd29f68e425d4249aac186434dc3536fe5f",
         deprecated=True,
     )
     version(
@@ -122,8 +123,8 @@ class Git(AutotoolsPackage):
         deprecated=True,
     )
     version(
-        "2.32.0",
-        sha256="6038f06d396ba9dab2eee541c7db6e7f9f847f181ec62f3d8441893f8c469398",
+        "2.31.6",
+        sha256="73971208dccdd6d87639abe50ee3405421ec4ba05dec9f8aa90b4e7f1985e15c",
         deprecated=True,
     )
     version(
@@ -132,13 +133,8 @@ class Git(AutotoolsPackage):
         deprecated=True,
     )
     version(
-        "2.31.1",
-        sha256="46d37c229e9d786510e0c53b60065704ce92d5aedc16f2c5111e3ed35093bfa7",
-        deprecated=True,
-    )
-    version(
-        "2.31.0",
-        sha256="bc6168777883562569144d536e8a855b12d25d46870d95188a3064260d7784ee",
+        "2.30.7",
+        sha256="c98bf38a296f23ad5619a097df928044b31859df8f89b3ae5a8ea109d3ebd88e",
         deprecated=True,
     )
     version(
@@ -146,257 +142,41 @@ class Git(AutotoolsPackage):
         sha256="a6130b38843a5c80e80fb4f7ac4864d361cbf103d262b64e267264e49440d24a",
         deprecated=True,
     )
-    version(
-        "2.30.1",
-        sha256="23a3e53f0d2dd3e62a8147b24a1a91d6ffe95b92123ef4dbae04e9a6205e71c0",
-        deprecated=True,
-    )
-    version(
-        "2.30.0",
-        sha256="d24c4fa2a658318c2e66e25ab67cc30038a35696d2d39e6b12ceccf024de1e5e",
-        deprecated=True,
-    )
-    version(
-        "2.29.2",
-        sha256="869a121e1d75e4c28213df03d204156a17f02fce2dc77be9795b327830f54195",
-        deprecated=True,
-    )
-    version(
-        "2.29.0",
-        sha256="fa08dc8424ef80c0f9bf307877f9e2e49f1a6049e873530d6747c2be770742ff",
-        deprecated=True,
-    )
-    version(
-        "2.28.0",
-        sha256="f914c60a874d466c1e18467c864a910dd4ea22281ba6d4d58077cb0c3f115170",
-        deprecated=True,
-    )
-    version(
-        "2.27.0",
-        sha256="77ded85cbe42b1ffdc2578b460a1ef5d23bcbc6683eabcafbb0d394dffe2e787",
-        deprecated=True,
-    )
-    version(
-        "2.26.0",
-        sha256="aa168c2318e7187cd295a645f7370cc6d71a324aafc932f80f00c780b6a26bed",
-        deprecated=True,
-    )
-    version(
-        "2.25.0",
-        sha256="a98c9b96d91544b130f13bf846ff080dda2867e77fe08700b793ab14ba5346f6",
-        deprecated=True,
-    )
-    version(
-        "2.23.0",
-        sha256="e3396c90888111a01bf607346db09b0fbf49a95bc83faf9506b61195936f0cfe",
-        deprecated=True,
-    )
-    version(
-        "2.22.0",
-        sha256="a4b7e4365bee43caa12a38d646d2c93743d755d1cea5eab448ffb40906c9da0b",
-        deprecated=True,
-    )
-    version(
-        "2.21.0",
-        sha256="85eca51c7404da75e353eba587f87fea9481ba41e162206a6f70ad8118147bee",
-        deprecated=True,
-    )
-    version(
-        "2.20.1",
-        sha256="edc3bc1495b69179ba4e272e97eff93334a20decb1d8db6ec3c19c16417738fd",
-        deprecated=True,
-    )
-    version(
-        "2.19.2",
-        sha256="db893ad69c9ac9498b09677c5839787eba2eb3b7ef2bc30bfba7e62e77cf7850",
-        deprecated=True,
-    )
-    version(
-        "2.19.1",
-        sha256="ec4dc96456612c65bf6d944cee9ac640145ec7245376832b781cb03e97cbb796",
-        deprecated=True,
-    )
-    version(
-        "2.18.0",
-        sha256="94faf2c0b02a7920b0b46f4961d8e9cad08e81418614102898a55f980fa3e7e4",
-        deprecated=True,
-    )
-    version(
-        "2.17.1",
-        sha256="ec6452f0c8d5c1f3bcceabd7070b8a8a5eea11d4e2a04955c139b5065fd7d09a",
-        deprecated=True,
-    )
-    version(
-        "2.17.0",
-        sha256="7a0cff35dbb14b77dca6924c33ac9fe510b9de35d5267172490af548ec5ee1b8",
-        deprecated=True,
-    )
-    version(
-        "2.15.1",
-        sha256="85fca8781a83c96ba6db384cc1aa6a5ee1e344746bafac1cbe1f0fe6d1109c84",
-        deprecated=True,
-    )
-    version(
-        "2.14.1",
-        sha256="01925349b9683940e53a621ee48dd9d9ac3f9e59c079806b58321c2cf85a4464",
-        deprecated=True,
-    )
-    version(
-        "2.13.0",
-        sha256="9f2fa8040ebafc0c2caae4a9e2cb385c6f16c0525bcb0fbd84938bc796372e80",
-        deprecated=True,
-    )
-    version(
-        "2.12.2",
-        sha256="d9c6d787a24670d7e5100db2367c250ad9756ef8084fb153a46b82f1d186f8d8",
-        deprecated=True,
-    )
-    version(
-        "2.12.1",
-        sha256="65d62d10caf317fc1daf2ca9975bdb09dbff874c92d24f9529d29a7784486b43",
-        deprecated=True,
-    )
-    version(
-        "2.12.0",
-        sha256="882f298daf582a07c597737eb4bbafb82c6208fe0e73c047defc12169c221a92",
-        deprecated=True,
-    )
-    version(
-        "2.11.1",
-        sha256="a1cdd7c820f92c44abb5003b36dc8cb7201ba38e8744802399f59c97285ca043",
-        deprecated=True,
-    )
-    version(
-        "2.11.0",
-        sha256="d3be9961c799562565f158ce5b836e2b90f38502d3992a115dfb653d7825fd7e",
-        deprecated=True,
-    )
-    version(
-        "2.9.3",
-        sha256="a252b6636b12d5ba57732c8469701544c26c2b1689933bd1b425e603cbb247c0",
-        deprecated=True,
-    )
-    version(
-        "2.9.2",
-        sha256="3cb09a3917c2d8150fc1708f3019cf99a8f0feee6cd61bba3797e3b2a85be9dc",
-        deprecated=True,
-    )
-    version(
-        "2.9.1",
-        sha256="c2230873bf77f93736473e6a06501bf93eed807d011107de6983dc015424b097",
-        deprecated=True,
-    )
-    version(
-        "2.9.0",
-        sha256="bff7560f5602fcd8e37669e0f65ef08c6edc996e4f324e4ed6bb8a84765e30bd",
-        deprecated=True,
-    )
-    version(
-        "2.8.4",
-        sha256="626e319f8a24fc0866167ea5f6bf3e2f38f69d6cb2e59e150f13709ca3ebf301",
-        deprecated=True,
-    )
-    version(
-        "2.8.3",
-        sha256="2dad50c758339d6f5235309db620e51249e0000ff34aa2f2acbcb84c2123ed09",
-        deprecated=True,
-    )
-    version(
-        "2.8.2",
-        sha256="a029c37ee2e0bb1efea5c4af827ff5afdb3356ec42fc19c1d40216d99e97e148",
-        deprecated=True,
-    )
-    version(
-        "2.8.1",
-        sha256="cfc66324179b9ed62ee02833f29d39935f4ab66874125a3ab9d5bb9055c0cb67",
-        deprecated=True,
-    )
-    version(
-        "2.8.0",
-        sha256="2c6eee5506237e0886df9973fd7938a1b2611ec93d07f64ed3447493ebac90d1",
-        deprecated=True,
-    )
-    version(
-        "2.7.3",
-        sha256="30d067499b61caddedaf1a407b4947244f14d10842d100f7c7c6ea1c288280cd",
-        deprecated=True,
-    )
-    version(
-        "2.7.1",
-        sha256="b4ab42798b7fb038eaefabb0c32ce9dbde2919103e5e2a35adc35dd46258a66f",
-        deprecated=True,
-    )
 
     for _version, _sha256_manpage in {
+        "2.42.0": "51643c53d70ce15dde83b6da2bad76ba0c7bbcd4f944d7c378f03a15b9f2e1de",
+        "2.41.0": "7b77c646b36d33c5c0f62677a147142011093270d6fd628ca38c42d5301f3888",
+        "2.40.1": "6bbde434121bd0bf8aa574c60fd9a162388383679bd5ddd99921505149ffd4c2",
         "2.40.0": "fda16047e9c1dd07d9585cc26bbf4002ebf8462ada54cb72b97a0e48135fd435",
+        "2.39.3": "c8377b5a3ff497d7e6377363c270931496e982509ff27a1e46956d6637671642",
         "2.39.2": "fd92e67fb750ceb2279dcee967a21303f2f8117834a21c1e0c9f312ebab6d254",
         "2.39.1": "b2d1b2c6cba2343934792c4409a370a8c684add1b3c0f9b757e71189b1a2e80e",
+        "2.38.5": "648f2b89c9a173c3a687b99629208222170a398c7b14ed92de128656123c73cd",
         "2.38.3": "9e5c924f6f1c961e09d1a8926c2775a158a0375a3311205d7a6176a3ed522272",
         "2.38.1": "fcb27484406b64419a9f9890e95ef29af08e1f911d9d368546eddc59a18e245d",
+        "2.37.7": "475a894584ecc8b278d592a2d99c5c4a4a863485f5126508bcef686cba4a4ac0",
         "2.37.5": "9fab559197891fc1b499cb57513effce7462383f861ac6a7791a46f5348dd7fe",
         "2.37.4": "06ed920949e717f3ab13c98327ee63cae5e3020ac657d14513ef8f843109b638",
-        "2.37.0": "69386ab0dcdbc8398ebb97487733166033f1c7711b02b8861b1ae8f4f46e6e4e",
+        "2.36.6": "08bded34c0ff49b7e8d5d0778511a07f191751c6edb98aaf2cee4c96962cc94c",
         "2.36.3": "c5f5385c2b46270a8ce062a9c510bfa4288d9cca54efe0dff48a12ca969cfc6f",
-        "2.36.6": "c5f5385c2b46270a8ce062a9c510bfa4288d9cca54efe0dff48a12ca969cfc6f",
-        "2.36.1": "3fcd315976f06b54b0abb9c14d38c3d484f431ea4de70a706cc5dddc1799f4f7",
+        "2.35.8": "f85e549d37936df744fd78c1ce670c1682bdd2f35d1f072883b82babe66e484a",
         "2.35.6": "5e44e05a97f49d7a170a7303f795063b19bc78560acd7458274882f19b631187",
         "2.35.5": "6cbd4d2185c7a757db21f873973fa1efb81069d8b8b8cc350ca6735cb98f45c5",
-        "2.35.2": "86e153bdd96edd8462cb7a5c57be1b2b670b033c18272b0aa2e6a102acce50be",
-        "2.35.1": "d90da8b28fe0088519e0dc3c9f4bc85e429c7d6ccbaadcfe94aed47fb9c95504",
-        "2.35.0": "c0408a1c944c8e481d7f507bd90a7ee43c34617a1a7af2d76a1898dcf44fa430",
+        "2.34.8": "e43e75edb8d339ceed4990b5054eb2302efc857d0feab690598e14dbdb9bcccc",
         "2.34.6": "70c784ced9c5ccbd4137d676b032e2ccffeea8aef3094626c2b44d6c843547df",
         "2.34.5": "897941be5b223b9d32217adb64ea8747db2ba57be5f68be598c44d747d1061b2",
-        "2.34.1": "220f1ed68582caeddf79c4db15e4eaa4808ec01fd11889e19232f0a74d7f31b0",
-        "2.34.0": "fe66a69244def488306c3e05c1362ea53d8626d2a7e57cd7311df2dab1ef8356",
+        "2.33.8": "9b49f931e58001d818b2cba7eb6d0242965cfb1eaa5194271b88fcc4529b4987",
         "2.33.6": "d7b9170dc7d6f461e00731cf5cf6e4b589e90c8d4eac440fd3e8b5e3d11f0b8f",
         "2.33.5": "34648ede9ac2869190083ee826065c36165e54d9e2906b10680261b243d89890",
-        "2.33.1": "292b08ca1b79422ff478a6221980099c5e3c0a38aba39d952063eedb68e27d93",
-        "2.33.0": "ba9cd0f29a3632a3b78f8ed2389f0780aa6e8fcbe258259d7c584920d19ed1f7",
+        "2.32.7": "dcce6d701f99190e081f74b539389cdf4674ddbcd4af143631034354a5db39fc",
         "2.32.5": "99b236824f1677e15b21514e310d7a0954586d031ffc3a873a4e2138ed073f15",
         "2.32.4": "fa73d0eac384e594efdd4c21343545e407267ab64e970a6b395c7f1874ddb0bf",
-        "2.32.0": "b5533c40ea1688231c0e2df51cc0d1c0272e17fe78a45ba6e60cb8f61fa4a53c",
+        "2.31.8": "73722b9487456d7605beec65a9fa9415410faa8b9f8a5fd209d75be47bf1a968",
         "2.31.6": "2e2f921d8ef8a839e05ba3a1cea8f864a49b04648378bf0253213a5d4f1642fe",
         "2.31.5": "18850fc8f1c34e51a0a98b9f974b8356a5d63a53c96fb9fe3dc2880ee84746ab",
-        "2.31.1": "d330498aaaea6928b0abbbbb896f6f605efd8d35f23cbbb2de38c87a737d4543",
-        "2.31.0": "a51b760c36be19113756839a9110b328a09abfff0d57f1c93ddac3974ccbc238",
+        "2.30.9": "a3f61fe08453dd88fdd84a28ee6d4c9fbd710a7b1ead7ce5c976146656714ece",
         "2.30.7": "4fc6063c229453de244a88c71f688a2508f30b80ebd47353cc68d730ea1b82aa",
         "2.30.6": "6c20ab10be233e8ff7838351fa5210e972c08005ec541a5241f626cfd4adebfe",
-        "2.30.1": "db323e1b242e9d0337363b1e538c8b879e4c46eedbf94d3bee9e65dab6d49138",
-        "2.30.0": "e23035ae232c9a5eda57db258bc3b7f1c1060cfd66920f92c7d388b6439773a6",
-        "2.29.2": "68b258e6d590cb78e02c0df741bbaeab94cbbac6d25de9da4fb3882ee098307b",
-        "2.29.0": "8f3bf70ddb515674ce2e19572920a39b1be96af12032b77f1dd57898981fb151",
-        "2.28.0": "3cfca28a88d5b8112ea42322b797a500a14d0acddea391aed0462aff1ab11bf7",
-        "2.27.0": "414e4b17133e54d846f6bfa2479f9757c50e16c013eb76167a492ae5409b8947",
-        "2.26.0": "c1ffaf0b4cd1e80a0eb0d4039e208c9d411ef94d5da44e38363804e1a7961218",
-        "2.25.0": "22b2380842ef75e9006c0358de250ead449e1376d7e5138070b9a3073ef61d44",
-        "2.23.0": "a5b0998f95c2290386d191d34780d145ea67e527fac98541e0350749bf76be75",
-        "2.22.0": "f6a5750dfc4a0aa5ec0c0cc495d4995d1f36ed47591c3941be9756c1c3a1aa0a",
-        "2.21.0": "14c76ebb4e31f9e55cf5338a04fd3a13bced0323cd51794ccf45fc74bd0c1080",
-        "2.20.1": "e9c123463abd05e142defe44a8060ce6e9853dfd8c83b2542e38b7deac4e6d4c",
-        "2.19.2": "60334ecd59ee10319af4a7815174d10991d1afabacd3b3129d589f038bf25542",
-        "2.19.1": "bd27f58dc90a661e3080b97365eb7322bfa185de95634fc59d98311925a7d894",
-        "2.18.0": "6cf38ab3ad43ccdcd6a73ffdcf2a016d56ab6b4b240a574b0bb96f520a04ff55",
-        "2.17.1": "9732053c1a618d2576c1751d0249e43702f632a571f84511331882beb360677d",
-        "2.17.0": "41b58c68e90e4c95265c75955ddd5b68f6491f4d57b2f17c6d68e60bbb07ba6a",
-        "2.15.1": "472454c494c9a7f50ad38060c3eec372f617de654b20f3eb3be59fc17a683fa1",
-        "2.14.1": "8c5810ce65d44cd333327d3a115c5b462712a2f81225d142e07bd889ad8dc0e0",
-        "2.13.0": "e764721796cad175a4cf9a4afe7fb4c4fc57582f6f9a6e214239498e0835355b",
-        "2.12.2": "6e7ed503f1190734e57c9427df356b42020f125fa36ab0478777960a682adf50",
-        "2.12.1": "35e46b8acd529ea671d94035232b1795919be8f3c3a363ea9698f1fd08d7d061",
-        "2.12.0": "1f7733a44c59f9ae8dd321d68a033499a76c82046025cc2a6792299178138d65",
-        "2.11.1": "ee567e7b0f95333816793714bb31c54e288cf8041f77a0092b85e62c9c2974f9",
-        "2.11.0": "437a0128acd707edce24e1a310ab2f09f9a09ee42de58a8e7641362012dcfe22",
-        "2.9.3": "8ea1a55b048fafbf0c0c6fcbca4b5b0f5e9917893221fc7345c09051d65832ce",
-        "2.9.2": "ac5c600153d1e4a1c6494e250cd27ca288e7667ad8d4ea2f2386f60ba1b78eec",
-        "2.9.1": "324f5f173f2bd50b0102b66e474b81146ccc078d621efeb86d7f75e3c1de33e6",
-        "2.9.0": "35ba69a8560529aa837e395a6d6c8d42f4d29b40a3c1cc6e3dc69bb1faadb332",
-        "2.8.4": "953a8eadaf4ae96dbad2c3ec12384c677416843917ef83d94b98367ffd55afc0",
-        "2.8.3": "2dad50c758339d6f5235309db620e51249e0000ff34aa2f2acbcb84c2123ed09",
-        "2.8.2": "82d322211aade626d1eb3bcf3b76730bfdd2fcc9c189950fb0a8bdd69c383e2f",
-        "2.8.1": "df46de0c172049f935cc3736361b263c5ff289b77077c73053e63ae83fcf43f4",
-        "2.8.0": "2c48902a69df3bec3b8b8f0350a65fd1b662d2f436f0e64d475ecd1c780767b6",
-        "2.7.3": "84b487c9071857ab0f15f11c4a102a583d59b524831cda0dc0954bd3ab73920b",
-        "2.7.1": "0313cf4d283336088883d8416692fb6c547512233e11dbf06e5b925b7e762d61",
     }.items():
         resource(
             name="git-manpages",
@@ -428,7 +208,7 @@ class Git(AutotoolsPackage):
     depends_on("pcre", when="@:2.13")
     depends_on("pcre2", when="@2.14:")
     depends_on("perl", when="+perl")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("openssh", type="run")
     depends_on("perl-alien-svn", type="run", when="+svn")
     depends_on("tk", type=("build", "link"), when="+tcltk")
@@ -475,8 +255,6 @@ def setup_build_environment(self, env):
                     extlib_bits.append(spec["gettext"].libs.search_flags)
                 extlib_bits.append("-lintl")
                 env.append_flags("EXTLIBS", " ".join(extlib_bits))
-            if not is_system_path(spec["gettext"].prefix):
-                env.append_flags("CFLAGS", spec["gettext"].headers.include_flags)
 
         if not self.spec["curl"].satisfies("libs=shared"):
             curlconfig = which(os.path.join(self.spec["curl"].prefix.bin, "curl-config"))
@@ -495,7 +273,7 @@ def configure_args(self):
             "--with-curl={0}".format(spec["curl"].prefix),
             "--with-expat={0}".format(spec["expat"].prefix),
             "--with-openssl={0}".format(spec["openssl"].prefix),
-            "--with-zlib={0}".format(spec["zlib"].prefix),
+            "--with-zlib={0}".format(spec["zlib-api"].prefix),
         ]
 
         if not self.spec["iconv"].name == "libc":
diff --git a/var/spack/repos/builtin/packages/gl2ps/package.py b/var/spack/repos/builtin/packages/gl2ps/package.py
index f70dcafd347f3a..9e2973a166c668 100644
--- a/var/spack/repos/builtin/packages/gl2ps/package.py
+++ b/var/spack/repos/builtin/packages/gl2ps/package.py
@@ -26,7 +26,7 @@ class Gl2ps(CMakePackage):
     depends_on("gl")
 
     depends_on("libpng", when="+png")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("texlive", type="build", when="+doc")
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/glab/package.py b/var/spack/repos/builtin/packages/glab/package.py
index e50ec0c0b5bc27..1f5d5fc20d6bad 100644
--- a/var/spack/repos/builtin/packages/glab/package.py
+++ b/var/spack/repos/builtin/packages/glab/package.py
@@ -14,7 +14,10 @@ class Glab(Package):
 
     maintainers("alecbcs")
 
+    version("1.33.0", sha256="447a9b76acb5377642a4975908f610a3082026c176329c7c8cfed1461d2e1570")
+    version("1.31.0", sha256="5648e88e7d6cc993227f5a4e80238af189bed09c7aed1eb12be7408e9a042747")
     version("1.30.0", sha256="d3c1a9ba723d94a0be10fc343717cf7b61732644f5c42922f1c8d81047164b99")
+    version("1.29.4", sha256="f6c628d376ea2db9872b1df20abc886281ba58b7bdf29f19dc179c541958640b")
     version("1.28.1", sha256="243a0f15e4400aab7b4d27ec71c6ae650bf782473c47520ffccd57af8d939c90")
     version("1.28.0", sha256="9a0b433c02033cf3d257405d845592e2b7c2e38741027769bb97a8fd763aeeac")
     version("1.27.0", sha256="26bf5fe24eeaeb0f861c89b31129498f029441ae11cc9958e14ad96ec1356d51")
diff --git a/var/spack/repos/builtin/packages/glew/package.py b/var/spack/repos/builtin/packages/glew/package.py
index 172756fd46ea56..a5d8d1ef3e6117 100644
--- a/var/spack/repos/builtin/packages/glew/package.py
+++ b/var/spack/repos/builtin/packages/glew/package.py
@@ -26,10 +26,10 @@ class Glew(CMakePackage):
         multi=False,
         description="The OpenGL provider to use",
     )
-    conflicts("osmesa", when="gl=glx")
-    conflicts("osmesa", when="gl=other")
-    conflicts("glx", when="gl=osmesa")
-    conflicts("glx", when="gl=other")
+    conflicts("^osmesa", when="gl=glx")
+    conflicts("^osmesa", when="gl=other")
+    conflicts("^glx", when="gl=osmesa")
+    conflicts("^glx", when="gl=other")
 
     depends_on("gl")
     depends_on("osmesa", when="gl=osmesa")
diff --git a/var/spack/repos/builtin/packages/glfmultiples/package.py b/var/spack/repos/builtin/packages/glfmultiples/package.py
index c8c8d2949f6e4c..b59997ed4b5740 100644
--- a/var/spack/repos/builtin/packages/glfmultiples/package.py
+++ b/var/spack/repos/builtin/packages/glfmultiples/package.py
@@ -19,7 +19,7 @@ class Glfmultiples(MakefilePackage):
         "2010-06-16", sha256="f7abef6f6b043e9052fb408bb2aae6d0d97d907aedc1b3e02dd0db08eb81b979"
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def edit(self, spec, prefix):
         makefile = FileFilter("Makefile")
diff --git a/var/spack/repos/builtin/packages/glib/package.py b/var/spack/repos/builtin/packages/glib/package.py
index 2452550f30e7c7..f51d636d7eccda 100644
--- a/var/spack/repos/builtin/packages/glib/package.py
+++ b/var/spack/repos/builtin/packages/glib/package.py
@@ -26,6 +26,9 @@ class Glib(MesonPackage, AutotoolsPackage):
 
     maintainers("michaelkuhn")
 
+    version("2.78.0", sha256="44eaab8b720877ce303c5540b657b126f12dc94972d9880b52959f43fb537b30")
+    version("2.76.6", sha256="1136ae6987dcbb64e0be3197a80190520f7acab81e2bfb937dc85c11c8aa9f04")
+    version("2.76.4", sha256="5a5a191c96836e166a7771f7ea6ca2b0069c603c7da3cba1cd38d1694a395dda")
     version("2.76.3", sha256="c0be444e403d7c3184d1f394f89f0b644710b5e9331b54fa4e8b5037813ad32a")
     version("2.76.2", sha256="24f3847857b1d8674cdb0389a36edec0f13c666cd3ce727ecd340eb9da8aca9e")
     version("2.76.1", sha256="43dc0f6a126958f5b454136c4398eab420249c16171a769784486e25f2fda19f")
@@ -135,10 +138,11 @@ class Glib(MesonPackage, AutotoolsPackage):
 
     depends_on("pkgconfig", type="build")
     depends_on("libffi")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("gettext")
     depends_on("perl", type=("build", "run"))
-    depends_on("python", type=("build", "run"), when="@2.53.4:")
+    # Uses distutils in gio/gdbus-2.0/codegen/utils.py
+    depends_on("python@:3.11", type=("build", "run"), when="@2.53.4:")
     depends_on("pcre2", when="@2.73.2:")
     depends_on("pcre2@10.34:", when="@2.74:")
     depends_on("pcre+utf", when="@2.48:2.73.1")
@@ -169,6 +173,13 @@ class Glib(MesonPackage, AutotoolsPackage):
     patch("meson-gettext-2.66.patch", when="@2.66:2.68,2.72")
     patch("meson-gettext-2.70.patch", when="@2.70")
 
+    # Don't use PTRACE_O_EXITKILL if it's not defined
+    patch(
+        "https://gitlab.gnome.org/GNOME/glib/-/commit/bda87264372c006c94e21ffb8ff9c50ecb3e14bd.diff",
+        sha256="2c25d7b3bf581b3ec992d7af997fa6c769174d49b9350e0320c33f5e048cba99",
+        when="@2.78.0",
+    )
+
     def url_for_version(self, version):
         """Handle glib's version-based custom URLs."""
         url = "https://download.gnome.org/sources/glib"
@@ -310,6 +321,12 @@ def meson_args(self):
                     args.append("-Diconv=external")
                 else:
                     args.append("-Diconv=gnu")
+
+        if self.spec.satisfies("^gettext ~shared"):
+            libs = self.spec["iconv"].libs.search_flags + " " + self.spec["iconv"].libs.link_flags
+            args.append(f"-Dc_link_args={libs}")
+            args.append(f"-Dcpp_link_args={libs}")
+
         return args
 
 
diff --git a/var/spack/repos/builtin/packages/glibc/32cf406.patch b/var/spack/repos/builtin/packages/glibc/32cf406.patch
new file mode 100644
index 00000000000000..223301de399b5f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/32cf406.patch
@@ -0,0 +1,21 @@
+From 32cf40699346d37fabfa887bbd95e95004799ae1 Mon Sep 17 00:00:00 2001
+From: Andreas Schwab 
+Date: Mon, 6 Sep 2010 14:55:59 +0200
+Subject: [PATCH] Don't mix pattern rules with normal rules
+
+diff --git a/manual/Makefile b/manual/Makefile
+index c5866eb9def..b1f5fa73e5e 100644
+--- a/manual/Makefile
++++ b/manual/Makefile
+@@ -232,7 +232,10 @@ ifdef objpfx
+ .PHONY: stubs
+ stubs: $(objpfx)stubs
+ endif
+-$(objpfx)stubs ../po/manual.pot $(objpfx)stamp%:
++$(objpfx)stubs ../po/manual.pot:
++	$(make-target-directory)
++	touch $@
++$(objpfx)stamp%:
+ 	$(make-target-directory)
+ 	touch $@
+ 
diff --git a/var/spack/repos/builtin/packages/glibc/39b1f61.patch b/var/spack/repos/builtin/packages/glibc/39b1f61.patch
new file mode 100644
index 00000000000000..44cc80e97288d4
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/39b1f61.patch
@@ -0,0 +1,13 @@
+diff --git a/malloc/obstack.c b/malloc/obstack.c
+index 25a90514f78..c3c7db4a96b 100644
+--- a/malloc/obstack.c
++++ b/malloc/obstack.c
+@@ -115,7 +115,7 @@ int obstack_exit_failure = EXIT_FAILURE;
+ /* A looong time ago (before 1994, anyway; we're not sure) this global variable
+    was used by non-GNU-C macros to avoid multiple evaluation.  The GNU C
+    library still exports it because somebody might use it.  */
+-struct obstack *_obstack_compat;
++struct obstack *_obstack_compat = 0;
+ compat_symbol (libc, _obstack_compat, _obstack, GLIBC_2_0);
+ #  endif
+ # endif
diff --git a/var/spack/repos/builtin/packages/glibc/4a531bb.patch b/var/spack/repos/builtin/packages/glibc/4a531bb.patch
new file mode 100644
index 00000000000000..ca4566df5a3a99
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/4a531bb.patch
@@ -0,0 +1,123 @@
+commit 4a531bb0b3b582cb693de9f76d2d97d970f9a5d5
+Author: H.J. Lu 
+Date:   Fri Dec 24 20:14:37 2010 -0500
+
+    Remove `.ctors' and `.dtors' output sections
+
+diff --git a/config.h.in b/config.h.in
+index 18bf01a38c..9e797eb5b7 100644
+--- a/config.h.in
++++ b/config.h.in
+@@ -201,6 +201,9 @@
+ /* Define if multi-arch DSOs should be generated.  */
+ #undef USE_MULTIARCH
+ 
++/* Define if `.ctors' and `.dtors' sections shouldn't be used.  */
++#define NO_CTORS_DTORS_SECTIONS
++
+ /*
+ ^L */
+ 
+diff --git a/elf/sofini.c b/elf/sofini.c
+index 5e06f0ca92..13e74b7903 100644
+--- a/elf/sofini.c
++++ b/elf/sofini.c
+@@ -1,12 +1,14 @@
+ /* Finalizer module for ELF shared C library.  This provides terminating
+    null pointer words in the `.ctors' and `.dtors' sections.  */
+ 
++#ifndef NO_CTORS_DTORS_SECTIONS
+ static void (*const __CTOR_END__[1]) (void)
+      __attribute__ ((used, section (".ctors")))
+      = { 0 };
+ static void (*const __DTOR_END__[1]) (void)
+      __attribute__ ((used, section (".dtors")))
+      = { 0 };
++#endif
+ 
+ /* Terminate the frame unwind info section with a 4byte 0 as a sentinel;
+    this would be the 'length' field in a real FDE.  */
+diff --git a/elf/soinit.c b/elf/soinit.c
+index 6fecbb5674..1db676af01 100644
+--- a/elf/soinit.c
++++ b/elf/soinit.c
+@@ -3,6 +3,7 @@
+    the `.ctors' and `.dtors' sections so the lists are terminated, and
+    calling those lists of functions.  */
+ 
++#ifndef NO_CTORS_DTORS_SECTIONS
+ #include 
+ #include 
+ 
+@@ -40,3 +41,4 @@ __libc_fini (void)
+ 
+ void (*_fini_ptr) (void) __attribute__ ((section (".fini_array")))
+      = &__libc_fini;
++#endif
+diff --git a/sysdeps/i386/init-first.c b/sysdeps/i386/init-first.c
+index c6355a8b7b..2af042fe4b 100644
+--- a/sysdeps/i386/init-first.c
++++ b/sysdeps/i386/init-first.c
+@@ -59,7 +59,9 @@ _init (int argc, ...)
+ {
+   init (&argc);
+ 
++#ifndef NO_CTORS_DTORS_SECTIONS
+   __libc_global_ctors ();
++#endif
+ }
+ #endif
+ 
+diff --git a/sysdeps/mach/hurd/i386/init-first.c b/sysdeps/mach/hurd/i386/init-first.c
+index f9a7a58deb..60823bd789 100644
+--- a/sysdeps/mach/hurd/i386/init-first.c
++++ b/sysdeps/mach/hurd/i386/init-first.c
+@@ -92,7 +92,7 @@ posixland_init (int argc, char **argv, char **envp)
+   __getopt_clean_environment (envp);
+ #endif
+ 
+-#ifdef SHARED
++#if defined SHARED && !defined NO_CTORS_DTORS_SECTIONS
+   __libc_global_ctors ();
+ #endif
+ }
+diff --git a/sysdeps/mach/hurd/powerpc/init-first.c b/sysdeps/mach/hurd/powerpc/init-first.c
+index 20fa1d4f12..21b5054b0a 100644
+--- a/sysdeps/mach/hurd/powerpc/init-first.c
++++ b/sysdeps/mach/hurd/powerpc/init-first.c
+@@ -82,7 +82,7 @@ posixland_init (int argc, char **argv, char **envp)
+   __getopt_clean_environment (__environ);
+ #endif
+ 
+-#ifdef SHARED
++#if defined SHARED && !defined NO_CTORS_DTORS_SECTIONS
+   __libc_global_ctors ();
+ #endif
+ }
+diff --git a/sysdeps/sh/init-first.c b/sysdeps/sh/init-first.c
+index d816625ef4..1f3a821fea 100644
+--- a/sysdeps/sh/init-first.c
++++ b/sysdeps/sh/init-first.c
+@@ -59,7 +59,9 @@ _init (int argc, ...)
+ {
+   init (&argc);
+ 
++#ifndef NO_CTORS_DTORS_SECTIONS
+   __libc_global_ctors ();
++#endif
+ }
+ #endif
+ 
+diff --git a/sysdeps/unix/sysv/linux/init-first.c b/sysdeps/unix/sysv/linux/init-first.c
+index 7b2333d4bf..a60212f4ed 100644
+--- a/sysdeps/unix/sysv/linux/init-first.c
++++ b/sysdeps/unix/sysv/linux/init-first.c
+@@ -93,7 +93,7 @@ _init (int argc, char **argv, char **envp)
+   __getopt_clean_environment (envp);
+ #endif
+ 
+-#ifdef SHARED
++#if defined SHARED && !defined NO_CTORS_DTORS_SECTIONS
+   __libc_global_ctors ();
+ #endif
+ }
diff --git a/var/spack/repos/builtin/packages/glibc/67fbfa5.patch b/var/spack/repos/builtin/packages/glibc/67fbfa5.patch
new file mode 100644
index 00000000000000..8e478ac0502423
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/67fbfa5.patch
@@ -0,0 +1,53 @@
+From 67fbfa5ca15f75774dcb3794aceb06aa42544c94 Mon Sep 17 00:00:00 2001
+From: Roland McGrath 
+Date: Fri, 1 Feb 2008 00:20:11 +0000
+Subject: [PATCH] [BZ #5442]
+
+	* configure.in: Use -print-file-name if it yields a directory,
+	for each of include and include-fixed.
+	* configure: Regenerated.
+---
+ ChangeLog      | 5 +++++
+ configure      | 8 ++++++--
+ configure.in   | 8 ++++++--
+ nptl/ChangeLog | 4 ++++
+ 4 files changed, 21 insertions(+), 4 deletions(-)
+
+diff --git a/configure b/configure
+index 1f2cd4fc533..4b969787e3d 100755
+--- a/configure
++++ b/configure
+@@ -5065,8 +5065,12 @@ echo "$as_me: WARNING:
+ # header directory and add that to the list.  NOTE: Only does the right
+ # thing on a system that doesn't need fixincludes.  (Not presently a problem.)
+ if test -n "$sysheaders"; then
+-  ccheaders=`$CC -print-file-name=include`
+-  SYSINCLUDES="-nostdinc -isystem $ccheaders \
++  SYSINCLUDES=-nostdinc
++  for d in include include-fixed; do
++    i=`$CC -print-file-name="$d"` && test "x$i" != "x$d" &&
++    SYSINCLUDES="$SYSINCLUDES -isystem $i"
++  done
++  SYSINCLUDES="$SYSINCLUDES \
+ -isystem `echo $sysheaders | sed 's/:/ -isystem /g'`"
+   if test -n "$CXX"; then
+     cxxversion=`$CXX -dumpversion 2>&5` &&
+diff --git a/configure.in b/configure.in
+index 6ea1b26ff34..3098ebfd3d7 100644
+--- a/configure.in
++++ b/configure.in
+@@ -914,8 +914,12 @@ test -n "$aux_missing" && AC_MSG_WARN([
+ # header directory and add that to the list.  NOTE: Only does the right
+ # thing on a system that doesn't need fixincludes.  (Not presently a problem.)
+ if test -n "$sysheaders"; then
+-  ccheaders=`$CC -print-file-name=include`
+-  SYSINCLUDES="-nostdinc -isystem $ccheaders \
++  SYSINCLUDES=-nostdinc
++  for d in include include-fixed; do
++    i=`$CC -print-file-name="$d"` && test "x$i" != x && test "x$i" != "x$d" &&
++    SYSINCLUDES="$SYSINCLUDES -isystem $i"
++  done
++  SYSINCLUDES="$SYSINCLUDES \
+ -isystem `echo $sysheaders | sed 's/:/ -isystem /g'`"
+   if test -n "$CXX"; then
+     cxxversion=`$CXX -dumpversion 2>&AS_MESSAGE_LOG_FD` &&
diff --git a/var/spack/repos/builtin/packages/glibc/7c8a673.patch b/var/spack/repos/builtin/packages/glibc/7c8a673.patch
new file mode 100644
index 00000000000000..0903ebab76cfc8
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/7c8a673.patch
@@ -0,0 +1,21 @@
+commit 7c8a67320e26b8c11108bf0a3410d3aef9cf3486
+Author: Ulrich Drepper 
+Date:   Sat Jan 31 00:21:15 2009 +0000
+
+    * elf/Makefile (ld.so): Adjust the sed script to insert _begin in to
+    
+            newer linker scripts.
+
+diff --git a/elf/Makefile b/elf/Makefile
+index 8079fe9f96..e44ff1d382 100644
+--- a/elf/Makefile
++++ b/elf/Makefile
+@@ -304,7 +304,7 @@ $(objpfx)ld.so: $(objpfx)librtld.os $(ld-map)
+ 		  $(LDFLAGS-rtld) -Wl,-z,defs -Wl,--verbose 2>&1 |	\
+ 		  LC_ALL=C \
+ 		  sed -e '/^=========/,/^=========/!d;/^=========/d'	\
+-		      -e 's/\. = 0 + SIZEOF_HEADERS;/& _begin = . - SIZEOF_HEADERS;/' \
++		      -e 's/\. = .* + SIZEOF_HEADERS;/& _begin = . - SIZEOF_HEADERS;/' \
+ 		  > $@.lds
+ 	$(LINK.o) -nostdlib -nostartfiles -shared -o $@			\
+ 		  $(LDFLAGS-rtld) -Wl,-z,defs $(z-now-$(bind-now))	\
diff --git a/var/spack/repos/builtin/packages/glibc/95f5a9a-2.11.patch b/var/spack/repos/builtin/packages/glibc/95f5a9a-2.11.patch
new file mode 100644
index 00000000000000..bb73cf44d76e00
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/95f5a9a-2.11.patch
@@ -0,0 +1,194 @@
+From 95f5a9a866695da4e038aa4e6ccbbfd5d9cf63b7 Mon Sep 17 00:00:00 2001
+From: Joseph Myers 
+Date: Tue, 3 Jul 2012 19:14:59 +0000
+Subject: [PATCH] Avoid use of libgcc_s and libgcc_eh when building glibc.
+
+diff --git a/Makeconfig b/Makeconfig
+index 47638b13c6..6384e22a72 100644
+--- a/Makeconfig
++++ b/Makeconfig
+@@ -423,7 +423,7 @@ endif
+ 
+ # Command for linking programs with the C library.
+ ifndef +link
+-+link = $(CC) -nostdlib -nostartfiles -o $@ \
+++link-before-libc = $(CC) -nostdlib -nostartfiles -o $@ \
+ 	      $(sysdep-LDFLAGS) $(config-LDFLAGS) $(LDFLAGS) $(LDFLAGS-$(@F)) \
+ 	      $(combreloc-LDFLAGS) $(relro-LDFLAGS) $(hashstyle-LDFLAGS) \
+ 	      $(addprefix $(csu-objpfx),$(start-installed-name)) \
+@@ -432,7 +432,10 @@ ifndef +link
+ 						     $(start-installed-name))\
+ 			   $(+preinit) $(link-extra-libs) \
+ 			   $(common-objpfx)libc% $(+postinit),$^) \
+-	      $(link-extra-libs) $(link-libc) $(+postctor) $(+postinit)
++	      $(link-extra-libs)
+++link-after-libc = $(+postctor) $(+postinit)
+++link = $(+link-before-libc) $(link-libc) $(+link-after-libc)
+++link-tests = $(+link-before-libc) $(link-libc-tests) $(+link-after-libc)
+ endif
+ # Command for linking PIE programs with the C library.
+ ifndef +link-pie
+@@ -449,7 +452,7 @@ ifndef +link-pie
+ endif
+ # Command for statically linking programs with the C library.
+ ifndef +link-static
+-+link-static = $(CC) -nostdlib -nostartfiles -static -o $@ \
+++link-static-before-libc = $(CC) -nostdlib -nostartfiles -static -o $@ \
+ 	      $(sysdep-LDFLAGS) $(LDFLAGS) $(LDFLAGS-$(@F))  \
+ 	      $(addprefix $(csu-objpfx),$(static-start-installed-name)) \
+ 	      $(+preinit) $(+prector) \
+@@ -457,7 +460,12 @@ ifndef +link-static
+ 						     $(start-installed-name))\
+ 			   $(+preinit) $(link-extra-libs-static) \
+ 			   $(common-objpfx)libc% $(+postinit),$^) \
+-	      $(link-extra-libs-static) $(link-libc-static) $(+postctor) $(+postinit)
++	      $(link-extra-libs-static) $(link-libc-static)
+++link-static-after-libc = $(+postctor) $(+postinit)
+++link-static = $(+link-static-before-libc) $(link-libc-static) \
++	      $(+link-static-after-libc)
+++link-static-tests = $(+link-static-before-libc) $(link-libc-static-tests) \
++	      $(+link-static-after-libc)
+ endif
+ # Command for statically linking bounded-pointer programs with the C library.
+ ifndef +link-bounded
+@@ -482,9 +490,11 @@ ifeq ($(elf),yes)
+ # We need the versioned name of libc.so in the deps of $(others) et al
+ # so that the symlink to libc.so is created before anything tries to
+ # run the linked programs.
+-link-libc = -Wl,-rpath-link=$(rpath-link) \
++link-libc-before-gnulib = -Wl,-rpath-link=$(rpath-link) \
+ 	    $(common-objpfx)libc.so$(libc.so-version) \
+-	    $(common-objpfx)$(patsubst %,$(libtype.oS),c) $(gnulib)
++	    $(common-objpfx)$(patsubst %,$(libtype.oS),c)
++link-libc = $(link-libc-before-gnulib) $(gnulib)
++link-libc-tests = $(link-libc-before-gnulib) $(gnulib-tests)
+ # This is how to find at build-time things that will be installed there.
+ rpath-dirs = math elf dlfcn nss nis rt resolv crypt
+ endif
+@@ -495,6 +505,7 @@ else
+ nssobjdir := $(patsubst ../$(subdir),.,$(common-objpfx)nss)
+ resolvobjdir := $(patsubst ../$(subdir),.,$(common-objpfx)resolv)
+ link-libc = $(common-objpfx)libc.a $(otherlibs) $(gnulib) $(common-objpfx)libc.a $(gnulib)
++link-libc-tests = $(common-objpfx)libc.a $(otherlibs) $(gnulib-tests) $(common-objpfx)libc.a $(gnulib-tests)
+ endif
+ endif
+ 
+@@ -523,6 +534,7 @@ endif
+ # The static libraries.
+ ifeq (yes,$(build-static))
+ link-libc-static = $(common-objpfx)libc.a $(static-gnulib) $(common-objpfx)libc.a
++link-libc-static-tests = $(common-objpfx)libc.a $(static-gnulib-tests) $(common-objpfx)libc.a
+ else
+ ifeq (yes,$(build-shared))
+ # We can try to link the programs with lib*_pic.a...
+@@ -542,8 +554,12 @@ ifneq ($(have-as-needed),yes)
+ else
+  libgcc_eh := -Wl,--as-needed -lgcc_s$(libgcc_s_suffix) $(libunwind) -Wl,--no-as-needed
+ endif
+-gnulib := -lgcc $(libgcc_eh)
+-static-gnulib := -lgcc -lgcc_eh $(libunwind)
++gnulib-arch =
++gnulib = -lgcc $(gnulib-arch)
++gnulib-tests := -lgcc $(libgcc_eh)
++static-gnulib-arch =
++static-gnulib = -lgcc $(static-gnulib-arch)
++static-gnulib-tests := -lgcc -lgcc_eh $(libunwind)
+ libc.so-gnulib := -lgcc
+ endif
+ ifeq ($(elf),yes)
+diff --git a/Rules b/Rules
+index 5ace24cee0..45eb7541dc 100644
+--- a/Rules
++++ b/Rules
+@@ -104,29 +104,52 @@ xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-bp.out)
+ endif
+ 
+ ifeq ($(build-programs),yes)
+-binaries-all = $(others) $(sysdep-others) $(tests) $(xtests) $(test-srcs)
+-binaries-static = $(others-static) $(tests-static) $(xtests-static)
++binaries-all-notests = $(others) $(sysdep-others)
++binaries-all-tests = $(tests) $(xtests) $(test-srcs)
++binaries-all = $(binaries-all-notests) $(binaries-all-tests)
++binaries-static-notests = $(others-static)
++binaries-static-tests = $(tests-static) $(xtests-static)
++binaries-static = $(binaries-static-notests) $(binaries-static-tests)
+ else
+-binaries-all = $(tests) $(xtests) $(test-srcs)
++binaries-all-notests =
++binaries-all-tests = $(tests) $(xtests) $(test-srcs)
++binaries-all = $(binaries-all-tests)
++binaries-static-notests =
++binaries-static-tests =
+ binaries-static =
+ endif
+ 
+-binaries-shared = $(filter-out $(binaries-static), $(binaries-all))
++binaries-shared-tests = $(filter-out $(binaries-static), $(binaries-all-tests))
++binaries-shared-notests = $(filter-out $(binaries-static), $(binaries-all-notests))
+ 
+-ifneq "$(strip $(binaries-shared))" ""
+-$(addprefix $(objpfx),$(binaries-shared)): %: %.o \
++ifneq "$(strip $(binaries-shared-notests))" ""
++$(addprefix $(objpfx),$(binaries-shared-notests)): %: %.o \
+   $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \
+   $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
+ 	$(+link)
+ endif
+ 
+-ifneq "$(strip $(binaries-static))" ""
+-$(addprefix $(objpfx),$(binaries-static)): %: %.o \
++ifneq "$(strip $(binaries-shared-tests))" ""
++$(addprefix $(objpfx),$(binaries-shared-tests)): %: %.o \
++  $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \
++  $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
++	$(+link-tests)
++endif
++
++ifneq "$(strip $(binaries-static-notests))" ""
++$(addprefix $(objpfx),$(binaries-static-notests)): %: %.o \
+   $(sort $(filter $(common-objpfx)lib%,$(link-libc-static))) \
+   $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
+ 	$(+link-static)
+ endif
+ 
++ifneq "$(strip $(binaries-static-tests))" ""
++$(addprefix $(objpfx),$(binaries-static-tests)): %: %.o \
++  $(sort $(filter $(common-objpfx)lib%,$(link-libc-static-tests))) \
++  $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
++	$(+link-static-tests)
++endif
++
+ ifeq ($(build-bounded),yes)
+ binaries-bounded = $(addsuffix -bp,$(tests) $(xtests) $(test-srcs))
+ $(addprefix $(objpfx),$(binaries-bounded)): %-bp: %.ob \
+diff --git a/elf/Makefile b/elf/Makefile
+index 34609a0f85..84709920e8 100644
+--- a/elf/Makefile
++++ b/elf/Makefile
+@@ -150,6 +150,8 @@ others		= sprof sln
+ install-bin	= sprof
+ others-static   = sln
+ install-rootsbin = sln
++sln-modules	:= static-stubs
++extra-objs	+= $(sln-modules:=.o)
+ 
+ ifeq (yes,$(use-ldconfig))
+ ifeq (yes,$(build-shared))
+@@ -157,7 +159,7 @@ others-static	+= ldconfig
+ others		+= ldconfig
+ install-rootsbin += ldconfig
+ 
+-ldconfig-modules := cache readlib xmalloc xstrdup chroot_canon
++ldconfig-modules := cache readlib xmalloc xstrdup chroot_canon static-stubs
+ extra-objs	+= $(ldconfig-modules:=.o)
+ 
+ # To find xmalloc.c and xstrdup.c
+@@ -448,6 +450,8 @@ $(objpfx)ldd: ldd.bash.in $(common-objpfx)soversions.mk \
+ 
+ $(objpfx)sprof: $(libdl)
+ 
++$(objpfx)sln: $(sln-modules:%=$(objpfx)%.o)
++
+ $(objpfx)ldconfig: $(ldconfig-modules:%=$(objpfx)%.o)
+ SYSCONF-FLAGS := -D'SYSCONFDIR="$(sysconfdir)"'
+ CFLAGS-ldconfig.c = $(SYSCONF-FLAGS) -D'LIBDIR="$(libdir)"' \
diff --git a/var/spack/repos/builtin/packages/glibc/95f5a9a-2.13.patch b/var/spack/repos/builtin/packages/glibc/95f5a9a-2.13.patch
new file mode 100644
index 00000000000000..346d87fa0e40b9
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/95f5a9a-2.13.patch
@@ -0,0 +1,195 @@
+From 95f5a9a866695da4e038aa4e6ccbbfd5d9cf63b7 Mon Sep 17 00:00:00 2001
+From: Joseph Myers 
+Date: Tue, 3 Jul 2012 19:14:59 +0000
+Subject: [PATCH] Avoid use of libgcc_s and libgcc_eh when building glibc.
+
+diff --git a/Makeconfig b/Makeconfig
+index e5cbf646fa..870a4bec4f 100644
+--- a/Makeconfig
++++ b/Makeconfig
+@@ -431,7 +431,7 @@ endif
+ 
+ # Command for linking programs with the C library.
+ ifndef +link
+-+link = $(CC) -nostdlib -nostartfiles -o $@ \
+++link-before-libc = $(CC) -nostdlib -nostartfiles -o $@ \
+ 	      $(sysdep-LDFLAGS) $(config-LDFLAGS) $(LDFLAGS) $(LDFLAGS-$(@F)) \
+ 	      $(combreloc-LDFLAGS) $(relro-LDFLAGS) $(hashstyle-LDFLAGS) \
+ 	      $(addprefix $(csu-objpfx),$(start-installed-name)) \
+@@ -440,7 +440,10 @@ ifndef +link
+ 						     $(start-installed-name))\
+ 			   $(+preinit) $(link-extra-libs) \
+ 			   $(common-objpfx)libc% $(+postinit),$^) \
+-	      $(link-extra-libs) $(link-libc) $(+postctor) $(+postinit)
++	      $(link-extra-libs)
+++link-after-libc = $(+postctor) $(+postinit)
+++link = $(+link-before-libc) $(link-libc) $(+link-after-libc)
+++link-tests = $(+link-before-libc) $(link-libc-tests) $(+link-after-libc)
+ endif
+ # Command for linking PIE programs with the C library.
+ ifndef +link-pie
+@@ -457,7 +460,7 @@ ifndef +link-pie
+ endif
+ # Command for statically linking programs with the C library.
+ ifndef +link-static
+-+link-static = $(CC) -nostdlib -nostartfiles -static -o $@ \
+++link-static-before-libc = $(CC) -nostdlib -nostartfiles -static -o $@ \
+ 	      $(sysdep-LDFLAGS) $(LDFLAGS) $(LDFLAGS-$(@F))  \
+ 	      $(addprefix $(csu-objpfx),$(static-start-installed-name)) \
+ 	      $(+preinit) $(+prector) \
+@@ -465,7 +468,12 @@ ifndef +link-static
+ 						     $(start-installed-name))\
+ 			   $(+preinit) $(link-extra-libs-static) \
+ 			   $(common-objpfx)libc% $(+postinit),$^) \
+-	      $(link-extra-libs-static) $(link-libc-static) $(+postctor) $(+postinit)
++	      $(link-extra-libs-static) $(link-libc-static)
+++link-static-after-libc = $(+postctor) $(+postinit)
+++link-static = $(+link-static-before-libc) $(link-libc-static) \
++	      $(+link-static-after-libc)
+++link-static-tests = $(+link-static-before-libc) $(link-libc-static-tests) \
++	      $(+link-static-after-libc)
+ endif
+ # Command for statically linking bounded-pointer programs with the C library.
+ ifndef +link-bounded
+@@ -490,10 +498,12 @@ ifeq ($(elf),yes)
+ # We need the versioned name of libc.so in the deps of $(others) et al
+ # so that the symlink to libc.so is created before anything tries to
+ # run the linked programs.
+-link-libc = -Wl,-rpath-link=$(rpath-link) \
++link-libc-before-gnulib = -Wl,-rpath-link=$(rpath-link) \
+ 	    $(common-objpfx)libc.so$(libc.so-version) \
+ 	    $(common-objpfx)$(patsubst %,$(libtype.oS),c) \
+-	    $(as-needed) $(common-objpfx)elf/ld.so $(no-as-needed) $(gnulib)
++	    $(as-needed) $(common-objpfx)elf/ld.so $(no-as-needed)
++link-libc = $(link-libc-before-gnulib) $(gnulib)
++link-libc-tests = $(link-libc-before-gnulib) $(gnulib-tests)
+ # This is how to find at build-time things that will be installed there.
+ rpath-dirs = math elf dlfcn nss nis rt resolv crypt
+ endif
+@@ -504,6 +514,7 @@ else
+ nssobjdir := $(patsubst ../$(subdir),.,$(common-objpfx)nss)
+ resolvobjdir := $(patsubst ../$(subdir),.,$(common-objpfx)resolv)
+ link-libc = $(common-objpfx)libc.a $(otherlibs) $(gnulib) $(common-objpfx)libc.a $(gnulib)
++link-libc-tests = $(common-objpfx)libc.a $(otherlibs) $(gnulib-tests) $(common-objpfx)libc.a $(gnulib-tests)
+ endif
+ endif
+ 
+@@ -532,6 +543,7 @@ endif
+ # The static libraries.
+ ifeq (yes,$(build-static))
+ link-libc-static = $(common-objpfx)libc.a $(static-gnulib) $(common-objpfx)libc.a
++link-libc-static-tests = $(common-objpfx)libc.a $(static-gnulib-tests) $(common-objpfx)libc.a
+ else
+ ifeq (yes,$(build-shared))
+ # We can try to link the programs with lib*_pic.a...
+@@ -551,8 +563,12 @@ ifneq ($(have-as-needed),yes)
+ else
+  libgcc_eh := -Wl,--as-needed -lgcc_s$(libgcc_s_suffix) $(libunwind) -Wl,--no-as-needed
+ endif
+-gnulib := -lgcc $(libgcc_eh)
+-static-gnulib := -lgcc -lgcc_eh $(libunwind)
++gnulib-arch =
++gnulib = -lgcc $(gnulib-arch)
++gnulib-tests := -lgcc $(libgcc_eh)
++static-gnulib-arch =
++static-gnulib = -lgcc $(static-gnulib-arch)
++static-gnulib-tests := -lgcc -lgcc_eh $(libunwind)
+ libc.so-gnulib := -lgcc
+ endif
+ ifeq ($(elf),yes)
+diff --git a/Rules b/Rules
+index 5ace24cee0..45eb7541dc 100644
+--- a/Rules
++++ b/Rules
+@@ -104,29 +104,52 @@ xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-bp.out)
+ endif
+ 
+ ifeq ($(build-programs),yes)
+-binaries-all = $(others) $(sysdep-others) $(tests) $(xtests) $(test-srcs)
+-binaries-static = $(others-static) $(tests-static) $(xtests-static)
++binaries-all-notests = $(others) $(sysdep-others)
++binaries-all-tests = $(tests) $(xtests) $(test-srcs)
++binaries-all = $(binaries-all-notests) $(binaries-all-tests)
++binaries-static-notests = $(others-static)
++binaries-static-tests = $(tests-static) $(xtests-static)
++binaries-static = $(binaries-static-notests) $(binaries-static-tests)
+ else
+-binaries-all = $(tests) $(xtests) $(test-srcs)
++binaries-all-notests =
++binaries-all-tests = $(tests) $(xtests) $(test-srcs)
++binaries-all = $(binaries-all-tests)
++binaries-static-notests =
++binaries-static-tests =
+ binaries-static =
+ endif
+ 
+-binaries-shared = $(filter-out $(binaries-static), $(binaries-all))
++binaries-shared-tests = $(filter-out $(binaries-static), $(binaries-all-tests))
++binaries-shared-notests = $(filter-out $(binaries-static), $(binaries-all-notests))
+ 
+-ifneq "$(strip $(binaries-shared))" ""
+-$(addprefix $(objpfx),$(binaries-shared)): %: %.o \
++ifneq "$(strip $(binaries-shared-notests))" ""
++$(addprefix $(objpfx),$(binaries-shared-notests)): %: %.o \
+   $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \
+   $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
+ 	$(+link)
+ endif
+ 
+-ifneq "$(strip $(binaries-static))" ""
+-$(addprefix $(objpfx),$(binaries-static)): %: %.o \
++ifneq "$(strip $(binaries-shared-tests))" ""
++$(addprefix $(objpfx),$(binaries-shared-tests)): %: %.o \
++  $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \
++  $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
++	$(+link-tests)
++endif
++
++ifneq "$(strip $(binaries-static-notests))" ""
++$(addprefix $(objpfx),$(binaries-static-notests)): %: %.o \
+   $(sort $(filter $(common-objpfx)lib%,$(link-libc-static))) \
+   $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
+ 	$(+link-static)
+ endif
+ 
++ifneq "$(strip $(binaries-static-tests))" ""
++$(addprefix $(objpfx),$(binaries-static-tests)): %: %.o \
++  $(sort $(filter $(common-objpfx)lib%,$(link-libc-static-tests))) \
++  $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
++	$(+link-static-tests)
++endif
++
+ ifeq ($(build-bounded),yes)
+ binaries-bounded = $(addsuffix -bp,$(tests) $(xtests) $(test-srcs))
+ $(addprefix $(objpfx),$(binaries-bounded)): %-bp: %.ob \
+diff --git a/elf/Makefile b/elf/Makefile
+index 2d2d568013..fe1924ebd4 100644
+--- a/elf/Makefile
++++ b/elf/Makefile
+@@ -150,6 +150,8 @@ others		= sprof sln
+ install-bin	= sprof
+ others-static   = sln
+ install-rootsbin = sln
++sln-modules	:= static-stubs
++extra-objs	+= $(sln-modules:=.o)
+ 
+ ifeq (yes,$(use-ldconfig))
+ ifeq (yes,$(build-shared))
+@@ -157,7 +159,7 @@ others-static	+= ldconfig
+ others		+= ldconfig
+ install-rootsbin += ldconfig
+ 
+-ldconfig-modules := cache readlib xmalloc xstrdup chroot_canon
++ldconfig-modules := cache readlib xmalloc xstrdup chroot_canon static-stubs
+ extra-objs	+= $(ldconfig-modules:=.o)
+ 
+ # To find xmalloc.c and xstrdup.c
+@@ -458,6 +460,8 @@ $(objpfx)ldd: ldd.bash.in $(common-objpfx)soversions.mk \
+ 
+ $(objpfx)sprof: $(libdl)
+ 
++$(objpfx)sln: $(sln-modules:%=$(objpfx)%.o)
++
+ $(objpfx)ldconfig: $(ldconfig-modules:%=$(objpfx)%.o)
+ SYSCONF-FLAGS := -D'SYSCONFDIR="$(sysconfdir)"'
+ CFLAGS-ldconfig.c = $(SYSCONF-FLAGS) -D'LIBDIR="$(libdir)"' \
diff --git a/var/spack/repos/builtin/packages/glibc/95f5a9a-2.15.patch b/var/spack/repos/builtin/packages/glibc/95f5a9a-2.15.patch
new file mode 100644
index 00000000000000..3fecf3a3c80dce
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/95f5a9a-2.15.patch
@@ -0,0 +1,211 @@
+From 95f5a9a866695da4e038aa4e6ccbbfd5d9cf63b7 Mon Sep 17 00:00:00 2001
+From: Joseph Myers 
+Date: Tue, 3 Jul 2012 19:14:59 +0000
+Subject: [PATCH] Avoid use of libgcc_s and libgcc_eh when building glibc.
+
+diff --git a/Makeconfig b/Makeconfig
+index 8195245052..b23cee8723 100644
+--- a/Makeconfig
++++ b/Makeconfig
+@@ -447,7 +447,7 @@ endif
+ 
+ # Command for linking programs with the C library.
+ ifndef +link
+-+link = $(CC) -nostdlib -nostartfiles -o $@ \
+++link-before-libc = $(CC) -nostdlib -nostartfiles -o $@ \
+ 	      $(sysdep-LDFLAGS) $(config-LDFLAGS) $(LDFLAGS) $(LDFLAGS-$(@F)) \
+ 	      $(combreloc-LDFLAGS) $(relro-LDFLAGS) $(hashstyle-LDFLAGS) \
+ 	      $(addprefix $(csu-objpfx),$(start-installed-name)) \
+@@ -456,7 +456,10 @@ ifndef +link
+ 						     $(start-installed-name))\
+ 			   $(+preinit) $(link-extra-libs) \
+ 			   $(common-objpfx)libc% $(+postinit),$^) \
+-	      $(link-extra-libs) $(link-libc) $(+postctor) $(+postinit)
++	      $(link-extra-libs)
+++link-after-libc = $(+postctor) $(+postinit)
+++link = $(+link-before-libc) $(link-libc) $(+link-after-libc)
+++link-tests = $(+link-before-libc) $(link-libc-tests) $(+link-after-libc)
+ endif
+ # Command for linking PIE programs with the C library.
+ ifndef +link-pie
+@@ -473,7 +476,7 @@ ifndef +link-pie
+ endif
+ # Command for statically linking programs with the C library.
+ ifndef +link-static
+-+link-static = $(CC) -nostdlib -nostartfiles -static -o $@ \
+++link-static-before-libc = $(CC) -nostdlib -nostartfiles -static -o $@ \
+ 	      $(sysdep-LDFLAGS) $(LDFLAGS) $(LDFLAGS-$(@F))  \
+ 	      $(addprefix $(csu-objpfx),$(static-start-installed-name)) \
+ 	      $(+preinit) $(+prector) \
+@@ -481,7 +484,12 @@ ifndef +link-static
+ 						     $(start-installed-name))\
+ 			   $(+preinit) $(link-extra-libs-static) \
+ 			   $(common-objpfx)libc% $(+postinit),$^) \
+-	      $(link-extra-libs-static) $(link-libc-static) $(+postctor) $(+postinit)
++	      $(link-extra-libs-static) $(link-libc-static)
+++link-static-after-libc = $(+postctor) $(+postinit)
+++link-static = $(+link-static-before-libc) $(link-libc-static) \
++	      $(+link-static-after-libc)
+++link-static-tests = $(+link-static-before-libc) $(link-libc-static-tests) \
++	      $(+link-static-after-libc)
+ endif
+ # Command for statically linking bounded-pointer programs with the C library.
+ ifndef +link-bounded
+@@ -506,10 +514,12 @@ ifeq ($(elf),yes)
+ # We need the versioned name of libc.so in the deps of $(others) et al
+ # so that the symlink to libc.so is created before anything tries to
+ # run the linked programs.
+-link-libc = -Wl,-rpath-link=$(rpath-link) \
++link-libc-before-gnulib = -Wl,-rpath-link=$(rpath-link) \
+ 	    $(common-objpfx)libc.so$(libc.so-version) \
+ 	    $(common-objpfx)$(patsubst %,$(libtype.oS),c) \
+-	    $(as-needed) $(common-objpfx)elf/ld.so $(no-as-needed) $(gnulib)
++	    $(as-needed) $(common-objpfx)elf/ld.so $(no-as-needed)
++link-libc = $(link-libc-before-gnulib) $(gnulib)
++link-libc-tests = $(link-libc-before-gnulib) $(gnulib-tests)
+ # This is how to find at build-time things that will be installed there.
+ rpath-dirs = math elf dlfcn nss nis rt resolv crypt
+ endif
+@@ -520,6 +530,7 @@ else
+ nssobjdir := $(patsubst ../$(subdir),.,$(common-objpfx)nss)
+ resolvobjdir := $(patsubst ../$(subdir),.,$(common-objpfx)resolv)
+ link-libc = $(common-objpfx)libc.a $(otherlibs) $(gnulib) $(common-objpfx)libc.a $(gnulib)
++link-libc-tests = $(common-objpfx)libc.a $(otherlibs) $(gnulib-tests) $(common-objpfx)libc.a $(gnulib-tests)
+ endif
+ endif
+ 
+@@ -548,6 +559,7 @@ endif
+ # The static libraries.
+ ifeq (yes,$(build-static))
+ link-libc-static = -Wl,--start-group $(common-objpfx)libc.a $(static-gnulib) -Wl,--end-group
++link-libc-static-tests = -Wl,--start-group $(common-objpfx)libc.a $(static-gnulib-tests) -Wl,--end-group
+ else
+ ifeq (yes,$(build-shared))
+ # We can try to link the programs with lib*_pic.a...
+@@ -567,8 +579,12 @@ ifneq ($(have-as-needed),yes)
+ else
+  libgcc_eh := -Wl,--as-needed -lgcc_s$(libgcc_s_suffix) $(libunwind) -Wl,--no-as-needed
+ endif
+-gnulib := -lgcc $(libgcc_eh)
+-static-gnulib := -lgcc -lgcc_eh $(libunwind)
++gnulib-arch =
++gnulib = -lgcc $(gnulib-arch)
++gnulib-tests := -lgcc $(libgcc_eh)
++static-gnulib-arch =
++static-gnulib = -lgcc $(static-gnulib-arch)
++static-gnulib-tests := -lgcc -lgcc_eh $(libunwind)
+ libc.so-gnulib := -lgcc
+ endif
+ ifeq ($(elf),yes)
+diff --git a/Rules b/Rules
+index 00f03df6da..4a31d2905c 100644
+--- a/Rules
++++ b/Rules
+@@ -104,29 +104,46 @@ xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-bp.out)
+ endif
+ 
+ ifeq ($(build-programs),yes)
+-binaries-all = $(others) $(sysdep-others) $(tests) $(xtests) $(test-srcs)
+-binaries-static = $(others-static) $(tests-static) $(xtests-static)
++binaries-all-notests = $(others) $(sysdep-others)
++binaries-all-tests = $(tests) $(xtests) $(test-srcs)
++binaries-all = $(binaries-all-notests) $(binaries-all-tests)
++binaries-static-notests = $(others-static)
++binaries-static-tests = $(tests-static) $(xtests-static)
++binaries-static = $(binaries-static-notests) $(binaries-static-tests)
+ ifeq (yesyes,$(have-fpie)$(build-shared))
+ binaries-pie = $(others-pie) $(tests-pie) $(xtests-pie)
+ else
+ binaries-pie =
+ endif
+ else
+-binaries-all = $(tests) $(xtests) $(test-srcs)
++binaries-all-notests =
++binaries-all-tests = $(tests) $(xtests) $(test-srcs)
++binaries-all = $(binaries-all-tests)
++binaries-static-notests =
++binaries-static-tests =
+ binaries-static =
+ binaries-pie =
+ endif
+ 
+-binaries-shared = $(filter-out $(binaries-pie) $(binaries-static), \
+-			       $(binaries-all))
++binaries-shared-tests = $(filter-out $(binaries-pie) $(binaries-static), \
++				     $(binaries-all-tests))
++binaries-shared-notests = $(filter-out $(binaries-pie) $(binaries-static), \
++				       $(binaries-all-notests))
+ 
+-ifneq "$(strip $(binaries-shared))" ""
+-$(addprefix $(objpfx),$(binaries-shared)): %: %.o \
++ifneq "$(strip $(binaries-shared-notests))" ""
++$(addprefix $(objpfx),$(binaries-shared-notests)): %: %.o \
+   $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \
+   $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
+ 	$(+link)
+ endif
+ 
++ifneq "$(strip $(binaries-shared-tests))" ""
++$(addprefix $(objpfx),$(binaries-shared-tests)): %: %.o \
++  $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \
++  $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
++	$(+link-tests)
++endif
++
+ ifneq "$(strip $(binaries-pie))" ""
+ $(addprefix $(objpfx),$(binaries-pie)): %: %.o \
+   $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \
+@@ -134,13 +151,20 @@ $(addprefix $(objpfx),$(binaries-pie)): %: %.o \
+ 	$(+link-pie)
+ endif
+ 
+-ifneq "$(strip $(binaries-static))" ""
+-$(addprefix $(objpfx),$(binaries-static)): %: %.o \
++ifneq "$(strip $(binaries-static-notests))" ""
++$(addprefix $(objpfx),$(binaries-static-notests)): %: %.o \
+   $(sort $(filter $(common-objpfx)lib%,$(link-libc-static))) \
+   $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
+ 	$(+link-static)
+ endif
+ 
++ifneq "$(strip $(binaries-static-tests))" ""
++$(addprefix $(objpfx),$(binaries-static-tests)): %: %.o \
++  $(sort $(filter $(common-objpfx)lib%,$(link-libc-static-tests))) \
++  $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
++	$(+link-static-tests)
++endif
++
+ ifeq ($(build-bounded),yes)
+ binaries-bounded = $(addsuffix -bp,$(tests) $(xtests) $(test-srcs))
+ $(addprefix $(objpfx),$(binaries-bounded)): %-bp: %.ob \
+diff --git a/elf/Makefile b/elf/Makefile
+index f20f52dee1..64555b9243 100644
+--- a/elf/Makefile
++++ b/elf/Makefile
+@@ -156,6 +156,8 @@ others		= sprof sln pldd
+ install-bin	= sprof pldd
+ others-static   = sln
+ install-rootsbin = sln
++sln-modules	:= static-stubs
++extra-objs	+= $(sln-modules:=.o)
+ 
+ ifeq (yes,$(use-ldconfig))
+ ifeq (yes,$(build-shared))
+@@ -163,7 +165,7 @@ others-static	+= ldconfig
+ others		+= ldconfig
+ install-rootsbin += ldconfig
+ 
+-ldconfig-modules := cache readlib xmalloc xstrdup chroot_canon
++ldconfig-modules := cache readlib xmalloc xstrdup chroot_canon static-stubs
+ extra-objs	+= $(ldconfig-modules:=.o)
+ 
+ pldd-modules := xmalloc
+@@ -495,6 +497,8 @@ $(objpfx)ldd: ldd.bash.in $(common-objpfx)soversions.mk \
+ 
+ $(objpfx)sprof: $(libdl)
+ 
++$(objpfx)sln: $(sln-modules:%=$(objpfx)%.o)
++
+ $(objpfx)ldconfig: $(ldconfig-modules:%=$(objpfx)%.o)
+ 
+ $(objpfx)pldd: $(pldd-modules:%=$(objpfx)%.o)
diff --git a/var/spack/repos/builtin/packages/glibc/95f5a9a-2.16.patch b/var/spack/repos/builtin/packages/glibc/95f5a9a-2.16.patch
new file mode 100644
index 00000000000000..4c2c77edb4a29b
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/95f5a9a-2.16.patch
@@ -0,0 +1,214 @@
+From 95f5a9a866695da4e038aa4e6ccbbfd5d9cf63b7 Mon Sep 17 00:00:00 2001
+From: Joseph Myers 
+Date: Tue, 3 Jul 2012 19:14:59 +0000
+Subject: [PATCH] Avoid use of libgcc_s and libgcc_eh when building glibc.
+
+diff --git a/Makeconfig b/Makeconfig
+index 417fa508a6..c860e68cdd 100644
+--- a/Makeconfig
++++ b/Makeconfig
+@@ -415,9 +415,9 @@ LDFLAGS.so += $(hashstyle-LDFLAGS)
+ LDFLAGS-rtld += $(hashstyle-LDFLAGS)
+ endif
+ 
+-# Command for linking programs with the C library.
++# Commands for linking programs with the C library.
+ ifndef +link
+-+link = $(CC) -nostdlib -nostartfiles -o $@ \
+++link-before-libc = $(CC) -nostdlib -nostartfiles -o $@ \
+ 	      $(sysdep-LDFLAGS) $(config-LDFLAGS) $(LDFLAGS) $(LDFLAGS-$(@F)) \
+ 	      $(combreloc-LDFLAGS) $(relro-LDFLAGS) $(hashstyle-LDFLAGS) \
+ 	      $(addprefix $(csu-objpfx),$(start-installed-name)) \
+@@ -426,7 +426,10 @@ ifndef +link
+ 						     $(start-installed-name))\
+ 			   $(+preinit) $(link-extra-libs) \
+ 			   $(common-objpfx)libc% $(+postinit),$^) \
+-	      $(link-extra-libs) $(link-libc) $(+postctor) $(+postinit)
++	      $(link-extra-libs)
+++link-after-libc = $(+postctor) $(+postinit)
+++link = $(+link-before-libc) $(link-libc) $(+link-after-libc)
+++link-tests = $(+link-before-libc) $(link-libc-tests) $(+link-after-libc)
+ endif
+ # Command for linking PIE programs with the C library.
+ ifndef +link-pie
+@@ -443,7 +446,7 @@ ifndef +link-pie
+ endif
+ # Command for statically linking programs with the C library.
+ ifndef +link-static
+-+link-static = $(CC) -nostdlib -nostartfiles -static -o $@ \
+++link-static-before-libc = $(CC) -nostdlib -nostartfiles -static -o $@ \
+ 	      $(sysdep-LDFLAGS) $(LDFLAGS) $(LDFLAGS-$(@F))  \
+ 	      $(addprefix $(csu-objpfx),$(static-start-installed-name)) \
+ 	      $(+preinit) $(+prector) \
+@@ -451,7 +454,12 @@ ifndef +link-static
+ 						     $(start-installed-name))\
+ 			   $(+preinit) $(link-extra-libs-static) \
+ 			   $(common-objpfx)libc% $(+postinit),$^) \
+-	      $(link-extra-libs-static) $(link-libc-static) $(+postctor) $(+postinit)
++	      $(link-extra-libs-static) $(link-libc-static)
+++link-static-after-libc = $(+postctor) $(+postinit)
+++link-static = $(+link-static-before-libc) $(link-libc-static) \
++	       $(+link-static-after-libc)
+++link-static-tests = $(+link-static-before-libc) $(link-libc-static-tests) \
++		     $(+link-static-after-libc)
+ endif
+ # Command for statically linking bounded-pointer programs with the C library.
+ ifndef +link-bounded
+@@ -475,10 +483,12 @@ ifeq (yes,$(build-shared))
+ # We need the versioned name of libc.so in the deps of $(others) et al
+ # so that the symlink to libc.so is created before anything tries to
+ # run the linked programs.
+-link-libc = -Wl,-rpath-link=$(rpath-link) \
++link-libc-before-gnulib = -Wl,-rpath-link=$(rpath-link) \
+ 	    $(common-objpfx)libc.so$(libc.so-version) \
+ 	    $(common-objpfx)$(patsubst %,$(libtype.oS),c) \
+-	    $(as-needed) $(common-objpfx)elf/ld.so $(no-as-needed) $(gnulib)
++	    $(as-needed) $(common-objpfx)elf/ld.so $(no-as-needed)
++link-libc = $(link-libc-before-gnulib) $(gnulib)
++link-libc-tests = $(link-libc-before-gnulib) $(gnulib-tests)
+ # This is how to find at build-time things that will be installed there.
+ rpath-dirs = math elf dlfcn nss nis rt resolv crypt
+ rpath-link = \
+@@ -488,6 +498,7 @@ else
+ nssobjdir := $(patsubst ../$(subdir),.,$(common-objpfx)nss)
+ resolvobjdir := $(patsubst ../$(subdir),.,$(common-objpfx)resolv)
+ link-libc = $(common-objpfx)libc.a $(otherlibs) $(gnulib) $(common-objpfx)libc.a $(gnulib)
++link-libc-tests = $(common-objpfx)libc.a $(otherlibs) $(gnulib-tests) $(common-objpfx)libc.a $(gnulib-tests)
+ endif
+ endif
+ 
+@@ -513,6 +524,7 @@ endif
+ 
+ # The static libraries.
+ link-libc-static = -Wl,--start-group $(common-objpfx)libc.a $(static-gnulib) -Wl,--end-group
++link-libc-static-tests = -Wl,--start-group $(common-objpfx)libc.a $(static-gnulib-tests) -Wl,--end-group
+ link-libc-bounded = $(common-objpfx)libc_b.a $(gnulib) $(common-objpfx)libc_b.a
+ 
+ ifndef gnulib
+@@ -522,8 +534,12 @@ else
+   libunwind = -lunwind
+ endif
+ libgcc_eh := -Wl,--as-needed -lgcc_s $(libunwind) -Wl,--no-as-needed
+-gnulib := -lgcc $(libgcc_eh)
+-static-gnulib := -lgcc -lgcc_eh $(libunwind)
++gnulib-arch =
++gnulib = -lgcc $(gnulib-arch)
++gnulib-tests := -lgcc $(libgcc_eh)
++static-gnulib-arch =
++static-gnulib = -lgcc $(static-gnulib-arch)
++static-gnulib-tests := -lgcc -lgcc_eh $(libunwind)
+ libc.so-gnulib := -lgcc
+ endif
+ +preinit = $(addprefix $(csu-objpfx),crti.o)
+diff --git a/Rules b/Rules
+index 3c61a2933b..be3a738574 100644
+--- a/Rules
++++ b/Rules
+@@ -103,29 +103,46 @@ xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-bp.out)
+ endif
+ 
+ ifeq ($(build-programs),yes)
+-binaries-all = $(others) $(sysdep-others) $(tests) $(xtests) $(test-srcs)
+-binaries-static = $(others-static) $(tests-static) $(xtests-static)
++binaries-all-notests = $(others) $(sysdep-others)
++binaries-all-tests = $(tests) $(xtests) $(test-srcs)
++binaries-all = $(binaries-all-notests) $(binaries-all-tests)
++binaries-static-notests = $(others-static)
++binaries-static-tests = $(tests-static) $(xtests-static)
++binaries-static = $(binaries-static-notests) $(binaries-static-tests)
+ ifeq (yesyes,$(have-fpie)$(build-shared))
+ binaries-pie = $(others-pie) $(tests-pie) $(xtests-pie)
+ else
+ binaries-pie =
+ endif
+ else
+-binaries-all = $(tests) $(xtests) $(test-srcs)
++binaries-all-notests =
++binaries-all-tests = $(tests) $(xtests) $(test-srcs)
++binaries-all = $(binaries-all-tests)
++binaries-static-notests =
++binaries-static-tests =
+ binaries-static =
+ binaries-pie =
+ endif
+ 
+-binaries-shared = $(filter-out $(binaries-pie) $(binaries-static), \
+-			       $(binaries-all))
++binaries-shared-tests = $(filter-out $(binaries-pie) $(binaries-static), \
++				     $(binaries-all-tests))
++binaries-shared-notests = $(filter-out $(binaries-pie) $(binaries-static), \
++				       $(binaries-all-notests))
+ 
+-ifneq "$(strip $(binaries-shared))" ""
+-$(addprefix $(objpfx),$(binaries-shared)): %: %.o \
++ifneq "$(strip $(binaries-shared-notests))" ""
++$(addprefix $(objpfx),$(binaries-shared-notests)): %: %.o \
+   $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \
+   $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
+ 	$(+link)
+ endif
+ 
++ifneq "$(strip $(binaries-shared-tests))" ""
++$(addprefix $(objpfx),$(binaries-shared-tests)): %: %.o \
++  $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \
++  $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
++	$(+link-tests)
++endif
++
+ ifneq "$(strip $(binaries-pie))" ""
+ $(addprefix $(objpfx),$(binaries-pie)): %: %.o \
+   $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \
+@@ -133,13 +150,20 @@ $(addprefix $(objpfx),$(binaries-pie)): %: %.o \
+ 	$(+link-pie)
+ endif
+ 
+-ifneq "$(strip $(binaries-static))" ""
+-$(addprefix $(objpfx),$(binaries-static)): %: %.o \
++ifneq "$(strip $(binaries-static-notests))" ""
++$(addprefix $(objpfx),$(binaries-static-notests)): %: %.o \
+   $(sort $(filter $(common-objpfx)lib%,$(link-libc-static))) \
+   $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
+ 	$(+link-static)
+ endif
+ 
++ifneq "$(strip $(binaries-static-tests))" ""
++$(addprefix $(objpfx),$(binaries-static-tests)): %: %.o \
++  $(sort $(filter $(common-objpfx)lib%,$(link-libc-static-tests))) \
++  $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit)
++	$(+link-static-tests)
++endif
++
+ ifeq ($(build-bounded),yes)
+ binaries-bounded = $(addsuffix -bp,$(tests) $(xtests) $(test-srcs))
+ $(addprefix $(objpfx),$(binaries-bounded)): %-bp: %.ob \
+diff --git a/elf/Makefile b/elf/Makefile
+index 0c26ce545a..90541991d2 100644
+--- a/elf/Makefile
++++ b/elf/Makefile
+@@ -71,6 +71,8 @@ others		= sprof sln pldd
+ install-bin	= sprof pldd
+ others-static   = sln
+ install-rootsbin = sln
++sln-modules	:= static-stubs
++extra-objs	+= $(sln-modules:=.o)
+ 
+ ifeq (yes,$(use-ldconfig))
+ ifeq (yes,$(build-shared))
+@@ -78,7 +80,7 @@ others-static	+= ldconfig
+ others		+= ldconfig
+ install-rootsbin += ldconfig
+ 
+-ldconfig-modules := cache readlib xmalloc xstrdup chroot_canon
++ldconfig-modules := cache readlib xmalloc xstrdup chroot_canon static-stubs
+ extra-objs	+= $(ldconfig-modules:=.o)
+ endif
+ endif
+@@ -411,6 +413,8 @@ $(objpfx)ldd: ldd.bash.in $(common-objpfx)soversions.mk \
+ 
+ $(objpfx)sprof: $(libdl)
+ 
++$(objpfx)sln: $(sln-modules:%=$(objpfx)%.o)
++
+ $(objpfx)ldconfig: $(ldconfig-modules:%=$(objpfx)%.o)
+ 
+ $(objpfx)pldd: $(pldd-modules:%=$(objpfx)%.o)
diff --git a/var/spack/repos/builtin/packages/glibc/95f5a9a-stub.patch b/var/spack/repos/builtin/packages/glibc/95f5a9a-stub.patch
new file mode 100644
index 00000000000000..da25ce77ade87a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/95f5a9a-stub.patch
@@ -0,0 +1,57 @@
+From 95f5a9a866695da4e038aa4e6ccbbfd5d9cf63b7 Mon Sep 17 00:00:00 2001
+From: Joseph Myers 
+Date: Tue, 3 Jul 2012 19:14:59 +0000
+Subject: [PATCH] Avoid use of libgcc_s and libgcc_eh when building glibc.
+
+diff --git a/elf/static-stubs.c b/elf/static-stubs.c
+new file mode 100644
+index 0000000000..6c5eebc3fb
+--- /dev/null
++++ b/elf/static-stubs.c
+@@ -0,0 +1,46 @@
++/* Stub implementations of functions to link into statically linked
++   programs without needing libgcc_eh.
++   Copyright (C) 2012 Free Software Foundation, Inc.
++   This file is part of the GNU C Library.
++
++   The GNU C Library is free software; you can redistribute it and/or
++   modify it under the terms of the GNU Lesser General Public
++   License as published by the Free Software Foundation; either
++   version 2.1 of the License, or (at your option) any later version.
++
++   The GNU C Library is distributed in the hope that it will be useful,
++   but WITHOUT ANY WARRANTY; without even the implied warranty of
++   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
++   Lesser General Public License for more details.
++
++   You should have received a copy of the GNU Lesser General Public
++   License along with the GNU C Library; if not, see
++   .  */
++
++/* Avoid backtrace (and so _Unwind_Backtrace) dependencies from
++   sysdeps/unix/sysv/linux/libc_fatal.c.  */
++#include 
++
++#include 
++#include 
++
++/* These programs do not use thread cancellation, so _Unwind_Resume
++   and the personality routine are never actually called.  */
++
++void
++_Unwind_Resume (struct _Unwind_Exception *exc __attribute__ ((unused)))
++{
++  abort ();
++}
++
++_Unwind_Reason_Code
++__gcc_personality_v0 (int version __attribute__ ((unused)),
++		      _Unwind_Action actions __attribute__ ((unused)),
++		      _Unwind_Exception_Class exception_class
++		      __attribute__ ((unused)),
++		      struct _Unwind_Exception *ue_header
++		      __attribute__ ((unused)),
++		      struct _Unwind_Context *context __attribute__ ((unused)))
++{
++  abort ();
++}
diff --git a/var/spack/repos/builtin/packages/glibc/965cb60-2.5.patch b/var/spack/repos/builtin/packages/glibc/965cb60-2.5.patch
new file mode 100644
index 00000000000000..e998379ce6010e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/965cb60-2.5.patch
@@ -0,0 +1,176 @@
+diff --git a/csu/libc-start.c b/csu/libc-start.c
+index 194db6b1ec..f85ec9604e 100644
+--- a/csu/libc-start.c
++++ b/csu/libc-start.c
+@@ -151,8 +151,8 @@ LIBC_START_MAIN (int (*main) (int, char **, char ** MAIN_AUXVEC_DECL),
+ 
+ # ifndef SHARED
+   /* Set up the stack checker's canary.  */
+-  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard ();
+-#  ifdef THREAD_SET_STACK_GUARD
++  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard (_dl_random);
++# ifdef THREAD_SET_STACK_GUARD
+   THREAD_SET_STACK_GUARD (stack_chk_guard);
+ #  else
+   __stack_chk_guard = stack_chk_guard;
+diff --git a/elf/dl-support.c b/elf/dl-support.c
+index 4b7be6bc27..f62867abf1 100644
+--- a/elf/dl-support.c
++++ b/elf/dl-support.c
+@@ -84,6 +84,9 @@ struct r_scope_elem _dl_initial_searchlist;
+ int _dl_starting_up = 1;
+ #endif
+ 
++/* Random data provided by the kernel.  */
++void *_dl_random;
++
+ /* Get architecture specific initializer.  */
+ #include 
+ 
+@@ -218,6 +221,9 @@ _dl_aux_init (ElfW(auxv_t) *av)
+ 	__libc_enable_secure = av->a_un.a_val;
+ 	__libc_enable_secure_decided = 1;
+ 	break;
++      case AT_RANDOM:
++	_dl_random = (void *) av->a_un.a_val;
++	break;
+       }
+   if (seen == 0xf)
+     {
+diff --git a/elf/dl-sysdep.c b/elf/dl-sysdep.c
+index 68e08f480a..133ba0d29f 100644
+--- a/elf/dl-sysdep.c
++++ b/elf/dl-sysdep.c
+@@ -62,6 +62,7 @@ int __libc_multiple_libcs = 0;	/* Defining this here avoids the inclusion
+ void *__libc_stack_end attribute_relro = NULL;
+ rtld_hidden_data_def(__libc_stack_end)
+ static ElfW(auxv_t) *_dl_auxv attribute_relro;
++void *_dl_random attribute_relro = NULL;
+ 
+ #ifndef DL_FIND_ARG_COMPONENTS
+ # define DL_FIND_ARG_COMPONENTS(cookie, argc, argv, envp, auxp)	\
+@@ -173,6 +174,9 @@ _dl_sysdep_start (void **start_argptr,
+ 	GLRO(dl_sysinfo_dso) = (void *) av->a_un.a_val;
+ 	break;
+ #endif
++      case AT_RANDOM:
++	_dl_random = (void *) av->a_un.a_val;
++	break;
+ #ifdef DL_PLATFORM_AUXV
+       DL_PLATFORM_AUXV
+ #endif
+@@ -293,6 +297,7 @@ _dl_show_auxv (void)
+ 	  [AT_SECURE - 2] =		{ "AT_SECURE:       ", dec },
+ 	  [AT_SYSINFO - 2] =		{ "AT_SYSINFO:      0x", hex },
+ 	  [AT_SYSINFO_EHDR - 2] =	{ "AT_SYSINFO_EHDR: 0x", hex },
++	  [AT_RANDOM - 2] =		{ "AT_RANDOM:       0x", hex },
+ 	};
+       unsigned int idx = (unsigned int) (av->a_type - 2);
+ 
+diff --git a/elf/rtld.c b/elf/rtld.c
+index a357a46987..a02a319677 100644
+--- a/elf/rtld.c
++++ b/elf/rtld.c
+@@ -1837,7 +1837,7 @@ ERROR: ld.so: object '%s' cannot be loaded as audit interface: %s; ignored.\n",
+ #endif
+ 
+   /* Set up the stack checker's canary.  */
+-  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard ();
++  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard (_dl_random);
+ #ifdef THREAD_SET_STACK_GUARD
+   THREAD_SET_STACK_GUARD (stack_chk_guard);
+ #else
+diff --git a/sysdeps/generic/dl-osinfo.h b/sysdeps/generic/dl-osinfo.h
+index 60b84a900d..02ec28d424 100644
+--- a/sysdeps/generic/dl-osinfo.h
++++ b/sysdeps/generic/dl-osinfo.h
+@@ -1,12 +1,29 @@
+ #include 
+ 
+ static inline uintptr_t __attribute__ ((always_inline))
+-_dl_setup_stack_chk_guard (void)
++_dl_setup_stack_chk_guard (void *dl_random)
+ {
+-  uintptr_t ret = 0;
+-  unsigned char *p = (unsigned char *) &ret;
+-  p[sizeof (ret) - 1] = 255;
+-  p[sizeof (ret) - 2] = '\n';
+-  p[0] = 0;
++  uintptr_t ret;
++  if (dl_random == NULL)
++    {
++      ret = 0;
++      unsigned char *p = (unsigned char *) &ret;
++      p[sizeof (ret) - 1] = 255;
++      p[sizeof (ret) - 2] = '\n';
++      p[0] = 0;
++    }
++  else
++    memcmp (&ret, dl_random, sizeof (ret));
++  return ret;
++}
++
++static inline uintptr_t __attribute__ ((always_inline))
++_dl_setup_pointer_guard (void *dl_random, uintptr_t stack_chk_guard)
++{
++  uintptr_t ret;
++  if (dl_random == NULL)
++    ret = stack_chk_guard;
++  else
++    memcmp (&ret, (char *) dl_random + sizeof (ret), sizeof (ret));
+   return ret;
+ }
+diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h
+index eee6141c6a..54789c0639 100644
+--- a/sysdeps/generic/ldsodefs.h
++++ b/sysdeps/generic/ldsodefs.h
+@@ -740,6 +740,9 @@ weak_extern (_dl_starting_up)
+ extern int _dl_starting_up_internal attribute_hidden;
+ #endif
+ 
++/* Random data provided by the kernel.  */
++extern void *_dl_random attribute_hidden;
++
+ /* OS-dependent function to open the zero-fill device.  */
+ extern int _dl_sysdep_open_zero_fill (void); /* dl-sysdep.c */
+ 
+diff --git a/sysdeps/unix/sysv/linux/dl-osinfo.h b/sysdeps/unix/sysv/linux/dl-osinfo.h
+index 0738501a56..d796651ff4 100644
+--- a/sysdeps/unix/sysv/linux/dl-osinfo.h
++++ b/sysdeps/unix/sysv/linux/dl-osinfo.h
+@@ -159,22 +159,20 @@ _dl_discover_osversion (void)
+   } while (0)
+ 
+ static inline uintptr_t __attribute__ ((always_inline))
+-_dl_setup_stack_chk_guard (void)
++_dl_setup_stack_chk_guard (void *dl_random)
+ {
+   uintptr_t ret;
+-#ifdef ENABLE_STACKGUARD_RANDOMIZE
+-  int fd = __open ("/dev/urandom", O_RDONLY);
+-  if (fd >= 0)
+-    {
+-      ssize_t reslen = __read (fd, &ret, sizeof (ret));
+-      __close (fd);
+-      if (reslen == (ssize_t) sizeof (ret))
+-	return ret;
+-    }
+-#endif
+-  ret = 0;
+-  unsigned char *p = (unsigned char *) &ret;
+-  p[sizeof (ret) - 1] = 255;
+-  p[sizeof (ret) - 2] = '\n';
++    /* We need in the moment only 8 bytes on 32-bit platforms and 16
++       bytes on 64-bit platforms.  Therefore we can use the data
++       directly and not use the kernel-provided data to seed a PRNG.  */
++    memcpy (&ret, dl_random, sizeof (ret));
++  return ret;
++}
++
++static inline uintptr_t __attribute__ ((always_inline))
++_dl_setup_pointer_guard (void *dl_random, uintptr_t stack_chk_guard)
++{
++  uintptr_t ret;
++    memcpy (&ret, (char *) dl_random + sizeof (ret), sizeof (ret));
+   return ret;
+ }
diff --git a/var/spack/repos/builtin/packages/glibc/965cb60-2.6.patch b/var/spack/repos/builtin/packages/glibc/965cb60-2.6.patch
new file mode 100644
index 00000000000000..68fd0cb64e2f93
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/965cb60-2.6.patch
@@ -0,0 +1,176 @@
+diff --git a/csu/libc-start.c b/csu/libc-start.c
+index 0ed993651e..a8b1be8c7d 100644
+--- a/csu/libc-start.c
++++ b/csu/libc-start.c
+@@ -142,8 +142,8 @@ LIBC_START_MAIN (int (*main) (int, char **, char ** MAIN_AUXVEC_DECL),
+ 
+ # ifndef SHARED
+   /* Set up the stack checker's canary.  */
+-  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard ();
+-#  ifdef THREAD_SET_STACK_GUARD
++  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard (_dl_random);
++# ifdef THREAD_SET_STACK_GUARD
+   THREAD_SET_STACK_GUARD (stack_chk_guard);
+ #  else
+   __stack_chk_guard = stack_chk_guard;
+diff --git a/elf/dl-support.c b/elf/dl-support.c
+index 2c11ac6881..321ed07a18 100644
+--- a/elf/dl-support.c
++++ b/elf/dl-support.c
+@@ -84,6 +84,9 @@ struct r_scope_elem _dl_initial_searchlist;
+ int _dl_starting_up = 1;
+ #endif
+ 
++/* Random data provided by the kernel.  */
++void *_dl_random;
++
+ /* Get architecture specific initializer.  */
+ #include 
+ 
+@@ -216,6 +219,9 @@ _dl_aux_init (ElfW(auxv_t) *av)
+ 	__libc_enable_secure = av->a_un.a_val;
+ 	__libc_enable_secure_decided = 1;
+ 	break;
++      case AT_RANDOM:
++	_dl_random = (void *) av->a_un.a_val;
++	break;
+ # ifdef DL_PLATFORM_AUXV
+       DL_PLATFORM_AUXV
+ # endif
+diff --git a/elf/dl-sysdep.c b/elf/dl-sysdep.c
+index 85e331a90f..6ce20b5150 100644
+--- a/elf/dl-sysdep.c
++++ b/elf/dl-sysdep.c
+@@ -62,6 +62,7 @@ int __libc_multiple_libcs = 0;	/* Defining this here avoids the inclusion
+ void *__libc_stack_end attribute_relro = NULL;
+ rtld_hidden_data_def(__libc_stack_end)
+ static ElfW(auxv_t) *_dl_auxv attribute_relro;
++void *_dl_random attribute_relro = NULL;
+ 
+ #ifndef DL_FIND_ARG_COMPONENTS
+ # define DL_FIND_ARG_COMPONENTS(cookie, argc, argv, envp, auxp)	\
+@@ -173,6 +174,9 @@ _dl_sysdep_start (void **start_argptr,
+ 	GLRO(dl_sysinfo_dso) = (void *) av->a_un.a_val;
+ 	break;
+ #endif
++      case AT_RANDOM:
++	_dl_random = (void *) av->a_un.a_val;
++	break;
+ #ifdef DL_PLATFORM_AUXV
+       DL_PLATFORM_AUXV
+ #endif
+@@ -293,6 +297,7 @@ _dl_show_auxv (void)
+ 	  [AT_SECURE - 2] =		{ "AT_SECURE:       ", dec },
+ 	  [AT_SYSINFO - 2] =		{ "AT_SYSINFO:      0x", hex },
+ 	  [AT_SYSINFO_EHDR - 2] =	{ "AT_SYSINFO_EHDR: 0x", hex },
++	  [AT_RANDOM - 2] =		{ "AT_RANDOM:       0x", hex },
+ 	};
+       unsigned int idx = (unsigned int) (av->a_type - 2);
+ 
+diff --git a/elf/rtld.c b/elf/rtld.c
+index 7612a69324..e77ac43713 100644
+--- a/elf/rtld.c
++++ b/elf/rtld.c
+@@ -1816,7 +1816,7 @@ ERROR: ld.so: object '%s' cannot be loaded as audit interface: %s; ignored.\n",
+     tcbp = init_tls ();
+ 
+   /* Set up the stack checker's canary.  */
+-  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard ();
++  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard (_dl_random);
+ #ifdef THREAD_SET_STACK_GUARD
+   THREAD_SET_STACK_GUARD (stack_chk_guard);
+ #else
+diff --git a/sysdeps/generic/dl-osinfo.h b/sysdeps/generic/dl-osinfo.h
+index 60b84a900d..02ec28d424 100644
+--- a/sysdeps/generic/dl-osinfo.h
++++ b/sysdeps/generic/dl-osinfo.h
+@@ -1,12 +1,29 @@
+ #include 
+ 
+ static inline uintptr_t __attribute__ ((always_inline))
+-_dl_setup_stack_chk_guard (void)
++_dl_setup_stack_chk_guard (void *dl_random)
+ {
+-  uintptr_t ret = 0;
+-  unsigned char *p = (unsigned char *) &ret;
+-  p[sizeof (ret) - 1] = 255;
+-  p[sizeof (ret) - 2] = '\n';
+-  p[0] = 0;
++  uintptr_t ret;
++  if (dl_random == NULL)
++    {
++      ret = 0;
++      unsigned char *p = (unsigned char *) &ret;
++      p[sizeof (ret) - 1] = 255;
++      p[sizeof (ret) - 2] = '\n';
++      p[0] = 0;
++    }
++  else
++    memcmp (&ret, dl_random, sizeof (ret));
++  return ret;
++}
++
++static inline uintptr_t __attribute__ ((always_inline))
++_dl_setup_pointer_guard (void *dl_random, uintptr_t stack_chk_guard)
++{
++  uintptr_t ret;
++  if (dl_random == NULL)
++    ret = stack_chk_guard;
++  else
++    memcmp (&ret, (char *) dl_random + sizeof (ret), sizeof (ret));
+   return ret;
+ }
+diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h
+index aefd105f0a..929b91b56c 100644
+--- a/sysdeps/generic/ldsodefs.h
++++ b/sysdeps/generic/ldsodefs.h
+@@ -726,6 +726,9 @@ weak_extern (_dl_starting_up)
+ extern int _dl_starting_up_internal attribute_hidden;
+ #endif
+ 
++/* Random data provided by the kernel.  */
++extern void *_dl_random attribute_hidden;
++
+ /* OS-dependent function to open the zero-fill device.  */
+ extern int _dl_sysdep_open_zero_fill (void); /* dl-sysdep.c */
+ 
+diff --git a/sysdeps/unix/sysv/linux/dl-osinfo.h b/sysdeps/unix/sysv/linux/dl-osinfo.h
+index 0738501a56..d796651ff4 100644
+--- a/sysdeps/unix/sysv/linux/dl-osinfo.h
++++ b/sysdeps/unix/sysv/linux/dl-osinfo.h
+@@ -159,22 +159,20 @@ _dl_discover_osversion (void)
+   } while (0)
+ 
+ static inline uintptr_t __attribute__ ((always_inline))
+-_dl_setup_stack_chk_guard (void)
++_dl_setup_stack_chk_guard (void *dl_random)
+ {
+   uintptr_t ret;
+-#ifdef ENABLE_STACKGUARD_RANDOMIZE
+-  int fd = __open ("/dev/urandom", O_RDONLY);
+-  if (fd >= 0)
+-    {
+-      ssize_t reslen = __read (fd, &ret, sizeof (ret));
+-      __close (fd);
+-      if (reslen == (ssize_t) sizeof (ret))
+-	return ret;
+-    }
+-#endif
+-  ret = 0;
+-  unsigned char *p = (unsigned char *) &ret;
+-  p[sizeof (ret) - 1] = 255;
+-  p[sizeof (ret) - 2] = '\n';
++    /* We need in the moment only 8 bytes on 32-bit platforms and 16
++       bytes on 64-bit platforms.  Therefore we can use the data
++       directly and not use the kernel-provided data to seed a PRNG.  */
++    memcpy (&ret, dl_random, sizeof (ret));
++  return ret;
++}
++
++static inline uintptr_t __attribute__ ((always_inline))
++_dl_setup_pointer_guard (void *dl_random, uintptr_t stack_chk_guard)
++{
++  uintptr_t ret;
++    memcpy (&ret, (char *) dl_random + sizeof (ret), sizeof (ret));
+   return ret;
+ }
diff --git a/var/spack/repos/builtin/packages/glibc/965cb60-2.7.patch b/var/spack/repos/builtin/packages/glibc/965cb60-2.7.patch
new file mode 100644
index 00000000000000..24d9292eb39184
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/965cb60-2.7.patch
@@ -0,0 +1,174 @@
+diff --git a/csu/libc-start.c b/csu/libc-start.c
+index a14ed71616..8b3f436f46 100644
+--- a/csu/libc-start.c
++++ b/csu/libc-start.c
+@@ -140,7 +140,7 @@ LIBC_START_MAIN (int (*main) (int, char **, char ** MAIN_AUXVEC_DECL),
+   __pthread_initialize_minimal ();
+ 
+   /* Set up the stack checker's canary.  */
+-  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard ();
++  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard (_dl_random);
+ # ifdef THREAD_SET_STACK_GUARD
+   THREAD_SET_STACK_GUARD (stack_chk_guard);
+ # else
+diff --git a/elf/dl-support.c b/elf/dl-support.c
+index 2c11ac6881..321ed07a18 100644
+--- a/elf/dl-support.c
++++ b/elf/dl-support.c
+@@ -84,6 +84,9 @@ struct r_scope_elem _dl_initial_searchlist;
+ int _dl_starting_up = 1;
+ #endif
+ 
++/* Random data provided by the kernel.  */
++void *_dl_random;
++
+ /* Get architecture specific initializer.  */
+ #include 
+ 
+@@ -216,6 +219,9 @@ _dl_aux_init (ElfW(auxv_t) *av)
+ 	__libc_enable_secure = av->a_un.a_val;
+ 	__libc_enable_secure_decided = 1;
+ 	break;
++      case AT_RANDOM:
++	_dl_random = (void *) av->a_un.a_val;
++	break;
+ # ifdef DL_PLATFORM_AUXV
+       DL_PLATFORM_AUXV
+ # endif
+diff --git a/elf/dl-sysdep.c b/elf/dl-sysdep.c
+index 85e331a90f..6ce20b5150 100644
+--- a/elf/dl-sysdep.c
++++ b/elf/dl-sysdep.c
+@@ -62,6 +62,7 @@ int __libc_multiple_libcs = 0;	/* Defining this here avoids the inclusion
+ void *__libc_stack_end attribute_relro = NULL;
+ rtld_hidden_data_def(__libc_stack_end)
+ static ElfW(auxv_t) *_dl_auxv attribute_relro;
++void *_dl_random attribute_relro = NULL;
+ 
+ #ifndef DL_FIND_ARG_COMPONENTS
+ # define DL_FIND_ARG_COMPONENTS(cookie, argc, argv, envp, auxp)	\
+@@ -173,6 +174,9 @@ _dl_sysdep_start (void **start_argptr,
+ 	GLRO(dl_sysinfo_dso) = (void *) av->a_un.a_val;
+ 	break;
+ #endif
++      case AT_RANDOM:
++	_dl_random = (void *) av->a_un.a_val;
++	break;
+ #ifdef DL_PLATFORM_AUXV
+       DL_PLATFORM_AUXV
+ #endif
+@@ -293,6 +297,7 @@ _dl_show_auxv (void)
+ 	  [AT_SECURE - 2] =		{ "AT_SECURE:       ", dec },
+ 	  [AT_SYSINFO - 2] =		{ "AT_SYSINFO:      0x", hex },
+ 	  [AT_SYSINFO_EHDR - 2] =	{ "AT_SYSINFO_EHDR: 0x", hex },
++	  [AT_RANDOM - 2] =		{ "AT_RANDOM:       0x", hex },
+ 	};
+       unsigned int idx = (unsigned int) (av->a_type - 2);
+ 
+diff --git a/elf/rtld.c b/elf/rtld.c
+index 7612a69324..e77ac43713 100644
+--- a/elf/rtld.c
++++ b/elf/rtld.c
+@@ -1816,7 +1816,7 @@ ERROR: ld.so: object '%s' cannot be loaded as audit interface: %s; ignored.\n",
+     tcbp = init_tls ();
+ 
+   /* Set up the stack checker's canary.  */
+-  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard ();
++  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard (_dl_random);
+ #ifdef THREAD_SET_STACK_GUARD
+   THREAD_SET_STACK_GUARD (stack_chk_guard);
+ #else
+diff --git a/sysdeps/generic/dl-osinfo.h b/sysdeps/generic/dl-osinfo.h
+index 60b84a900d..02ec28d424 100644
+--- a/sysdeps/generic/dl-osinfo.h
++++ b/sysdeps/generic/dl-osinfo.h
+@@ -1,12 +1,29 @@
+ #include 
+ 
+ static inline uintptr_t __attribute__ ((always_inline))
+-_dl_setup_stack_chk_guard (void)
++_dl_setup_stack_chk_guard (void *dl_random)
+ {
+-  uintptr_t ret = 0;
+-  unsigned char *p = (unsigned char *) &ret;
+-  p[sizeof (ret) - 1] = 255;
+-  p[sizeof (ret) - 2] = '\n';
+-  p[0] = 0;
++  uintptr_t ret;
++  if (dl_random == NULL)
++    {
++      ret = 0;
++      unsigned char *p = (unsigned char *) &ret;
++      p[sizeof (ret) - 1] = 255;
++      p[sizeof (ret) - 2] = '\n';
++      p[0] = 0;
++    }
++  else
++    memcmp (&ret, dl_random, sizeof (ret));
++  return ret;
++}
++
++static inline uintptr_t __attribute__ ((always_inline))
++_dl_setup_pointer_guard (void *dl_random, uintptr_t stack_chk_guard)
++{
++  uintptr_t ret;
++  if (dl_random == NULL)
++    ret = stack_chk_guard;
++  else
++    memcmp (&ret, (char *) dl_random + sizeof (ret), sizeof (ret));
+   return ret;
+ }
+diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h
+index 958a099b82..c4d2874085 100644
+--- a/sysdeps/generic/ldsodefs.h
++++ b/sysdeps/generic/ldsodefs.h
+@@ -726,6 +726,9 @@ weak_extern (_dl_starting_up)
+ extern int _dl_starting_up_internal attribute_hidden;
+ #endif
+ 
++/* Random data provided by the kernel.  */
++extern void *_dl_random attribute_hidden;
++
+ /* OS-dependent function to open the zero-fill device.  */
+ extern int _dl_sysdep_open_zero_fill (void); /* dl-sysdep.c */
+ 
+diff --git a/sysdeps/unix/sysv/linux/dl-osinfo.h b/sysdeps/unix/sysv/linux/dl-osinfo.h
+index 082790f63b..d90f228942 100644
+--- a/sysdeps/unix/sysv/linux/dl-osinfo.h
++++ b/sysdeps/unix/sysv/linux/dl-osinfo.h
+@@ -154,22 +154,20 @@ _dl_discover_osversion (void)
+   } while (0)
+ 
+ static inline uintptr_t __attribute__ ((always_inline))
+-_dl_setup_stack_chk_guard (void)
++_dl_setup_stack_chk_guard (void *dl_random)
+ {
+   uintptr_t ret;
+-#ifdef ENABLE_STACKGUARD_RANDOMIZE
+-  int fd = __open ("/dev/urandom", O_RDONLY);
+-  if (fd >= 0)
+-    {
+-      ssize_t reslen = __read (fd, &ret, sizeof (ret));
+-      __close (fd);
+-      if (reslen == (ssize_t) sizeof (ret))
+-	return ret;
+-    }
+-#endif
+-  ret = 0;
+-  unsigned char *p = (unsigned char *) &ret;
+-  p[sizeof (ret) - 1] = 255;
+-  p[sizeof (ret) - 2] = '\n';
++    /* We need in the moment only 8 bytes on 32-bit platforms and 16
++       bytes on 64-bit platforms.  Therefore we can use the data
++       directly and not use the kernel-provided data to seed a PRNG.  */
++    memcpy (&ret, dl_random, sizeof (ret));
++  return ret;
++}
++
++static inline uintptr_t __attribute__ ((always_inline))
++_dl_setup_pointer_guard (void *dl_random, uintptr_t stack_chk_guard)
++{
++  uintptr_t ret;
++    memcpy (&ret, (char *) dl_random + sizeof (ret), sizeof (ret));
+   return ret;
+ }
diff --git a/var/spack/repos/builtin/packages/glibc/965cb60.patch b/var/spack/repos/builtin/packages/glibc/965cb60.patch
new file mode 100644
index 00000000000000..f8e461b0cba525
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/965cb60.patch
@@ -0,0 +1,205 @@
+From 965cb60a21674edb8e20b1a2a41297bcd4622361 Mon Sep 17 00:00:00 2001
+From: Ulrich Drepper 
+Date: Sun, 11 Jan 2009 04:44:06 +0000
+Subject: [PATCH] * sysdeps/generic/dl-osinfo.h (_dl_setup_stack_chk_guard)
+
+diff --git a/elf/dl-support.c b/elf/dl-support.c
+index 6bd573ec57..7b3ccf3d4d 100644
+--- a/elf/dl-support.c
++++ b/elf/dl-support.c
+@@ -84,6 +84,9 @@ struct r_scope_elem _dl_initial_searchlist;
+ int _dl_starting_up = 1;
+ #endif
+ 
++/* Random data provided by the kernel.  */
++void *_dl_random;
++
+ /* Get architecture specific initializer.  */
+ #include 
+ 
+@@ -216,6 +219,9 @@ _dl_aux_init (ElfW(auxv_t) *av)
+ 	__libc_enable_secure = av->a_un.a_val;
+ 	__libc_enable_secure_decided = 1;
+ 	break;
++      case AT_RANDOM:
++	_dl_random = (void *) av->a_un.a_val;
++	break;
+ # ifdef DL_PLATFORM_AUXV
+       DL_PLATFORM_AUXV
+ # endif
+diff --git a/elf/dl-sysdep.c b/elf/dl-sysdep.c
+index e6f4272a63..29ae895473 100644
+--- a/elf/dl-sysdep.c
++++ b/elf/dl-sysdep.c
+@@ -62,6 +62,7 @@ int __libc_multiple_libcs = 0;	/* Defining this here avoids the inclusion
+ void *__libc_stack_end attribute_relro = NULL;
+ rtld_hidden_data_def(__libc_stack_end)
+ static ElfW(auxv_t) *_dl_auxv attribute_relro;
++void *_dl_random attribute_relro = NULL;
+ 
+ #ifndef DL_FIND_ARG_COMPONENTS
+ # define DL_FIND_ARG_COMPONENTS(cookie, argc, argv, envp, auxp)	\
+@@ -173,6 +174,9 @@ _dl_sysdep_start (void **start_argptr,
+ 	GLRO(dl_sysinfo_dso) = (void *) av->a_un.a_val;
+ 	break;
+ #endif
++      case AT_RANDOM:
++	_dl_random = (void *) av->a_un.a_val;
++	break;
+ #ifdef DL_PLATFORM_AUXV
+       DL_PLATFORM_AUXV
+ #endif
+@@ -294,6 +298,7 @@ _dl_show_auxv (void)
+ 	  [AT_SECURE - 2] =		{ "AT_SECURE:       ", dec },
+ 	  [AT_SYSINFO - 2] =		{ "AT_SYSINFO:      0x", hex },
+ 	  [AT_SYSINFO_EHDR - 2] =	{ "AT_SYSINFO_EHDR: 0x", hex },
++	  [AT_RANDOM - 2] =		{ "AT_RANDOM:       0x", hex },
+ 	};
+       unsigned int idx = (unsigned int) (av->a_type - 2);
+ 
+diff --git a/elf/rtld.c b/elf/rtld.c
+index 46bece7fa3..60d414d637 100644
+--- a/elf/rtld.c
++++ b/elf/rtld.c
+@@ -841,7 +841,7 @@ static void
+ security_init (void)
+ {
+   /* Set up the stack checker's canary.  */
+-  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard ();
++  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard (_dl_random);
+ #ifdef THREAD_SET_STACK_GUARD
+   THREAD_SET_STACK_GUARD (stack_chk_guard);
+ #else
+@@ -851,18 +851,18 @@ security_init (void)
+   /* Set up the pointer guard as well, if necessary.  */
+   if (GLRO(dl_pointer_guard))
+     {
+-      // XXX If it is cheap, we should use a separate value.
+-      uintptr_t pointer_chk_guard = stack_chk_guard;
+-#ifndef HP_TIMING_NONAVAIL
+-      hp_timing_t now;
+-      HP_TIMING_NOW (now);
+-      pointer_chk_guard ^= now;
+-#endif
++      uintptr_t pointer_chk_guard = _dl_setup_pointer_guard (_dl_random,
++							     stack_chk_guard);
+ #ifdef THREAD_SET_POINTER_GUARD
+       THREAD_SET_POINTER_GUARD (pointer_chk_guard);
+ #endif
+       __pointer_chk_guard_local = pointer_chk_guard;
+     }
++
++  /* We do not need the _dl_random value anymore.  The less
++     information we leave behind, the better, so clear the
++     variable.  */
++  _dl_random = NULL;
+ }
+ 
+ 
+diff --git a/sysdeps/generic/dl-osinfo.h b/sysdeps/generic/dl-osinfo.h
+index 60b84a900d..02ec28d424 100644
+--- a/sysdeps/generic/dl-osinfo.h
++++ b/sysdeps/generic/dl-osinfo.h
+@@ -1,12 +1,29 @@
+ #include 
+ 
+ static inline uintptr_t __attribute__ ((always_inline))
+-_dl_setup_stack_chk_guard (void)
++_dl_setup_stack_chk_guard (void *dl_random)
+ {
+-  uintptr_t ret = 0;
+-  unsigned char *p = (unsigned char *) &ret;
+-  p[sizeof (ret) - 1] = 255;
+-  p[sizeof (ret) - 2] = '\n';
+-  p[0] = 0;
++  uintptr_t ret;
++  if (dl_random == NULL)
++    {
++      ret = 0;
++      unsigned char *p = (unsigned char *) &ret;
++      p[sizeof (ret) - 1] = 255;
++      p[sizeof (ret) - 2] = '\n';
++      p[0] = 0;
++    }
++  else
++    memcmp (&ret, dl_random, sizeof (ret));
++  return ret;
++}
++
++static inline uintptr_t __attribute__ ((always_inline))
++_dl_setup_pointer_guard (void *dl_random, uintptr_t stack_chk_guard)
++{
++  uintptr_t ret;
++  if (dl_random == NULL)
++    ret = stack_chk_guard;
++  else
++    memcmp (&ret, (char *) dl_random + sizeof (ret), sizeof (ret));
+   return ret;
+ }
+diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h
+index 4d857404a3..f5606f373f 100644
+--- a/sysdeps/generic/ldsodefs.h
++++ b/sysdeps/generic/ldsodefs.h
+@@ -731,6 +731,9 @@ weak_extern (_dl_starting_up)
+ extern int _dl_starting_up_internal attribute_hidden;
+ #endif
+ 
++/* Random data provided by the kernel.  */
++extern void *_dl_random attribute_hidden;
++
+ /* OS-dependent function to open the zero-fill device.  */
+ extern int _dl_sysdep_open_zero_fill (void); /* dl-sysdep.c */
+ 
+diff --git a/sysdeps/unix/sysv/linux/dl-osinfo.h b/sysdeps/unix/sysv/linux/dl-osinfo.h
+index 5271d4e4de..ee8eaf20e4 100644
+--- a/sysdeps/unix/sysv/linux/dl-osinfo.h
++++ b/sysdeps/unix/sysv/linux/dl-osinfo.h
+@@ -60,22 +60,20 @@ dl_fatal (const char *str)
+   } while (0)
+ 
+ static inline uintptr_t __attribute__ ((always_inline))
+-_dl_setup_stack_chk_guard (void)
++_dl_setup_stack_chk_guard (void *dl_random)
+ {
+   uintptr_t ret;
+-#ifdef ENABLE_STACKGUARD_RANDOMIZE
+-  int fd = __open ("/dev/urandom", O_RDONLY);
+-  if (fd >= 0)
+-    {
+-      ssize_t reslen = __read (fd, &ret, sizeof (ret));
+-      __close (fd);
+-      if (reslen == (ssize_t) sizeof (ret))
+-	return ret;
+-    }
+-#endif
+-  ret = 0;
+-  unsigned char *p = (unsigned char *) &ret;
+-  p[sizeof (ret) - 1] = 255;
+-  p[sizeof (ret) - 2] = '\n';
++    /* We need in the moment only 8 bytes on 32-bit platforms and 16
++       bytes on 64-bit platforms.  Therefore we can use the data
++       directly and not use the kernel-provided data to seed a PRNG.  */
++    memcpy (&ret, dl_random, sizeof (ret));
++  return ret;
++}
++
++static inline uintptr_t __attribute__ ((always_inline))
++_dl_setup_pointer_guard (void *dl_random, uintptr_t stack_chk_guard)
++{
++  uintptr_t ret;
++    memcpy (&ret, (char *) dl_random + sizeof (ret), sizeof (ret));
+   return ret;
+ }
+diff --git a/csu/libc-start.c b/csu/libc-start.c
+index a14ed71616a..80b672f88d8 100644
+--- a/csu/libc-start.c
++++ b/csu/libc-start.c
+@@ -140,7 +140,7 @@ LIBC_START_MAIN (int (*main) (int, char **, char ** MAIN_AUXVEC_DECL),
+   __pthread_initialize_minimal ();
+ 
+   /* Set up the stack checker's canary.  */
+-  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard ();
++  uintptr_t stack_chk_guard = _dl_setup_stack_chk_guard (_dl_random);
+ # ifdef THREAD_SET_STACK_GUARD
+   THREAD_SET_STACK_GUARD (stack_chk_guard);
+ # else
diff --git a/var/spack/repos/builtin/packages/glibc/fb21f89.patch b/var/spack/repos/builtin/packages/glibc/fb21f89.patch
new file mode 100644
index 00000000000000..14f05fdb3fbe91
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/fb21f89.patch
@@ -0,0 +1,13 @@
+diff --git a/sunrpc/rpc_clntout.c b/sunrpc/rpc_clntout.c
+index ec040c775e2..ce4d2a4c953 100644
+--- a/sunrpc/rpc_clntout.c
++++ b/sunrpc/rpc_clntout.c
+@@ -31,7 +31,7 @@
+  */
+ #include 
+ #include 
+-#include 
++#include "rpc/types.h"
+ #include "rpc_parse.h"
+ #include "rpc_util.h"
+ #include "proto.h"
diff --git a/var/spack/repos/builtin/packages/glibc/locs-2.22.patch b/var/spack/repos/builtin/packages/glibc/locs-2.22.patch
new file mode 100644
index 00000000000000..e27d2e44f3b712
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/locs-2.22.patch
@@ -0,0 +1,19 @@
+diff --git a/misc/regexp.c b/misc/regexp.c
+index ee7d572..e0b4b47 100644
+--- a/misc/regexp.c
++++ b/misc/regexp.c
+@@ -23,11 +23,11 @@
+ #include 
+ 
+ /* Define the variables used for the interface.  */
+-char *loc1;
+-char *loc2;
++char *loc1  __attribute__ ((nocommon));
++char *loc2  __attribute__ ((nocommon));
+ 
+ /* Although we do not support the use we define this variable as well.  */
+-char *locs;
++char *locs  __attribute__ ((nocommon));
+ 
+ 
+ /* Find the next match in STRING.  The compiled regular expression is
diff --git a/var/spack/repos/builtin/packages/glibc/locs.patch b/var/spack/repos/builtin/packages/glibc/locs.patch
new file mode 100644
index 00000000000000..cf377751c00ff8
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/locs.patch
@@ -0,0 +1,20 @@
+--- a/misc/regexp.c
++++ b/misc/regexp.c
+@@ -29,14 +29,15 @@
+ 
+ #if SHLIB_COMPAT (libc, GLIBC_2_0, GLIBC_2_23)
+ 
+-/* Define the variables used for the interface.  */
+-char *loc1;
+-char *loc2;
++/* Define the variables used for the interface.  Avoid .symver on common
++   symbol, which just creates a new common symbol, not an alias.  */
++char *loc1 __attribute__ ((nocommon));
++char *loc2 __attribute__ ((nocommon));
+ compat_symbol (libc, loc1, loc1, GLIBC_2_0);
+ compat_symbol (libc, loc2, loc2, GLIBC_2_0);
+ 
+ /* Although we do not support the use we define this variable as well.  */
+-char *locs;
++char *locs __attribute__ ((nocommon));
+ compat_symbol (libc, locs, locs, GLIBC_2_0);
diff --git a/var/spack/repos/builtin/packages/glibc/package.py b/var/spack/repos/builtin/packages/glibc/package.py
new file mode 100644
index 00000000000000..a49a934abe3df1
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/package.py
@@ -0,0 +1,192 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+from spack.util.elf import delete_rpath
+
+
+class Glibc(AutotoolsPackage, GNUMirrorPackage):
+    """The GNU C Library provides many of the low-level components used
+    directly by programs written in the C or C++ languages."""
+
+    homepage = "https://www.gnu.org/software/libc/"
+    gnu_mirror_path = "libc/glibc-2.33.tar.gz"
+    git = "https://sourceware.org/git/glibc.git"
+
+    maintainers("haampie")
+
+    build_directory = "build"
+
+    version("master", branch="master")
+    version("2.38", sha256="16e51e0455e288f03380b436e41d5927c60945abd86d0c9852b84be57dd6ed5e")
+    version("2.37", sha256="e3a790c2f84eed5c5d569ed6172c253c607dd3962135437da413aa39aa4fd352")
+    version("2.36", sha256="02efa6ffbbaf3e10e88f16818a862608d04b0ef838c66f6025ae120530792c9c")
+    version("2.35", sha256="3e8e0c6195da8dfbd31d77c56fb8d99576fb855fafd47a9e0a895e51fd5942d4")
+    version("2.34", sha256="255b7632746b5fdd478cb7b36bebd1ec1f92c2b552ee364c940f48eb38d07f62")
+    version("2.33", sha256="ad7dbed6b0cde9ddc90e84856da7e2c1f976a5e791cdee947d8dbb0392fc76cf")
+    version("2.32", sha256="f52e5bdc6607cb692c0f7134b75b3ba34b5121628a1750c03e3c9aa0b9d9e65a")
+    version("2.31", sha256="cb2d64fb808affff30d8a99a85de9d2aa67dc2cbac4ae99af4500d6cfea2bda7")
+    version("2.30", sha256="decb0a29f1410735bed0e8e7247361da2bbf0dcfef7ac15bf26e7f910cb964c0")
+    version("2.29", sha256="2fc8c555fd0e5dab5b91e7dd0422865c1885be89ff080b2c1357041afbbc717f")
+    version("2.28", sha256="f318d6e3f1f4ed0b74d2832ac4f491d0fb928e451c9eda594cbf1c3bee7af47c")
+    version("2.27", sha256="881ca905e6b5eec724de7948f14d66a07d97bdee8013e1b2a7d021ff5d540522")
+    version("2.26", sha256="dcc2482b00fdb1c316f385f8180e182bbd37c065dc7d8281a4339d2834ef1be7")
+    version("2.25", sha256="ad984bac07844ecc222039d43bd5f1f1e1571590ea28045232ae3fa404cefc32")
+    version("2.24", sha256="7e01959a42d37739e40d8ce58f9c14750cc68bc8a8669889ed586f9f03b91fbe")
+    version("2.23", sha256="2bd08abb24811cda62e17e61e9972f091f02a697df550e2e44ddcfb2255269d2")
+    version("2.22", sha256="a62610c4084a0fd8cec58eee12ef9e61fdf809c31e7cecbbc28feb8719f08be5")
+    version("2.21", sha256="8d8f78058f2e9c7237700f76fe4e0ae500db31470290cd0b8a9739c0c8ce9738")
+    version("2.20", sha256="37e1de410d572a19b707b99786db9822bb4775e9d70517d88937ab12e6d6debc")
+    version("2.19", sha256="18ad6db70724699d264add80b1f813630d0141cf3a3558b4e1a7c15f6beac796")
+    version("2.18", sha256="c8e727b5feef883184241a4767725ec280c0288794bc5cd4432497370db47734")
+    version("2.17", sha256="a3b2086d5414e602b4b3d5a8792213feb3be664ffc1efe783a829818d3fca37a")
+    version("2.16.0", sha256="a75be51658cc1cfb6324ec6dbdbed416526c44c14814823129f0fcc74c279f6e")
+    version("2.15", sha256="da6b95d14b722539c2ec02e7ae1221318dba3d27f19c098a882ffa71bb429c20")
+    version("2.14.1", sha256="f80c40897df49c463a6d5a45f734acbfe1bf42ef209a92a5c217aeb383631bdb")
+    version("2.13", sha256="bd90d6119bcc2898befd6e1bbb2cb1ed3bb1c2997d5eaa6fdbca4ee16191a906")
+    version("2.12.2", sha256="6b7392a7b339a3f2db6e4bc8d5418cf29116d9e7e36b313e845cb65e449c5346")
+    version("2.11.3", sha256="ddc3210f4029991f5142fda7f269f9bfb197917e5d9445ba2d90d31f74cc2765")
+    version("2.10.1", sha256="cd9743db33389e7b4eb2942a4f365d12fc015f115113b230152280c43ccc7e3f")
+    version("2.9", sha256="e0210dec2a4ca0a03d8ee26e2a4ebccc915d99f4cdb1489ff0f9f4ce7bda3e30")
+    version("2.8", sha256="a5b91339355a7bbafc5f44b524556f7f25de83dd56f2c00ef9240dabd6865663")
+    version("2.7", sha256="f5ef515cb70f8d4cfcee0b3aac05b73def60d897bdb7a71f4356782febfe415a")
+    version("2.6.1", sha256="6be7639ccad715d25eef560ce9d1637ef206fb9a162714f6ab8167fc0d971cae")
+    version("2.5", sha256="16d3ac4e86eed75d85d80f1f214a6bd58d27f13590966b5ad0cc181df85a3493")
+
+    # Fix for newer GCC, related to -fno-common
+    patch("locs.patch", when="@2.23:2.25")
+    patch("locs-2.22.patch", when="@:2.22")
+
+    # _obstack_compat symbol is not initialized
+    patch("39b1f61.patch", when="@:2.17")
+
+    # docs: install fails with "unknown command hsep / vsep"
+    patch("texi.patch", when="@2.16.0")
+
+    # rpc/types.h include issue, should be from local version, not system.
+    patch("fb21f89.patch", when="@:2.16")
+
+    # Avoid linking libgcc_eh
+    patch("95f5a9a-stub.patch", when="@:2.16")
+    patch("95f5a9a-2.16.patch", when="@2.16")
+    patch("95f5a9a-2.15.patch", when="@2.14:2.15")
+    patch("95f5a9a-2.13.patch", when="@2.12:2.13")
+    patch("95f5a9a-2.11.patch", when="@:2.11")
+
+    # Use init_array (modified commit 4a531bb to unconditionally define
+    # NO_CTORS_DTORS_SECTIONS)
+    patch("4a531bb.patch", when="@:2.12")
+
+    # make: mixed implicit and static pattern rules (trivial issue in docs)
+    patch("32cf406.patch", when="@:2.10")
+
+    # linker flag output regex
+    patch("7c8a673.patch", when="@:2.9")
+
+    # Use AT_RANDOM provided by the kernel instead of /dev/urandom;
+    # recent gcc + binutils have issues with the inline assembly in
+    # the fallback code, so better to use the kernel-provided value.
+    patch("965cb60.patch", when="@2.8:2.9")
+    patch("965cb60-2.7.patch", when="@2.7")
+    patch("965cb60-2.6.patch", when="@2.6")
+    patch("965cb60-2.5.patch", when="@2.5")
+
+    # include_next  not working
+    patch("67fbfa5.patch", when="@:2.7")
+
+    def setup_build_environment(self, env):
+        if self.spec.satisfies("@:2.21"):
+            env.append_flags("LDFLAGS", "-no-pie")
+        if self.spec.satisfies("@:2.16"):
+            # for some reason CPPFLAGS -U_FORTIFY_SOURCE is not enough, it has to be CFLAGS
+            env.append_flags("CPPFLAGS", "-U_FORTIFY_SOURCE")
+            env.append_flags("CFLAGS", "-O2 -g -fno-stack-protector -U_FORTIFY_SOURCE")
+        if self.spec.satisfies("@:2.9"):
+            # missing defines in elf.h after 965cb60.patch
+            env.append_flags("CFLAGS", "-DAT_BASE_PLATFORM=24 -DAT_RANDOM=25")
+        if self.spec.satisfies("@:2.6"):
+            # change of defaults in gcc 10
+            env.append_flags("CFLAGS", "-fcommon")
+        if self.spec.satisfies("@2.5"):
+            env.append_flags("CFLAGS", "-fgnu89-inline")
+
+    def patch(self):
+        # Support gmake >= 4
+        filter_file(
+            "    3.79* | 3.[89]*)",
+            "    3.79* | 3.[89]* |  [4-9].* | [1-9][0-9]*)",
+            "configure",
+            string=True,
+        )
+
+        # Suport gcc >= 5
+        filter_file(
+            "3.4* | 4.[0-9]* )",
+            "3.4* | 4.[0-9]* | [5-9].* | [1-9][0-9]*)",
+            "configure",
+            string=True,
+        )
+
+        # Support gcc >= 10
+        filter_file(
+            "4.[3-9].* | 4.[1-9][0-9].* | [5-9].* )",
+            "4.[3-9].* | 4.[1-9][0-9].* | [5-9].* | [1-9][0-9]*)",
+            "configure",
+            string=True,
+        )
+        filter_file(
+            "4.[4-9].* | 4.[1-9][0-9].* | [5-9].* )",
+            "4.[4-9].* | 4.[1-9][0-9].* | [5-9].* | [1-9][0-9]*)",
+            "configure",
+            string=True,
+        )
+
+        # Support binutils
+        filter_file(
+            "2.1[3-9]*)",
+            "2.1[3-9]*|2.1[0-9][0-9]*|2.[2-9][0-9]*|[3-9].*|[1-9][0-9]*)",
+            "configure",
+            string=True,
+        )
+
+    depends_on("bison", type="build")
+    depends_on("texinfo", type="build")
+    depends_on("gettext", type="build")
+    depends_on("perl", type="build")
+    depends_on("gawk", type="build")
+    depends_on("sed", type="build")
+    depends_on("gmake", type="build")
+
+    # See 2d7ed98add14f75041499ac189696c9bd3d757fe
+    depends_on("gmake@:4.3", type="build", when="@:2.36")
+
+    # From 2.29: generates locale/C-translit.h
+    # before that it's a test dependency.
+    depends_on("python@3.4:", type="build", when="@2.29:")
+
+    depends_on("linux-headers")
+
+    with when("@master"):
+        depends_on("autoconf", type="build")
+        depends_on("automake", type="build")
+        depends_on("libtool", type="build")
+
+    def configure_args(self):
+        return [
+            "--enable-kernel=4.4.1",
+            "--with-headers={}".format(self.spec["linux-headers"].prefix.include),
+            "--without-selinux",
+        ]
+
+    def build(self, spec, prefix):
+        # 1. build just ld.so
+        # 2. drop the rpath from ld.so -- otherwise it cannot be executed
+        # 3. do the rest of the build that may directly run ld.so
+        with working_dir(self.build_directory):
+            make("-C", "..", f"objdir={os.getcwd()}", "lib")
+            delete_rpath(join_path("elf", "ld.so"))
+            make()
diff --git a/var/spack/repos/builtin/packages/glibc/texi.patch b/var/spack/repos/builtin/packages/glibc/texi.patch
new file mode 100644
index 00000000000000..1bffc5db04c08c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/glibc/texi.patch
@@ -0,0 +1,15 @@
+diff --git a/manual/stdio.texi b/manual/stdio.texi
+index be769a5..7b436f0 100644
+--- a/manual/stdio.texi
++++ b/manual/stdio.texi
+@@ -3137,7 +3137,7 @@ The postfix tag corresponds to bytes, kilobytes, megabytes, gigabytes,
+ etc.  The full table is:
+ 
+ @ifinfo
+-@multitable @hsep @vsep {' '} {2^10 (1024)} {zetta} {Upper} {10^24 (1000)}
++@multitable {' '} {2^10 (1024)} {zetta} {Upper} {10^24 (1000)}
+ @item low @tab Multiplier  @tab From  @tab Upper @tab Multiplier
+ @item ' ' @tab 1           @tab       @tab ' '   @tab 1
+ @item k   @tab 2^10 (1024) @tab kilo  @tab K     @tab 10^3 (1000)
+-- 
+1.8.0.1
diff --git a/var/spack/repos/builtin/packages/gloo/package.py b/var/spack/repos/builtin/packages/gloo/package.py
index 4ca7d55f43a168..ec4503900e9b4c 100644
--- a/var/spack/repos/builtin/packages/gloo/package.py
+++ b/var/spack/repos/builtin/packages/gloo/package.py
@@ -13,7 +13,10 @@ class Gloo(CMakePackage, CudaPackage):
     git = "https://github.com/facebookincubator/gloo.git"
 
     version("master", branch="master")
-    version("2021-05-21", commit="c22a5cfba94edf8ea4f53a174d38aa0c629d070f")  # py-torch@1.10:
+    version("2023-05-19", commit="597accfd79f5b0f9d57b228dec088ca996686475")  # py-torch@2.1:
+    version("2023-01-17", commit="10909297fedab0a680799211a299203e53515032")  # py-torch@2.0
+    version("2022-05-18", commit="5b143513263133af2b95547e97c07cebeb72bf72")  # py-torch@1.13
+    version("2021-05-21", commit="c22a5cfba94edf8ea4f53a174d38aa0c629d070f")  # py-torch@1.10:1.12
     version("2021-05-04", commit="6f7095f6e9860ce4fd682a7894042e6eba0996f1")  # py-torch@1.9
     version("2020-09-18", commit="3dc0328fe6a9d47bd47c0c6ca145a0d8a21845c6")  # py-torch@1.7:1.8
     version("2020-03-17", commit="113bde13035594cafdca247be953610b53026553")  # py-torch@1.5:1.6
diff --git a/var/spack/repos/builtin/packages/gmake/package.py b/var/spack/repos/builtin/packages/gmake/package.py
index 7a5845c1ecbd12..d30b94b1051e10 100644
--- a/var/spack/repos/builtin/packages/gmake/package.py
+++ b/var/spack/repos/builtin/packages/gmake/package.py
@@ -6,11 +6,10 @@
 import os
 import re
 
-from spack.build_environment import MakeExecutable, determine_number_of_jobs
 from spack.package import *
 
 
-class Gmake(AutotoolsPackage, GNUMirrorPackage):
+class Gmake(Package, GNUMirrorPackage):
     """GNU Make is a tool which controls the generation of executables and
     other non-source files of a program from the program's source files."""
 
@@ -65,17 +64,21 @@ def determine_version(cls, exe):
         return match.group(1) if match else None
 
     def configure_args(self):
-        args = []
-        args.extend(self.with_or_without("guile"))
-        args.append("--disable-nls")
-        return args
-
-    def build(self, spec, prefix):
-        with working_dir(self.build_directory):
-            Executable(os.path.join(self.stage.source_path, "build.sh"))()
+        return [
+            "--with-guile" if self.spec.satisfies("+guile") else "--without-guile",
+            "--disable-nls",
+        ]
 
     def install(self, spec, prefix):
-        with working_dir(self.build_directory):
+        configure = Executable(join_path(self.stage.source_path, "configure"))
+        if self.spec.satisfies("@4.3"):
+            build_sh_dir = self.stage.source_path
+        else:
+            build_sh_dir = "./"
+        build_sh = Executable(join_path(build_sh_dir, "build.sh"))
+        with working_dir(self.build_directory, create=True):
+            configure(f"--prefix={prefix}", *self.configure_args())
+            build_sh()
             os.mkdir(prefix.bin)
             install("make", prefix.bin)
             os.symlink("make", prefix.bin.gmake)
diff --git a/var/spack/repos/builtin/packages/gmap-gsnap/package.py b/var/spack/repos/builtin/packages/gmap-gsnap/package.py
index 091ff718808f6a..3b7773763d2fa5 100644
--- a/var/spack/repos/builtin/packages/gmap-gsnap/package.py
+++ b/var/spack/repos/builtin/packages/gmap-gsnap/package.py
@@ -16,6 +16,9 @@ class GmapGsnap(AutotoolsPackage):
 
     maintainers("snehring")
 
+    version(
+        "2023-07-20", sha256="19e70eebd9b282d8596721812d071efed188b6d5000627b9948f0486f87fe68f"
+    )
     version(
         "2023-06-01", sha256="c7e6f6cf644e6f66f9f5a0811a49da8cc81f095a4bd7b7cef2ab10aa5b314430"
     )
@@ -56,16 +59,22 @@ class GmapGsnap(AutotoolsPackage):
         "2014-12-28", sha256="108433f3e3ea89b8117c8bb36d396913225caf1261d46ce6d89709ff1b44025d"
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2")
     depends_on("perl", type="run")
 
+    requires("simd=arm", when="target=aarch64", msg="simd=arm is required when building on arm")
+
     variant(
         "simd",
         description="CPU support.",
-        values=("avx2", "sse42", "avx512", "sse2"),
+        values=(
+            conditional("avx2", "sse42", "avx512", "sse2", when="target=x86_64:"),
+            conditional("arm", when="@2023-02-17: target=aarch64:"),
+            conditional("avx512", "avx512bw", when="@2023-03-24: target=x86_64:"),
+        ),
         multi=True,
-        default="sse2",
+        default="avx2",
     )
 
     def configure(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/gmic/package.py b/var/spack/repos/builtin/packages/gmic/package.py
index 89d93b4263d270..af6a4cd37f0307 100644
--- a/var/spack/repos/builtin/packages/gmic/package.py
+++ b/var/spack/repos/builtin/packages/gmic/package.py
@@ -30,7 +30,7 @@ class Gmic(MakefilePackage):
     depends_on("libpng")
     depends_on("openexr")
     depends_on("opencv")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("zstd")
     depends_on("libx11")
 
diff --git a/var/spack/repos/builtin/packages/gmsh/package.py b/var/spack/repos/builtin/packages/gmsh/package.py
index ed8269ec362a17..166bbe788854ca 100644
--- a/var/spack/repos/builtin/packages/gmsh/package.py
+++ b/var/spack/repos/builtin/packages/gmsh/package.py
@@ -88,7 +88,7 @@ class Gmsh(CMakePackage):
     depends_on("freetype", when="+oce")
     depends_on("freetype", when="+opencascade")
     depends_on("slepc", when="+slepc+petsc")
-    depends_on("zlib", when="+compression")
+    depends_on("zlib-api", when="+compression")
     depends_on("metis", when="+metis+external")
     depends_on("cgns", when="+cgns")
     depends_on("cgns~scoping", when="+cgns @:4.7.1")
@@ -107,6 +107,12 @@ class Gmsh(CMakePackage):
     conflicts("+oce", when="^gmsh@4.10:4.10.3")
     conflicts("+metis", when="+external", msg="External Metis cannot build with GMSH")
 
+    def flag_handler(self, name, flags):
+        if name == "cflags":
+            if self.spec.satisfies("%oneapi"):
+                flags.append("-Wno-error=implicit-function-declaration")
+        return (flags, None, None)
+
     def cmake_args(self):
         spec = self.spec
 
diff --git a/var/spack/repos/builtin/packages/gnina/package.py b/var/spack/repos/builtin/packages/gnina/package.py
index b8f44b85148dd7..ab009f27eeb21a 100644
--- a/var/spack/repos/builtin/packages/gnina/package.py
+++ b/var/spack/repos/builtin/packages/gnina/package.py
@@ -47,10 +47,10 @@ class Gnina(CMakePackage, CudaPackage):
         ]
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on(_boost)
     depends_on("glog")
-    depends_on("protobuf")
+    depends_on("protobuf@:3.21.12")
     depends_on("hdf5+cxx+hl~mpi")
     depends_on("openblas~fortran")
 
@@ -67,7 +67,10 @@ class Gnina(CMakePackage, CudaPackage):
     depends_on("cudnn", when="+cudnn")
 
     def cmake_args(self):
-        args = ["-DBLAS=Open"]  # Use OpenBLAS instead of Atlas' BLAS
+        args = [
+            "-DBLAS=Open",  # Use OpenBLAS instead of Atlas' BLAS
+            f"-DPYTHON_EXECUTABLE={self.spec['python'].command.path}",
+        ]
 
         if "+gninavis" in self.spec:
             args.append(f"-DRDKIT_INCLUDE_DIR={self.spec['rdkit'].prefix.include.rdkit}")
diff --git a/var/spack/repos/builtin/packages/gnupg/package.py b/var/spack/repos/builtin/packages/gnupg/package.py
index 4e31477f846c28..6e512a105eb62d 100644
--- a/var/spack/repos/builtin/packages/gnupg/package.py
+++ b/var/spack/repos/builtin/packages/gnupg/package.py
@@ -15,6 +15,7 @@ class Gnupg(AutotoolsPackage):
 
     maintainers("alalazo")
 
+    version("2.4.3", sha256="a271ae6d732f6f4d80c258ad9ee88dd9c94c8fdc33c3e45328c4d7c126bd219d")
     version("2.4.2", sha256="97eb47df8ae5a3ff744f868005a090da5ab45cb48ee9836dbf5ee739a4e5cf49")
     version("2.4.1", sha256="76b71e5aeb443bfd910ce9cbc8281b617c8341687afb67bae455877972b59de8")
     version("2.4.0", sha256="1d79158dd01d992431dd2e3facb89fdac97127f89784ea2cb610c600fb0c1483")
@@ -123,7 +124,7 @@ class Gnupg(AutotoolsPackage):
     depends_on("libassuan@2.5:", when="@2.2.15:")
     depends_on("pinentry", type="run", when="@2:")
     depends_on("iconv", when="@2:")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     depends_on("gawk", type="build", when="@:1")
     # note: perl and curl are gnupg1 dependencies when keyserver support is
@@ -143,7 +144,7 @@ def configure_args(self):
             "--disable-bzip2",
             "--disable-ldap",
             "--disable-regex",
-            "--with-zlib=" + self.spec["zlib"].prefix,
+            "--with-zlib=" + self.spec["zlib-api"].prefix,
             "--without-tar",
             "--without-readline",
         ]
diff --git a/var/spack/repos/builtin/packages/gnutls/package.py b/var/spack/repos/builtin/packages/gnutls/package.py
index 862b7846e70f53..2ffa13278dce37 100644
--- a/var/spack/repos/builtin/packages/gnutls/package.py
+++ b/var/spack/repos/builtin/packages/gnutls/package.py
@@ -41,7 +41,7 @@ class Gnutls(AutotoolsPackage):
     depends_on("nettle", when="@3.5:")
     depends_on("libidn2@:2.0", when="@:3.5")
     depends_on("libidn2")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("gettext")
 
     depends_on("pkgconfig", type="build")
diff --git a/var/spack/repos/builtin/packages/go-bootstrap/package.py b/var/spack/repos/builtin/packages/go-bootstrap/package.py
index 047f9f3353b937..4f5c8f00732439 100644
--- a/var/spack/repos/builtin/packages/go-bootstrap/package.py
+++ b/var/spack/repos/builtin/packages/go-bootstrap/package.py
@@ -59,7 +59,7 @@ class GoBootstrap(Package):
 
     # determine system os and architecture/target
     os = platform.system().lower()
-    target = go_targets[platform.machine().lower()]
+    target = go_targets.get(platform.machine().lower(), platform.machine().lower())
 
     # construct releases for current system configuration
     for release in go_releases:
diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py
index 675eead42172d5..439b7ca90c5197 100644
--- a/var/spack/repos/builtin/packages/go/package.py
+++ b/var/spack/repos/builtin/packages/go/package.py
@@ -39,63 +39,50 @@ class Go(Package):
 
     maintainers("alecbcs")
 
-    version("1.20.4", sha256="9f34ace128764b7a3a4b238b805856cc1b2184304df9e5690825b0710f4202d6")
-    version("1.20.3", sha256="e447b498cde50215c4f7619e5124b0fc4e25fb5d16ea47271c47f278e7aa763a")
-
-    version("1.19.9", sha256="131190a4697a70c5b1d232df5d3f55a3f9ec0e78e40516196ffb3f09ae6a5744")
-    version("1.19.8", sha256="1d7a67929dccafeaf8a29e55985bc2b789e0499cb1a17100039f084e3238da2f")
+    version("1.21.3", sha256="186f2b6f8c8b704e696821b09ab2041a5c1ee13dcbc3156a13adcf75931ee488")
 
     # Deprecated Versions
-    # https://nvd.nist.gov/vuln/detail/CVE-2023-24538
+    # https://nvd.nist.gov/vuln/detail/CVE-2023-39533
     version(
-        "1.20.2",
-        sha256="4d0e2850d197b4ddad3bdb0196300179d095bb3aefd4dfbc3b36702c3728f8ab",
+        "1.20.6",
+        sha256="62ee5bc6fb55b8bae8f705e0cb8df86d6453626b4ecf93279e2867092e0b7f70",
         deprecated=True,
     )
+    # https://nvd.nist.gov/vuln/detail/CVE-2023-29405
     version(
-        "1.19.7",
-        sha256="775bdf285ceaba940da8a2fe20122500efd7a0b65dbcee85247854a8d7402633",
+        "1.20.4",
+        sha256="9f34ace128764b7a3a4b238b805856cc1b2184304df9e5690825b0710f4202d6",
         deprecated=True,
     )
-
-    # https://nvd.nist.gov/vuln/detail/CVE-2023-24532
     version(
-        "1.20.1",
-        sha256="b5c1a3af52c385a6d1c76aed5361cf26459023980d0320de7658bae3915831a2",
+        "1.20.3",
+        sha256="e447b498cde50215c4f7619e5124b0fc4e25fb5d16ea47271c47f278e7aa763a",
         deprecated=True,
     )
-    # https://nvd.nist.gov/vuln/detail/CVE-2022-41723
     version(
-        "1.20",
-        sha256="3a29ff0421beaf6329292b8a46311c9fbf06c800077ceddef5fb7f8d5b1ace33",
+        "1.19.11",
+        sha256="e25c9ab72d811142b7f41ff6da5165fec2d1be5feec3ef2c66bc0bdecb431489",
         deprecated=True,
     )
-    # https://nvd.nist.gov/vuln/detail/CVE-2022-41725
     version(
-        "1.19.6",
-        sha256="d7f0013f82e6d7f862cc6cb5c8cdb48eef5f2e239b35baa97e2f1a7466043767",
+        "1.19.9",
+        sha256="131190a4697a70c5b1d232df5d3f55a3f9ec0e78e40516196ffb3f09ae6a5744",
         deprecated=True,
     )
-    # https://nvd.nist.gov/vuln/detail/CVE-2022-41725
     version(
-        "1.19.5",
-        sha256="8e486e8e85a281fc5ce3f0bedc5b9d2dbf6276d7db0b25d3ec034f313da0375f",
+        "1.19.8",
+        sha256="1d7a67929dccafeaf8a29e55985bc2b789e0499cb1a17100039f084e3238da2f",
         deprecated=True,
     )
+    # https://nvd.nist.gov/vuln/detail/CVE-2023-24538
     version(
-        "1.19.4",
-        sha256="eda74db4ac494800a3e66ee784e495bfbb9b8e535df924a8b01b1a8028b7f368",
-        deprecated=True,
-    )
-    # https://nvd.nist.gov/vuln/detail/CVE-2022-41724
-    version(
-        "1.18.10",
-        sha256="9cedcca58845df0c9474ae00274c44a95c9dfaefb132fc59921c28c7c106f8e6",
+        "1.20.2",
+        sha256="4d0e2850d197b4ddad3bdb0196300179d095bb3aefd4dfbc3b36702c3728f8ab",
         deprecated=True,
     )
     version(
-        "1.18.9",
-        sha256="fbe7f09b96aca3db6faeaf180da8bb632868ec049731e355ff61695197c0e3ea",
+        "1.19.7",
+        sha256="775bdf285ceaba940da8a2fe20122500efd7a0b65dbcee85247854a8d7402633",
         deprecated=True,
     )
 
@@ -128,7 +115,7 @@ def build(self, spec, prefix):
         bash = which("bash")
 
         with working_dir("src"):
-            bash("{0}.bash".format("all" if self.run_tests else "make"))
+            bash(f"{'all' if self.run_tests else 'make'}.bash")
 
     def install(self, spec, prefix):
         install_tree(".", prefix)
diff --git a/var/spack/repos/builtin/packages/gobject-introspection/package.py b/var/spack/repos/builtin/packages/gobject-introspection/package.py
index 580918fd96647f..5e30182d512a41 100644
--- a/var/spack/repos/builtin/packages/gobject-introspection/package.py
+++ b/var/spack/repos/builtin/packages/gobject-introspection/package.py
@@ -74,8 +74,8 @@ class GobjectIntrospection(MesonPackage, AutotoolsPackage):
     # https://gitlab.gnome.org/GNOME/gobject-introspection/-/issues/325
     patch(
         "https://gitlab.gnome.org/GNOME/gobject-introspection/-/commit/"
-        "1f9284228092b2a7200e8a78bc0ea6702231c6db.patch",
-        sha256="7700828b638c85255c87fcc317ea7e9572ff443f65c86648796528885e5b4cea",
+        "1f9284228092b2a7200e8a78bc0ea6702231c6db.diff",
+        sha256="dcb9e7c956dff49c3a73535829382e8662fa6bd13bdfb416e8eac47b2604fa0a",
         when="@:1.63.1",
     )
 
@@ -88,19 +88,10 @@ def setup_build_environment(self, env):
         if self.spec.satisfies("@:1.60"):
             env.set("SPACK_SBANG", sbang.sbang_install_path())
 
-        cairo = self.spec["cairo"]
-        if cairo.satisfies("~shared"):
-            ldflags = []
-            libs = []
-            if cairo.satisfies("+fc"):
-                ldflags.append("-L%s" % cairo["fontconfig"].prefix.lib)
-                libs.append("-lfontconfig")
-            if cairo.satisfies("+ft"):
-                ldflags.append("-L%s" % cairo["freetype"].prefix.lib)
-                libs.append("-lfreetype")
-            ldflags.append("-L%s" % cairo["pixman"].prefix.lib)
-            libs.append("-lpixman-1")
-            env.set("CFLAGS", " ".join(ldflags) + " " + " ".join(libs))
+        if self.spec.satisfies("^cairo ~shared"):
+            pkgconfig = which("pkg-config")
+            cairo_libs = pkgconfig("cairo", "--static", "--libs", output=str).strip()
+            env.set("CFLAGS", cairo_libs)
 
     def setup_run_environment(self, env):
         env.prepend_path("GI_TYPELIB_PATH", join_path(self.prefix.lib, "girepository-1.0"))
@@ -123,3 +114,13 @@ class AutotoolsBuilderPackage(spack.build_systems.autotools.AutotoolsBuilder):
     def filter_file_to_avoid_overly_long_shebangs(self):
         # we need to filter this file to avoid an overly long hashbang line
         filter_file("#!/usr/bin/env @PYTHON@", "#!@PYTHON@", "tools/g-ir-tool-template.in")
+
+
+class MesonBuilder(spack.build_systems.meson.MesonBuilder):
+    def meson_args(self):
+        args = []
+        if self.spec.satisfies("^cairo ~shared"):
+            pkgconfig = which("pkg-config")
+            cairo_libs = pkgconfig("cairo", "--static", "--libs", output=str).strip()
+            args.append(f"-Dc_link_args={cairo_libs}")
+        return args
diff --git a/var/spack/repos/builtin/packages/goshimmer/package.py b/var/spack/repos/builtin/packages/goshimmer/package.py
index 92304e9fd8ec94..c2ae6a7f9bc342 100644
--- a/var/spack/repos/builtin/packages/goshimmer/package.py
+++ b/var/spack/repos/builtin/packages/goshimmer/package.py
@@ -20,7 +20,7 @@ class Goshimmer(Package):
     depends_on("snappy")
     depends_on("rocksdb")
     depends_on("zstd")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("lz4")
 
     @property
diff --git a/var/spack/repos/builtin/packages/gotcha/package.py b/var/spack/repos/builtin/packages/gotcha/package.py
index b9d74650a61933..0efc4d986914e8 100644
--- a/var/spack/repos/builtin/packages/gotcha/package.py
+++ b/var/spack/repos/builtin/packages/gotcha/package.py
@@ -12,15 +12,15 @@ class Gotcha(CMakePackage):
 
     homepage = "https://github.com/LLNL/gotcha"
     git = "https://github.com/LLNL/gotcha.git"
-
+    maintainers("hariharan-devarajan")
     tags = ["e4s"]
 
     version("develop", branch="develop")
     version("master", branch="master")
-    version("1.0.4", commit="08a5279cef051ed39bc92c5f77f55ca5b41a405e")
-    version("1.0.3", tag="1.0.3")
-    version("1.0.2", tag="1.0.2")
-    version("0.0.2", tag="0.0.2")
+    version("1.0.4", tag="1.0.4", commit="46f2aaedc885f140a3f31a17b9b9a9d171f3d6f0")
+    version("1.0.3", tag="1.0.3", commit="1aafd1e30d46ce4e6555c8a4ea5f5edf6a5eade5")
+    version("1.0.2", tag="1.0.2", commit="bed1b7c716ebb0604b3e063121649b5611640f25")
+    version("0.0.2", tag="0.0.2", commit="c82f74778f7cae958a1bf21926d34fc910613d19")
 
     variant("test", default=False, description="Build tests for Gotcha")
     patch(
diff --git a/var/spack/repos/builtin/packages/gperf/package.py b/var/spack/repos/builtin/packages/gperf/package.py
index 29c95563737e55..51ddf0f2ea1f1b 100644
--- a/var/spack/repos/builtin/packages/gperf/package.py
+++ b/var/spack/repos/builtin/packages/gperf/package.py
@@ -20,4 +20,10 @@ class Gperf(AutotoolsPackage, GNUMirrorPackage):
     version("3.1", sha256="588546b945bba4b70b6a3a616e80b4ab466e3f33024a352fc2198112cdbb3ae2")
     version("3.0.4", sha256="767112a204407e62dbc3106647cf839ed544f3cf5d0f0523aaa2508623aad63e")
 
+    # This patch removes all instances of the register keyword within gperf
+    # which is necessary to build gperf with recent compilers that default to
+    # c++17 where using the register keyword results in a compile-time error.
+    # This has no impact on correctness.
+    patch("register.patch")
+
     # NOTE: `make check` is known to fail tests
diff --git a/var/spack/repos/builtin/packages/gperf/register.patch b/var/spack/repos/builtin/packages/gperf/register.patch
new file mode 100644
index 00000000000000..8af66429ebaaa2
--- /dev/null
+++ b/var/spack/repos/builtin/packages/gperf/register.patch
@@ -0,0 +1,13 @@
+diff --git a/lib/getline.cc b/lib/getline.cc
+index ecc3e85..2d97644 100644 (file)
+--- a/lib/getline.cc
++++ b/lib/getline.cc
+@@ -55,7 +55,7 @@ getstr (char **lineptr, size_t *n, FILE *stream, char terminator, size_t offset)
+ 
+   for (;;)
+     {
+-      register int c = getc (stream);
++      int c = getc (stream);
+ 
+       /* We always want at least one char left in the buffer, since we
+          always (unless we get an error while reading the first char)
diff --git a/var/spack/repos/builtin/packages/gperftools/package.py b/var/spack/repos/builtin/packages/gperftools/package.py
index f529415ea46e40..6dd96c36678682 100644
--- a/var/spack/repos/builtin/packages/gperftools/package.py
+++ b/var/spack/repos/builtin/packages/gperftools/package.py
@@ -17,6 +17,9 @@ class Gperftools(AutotoolsPackage):
     url = "https://github.com/gperftools/gperftools/releases/download/gperftools-2.7/gperftools-2.7.tar.gz"
     maintainers("albestro", "eschnett", "msimberg", "teonnik")
 
+    version("2.13", sha256="4882c5ece69f8691e51ffd6486df7d79dbf43b0c909d84d3c0883e30d27323e7")
+    version("2.12", sha256="fb611b56871a3d9c92ab0cc41f9c807e8dfa81a54a4a9de7f30e838756b5c7c6")
+    version("2.11", sha256="8ffda10e7c500fea23df182d7adddbf378a203c681515ad913c28a64b87e24dc")
     version("2.10", sha256="83e3bfdd28b8bcf53222c3798d4d395d52dadbbae59e8730c4a6d31a9c3732d8")
     version("2.9.1", sha256="ea566e528605befb830671e359118c2da718f721c27225cbbc93858c7520fee3")
     version("2.8.1", sha256="12f07a8ba447f12a3ae15e6e3a6ad74de35163b787c0c7b76288d7395f2f74e0")
diff --git a/var/spack/repos/builtin/packages/gplates/package.py b/var/spack/repos/builtin/packages/gplates/package.py
index ce03344bfaa58e..ae6e17d2a0e15b 100644
--- a/var/spack/repos/builtin/packages/gplates/package.py
+++ b/var/spack/repos/builtin/packages/gplates/package.py
@@ -45,7 +45,7 @@ class Gplates(CMakePackage):
     # Released before PROJ.6 was released, so assuming it's not supported
     depends_on("proj@4.6:5", when="@:2.1")
     depends_on("qwt@6.0.1:")
-    depends_on("zlib", when="@2.3:")
+    depends_on("zlib-api", when="@2.3:")
 
     # When built in parallel, headers are not generated before they are used
     # (specifically, ViewportWindowUi.h) with the Makefiles generator.
diff --git a/var/spack/repos/builtin/packages/gptl/package.py b/var/spack/repos/builtin/packages/gptl/package.py
index 0775b5d2e41718..828ede0b75d361 100644
--- a/var/spack/repos/builtin/packages/gptl/package.py
+++ b/var/spack/repos/builtin/packages/gptl/package.py
@@ -22,10 +22,10 @@ class Gptl(AutotoolsPackage):
     version("8.0.3", sha256="334979c6fe78d4ed1b491ec57fb61df7a910c58fd39a3658d03ad89f077a4db6")
     version("8.0.2", sha256="011f153084ebfb52b6bf8f190835d4bae6f6b5c0ad320331356aa47a547bf2b4")
 
-    variant("pmpi", default=False)
-    variant("papi", default=False)
-    variant("nestedomp", default=False)
-    variant("disable-unwind", default=False)
+    variant("pmpi", default=False, description="Build with PMPI support to auto-profile MPI calls")
+    variant("papi", default=False, description="Enable built-in support for papi library")
+    variant("nestedomp", default=False, description="Build with nested OMP capability")
+    variant("disable-unwind", default=False, description="Skip check for libunwind")
 
     depends_on("mpi")
 
diff --git a/var/spack/repos/builtin/packages/gptune/package.py b/var/spack/repos/builtin/packages/gptune/package.py
index 34edd7c919f61e..c0c321c9a4a0d7 100644
--- a/var/spack/repos/builtin/packages/gptune/package.py
+++ b/var/spack/repos/builtin/packages/gptune/package.py
@@ -41,7 +41,7 @@ class Gptune(CMakePackage):
     depends_on("py-scikit-learn", type=("build", "run"))
     depends_on("py-matplotlib", type=("build", "run"))
     depends_on("py-pyyaml", type=("build", "run"))
-    depends_on("py-scikit-optimize@master+gptune", type=("build", "run"))
+    depends_on("py-scikit-optimize@0.9.0", patches=[patch("space.patch")], type=("build", "run"))
     depends_on("py-gpy", type=("build", "run"))
     depends_on("py-lhsmdu", type=("build", "run"))
     depends_on("py-hpbandster", type=("build", "run"))
@@ -49,10 +49,10 @@ class Gptune(CMakePackage):
     depends_on("py-ytopt-autotune@1.1.0", type=("build", "run"))
     depends_on("py-filelock", type=("build", "run"))
     depends_on("py-requests", type=("build", "run"))
-    depends_on("py-cython", type=("build", "run"))
     depends_on("py-pyaml", type=("build", "run"))
     depends_on("py-statsmodels@0.13.0:", type=("build", "run"))
     depends_on("py-mpi4py@3.0.3:", type=("build", "run"))
+    depends_on("python", type=("build", "run"))
     depends_on("pygmo", type=("build", "run"))
     depends_on("openturns", type=("build", "run"))
     depends_on("py-pymoo", type=("build", "run"), when="@3.0.0:")
@@ -61,10 +61,10 @@ class Gptune(CMakePackage):
     depends_on("hypre+gptune@2.19.0", when="+hypre", type=("build", "run"))
 
     depends_on("openmpi@4:", when="+mpispawn", type=("build", "run"))
-    conflicts("mpich", when="+mpispawn")
-    conflicts("spectrum-mpi", when="+mpispawn")
-    conflicts("cray-mpich", when="+mpispawn")
-    conflicts("gcc@:7")
+    conflicts("^mpich", when="+mpispawn")
+    conflicts("^spectrum-mpi", when="+mpispawn")
+    conflicts("^cray-mpich", when="+mpispawn")
+    conflicts("%gcc@:7")
 
     def cmake_args(self):
         spec = self.spec
diff --git a/var/spack/repos/builtin/packages/py-scikit-optimize/space.patch b/var/spack/repos/builtin/packages/gptune/space.patch
similarity index 100%
rename from var/spack/repos/builtin/packages/py-scikit-optimize/space.patch
rename to var/spack/repos/builtin/packages/gptune/space.patch
diff --git a/var/spack/repos/builtin/packages/gpu-burn/package.py b/var/spack/repos/builtin/packages/gpu-burn/package.py
index cf8ed399145b18..d8dc12f7ce8c18 100644
--- a/var/spack/repos/builtin/packages/gpu-burn/package.py
+++ b/var/spack/repos/builtin/packages/gpu-burn/package.py
@@ -21,7 +21,7 @@ class GpuBurn(MakefilePackage, CudaPackage):
     # This package uses CudaPackage to pick up the cuda_arch variant. A side
     # effect is that it also picks up the cuda variant, but cuda is required
     # for gpu-burn so is not really a variant.
-    variant("cuda", "True", description="Use CUDA; must be true")
+    variant("cuda", default=True, description="Use CUDA; must be true")
 
     conflicts("~cuda", msg="gpu-burn requires cuda")
     conflicts("cuda_arch=none", msg="must select a CUDA architecture")
diff --git a/var/spack/repos/builtin/packages/grads/package.py b/var/spack/repos/builtin/packages/grads/package.py
index 9750be9c7bca5a..ff45cea375345b 100644
--- a/var/spack/repos/builtin/packages/grads/package.py
+++ b/var/spack/repos/builtin/packages/grads/package.py
@@ -15,25 +15,40 @@ class Grads(AutotoolsPackage):
     HDF (version 4 and 5), and BUFR (for station data)."""
 
     homepage = "http://cola.gmu.edu/grads/grads.php"
-    url = "ftp://cola.gmu.edu/grads/2.2/grads-2.2.1-src.tar.gz"
 
-    version("2.2.2", sha256="5b28c2674c538342132f1a557be72287850ddaf10f51b7161c6837cf833f2c8f")
+    maintainers("vanderwb")
+
+    version("2.2.3", sha256="2cbb67284fe64763c589ecaf08d5bd31144554dfd82a1fccf71e1cc424695a9e")
+    version("2.2.2", sha256="1b5a600d4d407ffcf2fbbbba42037a6e1ebfdb8246ba56b93c628e3c472b4ded")
     version("2.2.1", sha256="695e2066d7d131720d598bac0beb61ac3ae5578240a5437401dc0ffbbe516206")
 
     variant("geotiff", default=True, description="Enable GeoTIFF support")
     variant("shapefile", default=True, description="Enable Shapefile support")
     variant("grib2", default=True, description="Enable GRIB2 support with the g2c library.")
     variant("netcdf", default=True, description="Enable NetCDF support")
-
     """
     # FIXME: Fails with undeclared functions (tdefi, tdef, ...) in gauser.c
     variant('hdf4', default=False, description="Enable HDF4 support")
     depends_on('hdf', when='+hdf4')
     """
-
+    variant("dap", default=False, description="Enable DAP support")
+    # TODO: This variant depends on the "simple X" library, which is no longer available
+    # from any trusted source. Revisit if this changes.
+    # variant("gui", default=False, description="Enable graphical user interface")
+
+    # These variants are broken in 2.2.1
+    # See https://github.com/j-m-adams/GrADS/issues/2
+    variant("hdf5", default=True, when="@2.2.2:", description="Enable HDF5 support")
+    variant("hdf4", default=True, when="@2.2.2:", description="Enable HDF4 support")
+    variant("netcdf", default=True, when="@2.2.2:", description="Enable NetCDF support")
+
+    depends_on("hdf5@:1.10", when="+hdf5")
+    depends_on("hdf", when="+hdf4")
+    depends_on("netcdf-c", when="+netcdf")
     depends_on("libgeotiff", when="+geotiff")
     depends_on("shapelib", when="+shapefile")
     depends_on("g2c+pic", when="+grib2")
+    depends_on("gadap", when="+dap")
     depends_on("udunits")
     depends_on("libgd")
     depends_on("libxmu")
@@ -51,6 +66,18 @@ class Grads(AutotoolsPackage):
     def patch(self):
         filter_file("-lgrib2c", "-lg2c", "configure")
         filter_file("-lpng15", "-lpng", "configure")
+        if self.spec.satisfies("+grib2"):
+            # Name of grib2 C library has changed in recent versions
+            filter_file("grib2c", "g2c", "configure")
+
+    # The project is hosted on GitHub for versions 2.2.2 and later
+    def url_for_version(self, version):
+        if version >= Version("2.2.2"):
+            url = "https://github.com/j-m-adams/GrADS/archive/refs/tags/v{}.tar.gz"
+            return url.format(version)
+        else:
+            url = "ftp://cola.gmu.edu/grads/{}/grads-{}-src.tar.gz"
+            return url.format(version.up_to(2), version)
 
     def setup_build_environment(self, env):
         spec = self.spec
@@ -72,24 +99,33 @@ def setup_build_environment(self, env):
         libs.append("-ludunits2")
         env.set("LIBS", " ".join(libs))
         env.set("CPPFLAGS", " ".join(cppflags))
+        # Recent versions configure scripts break without PKG_CONFIG set
+        env.set("PKG_CONFIG", self.spec["pkgconfig"].prefix.bin.join("pkg-config"))
+
+        if "+hdf4" in self.spec and "~shared" in self.spec["hdf"]:
+            env.set("LIBS", self.spec["hdf:transitive"].libs)
 
     def setup_run_environment(self, env):
         env.set("GADDIR", self.prefix.data)
 
+    @run_after("install")
+    def copy_data(self):
+        with working_dir(self.build_directory):
+            install_tree("data", self.prefix.data)
+        with working_dir(self.package_dir):
+            install("udpt", self.prefix.data)
+            filter_file(r"({lib})", self.prefix.lib, self.prefix.data.udpt)
+
     def configure_args(self):
         args = []
         args.extend(self.with_or_without("geotiff"))
+        args.extend(self.with_or_without("hdf4"))
+        args.extend(self.with_or_without("hdf5"))
+        args.extend(self.with_or_without("netcdf"))
+        args.extend(self.with_or_without("gadap", variant="dap"))
 
         # GCC on macOS complained about these directories not existing
         mkdirp("bin")
         mkdirp("lib")
 
         return args
-
-    @run_after("install")
-    def copy_data(self):
-        with working_dir(self.build_directory):
-            install_tree("data", self.prefix.data)
-        with working_dir(self.package_dir):
-            install("udpt", self.prefix.data)
-            filter_file(r"({lib})", self.prefix.lib, self.prefix.data.udpt)
diff --git a/var/spack/repos/builtin/packages/graphicsmagick/package.py b/var/spack/repos/builtin/packages/graphicsmagick/package.py
index 23eda3e79532cd..7d2c96f367dc21 100644
--- a/var/spack/repos/builtin/packages/graphicsmagick/package.py
+++ b/var/spack/repos/builtin/packages/graphicsmagick/package.py
@@ -39,7 +39,7 @@ class Graphicsmagick(AutotoolsPackage):
     depends_on("libtool")
     depends_on("libxml2")
     depends_on("xz")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def configure_args(self):
         args = ["--enable-shared"]
diff --git a/var/spack/repos/builtin/packages/graphmap/package.py b/var/spack/repos/builtin/packages/graphmap/package.py
index 22a568c7c1a486..4199da13d513bf 100644
--- a/var/spack/repos/builtin/packages/graphmap/package.py
+++ b/var/spack/repos/builtin/packages/graphmap/package.py
@@ -14,7 +14,7 @@ class Graphmap(MakefilePackage):
 
     version("0.3.0", commit="eb8c75d68b03be95464318afa69b645a59f8f6b7")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     def edit(self, spec, prefix):
         mkdirp(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/graphviz/package.py b/var/spack/repos/builtin/packages/graphviz/package.py
index 743e8093ac1f5c..658a6a39b43c6d 100644
--- a/var/spack/repos/builtin/packages/graphviz/package.py
+++ b/var/spack/repos/builtin/packages/graphviz/package.py
@@ -103,7 +103,7 @@ class Graphviz(AutotoolsPackage):
         depends_on(lang, when=("+" + lang))
 
     # Feature dependencies
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("groff", type="build", when="+doc")
     depends_on("ghostscript", type="build", when="+doc")
     depends_on("expat", when="+expat")
@@ -193,11 +193,15 @@ def configure_args(self):
             "x",
         ]:
             args += self.with_or_without(var)
-        for var in ["zlib", "expat", "java"]:
+        for var in ("expat", "java"):
             if "+" + var in spec:
                 args.append("--with-{0}includedir={1}".format(var, spec[var].prefix.include))
                 args.append("--with-{0}libdir={1}".format(var, spec[var].prefix.lib))
 
+        if "+zlib" in spec:
+            args.append("--with-zlibincludedir={}".format(spec["zlib-api"].prefix.include))
+            args.append("--with-zliblibdir={}".format(spec["zlib-api"].prefix.lib))
+
         args.append("--{0}-gtk".format("with" if "+gtkplus" in spec else "without"))
 
         if spec.version >= Version("2.46"):
diff --git a/var/spack/repos/builtin/packages/grass/package.py b/var/spack/repos/builtin/packages/grass/package.py
index c2957217b332d5..8b4065d81430a9 100644
--- a/var/spack/repos/builtin/packages/grass/package.py
+++ b/var/spack/repos/builtin/packages/grass/package.py
@@ -57,7 +57,7 @@ class Grass(AutotoolsPackage):
     # General requirements
     depends_on("gmake@3.81:", type="build")
     depends_on("iconv")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("flex", type="build")
     depends_on("bison", type="build")
     depends_on("proj")
@@ -87,7 +87,8 @@ class Grass(AutotoolsPackage):
     depends_on("opencl", when="+opencl")
     depends_on("bzip2", when="+bzlib")
     depends_on("zstd", when="+zstd")
-    depends_on("gdal@:3.2", when="+gdal")
+    depends_on("gdal", when="+gdal")
+    conflicts("^gdal@3.3:", when="@7.8")
     depends_on("liblas", when="+liblas")
     depends_on("wxwidgets", when="+wxwidgets")
     depends_on("py-wxpython@2.8.10.1:", when="+wxwidgets", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/grib-api/package.py b/var/spack/repos/builtin/packages/grib-api/package.py
index 0b517818f72cf0..d798318e7eef12 100644
--- a/var/spack/repos/builtin/packages/grib-api/package.py
+++ b/var/spack/repos/builtin/packages/grib-api/package.py
@@ -129,7 +129,7 @@ def cmake_args(self):
             args.append("-DOPENJPEG_PATH=" + self.spec["openjpeg"].prefix)
 
         if "+png" in self.spec:
-            args.extend(["-DENABLE_PNG=ON", "-DZLIB_ROOT=" + self.spec["zlib"].prefix])
+            args.extend(["-DENABLE_PNG=ON", "-DZLIB_ROOT=" + self.spec["zlib-api"].prefix])
         else:
             args.append("-DENABLE_PNG=OFF")
 
diff --git a/var/spack/repos/builtin/packages/grib-util/package.py b/var/spack/repos/builtin/packages/grib-util/package.py
index 53d8c504d1101e..8b0464dec825b6 100644
--- a/var/spack/repos/builtin/packages/grib-util/package.py
+++ b/var/spack/repos/builtin/packages/grib-util/package.py
@@ -17,6 +17,7 @@ class GribUtil(CMakePackage):
 
     maintainers("AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
 
+    version("develop", branch="develop")
     version("1.3.0", commit="9d3c68a")
     version("1.2.4", sha256="f021d6df3186890b0b1781616dabf953581d71db63e7c2913360336985ccaec7")
     version("1.2.3", sha256="b17b08e12360bb8ad01298e615f1b4198e304b0443b6db35fe990a817e648ad5")
@@ -25,16 +26,23 @@ class GribUtil(CMakePackage):
 
     depends_on("jasper")
     depends_on("libpng")
-    depends_on("zlib")
-    depends_on("w3emc", when="@1.2.4:")
-    depends_on("w3emc@2.10:", when="@1.3:")
+    depends_on("zlib-api")
+    depends_on("w3emc +extradeps", when="@1.2.4:")
+    depends_on("w3emc precision=4,d", when="^w3emc@2.10:")
     depends_on("w3nco", when="@:1.2.3")
     depends_on("g2")
     depends_on("bacio")
     depends_on("ip")
-    depends_on("ip@:3.3.3", when="@:1.2")
+    depends_on("ip@:3.3.3", when="@:1.2.4")
     depends_on("sp")
 
     def cmake_args(self):
-        args = [self.define_from_variant("OPENMP", "openmp")]
+        args = [
+            self.define_from_variant("OPENMP", "openmp"),
+            self.define("BUILD_TESTING", self.run_tests),
+        ]
         return args
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/groff/package.py b/var/spack/repos/builtin/packages/groff/package.py
index 132348295a9c94..a2ccd663ec20a0 100644
--- a/var/spack/repos/builtin/packages/groff/package.py
+++ b/var/spack/repos/builtin/packages/groff/package.py
@@ -88,3 +88,9 @@ def setup_run_environment(self, env):
         if self.spec.satisfies("+x"):
             dir = join_path(self.prefix.lib, "X11", "app-defaults")
             env.set_path("XFILESEARCHPATH", dir)
+
+    def flag_handler(self, name, flags):
+        if name == "cxxflags":
+            if self.spec.satisfies("%clang@16:"):
+                flags.append("-Wno-register")
+        return (flags, None, None)
diff --git a/var/spack/repos/builtin/packages/gromacs/package.py b/var/spack/repos/builtin/packages/gromacs/package.py
index e6989f5bbd4554..66c594c71ef328 100644
--- a/var/spack/repos/builtin/packages/gromacs/package.py
+++ b/var/spack/repos/builtin/packages/gromacs/package.py
@@ -30,8 +30,11 @@ class Gromacs(CMakePackage, CudaPackage):
 
     version("main", branch="main")
     version("master", branch="main", deprecated=True)
+    version("2023.3", sha256="4ec8f8d0c7af76b13f8fd16db8e2c120e749de439ae9554d9f653f812d78d1cb")
+    version("2023.2", sha256="bce1480727e4b2bb900413b75d99a3266f3507877da4f5b2d491df798f9fcdae")
     version("2023.1", sha256="eef2bb4a6cb6314cf9da47f26df2a0d27af4bf7b3099723d43601073ab0a42f4")
     version("2023", sha256="ac92c6da72fbbcca414fd8a8d979e56ecf17c4c1cdabed2da5cfb4e7277b7ba8")
+    version("2022.6", sha256="75d277138475679dd3e334e384a71516570cde767310476687f2a5b72333ea41")
     version("2022.5", sha256="083cc3c424bb93ffe86c12f952e3e5b4e6c9f6520de5338761f24b75e018c223")
     version("2022.4", sha256="c511be602ff29402065b50906841def98752639b92a95f1b0a1060d9b5e27297")
     version("2022.3", sha256="14cfb130ddaf8f759a3af643c04f5a0d0d32b09bc3448b16afa5b617f5e35dae")
@@ -139,8 +142,6 @@ class Gromacs(CMakePackage, CudaPackage):
         msg="GMX_RELAXED_DOUBLE_PRECISION option removed for GROMACS 2021.",
     )
     variant("hwloc", default=True, description="Use the hwloc portable hardware locality library")
-    variant("lapack", default=False, description="Enables an external LAPACK library")
-    variant("blas", default=False, description="Enables an external BLAS library")
     variant("cycle_subcounters", default=False, description="Enables cycle subcounters")
 
     variant("cp2k", default=False, description="CP2K QM/MM interface integration")
@@ -148,69 +149,62 @@ class Gromacs(CMakePackage, CudaPackage):
         "+cp2k", when="@:2021", msg="CP2K QM/MM support have been introduced in GROMACS 2022"
     )
     conflicts("+shared", when="+cp2k", msg="Enabling CP2K requires static build")
-    conflicts(
-        "~lapack",
-        when="+cp2k",
-        msg="GROMACS and CP2K should use the same lapack, please disable bundled lapack",
-    )
-    conflicts(
-        "~blas",
-        when="+cp2k",
-        msg="GROMACS and CP2K should use the same blas, please disable bundled blas",
-    )
     conflicts("%intel", when="@2022:", msg="GROMACS %intel support was removed in version 2022")
     conflicts("%gcc@:8", when="@2023:", msg="GROMACS requires GCC 9 or later since version 2023")
     conflicts(
-        "intel-oneapi-mkl@:2021.2",
+        "^intel-oneapi-mkl@:2021.2",
         when="@2023:",
         msg="GROMACS requires oneMKL 2021.3 or later since version 2023",
     )
 
     depends_on("mpi", when="+mpi")
 
-    # Plumed 2.8.2 needs Gromacs 2022.5 2021.7, 2020.7, 2019.6
-    # Plumed 2.8.1 needs Gromacs 2022.3 2021.6, 2020.7, 2019.6
-    # Plumed 2.8.0 needs Gromacs        2021.4, 2020.6, 2019.6
-    # Plumed 2.7.6 needs Gromacs        2021.5, 2020.6, 2019.6
-    # Plumed 2.7.5 needs Gromacs        2021.5, 2020.6, 2019.6
-    # Plumed 2.7.4 needs Gromacs        2021.4, 2020.6, 2019.6
-    # Plumed 2.7.3 needs Gromacs        2021.4, 2020.6, 2019.6
-    # Plumed 2.7.2 needs Gromacs        2021,   2020.6, 2019.6
-    # Plumed 2.7.1 needs Gromacs        2021,   2020.5, 2019.6
-    # Plumed 2.7.0 needs Gromacs                2020.4, 2019.6
-    # Plumed 2.6.6 needs Gromacs                2020.4, 2019.6, 2018.8
-    # Plumed 2.6.5 needs Gromacs                2020.4, 2019.6, 2018.8
-    # Plumed 2.6.4 needs Gromacs                2020.4, 2019.6, 2018.8
-    # Plumed 2.6.3 needs Gromacs                2020.4, 2019.6, 2018.8
-    # Plumed 2.6.2 needs Gromacs                2020.4, 2019.6, 2018.8
-    # Plumed 2.6.1 needs Gromacs                2020.2, 2019.6, 2018.8
-    # Plumed 2.6.0 needs Gromacs                        2019.4, 2018.8
-    # Plumed 2.5.7 needs Gromacs                        2019.4, 2018.8, 2016.6
-    # Plumed 2.5.6 needs Gromacs                        2019.4, 2018.8, 2016.6
-    # Plumed 2.5.5 needs Gromacs                        2019.4, 2018.8, 2016.6
-    # Plumed 2.5.4 needs Gromacs                        2019.4, 2018.8, 2016.6
-    # Plumed 2.5.3 needs Gromacs                        2019.4, 2018.8, 2016.6
-    # Plumed 2.5.2 needs Gromacs                        2019.2, 2018.6, 2016.6
-    # Plumed 2.5.1 needs Gromacs                                2018.6, 2016.6
-    # Plumed 2.5.0 needs Gromacs                                2018.4, 2016.5
+    # Plumed 2.9.0 needs Gromacs 2023,  2022.5, 2021.7, 2020.7
+    # Plumed 2.8.3 needs Gromacs        2022.5, 2021.7, 2020.7, 2019.6
+    # Plumed 2.8.2 needs Gromacs        2022.5, 2021.7, 2020.7, 2019.6
+    # Plumed 2.8.1 needs Gromacs        2022.3, 2021.6, 2020.7, 2019.6
+    # Plumed 2.8.0 needs Gromacs                2021.4, 2020.6, 2019.6
+    # Plumed 2.7.6 needs Gromacs                2021.5, 2020.6, 2019.6
+    # Plumed 2.7.5 needs Gromacs                2021.5, 2020.6, 2019.6
+    # Plumed 2.7.4 needs Gromacs                2021.4, 2020.6, 2019.6
+    # Plumed 2.7.3 needs Gromacs                2021.4, 2020.6, 2019.6
+    # Plumed 2.7.2 needs Gromacs                2021,   2020.6, 2019.6
+    # Plumed 2.7.1 needs Gromacs                2021,   2020.5, 2019.6
+    # Plumed 2.7.0 needs Gromacs                        2020.4, 2019.6
+    # Plumed 2.6.6 needs Gromacs                        2020.4, 2019.6, 2018.8
+    # Plumed 2.6.5 needs Gromacs                        2020.4, 2019.6, 2018.8
+    # Plumed 2.6.4 needs Gromacs                        2020.4, 2019.6, 2018.8
+    # Plumed 2.6.3 needs Gromacs                        2020.4, 2019.6, 2018.8
+    # Plumed 2.6.2 needs Gromacs                        2020.4, 2019.6, 2018.8
+    # Plumed 2.6.1 needs Gromacs                        2020.2, 2019.6, 2018.8
+    # Plumed 2.6.0 needs Gromacs                                2019.4, 2018.8
+    # Plumed 2.5.7 needs Gromacs                                2019.4, 2018.8, 2016.6
+    # Plumed 2.5.6 needs Gromacs                                2019.4, 2018.8, 2016.6
+    # Plumed 2.5.5 needs Gromacs                                2019.4, 2018.8, 2016.6
+    # Plumed 2.5.4 needs Gromacs                                2019.4, 2018.8, 2016.6
+    # Plumed 2.5.3 needs Gromacs                                2019.4, 2018.8, 2016.6
+    # Plumed 2.5.2 needs Gromacs                                2019.2, 2018.6, 2016.6
+    # Plumed 2.5.1 needs Gromacs                                        2018.6, 2016.6
+    # Plumed 2.5.0 needs Gromacs                                        2018.4, 2016.5
 
     # Above dependencies can be verified, and new versions added, by going to
-    # https://github.com/plumed/plumed2/tree/v2.7.1/patches
+    # https://github.com/plumed/plumed2/tree/v2.9.0/patches
     # and switching tags.
     plumed_patches = {
-        "2022.5": "2.8.2",
+        "2023": "2.9.0",
+        "2022.5": "2.8.2:2.9.0",
         "2022.3": "2.8.1",
-        "2021.7": "2.8.2",
+        "2021.7": "2.8.2:2.9.0",
         "2021.6": "2.8.1",
         "2021.5": "2.7.5:2.7.6",
         "2021.4": "2.7.3:2.8.0",
         "2021": "2.7.1:2.7.2",
-        "2020.7": "2.8.1:2.8.2",
+        "2020.7": "2.8.1:2.9.0",
         "2020.6": "2.7.2:2.8.0",
         "2020.5": "2.7.1",
         "2020.4": "2.6.2:2.7.0",
         "2020.2": "2.6.1",
-        "2019.6": "2.6.1:2.8.2",
+        "2019.6": "2.6.1:2.8.3",
         "2019.4": "2.5.3:2.6.0",
         "2019.2": "2.5.2",
         "2018.8": "2.5.3:2.6",
@@ -249,8 +243,8 @@ class Gromacs(CMakePackage, CudaPackage):
     depends_on("cmake@3.16.0:3", type="build", when="%fj")
     depends_on("cuda", when="+cuda")
     depends_on("sycl", when="+sycl")
-    depends_on("lapack", when="+lapack")
-    depends_on("blas", when="+blas")
+    depends_on("lapack")
+    depends_on("blas")
     depends_on("gcc", when="%oneapi ~intel_provided_gcc")
     depends_on("gcc", when="%intel ~intel_provided_gcc")
 
@@ -258,7 +252,6 @@ class Gromacs(CMakePackage, CudaPackage):
     depends_on("hwloc", when="+hwloc@2019:")
 
     depends_on("cp2k@8.1:", when="+cp2k")
-    depends_on("dbcsr", when="+cp2k")
 
     depends_on("nvhpc", when="+cufftmp")
 
@@ -270,8 +263,19 @@ class Gromacs(CMakePackage, CudaPackage):
         msg="Only attempt to find gcc libs for Intel compiler if Intel compiler is used.",
     )
 
+    # If the Intel suite is used for Lapack, it must be used for fftw and vice-versa
+    for _intel_pkg in INTEL_MATH_LIBRARIES:
+        requires(f"^[virtuals=fftw-api] {_intel_pkg}", when=f"^[virtuals=lapack]   {_intel_pkg}")
+        requires(f"^[virtuals=lapack]   {_intel_pkg}", when=f"^[virtuals=fftw-api] {_intel_pkg}")
+
     patch("gmxDetectCpu-cmake-3.14.patch", when="@2018:2019.3^cmake@3.14.0:")
     patch("gmxDetectSimd-cmake-3.14.patch", when="@5.0:2017^cmake@3.14.0:")
+    # 2021.2 will always try to build tests (see https://gromacs.bioexcel.eu/t/compilation-failure-for-gromacs-2021-1-and-2021-2-with-cmake-3-20-2/2129)
+    patch(
+        "https://gitlab.com/gromacs/gromacs/-/commit/10262892e11a87fda0f59e633c89ed5ab1100509.diff",
+        sha256="2c30d00404b76421c13866cc42afa5e63276f7926c862838751b158df8727b1b",
+        when="@2021.1:2021.2",
+    )
 
     filter_compiler_wrappers(
         "*.cmake", relative_root=os.path.join("share", "cmake", "gromacs_mpi")
@@ -493,21 +497,13 @@ def cmake_args(self):
         if "+cuda" in self.spec:
             options.append("-DCUDA_TOOLKIT_ROOT_DIR:STRING=" + self.spec["cuda"].prefix)
 
-        if "+lapack" in self.spec:
-            options.append("-DGMX_EXTERNAL_LAPACK:BOOL=ON")
-            if self.spec["lapack"].libs:
-                options.append(
-                    "-DGMX_LAPACK_USER={0}".format(self.spec["lapack"].libs.joined(";"))
-                )
-        else:
-            options.append("-DGMX_EXTERNAL_LAPACK:BOOL=OFF")
+        options.append("-DGMX_EXTERNAL_LAPACK:BOOL=ON")
+        if self.spec["lapack"].libs:
+            options.append("-DGMX_LAPACK_USER={0}".format(self.spec["lapack"].libs.joined(";")))
 
-        if "+blas" in self.spec:
-            options.append("-DGMX_EXTERNAL_BLAS:BOOL=ON")
-            if self.spec["blas"].libs:
-                options.append("-DGMX_BLAS_USER={0}".format(self.spec["blas"].libs.joined(";")))
-        else:
-            options.append("-DGMX_EXTERNAL_BLAS:BOOL=OFF")
+        options.append("-DGMX_EXTERNAL_BLAS:BOOL=ON")
+        if self.spec["blas"].libs:
+            options.append("-DGMX_BLAS_USER={0}".format(self.spec["blas"].libs.joined(";")))
 
         if "+cp2k" in self.spec:
             options.append("-DGMX_CP2K:BOOL=ON")
@@ -603,11 +599,11 @@ def cmake_args(self):
                 "-DGMX_OPENMP_MAX_THREADS=%s" % self.spec.variants["openmp_max_threads"].value
             )
 
-        if "^mkl" in self.spec:
+        if self.spec["lapack"].name in INTEL_MATH_LIBRARIES:
             # fftw-api@3 is provided by intel-mkl or intel-parllel-studio
             # we use the mkl interface of gromacs
             options.append("-DGMX_FFT_LIBRARY=mkl")
-            if not self.spec["mkl"].satisfies("@2023:"):
+            if self.spec.satisfies("@:2022"):
                 options.append(
                     "-DMKL_INCLUDE_DIR={0}".format(self.spec["mkl"].headers.directories[0])
                 )
diff --git a/var/spack/repos/builtin/packages/grpc/package.py b/var/spack/repos/builtin/packages/grpc/package.py
index b197f8e0778d03..58e64427ec23fb 100644
--- a/var/spack/repos/builtin/packages/grpc/package.py
+++ b/var/spack/repos/builtin/packages/grpc/package.py
@@ -12,7 +12,11 @@ class Grpc(CMakePackage):
 
     homepage = "https://grpc.io"
     url = "https://github.com/grpc/grpc/archive/v1.39.0.tar.gz"
-
+    version("1.55.0", sha256="9cf1a69a921534ac0b760dcbefb900f3c2f735f56070bf0536506913bb5bfd74")
+    version("1.50.0", sha256="76900ab068da86378395a8e125b5cc43dfae671e09ff6462ddfef18676e2165a")
+    version("1.47.0", sha256="271bdc890bf329a8de5b65819f0f9590a5381402429bca37625b63546ed19e54")
+    version("1.46.0", sha256="67423a4cd706ce16a88d1549297023f0f9f0d695a96dd684adc21e67b021f9bc")
+    version("1.45.0", sha256="ec19657a677d49af59aa806ec299c070c882986c9fcc022b1c22c2a3caf01bcd")
     version("1.44.0", sha256="8c05641b9f91cbc92f51cc4a5b3a226788d7a63f20af4ca7aaca50d92cc94a0d")
     version("1.39.0", sha256="b16992aa1c949c10d5d5ce2a62f9d99fa7de77da2943e643fb66dcaf075826d6")
     version("1.38.1", sha256="f60e5b112913bf776a22c16a3053cc02cf55e60bf27a959fd54d7aaf8e2da6e8")
@@ -52,10 +56,10 @@ class Grpc(CMakePackage):
 
     depends_on("protobuf")
     depends_on("openssl")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("c-ares")
     depends_on("abseil-cpp", when="@1.27:")
-    depends_on("re2+pic", when="@1.33.1:")
+    depends_on("re2+pic@2023-09-01", when="@1.33.1:")
 
     def cmake_args(self):
         args = [
diff --git a/var/spack/repos/builtin/packages/gsibec/package.py b/var/spack/repos/builtin/packages/gsibec/package.py
index 35dd3c8b6a1486..cf3b99e2b2cb3a 100644
--- a/var/spack/repos/builtin/packages/gsibec/package.py
+++ b/var/spack/repos/builtin/packages/gsibec/package.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -33,13 +33,16 @@ class Gsibec(CMakePackage):
     depends_on("netcdf-c +mpi", type=("build", "run"))
     depends_on("netcdf-fortran", type=("build", "run"))
 
-    depends_on("mkl", when="+mkl", type=("build", "run"))
-    depends_on("lapack", when="~mkl", type=("build", "run"))
+    depends_on("lapack", type=("build", "run"))
 
     depends_on("ecbuild", type=("build"))
     depends_on("jedi-cmake", type=("build"))
     depends_on("sp", type=("build"))
 
     def cmake_args(self):
-        args = [self.define_from_variant("ENABLE_MKL", "mkl")]
+        args = []
+
+        mkl_providers = ["intel-mkl", "intel-oneapi-mkl", "intel-parallel-studio"]
+        args.append(self.define("ENABLE_MKL", self.spec["lapack"].name in mkl_providers))
+
         return args
diff --git a/var/spack/repos/builtin/packages/gsl-lite/package.py b/var/spack/repos/builtin/packages/gsl-lite/package.py
index e0fbc507ae4780..8f819f25ed8c96 100644
--- a/var/spack/repos/builtin/packages/gsl-lite/package.py
+++ b/var/spack/repos/builtin/packages/gsl-lite/package.py
@@ -25,13 +25,27 @@ class GslLite(CMakePackage):
     version("0.36.0", sha256="c052cc4547b33cedee6f000393a7005915c45c6c06b35518d203db117f75c71c")
     version("0.34.0", sha256="a7d5b2672b78704ca03df9ef65bc274d8f8cacad3ca950365eef9e25b50324c5")
 
-    variant("tests", default=False)
-    variant("cuda_tests", default=False)
-    variant("examples", default=False)
-    variant("static_analysis_demos", default=False)
-    variant("cmake_export_package_registry", default=False)
-    variant("compat_header", default=False)
-    variant("legacy_headers", default=False)
+    variant("tests", default=False, description="Build and perform gsl-lite tests")
+    variant("cuda_tests", default=False, description="Build and perform gsl-lite CUDA tests")
+    variant("examples", default=False, description="Build gsl-lite examples")
+    variant(
+        "static_analysis_demos",
+        default=False,
+        description="Build and perform gsl-lite static analysis demos",
+    )
+    variant(
+        "cmake_export_package_registry",
+        default=False,
+        description="Export build directory to CMake user package registry",
+    )
+    variant(
+        "compat_header", default=False, description="Install MS-GSL compatibility header "
+    )
+    variant(
+        "legacy_headers",
+        default=False,
+        description="Install legacy headers , , ",
+    )
 
     def cmake_args(self):
         args = [
diff --git a/var/spack/repos/builtin/packages/gslib/package.py b/var/spack/repos/builtin/packages/gslib/package.py
index 79f1d3e7e29a55..fff12c49c4f020 100644
--- a/var/spack/repos/builtin/packages/gslib/package.py
+++ b/var/spack/repos/builtin/packages/gslib/package.py
@@ -13,14 +13,14 @@ class Gslib(Package):
     git = "https://github.com/gslib/gslib.git"
 
     version("develop", branch="master")
-    version("1.0.7", tag="v1.0.7")
-    version("1.0.6", tag="v1.0.6")
-    version("1.0.5", tag="v1.0.5")
-    version("1.0.4", tag="v1.0.4")
-    version("1.0.3", tag="v1.0.3")
-    version("1.0.2", tag="v1.0.2")
-    version("1.0.1", tag="v1.0.1")
-    version("1.0.0", tag="v1.0.0")
+    version("1.0.7", tag="v1.0.7", commit="88f90cb96953527e3e833f8dbf2719273fc8346d")
+    version("1.0.6", tag="v1.0.6", commit="1c2f74420fec36d5abe1d75f194a457c61f0df53")
+    version("1.0.5", tag="v1.0.5", commit="1de2fba1d94e27e20f3bc3af6a3a35901e223ecd")
+    version("1.0.4", tag="v1.0.4", commit="00a074c15a13fdfd121ac5781ae450af809dde3b")
+    version("1.0.3", tag="v1.0.3", commit="e2df99fad9480a981034fd0e4b3a7fe8f3cf9ae3")
+    version("1.0.2", tag="v1.0.2", commit="e53419c32a4a326e55e1c3e0d7de14ce665c1788")
+    version("1.0.1", tag="v1.0.1", commit="d16685f24551b7efd69e58d96dc76aec75239ea3")
+    version("1.0.0", tag="v1.0.0", commit="9533e652320a3b26a72c36487ae265b02072cd48")
 
     variant("mpi", default=True, description="Build with MPI")
     variant("mpiio", default=True, description="Build with MPI I/O")
diff --git a/var/spack/repos/builtin/packages/gtkplus/package.py b/var/spack/repos/builtin/packages/gtkplus/package.py
index 63a55f52e089e3..52aa981d3cde8a 100644
--- a/var/spack/repos/builtin/packages/gtkplus/package.py
+++ b/var/spack/repos/builtin/packages/gtkplus/package.py
@@ -36,7 +36,7 @@ class Gtkplus(MesonPackage):
         deprecated=True,
     )
 
-    variant("cups", default="False", description="enable cups support")
+    variant("cups", default=False, description="enable cups support")
 
     # See meson.build for version requirements
     depends_on("meson@0.48.0:", when="@3.24:", type="build")
diff --git a/var/spack/repos/builtin/packages/gunrock/package.py b/var/spack/repos/builtin/packages/gunrock/package.py
index 91f0954a39abd1..0c192654b4dd82 100644
--- a/var/spack/repos/builtin/packages/gunrock/package.py
+++ b/var/spack/repos/builtin/packages/gunrock/package.py
@@ -14,18 +14,22 @@ class Gunrock(CMakePackage, CudaPackage):
     git = "https://github.com/gunrock/gunrock.git"
 
     version("master", submodules=True)
-    version("1.2", submodules=True, tag="v1.2")
+    version("1.2", submodules=True, tag="v1.2", commit="5ee3df50c45f702eb247ef1abcea7a490b60b2ea")
     # v1.1 build is broken. See:
     # https://github.com/gunrock/gunrock/issues/777
-    version("1.1", submodules=True, tag="v1.1")
-    version("1.0", submodules=True, tag="v1.0")
-    version("0.5.1", submodules=True, tag="v0.5.1")
-    version("0.5", submodules=True, tag="v0.5")
-    version("0.4", submodules=True, tag="v0.4")
-    version("0.3.1", submodules=True, tag="v0.3.1")
-    version("0.3", submodules=True, tag="v0.3")
-    version("0.2", submodules=True, tag="v0.2")
-    version("0.1", submodules=True, tag="v0.1")
+    version("1.1", submodules=True, tag="v1.1", commit="7c197d6a498806fcfffd1f9304c663379a77f5e4")
+    version("1.0", submodules=True, tag="v1.0", commit="04279c89f394b97c81c63ad286048893e02f769e")
+    version(
+        "0.5.1", submodules=True, tag="v0.5.1", commit="0c9a96dd64ddeecae9208644631983fc889b32b4"
+    )
+    version("0.5", submodules=True, tag="v0.5", commit="91a6218868ecdab7986ef42b4b76ff17eec61ca3")
+    version("0.4", submodules=True, tag="v0.4", commit="4c33a0d5a7ad7e468362c9c081a567450a78ab97")
+    version(
+        "0.3.1", submodules=True, tag="v0.3.1", commit="897f170e3006e58c9a602201e5f9fc56162a3cb9"
+    )
+    version("0.3", submodules=True, tag="v0.3", commit="0b146a70f52f699a2a50d1b1aa26b92c45e834d7")
+    version("0.2", submodules=True, tag="v0.2", commit="f9d85343ee68c65567184d74021b9483cd142ea0")
+    version("0.1", submodules=True, tag="v0.1", commit="4c00284f6b7d490a83fa7afe5cdff60923316448")
 
     variant("cuda", default=True, description="Build with Cuda support")
 
diff --git a/var/spack/repos/builtin/packages/h5bench/package.py b/var/spack/repos/builtin/packages/h5bench/package.py
index 3829b02bb9f753..9457040c9564fc 100644
--- a/var/spack/repos/builtin/packages/h5bench/package.py
+++ b/var/spack/repos/builtin/packages/h5bench/package.py
@@ -14,9 +14,12 @@ class H5bench(CMakePackage):
 
     maintainers("jeanbez", "sbyna")
 
+    tags = ["e4s"]
+
     version("latest", branch="master", submodules=True)
     version("develop", branch="develop", submodules=True)
 
+    version("1.4", commit="67a3e6d91508e5ab77db79c6187b2eb3c96119e0", submodules=True)
     version("1.3", commit="ec75a466a77c337b4252c0afe1055c7fbe841e16", submodules=True)
     version(
         "1.2", commit="866af6777573d20740d02acc47a9080de093e4ad", submodules=True, deprecated=True
@@ -58,3 +61,30 @@ def cmake_args(self):
         ]
 
         return args
+
+    @run_after("install")
+    def setup_build_tests(self):
+        launcher = self.mpi_launcher()
+        filename = "samples/sync-write-1d-contig-contig.json"
+        filter_file("mpirun", f"{launcher}", filename)
+        filter_file(r"-n 2", "-n 1 --timeout 240", filename)
+
+        """Copy the example source files after the package is installed to an
+        install test subdirectory for use during `spack test run`."""
+        cache_extra_test_sources(self, ["tests", "samples"])
+
+    def mpi_launcher(self):
+        commands = ["mpirun", "mpiexec"]
+
+        return which(*commands, path=[self.spec["mpi"].prefix.bin]) or which(*commands)
+
+    def test_help(self):
+        """Run h5bench help."""
+        h5bench = which(self.prefix.bin.h5bench)
+        h5bench("-h")
+
+    def test_h5bench(self):
+        """Run h5bench synchronous write test."""
+        with working_dir(self.test_suite.current_test_cache_dir):
+            h5bench = which(self.prefix.bin.h5bench)
+            h5bench("--debug", "--abort", "samples/sync-write-1d-contig-contig.json")
diff --git a/var/spack/repos/builtin/packages/h5z-zfp/package.py b/var/spack/repos/builtin/packages/h5z-zfp/package.py
index a54fb0441d5fd6..f90a244ed8f832 100644
--- a/var/spack/repos/builtin/packages/h5z-zfp/package.py
+++ b/var/spack/repos/builtin/packages/h5z-zfp/package.py
@@ -12,9 +12,13 @@ class H5zZfp(MakefilePackage):
 
     homepage = "https://h5z-zfp.readthedocs.io/en/latest"
     git = "https://github.com/LLNL/H5Z-ZFP.git"
-    url = "https://github.com/LLNL/H5Z-ZFP/archive/refs/tags/v1.0.1.tar.gz"
+    url = "https://github.com/LLNL/H5Z-ZFP/archive/refs/tags/v1.1.1.tar.gz"
+
+    maintainers("markcmiller86", "brtnfld", "byrnHDF")
 
     version("develop", branch="master")
+    version("1.1.1", sha256="921af7b9d1c8c46c036b46544f2785f69d405c0701abe1c1ce3aca2bd5899171")
+    version("1.1.0", sha256="48a81e69d1f3b61d9a1eb07e868164fadf3b88690ec930efd849f5889681a893")
     version("1.0.1", sha256="b9ed91dab8e2ef82dc6706b4242c807fb352875e3b21c217dd00782dd1a22b24")
     version("0.8.0", sha256="a5eb089191369a5e929c51ec9e5da107afaee39c6ab3b7ad693c454319ab9217")
     version("0.7.0", sha256="f728b0bcb9e9cf8bafe05909ab02fec39415635d275e98b661176f69d34f87b3")
@@ -66,3 +70,8 @@ def build_targets(self):
     def install_targets(self):
         make_args = ["install"]
         return make_args + self.make_defs
+
+    @run_after("build")
+    @on_package_attributes(run_tests=True)
+    def check_build(self):
+        make("check", *self.make_defs, parallel=False)
diff --git a/var/spack/repos/builtin/packages/hacckernels/package.py b/var/spack/repos/builtin/packages/hacckernels/package.py
index b866ab6db85ee1..a16e1b7a11ed5d 100644
--- a/var/spack/repos/builtin/packages/hacckernels/package.py
+++ b/var/spack/repos/builtin/packages/hacckernels/package.py
@@ -13,8 +13,8 @@ class Hacckernels(CMakePackage):
     on diverse computing architectures and to scale to millions of
     cores and beyond."""
 
-    homepage = "https://xgitlab.cels.anl.gov/hacc/HACCKernels"
-    git = "https://xgitlab.cels.anl.gov/hacc/HACCKernels.git"
+    homepage = "https://git.cels.anl.gov/hacc/HACCKernels"
+    git = "https://git.cels.anl.gov/hacc/HACCKernels.git"
 
     tags = ["proxy-app"]
 
diff --git a/var/spack/repos/builtin/packages/half/package.py b/var/spack/repos/builtin/packages/half/package.py
index 7c90bbda00bd19..784dfe7043b3d9 100644
--- a/var/spack/repos/builtin/packages/half/package.py
+++ b/var/spack/repos/builtin/packages/half/package.py
@@ -28,3 +28,5 @@ class Half(Package):
     def install(self, spec, prefix):
         mkdirp(prefix.include)
         install_tree("include", prefix.include)
+        mkdirp(prefix.include.half)
+        symlink("../half.hpp", join_path(self.prefix.include.half, "half.hpp"))
diff --git a/var/spack/repos/builtin/packages/hapcut2/package.py b/var/spack/repos/builtin/packages/hapcut2/package.py
index 33b6c00807451c..c025fb3df8fc7c 100644
--- a/var/spack/repos/builtin/packages/hapcut2/package.py
+++ b/var/spack/repos/builtin/packages/hapcut2/package.py
@@ -28,7 +28,7 @@ class Hapcut2(MakefilePackage):
     depends_on("openssl")
     depends_on("xz")
     depends_on("bzip2")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     @when("@v1.3.1:")
     def edit(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/harfbuzz/package.py b/var/spack/repos/builtin/packages/harfbuzz/package.py
index fd68b37fb74586..fa637bd4763c9d 100644
--- a/var/spack/repos/builtin/packages/harfbuzz/package.py
+++ b/var/spack/repos/builtin/packages/harfbuzz/package.py
@@ -80,7 +80,7 @@ class Harfbuzz(MesonPackage, AutotoolsPackage):
     depends_on("icu4c")
     depends_on("freetype")
     depends_on("cairo+pdf+ft")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("graphite2", when="+graphite2")
 
     conflicts(
@@ -126,12 +126,23 @@ class MesonBuilder(spack.build_systems.meson.MesonBuilder, SetupEnvironment):
     def meson_args(self):
         graphite2 = "enabled" if self.pkg.spec.satisfies("+graphite2") else "disabled"
         coretext = "enabled" if self.pkg.spec.satisfies("+coretext") else "disabled"
-        return [
+        meson_args = [
             # disable building of gtk-doc files following #9885 and #9771
             "-Ddocs=disabled",
             "-Dgraphite2={0}".format(graphite2),
             "-Dcoretext={0}".format(coretext),
         ]
+        libs = []
+        pc = which("pkg-config")
+        deps = {"zlib": "~shared", "bzip": "~shared", "libpng": "libs=static", "cairo": "~shared"}
+        for dep in deps.keys():
+            if self.spec.satisfies(f"^{dep} {deps[dep]}"):
+                libs.append(pc(dep, "--static", "--libs", output=str).strip())
+        if libs:
+            meson_args.append("-Dc_link_args=%s" % " ".join(libs))
+            meson_args.append("-Dcpp_link_args=%s" % " ".join(libs))
+
+        return meson_args
 
 
 class AutotoolsBuilder(spack.build_systems.autotools.AutotoolsBuilder, SetupEnvironment):
@@ -148,29 +159,13 @@ def configure_args(self):
         args.extend(self.with_or_without("graphite2"))
         args.extend(self.with_or_without("coretext"))
 
-        ldflags = []
         libs = []
-        for lib in ["bzip2", "zlib", "libpng"]:
-            spec = self.spec[lib]
-            if spec.satisfies("+shared") or spec.satisfies("libs=shared"):
-                continue
-            ldflags.append(spec.libs.ld_flags)
-            libs.append(spec.libs.link_flags)
-        cairo = self.spec["cairo"]
-        if cairo.satisfies("~shared"):
-            if cairo.satisfies("+fc"):
-                ldflags.append("-L%s" % cairo["fontconfig"].prefix.lib)
-                libs.append("-lfontconfig")
-            if cairo.satisfies("+ft"):
-                ldflags.append("-L%s" % cairo["freetype"].prefix.lib)
-                libs.append("-lfreetype")
-            if cairo.satisfies("+png"):
-                ldflags.append("-L%s" % cairo["libpng"].prefix.lib)
-                libs.append("-lpng")
-            ldflags.append("-L%s" % cairo["pixman"].prefix.lib)
-            libs.append("-lpixman-1")
-        if ldflags:
-            args.append("LDFLAGS=%s" % " ".join(ldflags))
+        pc = which("pkg-config")
+        deps = {"zlib": "~shared", "bzip": "~shared", "libpng": "libs=static", "cairo": "~shared"}
+        for dep in deps.keys():
+            if self.spec.satisfies(f"^{dep} {deps[dep]}"):
+                libs.append(pc(dep, "--static", "--libs", output=str).strip())
+        if libs:
             args.append("LIBS=%s" % " ".join(libs))
 
         return args
diff --git a/var/spack/repos/builtin/packages/hdf-eos2/package.py b/var/spack/repos/builtin/packages/hdf-eos2/package.py
index f37d530c9c7012..e06e4d3684cdf5 100644
--- a/var/spack/repos/builtin/packages/hdf-eos2/package.py
+++ b/var/spack/repos/builtin/packages/hdf-eos2/package.py
@@ -21,6 +21,8 @@ class HdfEos2(AutotoolsPackage):
     # Template for url_for_version. 0 is sha256 checksum, 1 is filename
     url = "https://git.earthdata.nasa.gov/rest/git-lfs/storage/DAS/hdfeos/{0}?response-content-disposition=attachment%3B%20filename%3D%22{1}%22%3B%20filename*%3Dutf-8%27%27{1}"
 
+    maintainers("climbfuji")
+
     # Crazy URL scheme, differing with each version, and including the
     # sha256 checksum in the URL.  Yuck
     # The data in version_list is used to generate versions and urls
@@ -57,7 +59,7 @@ class HdfEos2(AutotoolsPackage):
     depends_on("hdf")
     # Because hdf always depends on zlib and jpeg in spack, the tests below in configure_args
     # (if "jpeg" in self.spec:) always returns true and hdf-eos2 wants zlib and jpeg, too.
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("jpeg")
     depends_on("szip", when="^hdf +szip")
 
@@ -81,18 +83,13 @@ def url_for_version(self, version):
                 "version/checksum not found in version_list".format(version)
             )
 
+    # spack patches the configure file unless autoconf is run,
+    # and this fails because configure has the wrong permissions (644)
     @run_before("configure")
     def fix_permissions(self):
         if not self.force_autoreconf:
             chmod(join_path(self.stage.source_path, "configure"), 0o755)
 
-    def flag_handler(self, name, flags):
-        if self.spec.compiler.name == "apple-clang":
-            if name == "cflags":
-                flags.append("-Wno-error=implicit-function-declaration")
-
-        return flags, None, None
-
     def configure_args(self):
         extra_args = []
 
@@ -114,6 +111,10 @@ def configure_args(self):
         if "szip" in self.spec:
             extra_args.append("--with-szlib={0}".format(self.spec["szip"].prefix))
         if "zlib" in self.spec:
-            extra_args.append("--with-zlib={0}".format(self.spec["zlib"].prefix))
+            extra_args.append("--with-zlib={0}".format(self.spec["zlib-api"].prefix))
+
+        # https://forum.hdfgroup.org/t/help-building-hdf4-with-clang-error-implicit-declaration-of-function-test-mgr-szip-is-invalid-in-c99/7680
+        if self.spec.satisfies("%apple-clang"):
+            extra_args.append("CFLAGS=-Wno-error=implicit-function-declaration")
 
         return extra_args
diff --git a/var/spack/repos/builtin/packages/hdf-eos5/package.py b/var/spack/repos/builtin/packages/hdf-eos5/package.py
index 12cdd06aff4cf8..258291ec3a4d54 100644
--- a/var/spack/repos/builtin/packages/hdf-eos5/package.py
+++ b/var/spack/repos/builtin/packages/hdf-eos5/package.py
@@ -99,7 +99,7 @@ def configure_args(self):
         extra_args.append("--with-hdf5={0}".format(self.spec["hdf5"].prefix))
         if "szip" in self.spec:
             extra_args.append("--with-szlib={0}".format(self.spec["szip"].prefix))
-        if "zlib" in self.spec:
-            extra_args.append("--with-zlib={0}".format(self.spec["zlib"].prefix))
+        if "zlib-api" in self.spec:
+            extra_args.append("--with-zlib={0}".format(self.spec["zlib-api"].prefix))
 
         return extra_args
diff --git a/var/spack/repos/builtin/packages/hdf/package.py b/var/spack/repos/builtin/packages/hdf/package.py
index d42b03fbb7bc86..4adb083fc1e8b7 100644
--- a/var/spack/repos/builtin/packages/hdf/package.py
+++ b/var/spack/repos/builtin/packages/hdf/package.py
@@ -35,7 +35,7 @@ class Hdf(AutotoolsPackage):
     variant("shared", default=False, description="Enable shared library")
     variant("pic", default=True, description="Produce position-independent code")
 
-    depends_on("zlib@1.1.4:")
+    depends_on("zlib-api")
     depends_on("jpeg")
     depends_on("szip", when="+szip")
     depends_on("rpc", when="+external-xdr")
@@ -144,17 +144,21 @@ def flag_handler(self, name, flags):
             elif name == "fflags":
                 flags.append(self.compiler.f77_pic_flag)
 
-        if self.spec.compiler.name == "apple-clang":
-            if name == "cflags":
+        if name == "cflags":
+            # https://forum.hdfgroup.org/t/help-building-hdf4-with-clang-error-implicit-declaration-of-function-test-mgr-szip-is-invalid-in-c99/7680
+            if self.spec.satisfies("@:4.2.15 %apple-clang") or self.spec.satisfies("%clang@16:"):
                 flags.append("-Wno-error=implicit-function-declaration")
 
+            if self.spec.satisfies("%clang@16:"):
+                flags.append("-Wno-error=implicit-int")
+
         return flags, None, None
 
     def configure_args(self):
         config_args = [
             "--enable-production",
             "--enable-static",
-            "--with-zlib=%s" % self.spec["zlib"].prefix,
+            "--with-zlib=%s" % self.spec["zlib-api"].prefix,
             "--with-jpeg=%s" % self.spec["jpeg"].prefix,
         ]
 
@@ -182,10 +186,6 @@ def configure_args(self):
                 ["FFLAGS=-fallow-argument-mismatch", "FCFLAGS=-fallow-argument-mismatch"]
             )
 
-        # https://forum.hdfgroup.org/t/help-building-hdf4-with-clang-error-implicit-declaration-of-function-test-mgr-szip-is-invalid-in-c99/7680
-        if self.spec.satisfies("@:4.2.15 %apple-clang"):
-            config_args.append("CFLAGS=-Wno-error=implicit-function-declaration")
-
         return config_args
 
     # Otherwise, we randomly get:
diff --git a/var/spack/repos/builtin/packages/hdf5-vol-async/package.py b/var/spack/repos/builtin/packages/hdf5-vol-async/package.py
index fd68cb7cd178b7..ab34a7f0b1a148 100644
--- a/var/spack/repos/builtin/packages/hdf5-vol-async/package.py
+++ b/var/spack/repos/builtin/packages/hdf5-vol-async/package.py
@@ -24,8 +24,9 @@ class Hdf5VolAsync(CMakePackage):
     tags = ["e4s"]
 
     version("develop", branch="develop")
-    version("1.6", tag="v1.6")
-    version("1.5", tag="v1.5")
+    version("1.7", tag="v1.7", commit="70a22cf9863a7c1386d97be865342deb751ca501")
+    version("1.6", tag="v1.6", commit="f3406d62ec055cdcfe077979a1068bd102c598a5")
+    version("1.5", tag="v1.5", commit="b917713ffcb207d9799c6d6863cf805ee54ccfea")
 
     variant("memcpy", default=False, description="Enable buffer copy for dataset write")
 
@@ -34,8 +35,8 @@ class Hdf5VolAsync(CMakePackage):
     depends_on("hdf5@1.14.0: +mpi +threadsafe")
 
     # Require MPI_THREAD_MULTIPLE.
-    depends_on("openmpi +thread_multiple", when="^openmpi")
-    depends_on("mvapich2 threads=multiple", when="^mvapich2")
+    depends_on("openmpi +thread_multiple", when="^[virtuals=mpi] openmpi@:2")
+    depends_on("mvapich2 threads=multiple", when="^[virtuals=mpi] mvapich2")
 
     def setup_run_environment(self, env):
         env.prepend_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib)
diff --git a/var/spack/repos/builtin/packages/hdf5-vol-cache/package.py b/var/spack/repos/builtin/packages/hdf5-vol-cache/package.py
index 84175558799526..fd9977b3f871f0 100644
--- a/var/spack/repos/builtin/packages/hdf5-vol-cache/package.py
+++ b/var/spack/repos/builtin/packages/hdf5-vol-cache/package.py
@@ -14,12 +14,18 @@ class Hdf5VolCache(CMakePackage):
     maintainers("hyoklee", "lrknox")
 
     version("default", branch="develop")
-    version("v1.1", tag="v1.1")
-    version("v1.0", tag="v1.0")
+    version("v1.1", tag="v1.1", commit="d886a17a381990b5949d95f5299461c39d7ac2bc")
+    version("v1.0", tag="v1.0", commit="a9b9704e74fa24af50b2a3bd0d63a40a69bde8fe")
 
     depends_on("hdf5@1.14: +mpi +threadsafe")
     depends_on("hdf5-vol-async")
 
+    def flag_handler(self, name, flags):
+        if name == "cflags":
+            if self.spec.satisfies("%oneapi"):
+                flags.append("-Wno-error=incompatible-function-pointer-types")
+        return (flags, None, None)
+
     def setup_run_environment(self, env):
         env.prepend_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib)
 
diff --git a/var/spack/repos/builtin/packages/hdf5-vol-log/package.py b/var/spack/repos/builtin/packages/hdf5-vol-log/package.py
index d5e057942c2ad4..fa1718d09168ea 100644
--- a/var/spack/repos/builtin/packages/hdf5-vol-log/package.py
+++ b/var/spack/repos/builtin/packages/hdf5-vol-log/package.py
@@ -17,7 +17,7 @@ class Hdf5VolLog(AutotoolsPackage):
 
     version("master-1.1", branch="master")
 
-    version("1.4.0", tag="logvol.1.4.0")
+    version("1.4.0", tag="logvol.1.4.0", commit="786d2cc4da8b4a0827ee00b1b0ab3968ef942f99")
 
     depends_on("hdf5@1.14.0:", when="@1.4.0:")
     depends_on("mpi")
diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py
index 55619b0bcbfb9d..210c9b08fea64f 100644
--- a/var/spack/repos/builtin/packages/hdf5/package.py
+++ b/var/spack/repos/builtin/packages/hdf5/package.py
@@ -20,7 +20,7 @@ class Hdf5(CMakePackage):
     """
 
     homepage = "https://portal.hdfgroup.org"
-    url = "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.8/src/hdf5-1.10.8.tar.gz"
+    url = "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.3/src/hdf5-1.14.3.tar.gz"
     list_url = "https://support.hdfgroup.org/ftp/HDF5/releases"
     list_depth = 3
     git = "https://github.com/HDFGroup/hdf5.git"
@@ -41,6 +41,16 @@ class Hdf5(CMakePackage):
 
     # Odd versions are considered experimental releases
     # Even versions are maintenance versions
+    version(
+        "1.14.3",
+        sha256="09cdb287aa7a89148c1638dd20891fdbae08102cf433ef128fd345338aa237c7",
+        preferred=True,
+    )
+    version(
+        "1.14.2",
+        sha256="1c342e634008284a8c2794c8e7608e2eaf26d01d445fb3dfd7f33cb2fb51ac53",
+        preferred=True,
+    )
     version(
         "1.14.1-2",
         sha256="cbe93f275d5231df28ced9549253793e40cd2b555e3d288df09d7b89a9967b07",
@@ -66,6 +76,11 @@ class Hdf5(CMakePackage):
         sha256="a62dcb276658cb78e6795dd29bf926ed7a9bc4edf6e77025cd2c689a8f97c17a",
         preferred=True,
     )
+    version(
+        "1.10.11",
+        sha256="341684c5c0976b8c7e6951735a400275a90693604464cac73e9f323c696fc79c",
+        preferred=True,
+    )
     version(
         "1.10.10",
         sha256="a6877ab7bd5d769d2d68618fdb54beb50263dcc2a8c157fe7e2186925cdb02db",
@@ -213,7 +228,7 @@ class Hdf5(CMakePackage):
     depends_on("mpi", when="+mpi")
     depends_on("java", type=("build", "run"), when="+java")
     depends_on("szip", when="+szip")
-    depends_on("zlib@1.1.2:")
+    depends_on("zlib-api")
 
     # The compiler wrappers (h5cc, h5fc, etc.) run 'pkg-config'.
     # Skip this on Windows since pkgconfig is autotools
@@ -658,7 +673,7 @@ def ensure_parallel_compiler_wrappers(self):
         # 1.10.6 and 1.12.0. The current develop versions do not produce 'h5pfc'
         # at all. Here, we make sure that 'h5pfc' is available when Fortran and
         # MPI support are enabled (only for versions that generate 'h5fc').
-        if self.spec.satisfies("@1.8.22:1.8," "1.10.6:1.10," "1.12.0:1.12" "+fortran+mpi"):
+        if self.spec.satisfies("@1.8.22:1.8," "1.10.6:1.10.9," "1.12.0:1.12" "+fortran+mpi"):
             with working_dir(self.prefix.bin):
                 # No try/except here, fix the condition above instead:
                 symlink("h5fc", "h5pfc")
diff --git a/var/spack/repos/builtin/packages/heaptrack/package.py b/var/spack/repos/builtin/packages/heaptrack/package.py
index 6cd2d96895e9e9..35e84890422224 100644
--- a/var/spack/repos/builtin/packages/heaptrack/package.py
+++ b/var/spack/repos/builtin/packages/heaptrack/package.py
@@ -20,7 +20,7 @@ class Heaptrack(CMakePackage):
     depends_on("cmake@2.8.9:", type="build")
     depends_on("elfutils")
     depends_on("libunwind")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("zstd")
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/heasoft/package.py b/var/spack/repos/builtin/packages/heasoft/package.py
index cad00f057d1600..750be26df89878 100644
--- a/var/spack/repos/builtin/packages/heasoft/package.py
+++ b/var/spack/repos/builtin/packages/heasoft/package.py
@@ -29,7 +29,7 @@ class Heasoft(AutotoolsPackage):
 
     variant("X", default=True, description="Enable X11 support")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("ncurses")
     depends_on("curl")
     depends_on("libxt", when="+X")
@@ -97,8 +97,8 @@ def patch(self):
     def configure_args(self):
         config_args = [
             "--with-png={0}".format(self.spec["libpng"].prefix),
-            "CPPFLAGS={0}".format(self.spec["zlib"].headers.include_flags),
-            "LDFLAGS={0}".format(self.spec["zlib"].libs.search_flags),
+            "CPPFLAGS={0}".format(self.spec["zlib-api"].headers.include_flags),
+            "LDFLAGS={0}".format(self.spec["zlib-api"].libs.search_flags),
         ]
 
         config_args += self.enable_or_disable("x", variant="X")
diff --git a/var/spack/repos/builtin/packages/heffte/package.py b/var/spack/repos/builtin/packages/heffte/package.py
index 0a502588fd8a92..01f1235771a744 100644
--- a/var/spack/repos/builtin/packages/heffte/package.py
+++ b/var/spack/repos/builtin/packages/heffte/package.py
@@ -10,7 +10,7 @@ class Heffte(CMakePackage, CudaPackage, ROCmPackage):
     """Highly Efficient FFT for Exascale"""
 
     homepage = "https://github.com/icl-utk-edu/heffte/"
-    url = "https://github.com/icl-utk-edu/heffte/archive/refs/tags/v2.3.0.tar.gz"
+    url = "https://github.com/icl-utk-edu/heffte/archive/refs/tags/v2.4.0.tar.gz"
     git = "https://github.com/icl-utk-edu/heffte/"
 
     maintainers("mkstoyanov", "G-Ragghianti")
@@ -19,6 +19,7 @@ class Heffte(CMakePackage, CudaPackage, ROCmPackage):
     test_requires_compiler = True
 
     version("develop", branch="master")
+    version("2.4.0", sha256="02310fb4f9688df02f7181667e61c3adb7e38baf79611d80919d47452ff7881d")
     version("2.3.0", sha256="63db8c9a8822211d23e29f7adf5aa88bb462c91d7a18c296c3ef3a06be8d6171")
     version("2.2.0", sha256="332346d5c1d1032288d09839134c79e4a9704e213a2d53051e96c3c414c74df0")
     version("2.1.0", sha256="63b8ea45a220afc4fa0b14769c0dd291e614a2fe9d5a91c50d28f16ee29b3f1c")
@@ -27,29 +28,13 @@ class Heffte(CMakePackage, CudaPackage, ROCmPackage):
         sha256="b575fafe19a635265904ca302d48e778341b1567c055ea7f2939c8c6718f7212",
         deprecated=True,
     )
-    version(
-        "1.0",
-        sha256="00e66cdff664ba90eeb26b4824f2a7341ba791b1d7220ece8180aba7623d36d5",
-        deprecated=True,
-    )
-    version(
-        "0.2",
-        sha256="6e606aa9de91912925ec49f463de4369459e509e0e21a97ca72dfa07651056e5",
-        deprecated=True,
-    )
-    version(
-        "0.1",
-        sha256="bcdc940c4cb254b178446d16c969b85ea6b5c69fdf4b6332bb3c8fbce00bccdf",
-        deprecated=True,
-    )
 
-    patch("threads10.patch", when="@1.0")
-    patch("fortran200.patch", when="@2.0.0")
     patch("cmake-magma-v230.patch", when="@2.3.0")
+    patch("fortran200.patch", when="@2.0.0")
 
-    depends_on("cmake@3.10:", type=("build", "run"))
-    depends_on("cmake@3.19:", when="@develop", type=("build", "run"))
-    depends_on("cmake@3.21:", when="@develop+rocm", type=("build", "run"))
+    depends_on("cmake@3.10:", when="@:2.3.0", type=("build", "run"))
+    depends_on("cmake@3.19:", when="@2.4.0:", type=("build", "run"))
+    depends_on("cmake@3.21:", when="@2.4.0:+rocm", type=("build", "run"))
 
     variant("shared", default=True, description="Builds with shared libraries")
     variant("fftw", default=False, description="Builds with support for FFTW backend")
@@ -64,14 +49,9 @@ class Heffte(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("py-numba", when="+python+cuda", type=("build", "run"))
     extends("python", when="+python", type=("build", "run"))
 
-    conflicts("~fftw", when="@:2.1.0~mkl~cuda")  # requires at least one backend
-    conflicts("+fftw", when="+mkl@:1.0")  # old API supports at most one CPU backend
     conflicts("^openmpi~cuda", when="+cuda")  # +cuda requires CUDA enabled OpenMPI
     conflicts("~cuda~rocm", when="+magma")  # magma requires CUDA or HIP
     conflicts("+rocm", when="@:2.1.0")  # heffte+rocm is in in development in spack
-    conflicts("+python", when="@:1.0")  # python support was added post v1.0
-    conflicts("+fortran", when="@:1.0")  # fortran support was added post v1.0
-    conflicts("+magma", when="@:1.0")  # magma support was added post v1.0
 
     depends_on("mpi", type=("build", "run"))
 
@@ -80,10 +60,12 @@ class Heffte(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("cuda@8.0:", when="+cuda", type=("build", "run"))
     depends_on("hip@3.8.0:", when="+rocm", type=("build", "run"))
     depends_on("rocfft@3.8.0:", when="+rocm", type=("build", "run"))
-    depends_on("hip@5.2.3:", when="@develop+rocm", type=("build", "run"))
-    depends_on("rocfft@5.2.3:", when="@develop+rocm", type=("build", "run"))
+    depends_on("hip@5.2.3:", when="@2.4.0:+rocm", type=("build", "run"))
+    depends_on("rocfft@5.2.3:", when="@2.4.0:+rocm", type=("build", "run"))
     depends_on("magma@2.5.3:", when="+cuda+magma", type=("build", "run"))
     depends_on("magma+rocm@2.6.1:", when="+magma+rocm @2.1:", type=("build", "run"))
+    depends_on("rocblas@3.8:", when="+magma+rocm", type=("build", "run"))
+    depends_on("rocsparse@3.8:", when="+magma+rocm", type=("build", "run"))
     depends_on("hipblas@3.8:", when="+magma+rocm", type=("build", "run"))
     depends_on("hipsparse@3.8:", when="+magma+rocm", type=("build", "run"))
 
@@ -92,6 +74,7 @@ class Heffte(CMakePackage, CudaPackage, ROCmPackage):
     def cmake_args(self):
         args = [
             "-DHeffte_SEQUENTIAL_TESTING=ON",
+            "-DHeffte_ENABLE_TESTING=ON",
             self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
             self.define_from_variant("Heffte_ENABLE_CUDA", "cuda"),
             self.define_from_variant("Heffte_ENABLE_ROCM", "rocm"),
@@ -128,32 +111,42 @@ def cmake_args(self):
 
     @run_after("install")
     def setup_smoke_test(self):
+        if self.spec.satisfies("@:2.2.0"):
+            return
         install_tree(
             self.prefix.share.heffte.testing, join_path(self.install_test_root, "testing")
         )
 
     def test_make_test(self):
         """build and run make(test)"""
+
+        if self.spec.satisfies("@:2.2.0"):
+            raise SkipTest("Test is not supported for versions @:2.2.0")
+
         # using the tests copied from /share/heffte/testing
         cmake_dir = self.test_suite.current_test_cache_dir.testing
 
         options = [cmake_dir]
+        options.append(self.define("Heffte_DIR", self.spec.prefix.lib.cmake.Heffte))
         if "+rocm" in self.spec:
+            # path name is 'hsa-runtime64' but python cannot have '-' in variable name
+            hsa_runtime = join_path(self.spec["hsa-rocr-dev"].prefix.lib.cmake, "hsa-runtime64")
             options.extend(
                 [
-                    f"-Dhip_DIR={self.spec['hip'].prefix.lib.cmake.hip}",
-                    "-DAMDDeviceLibs_DIR="
-                    + f"{self.spec['llvm-amdgpu'].prefix.lib.cmake.AMDDeviceLibs}",
-                    f"-Damd_comgr_DIR={self.spec['comgr'].prefix.lib.cmake.amd_comgr}",
-                    "-Dhsa-runtime64_DIR="
-                    + f"{self.spec['hsa-rocr-dev'].prefix.lib.cmake.hsa-runtime64}",
-                    "-DHSA_HEADER={self.spec['hsa-rocr-dev'].prefix.include}",
-                    "-Drocfft_DIR={self.spec['rocfft'].prefix.lib.cmake.rocfft}",
+                    self.define("hip_DIR", self.spec["hip"].prefix.lib.cmake.hip),
+                    self.define(
+                        "AMDDeviceLibs_DIR",
+                        self.spec["llvm-amdgpu"].prefix.lib.cmake.AMDDeviceLibs,
+                    ),
+                    self.define("amd_comgr_DIR", self.spec["comgr"].prefix.lib.cmake.amd_comgr),
+                    self.define("hsa-runtime64_DIR", hsa_runtime),
+                    self.define("HSA_HEADER", self.spec["hsa-rocr-dev"].prefix.include),
+                    self.define("rocfft_DIR", self.spec["rocfft"].prefix.lib.cmake.rocfft),
                 ]
             )
 
         # Provide the root directory of the MPI installation.
-        options.append(f"-DMPI_HOME={self.spec['mpi'].prefix}")
+        options.append(self.define("MPI_HOME", self.spec["mpi"].prefix))
 
         cmake = which(self.spec["cmake"].prefix.bin.cmake)
         cmake(*options)
diff --git a/var/spack/repos/builtin/packages/heffte/threads10.patch b/var/spack/repos/builtin/packages/heffte/threads10.patch
deleted file mode 100644
index 41d55d9bb80919..00000000000000
--- a/var/spack/repos/builtin/packages/heffte/threads10.patch
+++ /dev/null
@@ -1,13 +0,0 @@
-diff --git a/cmake/HeffteConfig.cmake b/cmake/HeffteConfig.cmake
-index bd67de9..ca06086 100644
---- a/cmake/HeffteConfig.cmake
-+++ b/cmake/HeffteConfig.cmake
-@@ -19,6 +19,8 @@ if (NOT TARGET MPI::MPI_CXX)
-     find_package(MPI REQUIRED)
- endif()
- 
-+find_package(Threads)
-+
- if ("@BUILD_SHARED_LIBS@")
-     set(Heffte_SHARED_FOUND "ON")
- else()
diff --git a/var/spack/repos/builtin/packages/henson/package.py b/var/spack/repos/builtin/packages/henson/package.py
index 34dc0a63bb5232..a66fb7552c9742 100644
--- a/var/spack/repos/builtin/packages/henson/package.py
+++ b/var/spack/repos/builtin/packages/henson/package.py
@@ -23,12 +23,17 @@ class Henson(CMakePackage):
     depends_on("py-mpi4py", when="+python", type=("build", "run"))
     variant("mpi-wrappers", default=False, description="Build MPI wrappers (PMPI)")
 
+    variant("boost", default=False, description="Use Boost for coroutine support")
+    depends_on("boost+context", when="+boost", type=("build", "run"))
+    conflicts("~boost", when="target=aarch64:")
+
     conflicts("^openmpi", when="+mpi-wrappers")
 
     def cmake_args(self):
         args = [
             self.define_from_variant("python", "python"),
             self.define_from_variant("mpi-wrappers", "mpi-wrappers"),
+            self.define_from_variant("use_boost", "boost"),
         ]
 
         if self.spec.satisfies("+python"):
diff --git a/var/spack/repos/builtin/packages/hepmc3/package.py b/var/spack/repos/builtin/packages/hepmc3/package.py
index d45df7d2e6dffb..de8936d52a3018 100644
--- a/var/spack/repos/builtin/packages/hepmc3/package.py
+++ b/var/spack/repos/builtin/packages/hepmc3/package.py
@@ -31,6 +31,7 @@ class Hepmc3(CMakePackage):
     # note that version 3.0.0 is not supported
     # conflicts with cmake configuration
 
+    variant("protobuf", default=False, description="Enable Protobuf I/O")
     variant("python", default=False, description="Enable Python bindings")
     variant("rootio", default=False, description="Enable ROOT I/O")
     variant(
@@ -41,6 +42,7 @@ class Hepmc3(CMakePackage):
 
     depends_on("cmake@2.8.9:", type="build")
     depends_on("root", when="+rootio")
+    depends_on("protobuf", when="+protobuf")
     depends_on("python", when="+python")
 
     conflicts("%gcc@9.3.0", when="@:3.1.1")
@@ -48,22 +50,25 @@ class Hepmc3(CMakePackage):
 
     def cmake_args(self):
         spec = self.spec
+        from_variant = self.define_from_variant
         args = [
-            "-DHEPMC3_ENABLE_PYTHON={0}".format(spec.satisfies("+python")),
-            "-DHEPMC3_ENABLE_ROOTIO={0}".format(spec.satisfies("+rootio")),
-            "-DHEPMC3_INSTALL_INTERFACES={0}".format(spec.satisfies("+interfaces")),
+            from_variant("HEPMC3_ENABLE_PROTOBUF", "protobuf"),
+            from_variant("HEPMC3_ENABLE_PYTHON", "python"),
+            from_variant("HEPMC3_ENABLE_ROOTIO", "rootio"),
+            from_variant("HEPMC3_INSTALL_INTERFACES", "interfaces"),
+            self.define("HEPMC3_ENABLE_TEST", self.run_tests),
         ]
 
-        if self.spec.satisfies("+python"):
+        if "+python" in spec:
             py_ver = spec["python"].version.up_to(2)
             args.extend(
                 [
-                    "-DHEPMC3_PYTHON_VERSIONS={0}".format(py_ver),
-                    "-DHEPMC3_Python_SITEARCH{0}={1}".format(py_ver.joined, python_platlib),
+                    from_variant("HEPMC3_PYTHON_VERSIONS", str(py_ver)),
+                    from_variant("HEPMC3_Python_SITEARCH" + py_ver.joined, python_platlib),
                 ]
             )
 
-        if self.spec.satisfies("+rootio"):
-            args.append("-DROOT_DIR={0}".format(self.spec["root"].prefix))
-        args.append("-DHEPMC3_ENABLE_TEST={0}".format(self.run_tests))
+        if "+rootio" in spec:
+            args.append(self.define("ROOT_DIR", spec["root"].prefix))
+
         return args
diff --git a/var/spack/repos/builtin/packages/highfive/package.py b/var/spack/repos/builtin/packages/highfive/package.py
index 78a6f337d5806f..02f0152bef5409 100644
--- a/var/spack/repos/builtin/packages/highfive/package.py
+++ b/var/spack/repos/builtin/packages/highfive/package.py
@@ -17,6 +17,7 @@ class Highfive(CMakePackage):
     maintainers("alkino")
 
     version("develop", branch="master")
+    version("2.8.0", sha256="cd2502cae61bfb00e32dd18c9dc75289e09ad1db5c2a46d3b0eefd32e0df983b")
     version("2.7.1", sha256="25b4c51a94d1e670dc93b9b73f51e79b65d8ff49bcd6e5d5582d5ecd2789a249")
     version("2.7.0", sha256="8e05672ddf81a59ce014b1d065bd9a8c5034dbd91a5c2578e805ef880afa5907")
     version("2.6.2", sha256="ab51b9fbb49e877dd1aa7b53b4b26875f41e4e0b8ee0fc2f1d735e0d1e43d708")
diff --git a/var/spack/repos/builtin/packages/highway/package.py b/var/spack/repos/builtin/packages/highway/package.py
index a708d3f3157df9..75f5398ab29717 100644
--- a/var/spack/repos/builtin/packages/highway/package.py
+++ b/var/spack/repos/builtin/packages/highway/package.py
@@ -12,7 +12,13 @@ class Highway(CMakePackage):
     homepage = "https://github.com/google/highway"
     url = "https://github.com/google/highway/archive/refs/tags/1.0.0.tar.gz"
 
+    version("1.0.7", sha256="5434488108186c170a5e2fca5e3c9b6ef59a1caa4d520b008a9b8be6b8abe6c5")
+    version("1.0.6", sha256="d89664a045a41d822146e787bceeefbf648cc228ce354f347b18f2b419e57207")
+    version("1.0.5", sha256="99b7dad98b8fa088673b720151458fae698ae5df9154016e39de4afdc23bb927")
     version("1.0.4", sha256="faccd343935c9e98afd1016e9d20e0b8b89d908508d1af958496f8c2d3004ac2")
+    version("1.0.3", sha256="566fc77315878473d9a6bd815f7de78c73734acdcb745c3dde8579560ac5440e")
+    version("1.0.2", sha256="e8ef71236ac0d97f12d553ec1ffc5b6375d57b5f0b860c7447dd69b6ed1072db")
+    version("1.0.1", sha256="7ca6af7dc2e3e054de9e17b9dfd88609a7fd202812b1c216f43cc41647c97311")
     version("1.0.0", sha256="ab4f5f864932268356f9f6aa86f612fa4430a7db3c8de0391076750197e876b8")
 
     depends_on("cmake@3.10:", type="build")
diff --git a/var/spack/repos/builtin/packages/hiop/package.py b/var/spack/repos/builtin/packages/hiop/package.py
index 624a29859a5ac3..4f68978ab640fa 100644
--- a/var/spack/repos/builtin/packages/hiop/package.py
+++ b/var/spack/repos/builtin/packages/hiop/package.py
@@ -19,34 +19,46 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage):
 
     homepage = "https://github.com/LLNL/hiop"
     git = "https://github.com/LLNL/hiop.git"
-    maintainers("ryandanehy", "CameronRutherford", "pelesh")
+    maintainers("ryandanehy", "cameronrutherford", "pelesh")
 
     # Most recent tagged snapshot is the preferred version when profiling.
-    version("0.7.2", commit="d0f57c880d4202a72c62dd1f5c92e3bc8acb9788", submodules=True)
-    version("0.7.1", commit="8064ef6b2249ad2feca92a9d1e90060bad3eebc7", submodules=True)
-    version("0.7.0", commit="5f42ab34b419b7cf64d0fffb29d443b009dbfd75", submodules=True)
-    version("0.6.2", commit="55652fbe923ab9107d002d0d070865bd22375b28")
-    version("0.6.1", commit="a9e2697b00aa13ecf0ae4783dd8a41dee11dc50e")
-    version("0.6.0", commit="21af7eb0d6427be73546cf303abc84e834a5a55d")
-    version("0.5.4", commit="a37a7a677884e95d1c0ad37936aef3778fc91c3e")
-    version("0.5.3", commit="698e8d0fdc0ff9975d8714339ff8c782b70d85f9")
-    version("0.5.2", commit="662ad76dee1f501f648a8bec9a490cb5881789e9")
-    version("0.5.1", commit="6789bbb55824e68e428c2df1009d647af81f9cf1")
-    version("0.5.0", commit="a39da8025037c7c8ae2eb31234eb80cc73bec2af")
-    version("0.4.6", commit="b72d163d52c9225c3196ceb2baebdc7cf09a69de")
-    version("0.4.5", commit="c353580456c4776c50811b97cf8ff802dc27b90c")
-    version("0.4.4", commit="e858eefa6b914f5c87c3717bbce811931ea69386")
-    version("0.4.3", commit="c0394af4d84ebb84b7d2b95283ad65ffd84e0d45")
-    version("0.4.2", commit="3fcb788d223eec24c0241680070c4a9a5ec71ef3")
-    version("0.4.1", commit="3f269560f76d5a89bcbd1d3c4f9f0e5acaa6fd64")
-    version("0.4", commit="91d21085a1149eacdb27cd738d4a74a7e412fcff")
-    version("0.3.99.3", commit="bed1dbef260e53a9d139ccfb77d2e83a98aab216")
-    version("0.3.99.2", commit="9eb026768bc5e0a2c1293d0487cc39913001ae19")
-    version("0.3.99.1", commit="220e32c0f318665d6d394ca3cd0735b9d26a65eb")
-    version("0.3.99.0", commit="589b9c76781447108fa55788d5fa1b83ff71a3d1")
-    version("0.3", commit="7e8adae9db757aed48e5c2bc448316307598258f")
-    version("0.2", commit="c52a6f6b9baaaa2d7f233a749aa98f901349723f")
-    version("0.1", commit="5f60e11b79d532115fb41694378b54c9c707aad9")
+    version(
+        "1.0.1", tag="v1.0.1", commit="c5e156c6f27d046f590dc35114980e3f9c573ca6", submodules=True
+    )
+    version(
+        "1.0.0", tag="v1.0.0", commit="10b7d3ee0a15cb4949ccee8c905d447b9528794f", submodules=True
+    )
+    version(
+        "0.7.2", tag="v0.7.2", commit="d0f57c880d4202a72c62dd1f5c92e3bc8acb9788", submodules=True
+    )
+    version(
+        "0.7.1", tag="v0.7.1", commit="8064ef6b2249ad2feca92a9d1e90060bad3eebc7", submodules=True
+    )
+    version(
+        "0.7.0", tag="v0.7.0", commit="5f42ab34b419b7cf64d0fffb29d443b009dbfd75", submodules=True
+    )
+    version("0.6.2", tag="v0.6.2", commit="55652fbe923ab9107d002d0d070865bd22375b28")
+    version("0.6.1", tag="v0.6.1", commit="a9e2697b00aa13ecf0ae4783dd8a41dee11dc50e")
+    version("0.6.0", tag="v0.6.0", commit="21af7eb0d6427be73546cf303abc84e834a5a55d")
+    version("0.5.4", tag="v0.5.4", commit="a37a7a677884e95d1c0ad37936aef3778fc91c3e")
+    version("0.5.3", tag="v0.5.3", commit="698e8d0fdc0ff9975d8714339ff8c782b70d85f9")
+    version("0.5.2", tag="v0.5.2", commit="662ad76dee1f501f648a8bec9a490cb5881789e9")
+    version("0.5.1", tag="v0.5.1", commit="6789bbb55824e68e428c2df1009d647af81f9cf1")
+    version("0.5.0", tag="v0.5.0", commit="a39da8025037c7c8ae2eb31234eb80cc73bec2af")
+    version("0.4.6", tag="v0.4.6", commit="b72d163d52c9225c3196ceb2baebdc7cf09a69de")
+    version("0.4.5", tag="v0.4.5", commit="c353580456c4776c50811b97cf8ff802dc27b90c")
+    version("0.4.4", tag="v0.4.4", commit="e858eefa6b914f5c87c3717bbce811931ea69386")
+    version("0.4.3", tag="v0.4.3", commit="c0394af4d84ebb84b7d2b95283ad65ffd84e0d45")
+    version("0.4.2", tag="v0.4.2", commit="3fcb788d223eec24c0241680070c4a9a5ec71ef3")
+    version("0.4.1", tag="v0.4.1", commit="3f269560f76d5a89bcbd1d3c4f9f0e5acaa6fd64")
+    version("0.4", tag="v0.4", commit="91d21085a1149eacdb27cd738d4a74a7e412fcff")
+    version("0.3.99.3", tag="v0.3.99.3", commit="bed1dbef260e53a9d139ccfb77d2e83a98aab216")
+    version("0.3.99.2", tag="v0.3.99.2", commit="9eb026768bc5e0a2c1293d0487cc39913001ae19")
+    version("0.3.99.1", tag="v0.3.99.1", commit="220e32c0f318665d6d394ca3cd0735b9d26a65eb")
+    version("0.3.99.0", tag="v0.3.99.0", commit="589b9c76781447108fa55788d5fa1b83ff71a3d1")
+    version("0.3", tag="v0.3", commit="7e8adae9db757aed48e5c2bc448316307598258f")
+    version("0.2", tag="v0.2", commit="c52a6f6b9baaaa2d7f233a749aa98f901349723f")
+    version("0.1", tag="v0.1", commit="5f60e11b79d532115fb41694378b54c9c707aad9")
 
     # Development branches
     version("master", branch="master")
@@ -101,7 +113,13 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage):
         depends_on("magma@{0}:".format(magma_v), when="@{0}:+cuda".format(hiop_v))
         depends_on("magma@{0}:".format(magma_v), when="@{0}:+rocm".format(hiop_v))
 
+    # 1.0.2 fixes bug with cuda 12 compatibility
+    # hiop@0.6.0 requires cusolver API in cuda@11
+    depends_on("cuda@11:11.9", when="@0.6.0:1.0.1+cuda")
     depends_on("cuda@11:", when="@develop:+cuda")
+    # Before hiop@0.6.0 only cuda requirement was magma
+    depends_on("cuda", when="@:0.5.4+cuda")
+
     depends_on("raja", when="+raja")
     depends_on("umpire", when="+raja")
     depends_on("raja+openmp", when="+raja~cuda~rocm")
diff --git a/var/spack/repos/builtin/packages/hip-examples/0001-add-inc-and-lib-paths-to-openmp-helloworld.patch b/var/spack/repos/builtin/packages/hip-examples/0001-add-inc-and-lib-paths-to-openmp-helloworld.patch
new file mode 100644
index 00000000000000..385b2d1116f67a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hip-examples/0001-add-inc-and-lib-paths-to-openmp-helloworld.patch
@@ -0,0 +1,12 @@
+diff --git a/openmp-helloworld/Makefile b/openmp-helloworld/Makefile
+index f048d26..3e0d5ac 100644
+--- a/openmp-helloworld/Makefile
++++ b/openmp-helloworld/Makefile
+@@ -6,6 +6,8 @@ endif
+ HIPCC=$(HIP_PATH)/bin/hipcc
+ CXX=$(HIPCC)
+ CXXFLAGS =-fopenmp
++CXXFLAGS += -I$(ROCM_OPENMP_EXTRAS_DIR)/include
++CXXFLAGS += -L$(ROCM_OPENMP_EXTRAS_DIR)/lib
+
+ SOURCES = openmp_helloworld.cpp
diff --git a/var/spack/repos/builtin/packages/hip-examples/0002-add-fpic-compile-to-add4.patch b/var/spack/repos/builtin/packages/hip-examples/0002-add-fpic-compile-to-add4.patch
new file mode 100644
index 00000000000000..a858fd44e028a4
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hip-examples/0002-add-fpic-compile-to-add4.patch
@@ -0,0 +1,9 @@
+diff --git a/add4/Makefile b/add4/Makefile
+index a0c6826..d7b26b1 100644
+--- a/add4/Makefile
++++ b/add4/Makefile
+@@ -1,4 +1,4 @@
+-CXXFLAGS += -std=c++11 -O3
++CXXFLAGS += -std=c++11 -O3 -fPIC
+
+ all:  gpu-stream-hip
diff --git a/var/spack/repos/builtin/packages/hip-examples/package.py b/var/spack/repos/builtin/packages/hip-examples/package.py
new file mode 100644
index 00000000000000..c625d0fe4b76c2
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hip-examples/package.py
@@ -0,0 +1,64 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+
+
+class HipExamples(Package):
+    """Examples for HIP"""
+
+    homepage = "https://github.com/ROCm-Developer-Tools/HIP-Examples/"
+    git = "https://github.com/ROCm-Developer-Tools/HIP-Examples.git"
+    url = "https://github.com/ROCm-Developer-Tools/HIP-Examples/archive/rocm-5.4.3.tar.gz"
+    tags = ["rocm"]
+
+    maintainers("srekolam", "renjithravindrankannath", "afzpatel")
+
+    version("master", branch="master")
+
+    version("5.4.3", sha256="053b8b7892e2929e3f90bd978d8bb1c9801e4803eadd7d97fc6692ce60af1d47")
+
+    patch("0001-add-inc-and-lib-paths-to-openmp-helloworld.patch")
+    patch("0002-add-fpic-compile-to-add4.patch")
+    depends_on("hip")
+    depends_on("rocm-openmp-extras")
+
+    def install(self, spec, prefix):
+        stage = os.getcwd()
+        os.environ["ROCM_OPENMP_EXTRAS_DIR"] = self.spec["rocm-openmp-extras"].prefix
+        os.environ["LD_LIBRARY_PATH"] = self.spec["rocm-openmp-extras"].prefix.lib
+        for root, dirs, files in os.walk(stage):
+            if "Makefile" in files:
+                with working_dir(root, create=True):
+                    make()
+                    # itterate through the files in dir to find the newly built binary
+                    for file in os.listdir("."):
+                        if file not in files and os.path.isfile(file) and os.access(file, os.X_OK):
+                            install(file, join_path(prefix, file))
+                            if file == "RecursiveGaussian":
+                                install(
+                                    "RecursiveGaussian_Input.bmp",
+                                    join_path(prefix, "RecursiveGaussian_Input.bmp"),
+                                )
+
+    def test_examples(self):
+        """
+        run all hip example binaries
+        """
+
+        os.environ["LD_LIBRARY_PATH"] = self.spec["rocm-openmp-extras"].prefix.lib
+        test_dir = self.prefix
+        for file_name in os.listdir(test_dir):
+            file_path = join_path(test_dir, file_name)
+            if os.path.isfile(file_path) and os.access(file_path, os.X_OK):
+                with test_part(
+                    self,
+                    "test_example_{0}".format(file_name),
+                    purpose="run installed {0}".format(file_name),
+                ):
+                    exe = which(file_path)
+                    exe()
diff --git a/var/spack/repos/builtin/packages/hip-rocclr/package.py b/var/spack/repos/builtin/packages/hip-rocclr/package.py
index febde10d8178b0..e6a4b3bbdfe1ae 100644
--- a/var/spack/repos/builtin/packages/hip-rocclr/package.py
+++ b/var/spack/repos/builtin/packages/hip-rocclr/package.py
@@ -27,7 +27,10 @@ def url_for_version(self, version):
         return url.format(version)
 
     version("master", branch="main")
-
+    version("5.6.1", sha256="cc9a99c7e4de3d9360c0a471b27d626e84a39c9e60e0aff1e8e1500d82391819")
+    version("5.6.0", sha256="864f87323e793e60b16905284fba381a7182b960dd4a37fb67420c174442c03c")
+    version("5.5.1", sha256="1375fc7723cfaa0ae22a78682186d4804188b0a54990bfd9c0b8eb421b85e37e")
+    version("5.5.0", sha256="efbae9a1ef2ab3de5ca44091e9bb78522e76759c43524c1349114f9596cc61d1")
     version("5.4.3", sha256="71d9668619ab57ec8a4564d11860438c5aad5bd161a3e58fbc49555fbd59182d")
     version("5.4.0", sha256="46a1579310b3ab9dc8948d0fb5bed4c6b312f158ca76967af7ab69e328d43138")
     version("5.3.3", sha256="f8133a5934f9c53b253d324876d74f08a19e2f5b073bc94a62fe64b0d2183a18")
@@ -108,13 +111,6 @@ def url_for_version(self, version):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3:", type="build")
     depends_on("gl@4.5:", type="link")
     depends_on("libelf", type="link", when="@3.7.0:3.8.0")
@@ -144,6 +140,10 @@ def url_for_version(self, version):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
     ]:
         depends_on("hsakmt-roct@" + ver, when="@" + ver)
@@ -166,6 +166,10 @@ def url_for_version(self, version):
 
     # Add opencl sources thru the below
     for d_version, d_shasum in [
+        ("5.6.1", "ec26049f7d93c95050c27ba65472736665ec7a40f25920a868616b2970f6b845"),
+        ("5.6.0", "52ab260d00d279c2a86c353901ffd88ee61b934ad89e9eb480f210656705f04e"),
+        ("5.5.1", "a8a62a7c6fc5398406d2203b8cb75621a24944688e545d917033d87de2724498"),
+        ("5.5.0", "0df9fa0b8aa0c8e6711d34eec0fdf1ed356adcd9625bc8f1ce9b3e72090f3e4f"),
         ("5.4.3", "b0f8339c844a2e62773bd85cd1e7c5ecddfe71d7c8e8d604e1a1d60900c30873"),
         ("5.4.0", "a294639478e76c75dac0e094b418f9bd309309b07faf6af126cdfad9aab3c5c7"),
         ("5.3.3", "cab394e6ef16c35bab8de29a66b96a7dc0e7d1297aaacba3718fa1d369233c9f"),
diff --git a/var/spack/repos/builtin/packages/hip/0014-remove-compiler-rt-linkage-for-host.5.5.0.patch b/var/spack/repos/builtin/packages/hip/0014-remove-compiler-rt-linkage-for-host.5.5.0.patch
new file mode 100644
index 00000000000000..b04f99403e9ad6
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hip/0014-remove-compiler-rt-linkage-for-host.5.5.0.patch
@@ -0,0 +1,58 @@
+From 6df478bfeeb646ecabba503b8170ced65a5e74cf Mon Sep 17 00:00:00 2001
+From: sreenivasa murthy kolam 
+Date: Mon, 22 May 2023 10:25:48 +0000
+Subject: [PATCH] update the llvm-path and rocm-info path based on install
+ prefix for llvm-amdgpu and rocminfo;remove compiler rt builtin linkage for
+ host
+
+---
+ bin/hipcc.pl               | 10 +++++++---
+ hipamd/hip-config.cmake.in |  1 -
+ 2 files changed, 7 insertions(+), 4 deletions(-)
+
+diff --git a/bin/hipcc.pl b/bin/hipcc.pl
+index 2cd3752..2af14d2 100644
+--- a/bin/hipcc.pl
++++ b/bin/hipcc.pl
+@@ -170,11 +170,14 @@ if ($HIP_PLATFORM eq "amd") {
+     if($isWindows) {
+         $execExtension = ".exe";
+     }
+-    $HIPCC="$HIP_CLANG_PATH/clang++" . $execExtension;
++    # llvm_path is set inside the hip recipe
++    $LLVM_PATH= $ENV{'LLVM_PATH'};
++    $HIPCC="${LLVM_PATH}/bin/clang++" . $execExtension;
+ 
+     # If $HIPCC clang++ is not compiled, use clang instead
+     if ( ! -e $HIPCC ) {
+-        $HIPCC="$HIP_CLANG_PATH/clang" . $execExtension;
++	$LLVM_PATH= $ENV{'LLVM_PATH'};
++        $HIPCC="${LLVM_PATH}/bin/clang" . $execExtension;
+         $HIPLDFLAGS = "--driver-mode=g++";
+     }
+     # to avoid using dk linker or MSVC linker
+@@ -493,7 +496,8 @@ if($HIP_PLATFORM eq "amd"){
+             $targetsStr = $ENV{HCC_AMDGPU_TARGET};
+         } elsif (not $isWindows) {
+             # Else try using rocm_agent_enumerator
+-            $ROCM_AGENT_ENUM = "${ROCM_PATH}/bin/rocm_agent_enumerator";
++	    $ROCMINFO_PATH = $ENV{'ROCMINFO_PATH'} // $ROCMINFO_PATH;
++            $ROCM_AGENT_ENUM = "${ROCMINFO_PATH}/bin/rocm_agent_enumerator";
+             $targetsStr = `${ROCM_AGENT_ENUM} -t GPU`;
+             $targetsStr =~ s/\n/,/g;
+         }
+diff --git a/hipamd/hip-config.cmake.in b/hipamd/hip-config.cmake.in
+index 7c4fe7f..4f2dc19 100755
+--- a/hipamd/hip-config.cmake.in
++++ b/hipamd/hip-config.cmake.in
+@@ -253,7 +253,6 @@ if(HIP_COMPILER STREQUAL "clang")
+   # Add support for __fp16 and _Float16, explicitly link with compiler-rt
+   if( "${CLANGRT_BUILTINS_FETCH_EXIT_CODE}" STREQUAL "0" )
+     # CLANG_RT Builtins found Successfully Set interface link libraries property
+-    set_property(TARGET hip::host APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${CLANGRT_BUILTINS}")
+     set_property(TARGET hip::device APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${CLANGRT_BUILTINS}")
+   else()
+     message(STATUS "clangrt builtins lib not found: ${CLANGRT_BUILTINS_FETCH_EXIT_CODE}")
+-- 
+2.17.1
+
diff --git a/var/spack/repos/builtin/packages/hip/0014-remove-compiler-rt-linkage-for-host.5.6.0.patch b/var/spack/repos/builtin/packages/hip/0014-remove-compiler-rt-linkage-for-host.5.6.0.patch
new file mode 100644
index 00000000000000..dfca3691f1731c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hip/0014-remove-compiler-rt-linkage-for-host.5.6.0.patch
@@ -0,0 +1,75 @@
+From cd4283eab943a3018237035afea61f1b5e0042cd Mon Sep 17 00:00:00 2001
+From: Renjith Ravindran 
+Date: Wed, 27 Sep 2023 06:38:18 +0000
+Subject: [PATCH] Remove-compiler-rt-linkage-for-host
+
+---
+ clr/hipamd/CMakeLists.txt      |  6 ++++--
+ clr/hipamd/hip-config.cmake.in |  1 -
+ hipcc/bin/hipcc.pl             | 11 ++++++++---
+ 3 files changed, 12 insertions(+), 6 deletions(-)
+
+diff --git a/clr/hipamd/CMakeLists.txt b/clr/hipamd/CMakeLists.txt
+index c14a9ad..ca49f7f 100755
+--- a/clr/hipamd/CMakeLists.txt
++++ b/clr/hipamd/CMakeLists.txt
+@@ -400,8 +400,10 @@ if (NOT ${HIPCC_BIN_DIR} STREQUAL "")
+   install(PROGRAMS ${HIPCC_BIN_DIR}/hipcc.pl DESTINATION bin)
+   install(PROGRAMS ${HIPCC_BIN_DIR}/hipconfig.pl DESTINATION bin)
+   install(PROGRAMS ${HIPCC_BIN_DIR}/hipvars.pm DESTINATION bin)
+-  install(PROGRAMS ${HIPCC_BIN_DIR}/hipcc.bat DESTINATION bin)
+-  install(PROGRAMS ${HIPCC_BIN_DIR}/hipconfig.bat DESTINATION bin)
++  if(WIN32)
++    install(PROGRAMS ${HIPCC_BIN_DIR}/hipcc.bat DESTINATION bin)
++    install(PROGRAMS ${HIPCC_BIN_DIR}/hipconfig.bat DESTINATION bin)
++  endif()
+ endif()
+ 
+ #############################
+diff --git a/clr/hipamd/hip-config.cmake.in b/clr/hipamd/hip-config.cmake.in
+index 537a599..7d10273 100755
+--- a/clr/hipamd/hip-config.cmake.in
++++ b/clr/hipamd/hip-config.cmake.in
+@@ -245,7 +245,6 @@ if(HIP_COMPILER STREQUAL "clang")
+   # Add support for __fp16 and _Float16, explicitly link with compiler-rt
+   if( "${CLANGRT_BUILTINS_FETCH_EXIT_CODE}" STREQUAL "0" )
+     # CLANG_RT Builtins found Successfully Set interface link libraries property
+-    set_property(TARGET hip::host APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${CLANGRT_BUILTINS}")
+     set_property(TARGET hip::device APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${CLANGRT_BUILTINS}")
+   else()
+     message(STATUS "clangrt builtins lib not found: ${CLANGRT_BUILTINS_FETCH_EXIT_CODE}")
+diff --git a/hipcc/bin/hipcc.pl b/hipcc/bin/hipcc.pl
+index 56dcda2..c7ae60b 100755
+--- a/hipcc/bin/hipcc.pl
++++ b/hipcc/bin/hipcc.pl
+@@ -155,11 +155,15 @@ if ($HIP_PLATFORM eq "amd") {
+     if($isWindows) {
+         $execExtension = ".exe";
+     }
+-    $HIPCC="$HIP_CLANG_PATH/clang++" . $execExtension;
++    # llvm_path is set inside the hip recipe
++    $LLVM_PATH= $ENV{'LLVM_PATH'};
++    $HIPCC="${LLVM_PATH}/bin/clang++" . $execExtension;
++
+ 
+     # If $HIPCC clang++ is not compiled, use clang instead
+     if ( ! -e $HIPCC ) {
+-        $HIPCC="$HIP_CLANG_PATH/clang" . $execExtension;
++       $LLVM_PATH= $ENV{'LLVM_PATH'};
++        $HIPCC="${LLVM_PATH}/bin/clang" . $execExtension;
+         $HIPLDFLAGS = "--driver-mode=g++";
+     }
+     # to avoid using dk linker or MSVC linker
+@@ -483,7 +487,8 @@ if($HIP_PLATFORM eq "amd"){
+             $targetsStr = $ENV{HCC_AMDGPU_TARGET};
+         } elsif (not $isWindows) {
+             # Else try using rocm_agent_enumerator
+-            $ROCM_AGENT_ENUM = "${ROCM_PATH}/bin/rocm_agent_enumerator";
++            $ROCMINFO_PATH = $ENV{'ROCMINFO_PATH'} // $ROCMINFO_PATH;
++            $ROCM_AGENT_ENUM = "${ROCMINFO_PATH}/bin/rocm_agent_enumerator";
+             $targetsStr = `${ROCM_AGENT_ENUM} -t GPU`;
+             $targetsStr =~ s/\n/,/g;
+         }
+-- 
+2.31.1
+
diff --git a/var/spack/repos/builtin/packages/hip/0015-reverting-operator-mixup-fix-for-slate.patch b/var/spack/repos/builtin/packages/hip/0015-reverting-operator-mixup-fix-for-slate.patch
new file mode 100644
index 00000000000000..36bfadfe94da6b
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hip/0015-reverting-operator-mixup-fix-for-slate.patch
@@ -0,0 +1,107 @@
+From 1d7f7eb9a52af2b83d3cb06bb4fe0f31eb47ce7f Mon Sep 17 00:00:00 2001
+From: Renjith Ravindran 
+Date: Wed, 27 Sep 2023 07:07:01 +0000
+Subject: [PATCH] Reverting operator mixup fix for slate
+
+---
+ .../include/hip/amd_detail/amd_hip_complex.h  | 17 ++++------
+ .../hip/amd_detail/amd_hip_vector_types.h     | 31 +++++++++++--------
+ 2 files changed, 24 insertions(+), 24 deletions(-)
+
+diff --git a/clr/hipamd/include/hip/amd_detail/amd_hip_complex.h b/clr/hipamd/include/hip/amd_detail/amd_hip_complex.h
+index 9d9dfd5..eba6eb5 100644
+--- a/clr/hipamd/include/hip/amd_detail/amd_hip_complex.h
++++ b/clr/hipamd/include/hip/amd_detail/amd_hip_complex.h
+@@ -106,20 +106,15 @@ THE SOFTWARE.
+         return lhs;                                                                                \
+     }
+ 
+-#define COMPLEX_MUL_PREOP_OVERLOAD(type)                                                            \
+-    __HOST_DEVICE__ static inline type& operator*=(type& lhs, const type& rhs) {                    \
+-        type temp{lhs};                                                                             \
+-        lhs.x = rhs.x * temp.x - rhs.y * temp.y;                                                    \
+-        lhs.y = rhs.y * temp.x + rhs.x * temp.y;                                                    \
+-        return lhs;                                                                                 \
++#define COMPLEX_MUL_PREOP_OVERLOAD(type)                                                           \
++    __HOST_DEVICE__ static inline type& operator*=(type& lhs, const type& rhs) {               \
++        lhs = lhs * rhs;                                                                           \
++        return lhs;                                                                                \
+     }
+ 
+ #define COMPLEX_DIV_PREOP_OVERLOAD(type)                                                           \
+-    __HOST_DEVICE__ static inline type& operator/=(type& lhs, const type& rhs) {                   \
+-        type temp;                                                                                 \
+-        temp.x = (lhs.x*rhs.x + lhs.y * rhs.y) / (rhs.x*rhs.x + rhs.y*rhs.y);                      \
+-        temp.y = (lhs.y * rhs.x - lhs.x * rhs.y) / (rhs.x*rhs.x + rhs.y*rhs.y);                    \
+-        lhs = temp;                                                                                \
++    __HOST_DEVICE__ static inline type& operator/=(type& lhs, const type& rhs) {               \
++        lhs = lhs / rhs;                                                                           \
+         return lhs;                                                                                \
+     }
+ 
+diff --git a/clr/hipamd/include/hip/amd_detail/amd_hip_vector_types.h b/clr/hipamd/include/hip/amd_detail/amd_hip_vector_types.h
+index 8215fb0..dfd3b39 100644
+--- a/clr/hipamd/include/hip/amd_detail/amd_hip_vector_types.h
++++ b/clr/hipamd/include/hip/amd_detail/amd_hip_vector_types.h
+@@ -544,13 +544,6 @@ template  struct is_scalar : public integral_constant struct is_scalar : public integral_constant struct is_scalar : public integral_constant{x} -= y;
+     }
+ 
++    template
++    __HOST_DEVICE__
++    inline
++    constexpr
++    HIP_vector_type operator*(
++        const HIP_vector_type& x, const HIP_vector_type& y) noexcept
++    {
++        return HIP_vector_type{x} *= y;
++    }
+     template
+     __HOST_DEVICE__
+     inline
+@@ -741,6 +737,15 @@ template  struct is_scalar : public integral_constant{x} *= y;
+     }
+ 
++    template
++    __HOST_DEVICE__
++    inline
++    constexpr
++    HIP_vector_type operator/(
++        const HIP_vector_type& x, const HIP_vector_type& y) noexcept
++    {
++        return HIP_vector_type{x} /= y;
++    }
+     template
+     __HOST_DEVICE__
+     inline
+-- 
+2.31.1
+
diff --git a/var/spack/repos/builtin/packages/hip/0016-hip-sample-fix-hipMalloc-call.patch b/var/spack/repos/builtin/packages/hip/0016-hip-sample-fix-hipMalloc-call.patch
new file mode 100644
index 00000000000000..80e981c86aa964
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hip/0016-hip-sample-fix-hipMalloc-call.patch
@@ -0,0 +1,55 @@
+diff --git a/samples/2_Cookbook/15_static_library/device_functions/hipMain2.cpp b/samples/2_Cookbook/15_static_library/device_functions/hipMain2.cpp
+index a3c3f8f..fafbf5a 100644
+--- a/samples/2_Cookbook/15_static_library/device_functions/hipMain2.cpp
++++ b/samples/2_Cookbook/15_static_library/device_functions/hipMain2.cpp
+@@ -23,8 +23,15 @@
+ #include 
+ #include 
+ #include 
++#include 
+
+-#define HIP_ASSERT(status) assert(status == hipSuccess)
++#define HIP_ASSERT(status)                                                                         \
++  {                                                                                                \
++    if ((status != hipSuccess)) {                                                                  \
++      std::cerr << "Failed in: " << __LINE__ << " on hip call: " #status << std::endl;             \
++      throw std::runtime_error("generic failure");                                                 \
++    }                                                                                              \
++  }
+ #define LEN 512
+
+ extern __device__ int square_me(int);
+diff --git a/samples/2_Cookbook/15_static_library/host_functions/CMakeLists.txt b/samples/2_Cookbook/15_static_library/host_functions/CMakeLists.txt
+index 3c7c306..8404ac5 100644
+--- a/samples/2_Cookbook/15_static_library/host_functions/CMakeLists.txt
++++ b/samples/2_Cookbook/15_static_library/host_functions/CMakeLists.txt
+@@ -37,7 +37,7 @@ endif()
+ add_library(HipOptLibrary STATIC ${CPP_SOURCES})
+
+ # Set-up the correct flags to generate the static library.
+-target_link_libraries(HipOptLibrary PRIVATE --emit-static-lib)
++target_link_options(HipOptLibrary PRIVATE --emit-static-lib)
+ target_include_directories(HipOptLibrary PRIVATE /opt/rocm/hsa/include)
+
+ # Create test executable that uses libHipOptLibrary.a
+diff --git a/samples/2_Cookbook/15_static_library/host_functions/hipOptLibrary.cpp b/samples/2_Cookbook/15_static_library/host_functions/hipOptLibrary.cpp
+index 68f5418..7e52ce3 100644
+--- a/samples/2_Cookbook/15_static_library/host_functions/hipOptLibrary.cpp
++++ b/samples/2_Cookbook/15_static_library/host_functions/hipOptLibrary.cpp
+@@ -23,8 +23,15 @@
+ #include 
+ #include 
+ #include 
++#include 
+
+-#define HIP_ASSERT(status) assert(status == hipSuccess)
++#define HIP_ASSERT(status)                                                                         \
++  {                                                                                                \
++    if ((status != hipSuccess)) {                                                                  \
++      std::cerr << "Failed in: " << __LINE__ << " on hip call: " #status << std::endl;             \
++      throw std::runtime_error("generic failure");                                                 \
++    }                                                                                              \
++  }
+ #define LEN 512
+
+ __global__ void copy(uint32_t* A, uint32_t* B) {
diff --git a/var/spack/repos/builtin/packages/hip/package.py b/var/spack/repos/builtin/packages/hip/package.py
index 21181f693ef3f5..1200cfdd2cb72d 100644
--- a/var/spack/repos/builtin/packages/hip/package.py
+++ b/var/spack/repos/builtin/packages/hip/package.py
@@ -18,14 +18,17 @@ class Hip(CMakePackage):
 
     homepage = "https://github.com/ROCm-Developer-Tools/HIP"
     git = "https://github.com/ROCm-Developer-Tools/HIP.git"
-    url = "https://github.com/ROCm-Developer-Tools/HIP/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCm-Developer-Tools/HIP/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath", "haampie")
     libraries = ["libamdhip64"]
 
     version("master", branch="master")
-
+    version("5.6.1", sha256="4b3c4dfcf8595da0e1b8c3e8067b1ccebeaac337762ff098db14375fa8dd4487")
+    version("5.6.0", sha256="a8237768c1ae70029d972376f8d279f4de18a1e6106fff6215d1e16847bc375e")
+    version("5.5.1", sha256="1f5f6bb72d8d64335ccc8242ef2e2ea8efeb380cce2997f475b1ee77528d9fb4")
+    version("5.5.0", sha256="5b0d0253e62f85cc21d043513f7c11c64e4a4ec416159668f0b160d732d09a3c")
     version("5.4.3", sha256="23e51d3af517cd63019f8d199e46b84d5a18251d148e727f3985e8d99ccb0e58")
     version("5.4.0", sha256="e290f835d69ef23e8b5833a7e616b0a989ff89ada4412d9742430819546efc6c")
     version("5.3.3", sha256="51d4049dc37d261afb9e1270e60e112708ff06b470721ff21023e16e040e4403")
@@ -106,13 +109,6 @@ class Hip(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     variant("rocm", default=True, description="Enable ROCm support")
     variant("cuda", default=False, description="Build with CUDA")
     conflicts("+cuda +rocm", msg="CUDA and ROCm support are mutually exclusive")
@@ -120,10 +116,12 @@ class Hip(CMakePackage):
 
     depends_on("cuda", when="+cuda")
 
-    depends_on("cmake@3.16.8:", type="build", when="@4.5.0:")
+    depends_on("cmake@3.16.8:", type=("build"), when="@4.5.0:")
     depends_on("cmake@3.4.3:", type="build")
     depends_on("perl@5.10:", type=("build", "run"))
 
+    test_requires_compiler = True
+
     with when("+rocm"):
         depends_on("gl@4.5:")
         depends_on("py-cppheaderparser", type="build", when="@5.3.3:")
@@ -164,6 +162,10 @@ class Hip(CMakePackage):
             "5.3.3",
             "5.4.0",
             "5.4.3",
+            "5.5.0",
+            "5.5.1",
+            "5.6.0",
+            "5.6.1",
         ]:
             depends_on("hsakmt-roct@" + ver, when="@" + ver)
             depends_on("hsa-rocr-dev@" + ver, when="@" + ver)
@@ -172,8 +174,11 @@ class Hip(CMakePackage):
             depends_on("rocminfo@" + ver, when="@" + ver)
             depends_on("roctracer-dev-api@" + ver, when="@" + ver)
 
-        for ver in ["5.4.0", "5.4.3"]:
+        for ver in ["5.4.0", "5.4.3", "5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
             depends_on("hipify-clang", when="@" + ver)
+
+        for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+            depends_on("rocm-core@" + ver, when="@" + ver)
         # hipcc likes to add `-lnuma` by default :(
         # ref https://github.com/ROCm-Developer-Tools/HIP/pull/2202
         depends_on("numactl", when="@3.7.0:")
@@ -184,6 +189,8 @@ class Hip(CMakePackage):
 
     # Add hip-amd sources thru the below
     for d_version, d_shasum in [
+        ("5.5.1", "9c8cb7611b3a496a0e9db92269143ee33b608eb69a8384957ace04e135ac90e9"),
+        ("5.5.0", "bf87ed3919987c1a3a3f293418d26b65b3f02b97464e48f0cfcdd8f35763a0b7"),
         ("5.4.3", "475edce0f29c4ccd82e5ee21d4cce4836f2b1e3b13cbc891475e423d38a0ebb9"),
         ("5.4.0", "c4b79738eb6e669160382b6c47d738ac59bd493fc681ca400ff012a2e8212955"),
         ("5.3.3", "36acce92af39b0fa06002e164f5a7f5a9c7daa19bf96645361325775a325499d"),
@@ -211,6 +218,8 @@ class Hip(CMakePackage):
         )
     # Add opencl sources thru the below
     for d_version, d_shasum in [
+        ("5.5.1", "a8a62a7c6fc5398406d2203b8cb75621a24944688e545d917033d87de2724498"),
+        ("5.5.0", "0df9fa0b8aa0c8e6711d34eec0fdf1ed356adcd9625bc8f1ce9b3e72090f3e4f"),
         ("5.4.3", "b0f8339c844a2e62773bd85cd1e7c5ecddfe71d7c8e8d604e1a1d60900c30873"),
         ("5.4.0", "a294639478e76c75dac0e094b418f9bd309309b07faf6af126cdfad9aab3c5c7"),
         ("5.3.3", "cab394e6ef16c35bab8de29a66b96a7dc0e7d1297aaacba3718fa1d369233c9f"),
@@ -237,6 +246,8 @@ class Hip(CMakePackage):
             when="@{0}".format(d_version),
         )
     for d_version, d_shasum in [
+        ("5.5.1", "1375fc7723cfaa0ae22a78682186d4804188b0a54990bfd9c0b8eb421b85e37e"),
+        ("5.5.0", "efbae9a1ef2ab3de5ca44091e9bb78522e76759c43524c1349114f9596cc61d1"),
         ("5.4.3", "71d9668619ab57ec8a4564d11860438c5aad5bd161a3e58fbc49555fbd59182d"),
         ("5.4.0", "46a1579310b3ab9dc8948d0fb5bed4c6b312f158ca76967af7ab69e328d43138"),
         ("5.3.3", "f8133a5934f9c53b253d324876d74f08a19e2f5b073bc94a62fe64b0d2183a18"),
@@ -262,6 +273,55 @@ class Hip(CMakePackage):
             placement="rocclr",
             when="@{0}".format(d_version),
         )
+    # Add hip-clr sources thru the below
+    for d_version, d_shasum in [
+        ("5.6.1", "0b88af1e99643899d11b1c8cf8a3c46601051b328a5e0ffbd44ee88b7eb0db33"),
+        ("5.6.0", "8dcd99110737a294f67a805639cf372890c8ca16c7603caaa793e71e84478fe4"),
+    ]:
+        resource(
+            name="clr",
+            url="https://github.com/ROCm-Developer-Tools/clr/archive/refs/tags/rocm-{0}.tar.gz".format(
+                d_version
+            ),
+            sha256=d_shasum,
+            expand=True,
+            destination="",
+            placement="clr",
+            when="@{0}".format(d_version),
+        )
+
+    # Add hipcc sources thru the below
+    for d_version, d_shasum in [
+        ("5.6.1", "5800fac92b841ef6f52acda78d9bf86f83970bec0fb848a6265d239bdb7eb51a"),
+        ("5.6.0", "fdb7fdc9e4648376120330f034ee8353038d34c8a015f9eb0c208c56eeddd097"),
+    ]:
+        resource(
+            name="hipcc",
+            url="https://github.com/ROCm-Developer-Tools/HIPCC/archive/refs/tags/rocm-{0}.tar.gz".format(
+                d_version
+            ),
+            sha256=d_shasum,
+            expand=True,
+            destination="",
+            placement="hipcc",
+            when="@{0}".format(d_version),
+        )
+    # Add hiptests sources thru the below
+    for d_version, d_shasum in [
+        ("5.6.1", "5b3002ddfafda162329e4d9e6ac1200eeb48ff08e666b342aa8aeca30750f48b"),
+        ("5.6.0", "8cf4509bf9c0747dab8ed8fec1365a9156792034b517207a0b2d63270429fd2e"),
+    ]:
+        resource(
+            name="hip-tests",
+            url="https://github.com/ROCm-Developer-Tools/hip-tests/archive/refs/tags/rocm-{0}.tar.gz".format(
+                d_version
+            ),
+            sha256=d_shasum,
+            expand=True,
+            destination="",
+            placement="hip-tests",
+            when="@{0}".format(d_version),
+        )
     # Note: the ROCm ecosystem expects `lib/` and `bin/` folders with symlinks
     # in the parent directory of the package, which is incompatible with spack.
     # In hipcc the ROCM_PATH variable is used to point to the parent directory
@@ -293,7 +353,7 @@ class Hip(CMakePackage):
         ".5.2.1.patch",
         when="@5.2.1:5.2.3",
     )
-    patch("0013-remove-compiler-rt-linkage-for-host.5.3.0.patch", when="@5.3.0:")
+    patch("0013-remove-compiler-rt-linkage-for-host.5.3.0.patch", when="@5.3.0:5.4")
 
     # See https://github.com/ROCm-Developer-Tools/HIP/pull/2141
     patch("0002-Fix-detection-of-HIP_CLANG_ROOT.patch", when="@:3.9.0")
@@ -324,14 +384,26 @@ class Hip(CMakePackage):
         patch("0005-Disable-tests-4.1.0.patch", when="@4.1.0:4.3.2")
 
     patch("Add_missing_open_cl_header_file_for_4.3.0.patch", when="@4.3.0:4.3.2")
-    patch("0014-hip-test-file-reorg-5.4.0.patch", when="@5.4.0:")
+    patch("0014-hip-test-file-reorg-5.4.0.patch", when="@5.4.0:5.5")
+    patch("0016-hip-sample-fix-hipMalloc-call.patch", when="@5.4.3:5.5")
+    patch("0014-remove-compiler-rt-linkage-for-host.5.5.0.patch", when="@5.5")
+    patch("0014-remove-compiler-rt-linkage-for-host.5.6.0.patch", when="@5.6:")
+    patch("0015-reverting-operator-mixup-fix-for-slate.patch", when="@5.6:")
+    # See https://github.com/ROCm-Developer-Tools/HIP/pull/3206
+    patch(
+        "https://github.com/ROCm-Developer-Tools/HIP/commit/50ee82f6bc4aad10908ce09198c9f7ebfb2a3561.patch?full_index=1",
+        sha256="c2ee21cdc55262c7c6ba65546b5ca5f65ea89730",
+        when="@5.2:",
+    )
 
     @property
     def root_cmakelists_dir(self):
         if self.spec.satisfies("@:4.3.2"):
             return self.stage.source_path
-        else:
+        elif self.spec.satisfies("@4.5:5.5"):
             return "hipamd"
+        else:
+            return "clr"
 
     def get_paths(self):
         if self.spec.external:
@@ -377,6 +449,7 @@ def get_paths(self):
                 "llvm-amdgpu": rocm_prefix.llvm,
                 "hsa-rocr-dev": rocm_prefix.hsa,
                 "rocminfo": rocm_prefix,
+                "comgr": rocm_prefix,
                 "rocm-device-libs": rocm_prefix,
             }
 
@@ -389,6 +462,7 @@ def get_paths(self):
                 "llvm-amdgpu": self.spec["llvm-amdgpu"].prefix,
                 "hsa-rocr-dev": self.spec["hsa-rocr-dev"].prefix,
                 "rocminfo": self.spec["rocminfo"].prefix,
+                "comgr": self.spec["comgr"].prefix,
                 "rocm-device-libs": self.spec["llvm-amdgpu"].prefix,
             }
 
@@ -460,6 +534,7 @@ def set_variables(self, env):
             # hiprtcCreateProgram:
             # https://github.com/RadeonOpenCompute/ROCm-CompilerSupport/blob/rocm-4.0.0/lib/comgr/src/comgr-env.cpp
             env.set("LLVM_PATH", paths["llvm-amdgpu"])
+            env.set("COMGR_PATH", paths["comgr"])
 
             # Finally we have to set --rocm-path= ourselves, which is not
             # the same as --hip-device-lib-path (set by hipcc). It's used to set
@@ -509,13 +584,20 @@ def patch(self):
                 "hip-config.cmake.in",
                 string=True,
             )
-        if self.spec.satisfies("@5.2: +rocm"):
+        if self.spec.satisfies("@5.2:5.4 +rocm"):
             filter_file(
                 '"${ROCM_PATH}/llvm"',
                 self.spec["llvm-amdgpu"].prefix,
                 "hipamd/hip-config.cmake.in",
                 string=True,
             )
+        if self.spec.satisfies("@5.6: +rocm"):
+            filter_file(
+                '"${ROCM_PATH}/llvm"',
+                self.spec["llvm-amdgpu"].prefix,
+                "clr/hipamd/hip-config.cmake.in",
+                string=True,
+            )
 
         perl = self.spec["perl"].command
         kwargs = {"ignore_absent": False, "backup": False, "string": False}
@@ -536,13 +618,13 @@ def patch(self):
                     "roc-obj-ls",
                     "hipvars.pm",
                 ]
-            elif self.spec.satisfies("@4.5.0:"):
+            elif self.spec.satisfies("@4.5.0:5.5"):
                 files = []
-            filter_file(match, substitute, *files, **kwargs)
-            # This guy is used during the cmake phase, so we have to fix the
-            # shebang already here in case it is too long.
-            filter_shebang("hipconfig")
-        if self.spec.satisfies("@4.5.0:"):
+                filter_file(match, substitute, *files, **kwargs)
+                # This guy is used during the cmake phase, so we have to fix the
+                # shebang already here in case it is too long.
+                filter_shebang("hipconfig")
+        if self.spec.satisfies("@4.5.0:5.5"):
             perl = self.spec["perl"].command
             kwargs = {"ignore_absent": False, "backup": False, "string": False}
             with working_dir("hipamd/bin"):
@@ -550,6 +632,18 @@ def patch(self):
                 substitute = "#!{perl}".format(perl=perl)
                 files = ["roc-obj-extract", "roc-obj-ls"]
                 filter_file(match, substitute, *files, **kwargs)
+        if self.spec.satisfies("@5.6.0:"):
+            perl = self.spec["perl"].command
+            kwargs = {"ignore_absent": False, "backup": False, "string": False}
+            match = "^#!/usr/bin/perl"
+            substitute = "#!{perl}".format(perl=perl)
+            with working_dir("clr/hipamd/bin"):
+                files = ["roc-obj-extract", "roc-obj-ls"]
+                filter_file(match, substitute, *files, **kwargs)
+            with working_dir("hipcc/bin"):
+                files = []
+                filter_file(match, substitute, *files, **kwargs)
+                filter_shebang("hipconfig")
         if "@3.7.0: +rocm" in self.spec:
             numactl = self.spec["numactl"].prefix.lib
             kwargs = {"ignore_absent": False, "backup": False, "string": False}
@@ -557,7 +651,16 @@ def patch(self):
             with working_dir("bin"):
                 match = " -lnuma"
                 substitute = " -L{numactl} -lnuma".format(numactl=numactl)
-                filter_file(match, substitute, "hipcc", **kwargs)
+                if self.spec.satisfies("@4.5.0:5.5"):
+                    filter_file(match, substitute, "hipcc", **kwargs)
+        if "@5.6.0: +rocm" in self.spec:
+            numactl = self.spec["numactl"].prefix.lib
+            kwargs = {"ignore_absent": False, "backup": False, "string": False}
+
+            with working_dir("hipcc/src"):
+                match = " -lnuma"
+                substitute = " -L{numactl} -lnuma".format(numactl=numactl)
+                filter_file(match, substitute, "hipBin_amd.h", **kwargs)
 
     def flag_handler(self, name, flags):
         if name == "cxxflags" and self.spec.satisfies("@3.7.0:4.3.2"):
@@ -593,9 +696,82 @@ def cmake_args(self):
         if "@4.5.0:" in self.spec:
             args.append(self.define("HIP_COMMON_DIR", self.stage.source_path))
             args.append(self.define("HIP_CATCH_TEST", "OFF"))
-            args.append(self.define("ROCCLR_PATH", self.stage.source_path + "/rocclr"))
-            args.append(self.define("AMD_OPENCL_PATH", self.stage.source_path + "/opencl"))
+        if "@4.5.0:5.5" in self.spec:
+            args.append(self.define("ROCCLR_PATH", self.stage.source_path + "rocclr"))
+            args.append(self.define("AMD_OPENCL_PATH", self.stage.source_path + "opencl"))
         if "@5.3.0:" in self.spec:
             args.append("-DCMAKE_INSTALL_LIBDIR=lib")
-
+        if "@5.6.0:" in self.spec:
+            args.append(self.define("ROCCLR_PATH", self.stage.source_path + "/clr/rocclr"))
+            args.append(self.define("AMD_OPENCL_PATH", self.stage.source_path + "/clr/opencl"))
+            args.append(self.define("HIPCC_BIN_DIR", self.stage.source_path + "/hipcc/bin")),
+            args.append(self.define("CLR_BUILD_HIP", True)),
+            args.append(self.define("CLR_BUILD_OCL", False)),
         return args
+
+    @run_after("install")
+    def cache_test_sources(self):
+        """Copy the tests source files after the package is installed to an
+        install test subdirectory for use during `spack test run`."""
+        if self.spec.satisfies("@:5.1.0"):
+            return
+        elif self.spec.satisfies("@5.1:5.5"):
+            self.test_src_dir = "samples"
+        elif self.spec.satisfies("@5.6:"):
+            self.test_src_dir = "hip-tests/samples"
+        self.cache_extra_test_sources([self.test_src_dir])
+
+    def test_samples(self):
+        # configure, build and run all hip samples
+        if self.spec.satisfies("@:5.1.0"):
+            raise SkipTest("Test is only available for specs after version 5.1.0")
+        test_dir = join_path(self.test_suite.current_test_cache_dir, self.test_src_dir)
+        prefixes = ";".join(
+            [
+                self.spec["hip"].prefix,
+                self.spec["llvm-amdgpu"].prefix,
+                self.spec["comgr"].prefix,
+                self.spec["hsa-rocr-dev"].prefix,
+            ]
+        )
+        cc_options = ["-DCMAKE_PREFIX_PATH=" + prefixes, ".."]
+
+        amdclang_path = join_path(self.spec["llvm-amdgpu"].prefix, "bin", "amdclang++")
+        os.environ["CXX"] = amdclang_path
+        os.environ["FC"] = "/usr/bin/gfortran"
+
+        cmake = which(self.spec["cmake"].prefix.bin.cmake)
+
+        for root, dirs, files in os.walk(test_dir):
+            dirs.sort()
+            if "CMakeLists.txt" in files or "Makefile" in files:
+                with working_dir(root, create=True):
+                    head, test_name = os.path.split(root)
+                    with test_part(
+                        self,
+                        "test_sample_{0}".format(test_name),
+                        purpose="configure, build and run test: {0}".format(test_name),
+                    ):
+                        if "CMakeLists.txt" in files:
+                            print("Configuring  test " + test_name)
+                            os.mkdir("build")
+                            os.chdir("build")
+                            cmake(*cc_options)
+
+                        print("Building test " + test_name)
+                        make(parallel=False)
+                        # iterate through the files in dir to find the newly built binary
+                        for file in os.listdir("."):
+                            if (
+                                file not in files
+                                and os.path.isfile(file)
+                                and os.access(file, os.X_OK)
+                                and not file.endswith(".o")
+                            ):
+                                print("Executing test binary: " + file)
+                                exe = which(file)
+                                if file == "hipDispatchEnqueueRateMT":
+                                    options = ["16", "0"]
+                                else:
+                                    options = []
+                                exe(*options)
diff --git a/var/spack/repos/builtin/packages/hipace/package.py b/var/spack/repos/builtin/packages/hipace/package.py
index fad5e37262c08b..044d4ecfa2ad40 100644
--- a/var/spack/repos/builtin/packages/hipace/package.py
+++ b/var/spack/repos/builtin/packages/hipace/package.py
@@ -12,12 +12,14 @@ class Hipace(CMakePackage):
     """
 
     homepage = "https://hipace.readthedocs.io"
-    url = "https://github.com/Hi-PACE/hipace/archive/refs/tags/v21.09.tar.gz"
+    url = "https://github.com/Hi-PACE/hipace/archive/refs/tags/v23.07.tar.gz"
     git = "https://github.com/Hi-PACE/hipace.git"
 
     maintainers("ax3l", "MaxThevenet", "SeverinDiederichs")
 
     version("develop", branch="development")
+    version("23.07", sha256="2b1f61c91d2543d7ee360eba3630c864107e29f7bcfd0221451beea88f414f21")
+    version("23.05", sha256="33a15cfeada3ca16c2a3af1538caa7ff731df13b48b884045a0fe7974382fcd1")
     version("21.09", sha256="5d27824fe6aac47ce26ca69759140ab4d7844f9042e436c343c03ea4852825f1")
 
     variant(
@@ -37,10 +39,11 @@ class Hipace(CMakePackage):
         description="Floating point precision (single/double)",
     )
 
+    depends_on("cmake@3.18.0:", type="build", when="@23.05:")
     depends_on("cmake@3.15.0:", type="build")
-    depends_on("cuda@9.2.88:", when="compute=cuda")
     depends_on("mpi", when="+mpi")
     with when("+openpmd"):
+        depends_on("openpmd-api@0.15.1:", when="@23.05:")
         depends_on("openpmd-api@0.14.2:")
         depends_on("openpmd-api ~mpi", when="~mpi")
         depends_on("openpmd-api +mpi", when="+mpi")
@@ -54,6 +57,13 @@ class Hipace(CMakePackage):
         depends_on("fftw +mpi", when="+mpi")
         depends_on("pkgconfig", type="build")
         depends_on("llvm-openmp", when="%apple-clang")
+    with when("compute=cuda"):
+        depends_on("cuda@:12.1", when="%gcc@:12.2")
+        depends_on("cuda@:12.0", when="%gcc@:12.1")
+        depends_on("cuda@:12.1", when="%gcc@:12.2")
+        depends_on("cuda@:11.8", when="%gcc@:9")
+        depends_on("cuda@:10", when="%gcc@:8")
+        depends_on("cuda@9.2.88:")
     with when("compute=hip"):
         depends_on("rocfft")
         depends_on("rocprim")
diff --git a/var/spack/repos/builtin/packages/hipblas/package.py b/var/spack/repos/builtin/packages/hipblas/package.py
index 7212facf073227..b0261bd5db545c 100644
--- a/var/spack/repos/builtin/packages/hipblas/package.py
+++ b/var/spack/repos/builtin/packages/hipblas/package.py
@@ -14,7 +14,7 @@ class Hipblas(CMakePackage, CudaPackage, ROCmPackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/hipBLAS"
     git = "https://github.com/ROCmSoftwarePlatform/hipBLAS.git"
-    url = "https://github.com/ROCmSoftwarePlatform/hipBLAS/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/hipBLAS/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath", "haampie")
@@ -22,7 +22,10 @@ class Hipblas(CMakePackage, CudaPackage, ROCmPackage):
 
     version("develop", branch="develop")
     version("master", branch="master")
-
+    version("5.6.1", sha256="f9da82fbefc68b84081ea0ed0139b91d2a540357fcf505c7f1d57eab01eb327c")
+    version("5.6.0", sha256="9453a31324e10ba528f8f4755d2c270d0ed9baa33e980d8f8383204d8e28a563")
+    version("5.5.1", sha256="5920c9a9c83cf7e2b42d1f99f5d5091cac7f6c0a040a737e869e57b92d7045a9")
+    version("5.5.0", sha256="b080c25cb61531228d26badcdca856c46c640035c058bfc1c9f63de65f418cd5")
     version("5.4.3", sha256="5acac147aafc15c249c2f24c19459135ed68b506403aa92e602b67cfc10c38b7")
     version("5.4.0", sha256="341d61adff8d08cbf70aa07bd11a088bcd0687fc6156870a1aee9eff74f3eb4f")
     version("5.3.3", sha256="1ce093fc6bc021ad4fe0b0b93f9501038a7a5a16b0fd4fc485d65cbd220a195e")
@@ -107,6 +110,7 @@ class Hipblas(CMakePackage, CudaPackage, ROCmPackage):
     amdgpu_targets = ROCmPackage.amdgpu_targets
     variant(
         "amdgpu_target",
+        description="AMD GPU architecture",
         values=spack.variant.DisjointSetsOfValues(("auto",), ("none",), amdgpu_targets)
         .with_default("auto")
         .with_error(
@@ -119,13 +123,6 @@ class Hipblas(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("+cuda +rocm", msg="CUDA and ROCm support are mutually exclusive")
     conflicts("~cuda ~rocm", msg="CUDA or ROCm support is required")
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3.5:", type="build")
 
     depends_on("googletest@1.10.0:", type="test")
@@ -170,6 +167,10 @@ def check(self):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
         "develop",
     ]:
diff --git a/var/spack/repos/builtin/packages/hipcub/package.py b/var/spack/repos/builtin/packages/hipcub/package.py
index 95364a2a540a00..cb878d18233df7 100644
--- a/var/spack/repos/builtin/packages/hipcub/package.py
+++ b/var/spack/repos/builtin/packages/hipcub/package.py
@@ -11,11 +11,14 @@ class Hipcub(CMakePackage, CudaPackage, ROCmPackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/hipCUB"
     git = "https://github.com/ROCmSoftwarePlatform/hipCUB.git"
-    url = "https://github.com/ROCmSoftwarePlatform/hipCUB/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/hipCUB/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
-
+    version("5.6.1", sha256="4b9479daa40424c9ddbc14ce967aa170680f8ca1ed01a514e6e30ccfa22552ce")
+    version("5.6.0", sha256="5e74ddbf833f39836bf9ec6c6750348c7386a85ca67aaf9bb54d16c9e1959031")
+    version("5.5.1", sha256="ad83f3f1ed85ead9e3012906957c125a896168be913f6fb6af298228fc571480")
+    version("5.5.0", sha256="3eec838119326a67eb4cc006c706e328f3a51a01e98bbfb518df8fe4a4707e13")
     version("5.4.3", sha256="cf528d9acb4f9b9c3aad439ae76bfc3d02be6e7a74d96099544e5d54e1a23675")
     version("5.4.0", sha256="78db2c2ea466a4c5d84beedc000ae934f6d0ff1793eae90bb8d02b2dbff8932c")
     version("5.3.3", sha256="b4fc3c05892729873dc098f111c31f83af7d33da572bdb7d87de100d4c238e6d")
@@ -100,6 +103,7 @@ class Hipcub(CMakePackage, CudaPackage, ROCmPackage):
     amdgpu_targets = ROCmPackage.amdgpu_targets
     variant(
         "amdgpu_target",
+        description="AMD GPU architecture",
         values=spack.variant.DisjointSetsOfValues(("auto",), ("none",), amdgpu_targets)
         .with_default("auto")
         .with_error(
@@ -112,13 +116,6 @@ class Hipcub(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("+cuda +rocm", msg="CUDA and ROCm support are mutually exclusive")
     conflicts("~cuda ~rocm", msg="CUDA or ROCm support is required")
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3.10.2:", type="build", when="@4.2.0:")
     depends_on("cmake@3.5.1:", type="build")
 
@@ -150,6 +147,10 @@ class Hipcub(CMakePackage, CudaPackage, ROCmPackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocprim@" + ver, when="+rocm @" + ver)
         depends_on("rocm-cmake@%s:" % ver, type="build", when="@" + ver)
diff --git a/var/spack/repos/builtin/packages/hipfft/package.py b/var/spack/repos/builtin/packages/hipfft/package.py
index c0c15611c4066f..046d908e3e3c84 100644
--- a/var/spack/repos/builtin/packages/hipfft/package.py
+++ b/var/spack/repos/builtin/packages/hipfft/package.py
@@ -16,13 +16,16 @@ class Hipfft(CMakePackage, CudaPackage, ROCmPackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/hipFFT"
     git = "https://github.com/ROCmSoftwarePlatform/hipFFT.git"
-    url = "https://github.com/ROCmSoftwarePlatform/hipfft/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/hipfft/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("renjithravindrankannath", "srekolam")
 
     version("master", branch="master")
-
+    version("5.6.1", sha256="d2ae36b8eacd39b865e8a7972b8eb86bcea2de4ac90711bba7e29b39b01eaa74")
+    version("5.6.0", sha256="c7f425b693caf9371b42226d86392335d993a117d23219b6ba1fd13523cb8261")
+    version("5.5.1", sha256="3addd15a459752ad657e84c2a7b6b6289600d1d0a5f90d6e0946ba11e8148fc0")
+    version("5.5.0", sha256="47ec6f7da7346c312b80daaa8f763e86c7bdc33ac8617cfa3344068e5b20dd9e")
     version("5.4.3", sha256="ae37f40b6019a11f10646ef193716836f366d269eab3c5cc2ed09af85355b945")
     version("5.4.0", sha256="d0a8e790182928b3d19774b8db1eece9b881a422f6a7055c051b12739fded624")
     version("5.3.3", sha256="fd1662cd5b1e1bce9db53b320c0fe614179cd196251efc2ef3365d38922b5cdc")
@@ -77,6 +80,7 @@ class Hipfft(CMakePackage, CudaPackage, ROCmPackage):
     amdgpu_targets = ROCmPackage.amdgpu_targets
     variant(
         "amdgpu_target",
+        description="AMD GPU architecture",
         values=spack.variant.DisjointSetsOfValues(("auto",), ("none",), amdgpu_targets)
         .with_default("auto")
         .with_error(
@@ -89,13 +93,6 @@ class Hipfft(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("+cuda +rocm", msg="CUDA and ROCm support are mutually exclusive")
     conflicts("~cuda ~rocm", msg="CUDA or ROCm support is required")
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3.5:", type="build")
 
     depends_on("hip +cuda", when="+cuda")
@@ -118,6 +115,10 @@ class Hipfft(CMakePackage, CudaPackage, ROCmPackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocm-cmake@%s:" % ver, type="build", when="@" + ver)
         depends_on("rocfft@" + ver, when="+rocm @" + ver)
diff --git a/var/spack/repos/builtin/packages/hipfort/package.py b/var/spack/repos/builtin/packages/hipfort/package.py
index 0231fd6f56d737..da688d9c1fb868 100644
--- a/var/spack/repos/builtin/packages/hipfort/package.py
+++ b/var/spack/repos/builtin/packages/hipfort/package.py
@@ -11,11 +11,14 @@ class Hipfort(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/hipfort"
     git = "https://github.com/ROCmSoftwarePlatform/hipfort.git"
-    url = "https://github.com/ROCmSoftwarePlatform/hipfort/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/hipfort/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath")
-
+    version("5.6.1", sha256="a55345cc9ccaf0cd69d306b8eb9ec2a02c220a57e9c396443cc7273aa3377adc")
+    version("5.6.0", sha256="03176a099bc81e212ad1bf9d86f35561f8f2d21a2f126732d7620e1ea59888d5")
+    version("5.5.1", sha256="abc59f7b81cbefbe3555cbf1bf0d80e8aa65901c70799748c40870fe6f3fea60")
+    version("5.5.0", sha256="cae75ffeac129639cabebfe2f95f254c83d6c0a6cffd98142ea3537a132e42bb")
     version("5.4.3", sha256="1954a1cba351d566872ced5549b2ced7ab6332221e2b98dba3c07180dce8f173")
     version("5.4.0", sha256="a781bc6d1dbb508a4bd6cc3df931696fac6c6361d4fd35efb12c9a04a72e112c")
     version("5.3.3", sha256="593be86502578b68215ffe767c26849fd27d4dbd92c8e76762275805f99e64f5")
@@ -86,13 +89,6 @@ class Hipfort(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3.0.2:", type="build")
 
     depends_on("rocm-cmake@3.8.0:", type="build")
@@ -121,6 +117,10 @@ class Hipfort(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hip@" + ver, type="build", when="@" + ver)
 
diff --git a/var/spack/repos/builtin/packages/hipify-clang/0002-install-hipify-clang-in-bin-dir-and-llvm-clangs-head.patch b/var/spack/repos/builtin/packages/hipify-clang/0002-install-hipify-clang-in-bin-dir-and-llvm-clangs-head.patch
new file mode 100644
index 00000000000000..c2fad6d3f25780
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hipify-clang/0002-install-hipify-clang-in-bin-dir-and-llvm-clangs-head.patch
@@ -0,0 +1,13 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 80c8a3f..d2b88c0 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -137,7 +137,7 @@ install(
+ # install all folders under clang/version/ in CMAKE_INSTALL_PREFIX path
+ install(
+     DIRECTORY ${LLVM_DIR}/../../clang/${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}/
+-    DESTINATION .
++    DESTINATION ${CMAKE_INSTALL_PREFIX}/include
+     COMPONENT clang-resource-headers
+     FILES_MATCHING
+     PATTERN "*.h"
diff --git a/var/spack/repos/builtin/packages/hipify-clang/package.py b/var/spack/repos/builtin/packages/hipify-clang/package.py
index 5c65c55e9685b9..dd6b99ee716059 100644
--- a/var/spack/repos/builtin/packages/hipify-clang/package.py
+++ b/var/spack/repos/builtin/packages/hipify-clang/package.py
@@ -12,13 +12,16 @@ class HipifyClang(CMakePackage):
 
     homepage = "https://github.com/ROCm-Developer-Tools/HIPIFY"
     git = "https://github.com/ROCm-Developer-Tools/HIPIFY.git"
-    url = "https://github.com/ROCm-Developer-Tools/HIPIFY/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCm-Developer-Tools/HIPIFY/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
 
     version("master", branch="master")
-
+    version("5.6.1", sha256="ec3a4f276556f9fd924ea3c89be11b6c6ddf999cdd4387f669e38e41ee0042e8")
+    version("5.6.0", sha256="a2572037a7d3bd0813bd6819a5e6c0e911678db5fd3ab15a65370601df91891b")
+    version("5.5.1", sha256="35b9c07a7afaf9cf6f3bbe9dd147fa81b1b297af3e5e26e60c55629e83feaa48")
+    version("5.5.0", sha256="1b75c702799ac93027337f8fb61d7c27ba960e8ece60d907fc8c5ab3f15c3fe9")
     version("5.4.3", sha256="79e27bd6c0a28e6a62b02dccc0b5d88a81f69fe58487e83f3b7ab47d6b64341b")
     version("5.4.0", sha256="9f51eb280671ae7f7e14eb593ee3ef099899221c4bdccfbdb7a78681ad17f37f")
     version("5.3.3", sha256="9d08e2896e52c10a0a189a5407567043f2510adc7bf618591c97a22a23699691")
@@ -99,16 +102,11 @@ class HipifyClang(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
     # the patch was added to install the targets in the correct directory structure
     # this will fix the issue https://github.com/spack/spack/issues/30711
 
-    patch("0001-install-hipify-clang-in-bin-dir-and-llvm-clangs-head.patch", when="@5.1.0:")
+    patch("0001-install-hipify-clang-in-bin-dir-and-llvm-clangs-head.patch", when="@5.1.0:5.5")
+    patch("0002-install-hipify-clang-in-bin-dir-and-llvm-clangs-head.patch", when="@5.6:")
 
     depends_on("cmake@3.5:", type="build")
     for ver in [
@@ -135,11 +133,24 @@ class HipifyClang(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
     ]:
         depends_on("llvm-amdgpu@" + ver, when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     def setup_run_environment(self, env):
         # The installer puts the binaries directly into the prefix
         # instead of prefix/bin, so add prefix to the PATH
         env.prepend_path("PATH", self.spec.prefix)
+
+    def cmake_args(self):
+        args = []
+        if self.spec.satisfies("@5.5"):
+            args.append(self.define("SWDEV_375013", "ON"))
+        return args
diff --git a/var/spack/repos/builtin/packages/hiprand/package.py b/var/spack/repos/builtin/packages/hiprand/package.py
index 477de53b49f32c..5f85c46cb91698 100644
--- a/var/spack/repos/builtin/packages/hiprand/package.py
+++ b/var/spack/repos/builtin/packages/hiprand/package.py
@@ -14,7 +14,7 @@ class Hiprand(CMakePackage, CudaPackage, ROCmPackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/hipRAND"
     git = "https://github.com/ROCmSoftwarePlatform/hipRAND.git"
-    url = "https://github.com/ROCmSoftwarePlatform/hipRAND/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/hipRAND/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath")
@@ -22,6 +22,10 @@ class Hiprand(CMakePackage, CudaPackage, ROCmPackage):
 
     version("develop", branch="develop")
     version("master", branch="master")
+    version("5.6.1", sha256="a73d5578bc7f8dff0b8960e4bff97bc4fc28f508a19ed6acd1cfd4d3e76b47ee")
+    version("5.6.0", sha256="8c214e2f90337a5317a69950026bf337b1e567d43bb9ae64f2a802af2228c313")
+    version("5.5.1", sha256="5df9d78eae0991be5ec9f60e8d3530fabc23793d9f9cf274b075d689675db04e")
+    version("5.5.0", sha256="7c7dde7b989d5da9c0b0251233245f955b477c090462c7d34e3e0284c5fca761")
     version("5.4.3", sha256="7d3d04476880ec90c088dff81f69aac8699eaef972476000e5c4726584ffa98f")
     version("5.4.0", sha256="9456d4b4d5fd5c0b728f4aa4f8c224f829fe6fbf08e397848475293f71029a22")
     version("5.3.3", sha256="f72626b00d61ed2925b3124b7f094ccfaf7750f02bee6bac6b79317e1c5576ef")
@@ -36,6 +40,7 @@ class Hiprand(CMakePackage, CudaPackage, ROCmPackage):
     amdgpu_targets = ROCmPackage.amdgpu_targets
     variant(
         "amdgpu_target",
+        description="AMD GPU architecture",
         values=spack.variant.DisjointSetsOfValues(("auto",), ("none",), amdgpu_targets)
         .with_default("auto")
         .with_error(
@@ -73,6 +78,10 @@ class Hiprand(CMakePackage, CudaPackage, ROCmPackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
         "develop",
     ]:
diff --git a/var/spack/repos/builtin/packages/hipsolver/package.py b/var/spack/repos/builtin/packages/hipsolver/package.py
index 58c54079542696..1ef38160a65979 100644
--- a/var/spack/repos/builtin/packages/hipsolver/package.py
+++ b/var/spack/repos/builtin/packages/hipsolver/package.py
@@ -18,7 +18,7 @@ class Hipsolver(CMakePackage, CudaPackage, ROCmPackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/hipSOLVER"
     git = "https://github.com/ROCmSoftwarePlatform/hipSOLVER.git"
-    url = "https://github.com/ROCmSoftwarePlatform/hipSOLVER/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/hipSOLVER/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath")
@@ -26,7 +26,10 @@ class Hipsolver(CMakePackage, CudaPackage, ROCmPackage):
 
     version("develop", branch="develop")
     version("master", branch="master")
-
+    version("5.6.1", sha256="2e546bc7771f7bf0aa7892b69cded725941573e8b70614759c3d03c21eb78dde")
+    version("5.6.0", sha256="11fa51d210853d93d24d55b20367738e49711793412f58e8d7689710b92ae16c")
+    version("5.5.1", sha256="826bd64a4887176595bb7319d9a3612e7327602efe1f42aa3f2ad0e783d1a180")
+    version("5.5.0", sha256="0f45be0f90907381ae3e82424599e2ca2112d6411b4a64c72558d63f00409b83")
     version("5.4.3", sha256="02a1bffecc494393f49f97174db7d2c101db557d32404923a44520876e682e3a")
     version("5.4.0", sha256="d53d81c55b458ba5e6ea0ec6bd24bcc79ab06789730391da82d8c33b936339d9")
     version("5.3.3", sha256="f5a487a1c7225ab748996ac4d837ac7ab26b43618c4ed97a124f8fac1d67786e")
@@ -61,6 +64,7 @@ class Hipsolver(CMakePackage, CudaPackage, ROCmPackage):
     amdgpu_targets = ROCmPackage.amdgpu_targets
     variant(
         "amdgpu_target",
+        description="AMD GPU architecture",
         values=spack.variant.DisjointSetsOfValues(("auto",), ("none",), amdgpu_targets)
         .with_default("auto")
         .with_error(
@@ -101,6 +105,10 @@ class Hipsolver(CMakePackage, CudaPackage, ROCmPackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
         "develop",
     ]:
diff --git a/var/spack/repos/builtin/packages/hipsparse/package.py b/var/spack/repos/builtin/packages/hipsparse/package.py
index 51399c59d51860..696094cb5af0de 100644
--- a/var/spack/repos/builtin/packages/hipsparse/package.py
+++ b/var/spack/repos/builtin/packages/hipsparse/package.py
@@ -14,12 +14,15 @@ class Hipsparse(CMakePackage, CudaPackage, ROCmPackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/hipSPARSE"
     git = "https://github.com/ROCmSoftwarePlatform/hipSPARSE.git"
-    url = "https://github.com/ROCmSoftwarePlatform/hipSPARSE/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/hipSPARSE/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath", "haampie")
     libraries = ["libhipsparse"]
-
+    version("5.6.1", sha256="d636d0c5d1e38cc0c09b1e95380199ec82bd465b94bd6661f0c8d9374d9b565d")
+    version("5.6.0", sha256="3a6931b744ebaa4469a4c50d059a008403e4dc2a4f04dd69c3c6d20916b4a491")
+    version("5.5.1", sha256="3d291e4fe2c611d555e54de66149b204fe7ac59f5dd00a9ad93bc6dca0528880")
+    version("5.5.0", sha256="8122c8f17d899385de83efb7ac0d8a4fabfcd2aa21bbed63e63ea7adf0d22df6")
     version("5.4.3", sha256="b373eccd03679a13fab4e740fc780da25cbd598abca3a1e5e3613ae14954f9db")
     version("5.4.0", sha256="47420d38483c8124813b744971e428a0352c83d9b62a5a50f74ffa8f9b785b20")
     version("5.3.3", sha256="d96d0e47594ab12e8c380da2300704c105736a0771940d7d2fae666f2869e457")
@@ -104,6 +107,7 @@ class Hipsparse(CMakePackage, CudaPackage, ROCmPackage):
     amdgpu_targets = ROCmPackage.amdgpu_targets
     variant(
         "amdgpu_target",
+        description="AMD GPU architecture",
         values=spack.variant.DisjointSetsOfValues(("auto",), ("none",), amdgpu_targets)
         .with_default("auto")
         .with_error(
@@ -116,13 +120,6 @@ class Hipsparse(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("+cuda +rocm", msg="CUDA and ROCm support are mutually exclusive")
     conflicts("~cuda ~rocm", msg="CUDA or ROCm support is required")
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("hip +cuda", when="+cuda")
 
     depends_on("cmake@3.5:", type="build")
@@ -152,6 +149,10 @@ class Hipsparse(CMakePackage, CudaPackage, ROCmPackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocm-cmake@%s:" % ver, type="build", when="@" + ver)
         depends_on("rocsparse@" + ver, when="+rocm @" + ver)
@@ -163,7 +164,7 @@ class Hipsparse(CMakePackage, CudaPackage, ROCmPackage):
 
     patch("e79985dccde22d826aceb3badfc643a3227979d2.patch", when="@3.5.0")
     patch("530047af4a0f437dafc02f76b3a17e3b1536c7ec.patch", when="@3.5.0")
-    patch("0a90ddc4c33ed409a938513b9dbdca8bfad65e06.patch")
+    patch("0a90ddc4c33ed409a938513b9dbdca8bfad65e06.patch", when="@:5.4")
 
     @classmethod
     def determine_version(cls, lib):
diff --git a/var/spack/repos/builtin/packages/hipsycl/package.py b/var/spack/repos/builtin/packages/hipsycl/package.py
index e4188df8ad75aa..b6b30c2e5cfa7a 100644
--- a/var/spack/repos/builtin/packages/hipsycl/package.py
+++ b/var/spack/repos/builtin/packages/hipsycl/package.py
@@ -39,8 +39,11 @@ class Hipsycl(CMakePackage):
     depends_on("python@3:")
     depends_on("llvm@8: +clang", when="~cuda")
     depends_on("llvm@9: +clang", when="+cuda")
+    # hipSYCL 0.8.0 supported only LLVM 8-10:
+    # (https://github.com/AdaptiveCpp/AdaptiveCpp/blob/v0.8.0/CMakeLists.txt#L29-L37)
+    depends_on("llvm@8:10", when="@0.8.0")
     # https://github.com/OpenSYCL/OpenSYCL/pull/918 was introduced after 0.9.4
-    conflicts("llvm@16:", when="@:0.9.4")
+    conflicts("^llvm@16:", when="@:0.9.4")
     # LLVM PTX backend requires cuda7:10.1 (https://tinyurl.com/v82k5qq)
     depends_on("cuda@9:10.1", when="@0.8.1: +cuda ^llvm@9")
     depends_on("cuda@9:", when="@0.8.1: +cuda ^llvm@10:")
diff --git a/var/spack/repos/builtin/packages/hiptt/bugfix_make.patch b/var/spack/repos/builtin/packages/hiptt/bugfix_make.patch
new file mode 100644
index 00000000000000..aff01b8af8395e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hiptt/bugfix_make.patch
@@ -0,0 +1,18 @@
+diff --git a/Makefile b/Makefile
+index 14a7aae..c7e39f2 100755
+--- a/Makefile
++++ b/Makefile
+@@ -93,11 +93,11 @@ CUDAROOT = $(subst /bin/,,$(dir $(shell which $(CUDAC))))
+ #CFLAGS = -I${CUDAROOT}/include -std=c++11 $(DEFS) $(OPTLEV) -fPIC -D__HIP_PLATFORM_NVCC__
+ CFLAGS = -I${CUDAROOT}/include -std=c++11 $(DEFS) $(OPTLEV) -fPIC -D__HIP_PLATFORM_HCC__ -D__HIP_ROCclr__
+ ifeq ($(CPU),x86_64)
+-CFLAGS += -march=native
++CFLAGS += -march=native -fPIC
+ endif
+ 
+ #CUDA_CFLAGS = -ccbin $(GPU_CC) -I${CUDAROOT}/include -std=c++11 $(OPTLEV) -Xptxas -dlcm=ca -lineinfo $(GENCODE_FLAGS) --resource-usage -Xcompiler -fPIC -D_FORCE_INLINES -x cu -Wno-deprecated-declarations
+-CUDA_CFLAGS = --amdgpu-target=gfx906,gfx908 -std=c++11 $(OPTLEV) -D_FORCE_INLINES
++CUDA_CFLAGS = --amdgpu-target=gfx906,gfx908,gfx90a -std=c++11 $(OPTLEV) -D_FORCE_INLINES -fPIC
+ 
+ ifeq ($(OS),osx)
+ CUDA_LFLAGS = -L$(CUDAROOT)/lib
diff --git a/var/spack/repos/builtin/packages/hiptt/package.py b/var/spack/repos/builtin/packages/hiptt/package.py
new file mode 100644
index 00000000000000..d2c9e118a72849
--- /dev/null
+++ b/var/spack/repos/builtin/packages/hiptt/package.py
@@ -0,0 +1,35 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Hiptt(MakefilePackage, ROCmPackage):
+    """hipTT - Fast GPU Tensor Transpose for NVIDIA and AMD GPU."""
+
+    homepage = "https://github.com/DmitryLyakh/hipTT"
+    url = "https://github.com/DmitryLyakh/hipTT"
+    git = "https://github.com/DmitryLyakh/hipTT.git"
+    tags = ["ecp", "radiuss"]
+
+    maintainers("bvanessen")
+
+    version("master", branch="master")
+
+    patch("bugfix_make.patch")
+
+    # To enable this package add it to the LD_LIBRARY_PATH
+    def setup_dependent_build_environment(self, env, dependent_spec):
+        hiptt_home = self.spec["hiptt"].prefix
+        env.prepend_path("cuTT_ROOT", hiptt_home)
+        env.prepend_path("cuTT_LIBRARY", hiptt_home.lib)
+        env.prepend_path("cuTT_INCLUDE_PATH", hiptt_home.include)
+
+    def install(self, spec, prefix):
+        mkdir(prefix.bin)
+        install("bin/cutt_test", prefix.bin)
+        install("bin/cutt_bench", prefix.bin)
+        install_tree("lib", prefix.lib)
+        install_tree("include", prefix.include)
diff --git a/var/spack/repos/builtin/packages/hmmer/package.py b/var/spack/repos/builtin/packages/hmmer/package.py
index 4891715899e2e6..6753c331a8dc25 100644
--- a/var/spack/repos/builtin/packages/hmmer/package.py
+++ b/var/spack/repos/builtin/packages/hmmer/package.py
@@ -15,6 +15,7 @@ class Hmmer(Package):
     homepage = "http://www.hmmer.org"
     url = "http://eddylab.org/software/hmmer/hmmer-3.3.tar.gz"
 
+    version("3.4", sha256="ca70d94fd0cf271bd7063423aabb116d42de533117343a9b27a65c17ff06fbf3")
     version("3.3.2", sha256="92fee9b5efe37a5276352d3502775e7c46e9f7a0ee45a331eacb2a0cac713c69")
     version("3.3", sha256="0186bf40af67032666014971ed8ddc3cf2834bebc2be5b3bc0304a93e763736c")
     version("3.2.1", sha256="a56129f9d786ec25265774519fc4e736bbc16e4076946dcbd7f2c16efc8e2b9c")
@@ -31,7 +32,9 @@ class Hmmer(Package):
     depends_on("gsl", when="+gsl")
 
     # https://github.com/EddyRivasLab/hmmer/issues/283
-    conflicts("target=aarch64:", msg="hmmer is only available for x86_64 and PowerPC")
+    conflicts(
+        "target=aarch64:", when="@:3.3", msg="hmmer is only available for x86_64 and PowerPC"
+    )
 
     def install(self, spec, prefix):
         configure_args = ["--prefix={0}".format(prefix)]
diff --git a/var/spack/repos/builtin/packages/hoomd-blue/package.py b/var/spack/repos/builtin/packages/hoomd-blue/package.py
index a1b108e10f9160..9d58a05ac67195 100644
--- a/var/spack/repos/builtin/packages/hoomd-blue/package.py
+++ b/var/spack/repos/builtin/packages/hoomd-blue/package.py
@@ -28,8 +28,12 @@ class HoomdBlue(CMakePackage):
     # with a .git directory, causing the build to fail. As a workaround,
     # clone a specific tag from Bitbucket instead of using the tarballs.
     # https://bitbucket.org/glotzer/hoomd-blue/issues/238
-    version("2.2.2", tag="v2.2.2", submodules=True)
-    version("2.1.6", tag="v2.1.6", submodules=True)
+    version(
+        "2.2.2", tag="v2.2.2", commit="5007cf262c48aa72210bf93266aa175898ebb254", submodules=True
+    )
+    version(
+        "2.1.6", tag="v2.1.6", commit="aa650aaf13721f2abf945e868f65b806fcc54fea", submodules=True
+    )
 
     variant("mpi", default=True, description="Compile with MPI enabled")
     variant("cuda", default=True, description="Compile with CUDA Toolkit")
diff --git a/var/spack/repos/builtin/packages/hpcc/package.py b/var/spack/repos/builtin/packages/hpcc/package.py
index 4b281cf42426eb..05d08014482541 100644
--- a/var/spack/repos/builtin/packages/hpcc/package.py
+++ b/var/spack/repos/builtin/packages/hpcc/package.py
@@ -118,7 +118,10 @@ def edit(self, spec, prefix):
                 lin_alg_libs.append(join_path(spec["fftw-api"].prefix.lib, "libsfftw_mpi.so"))
                 lin_alg_libs.append(join_path(spec["fftw-api"].prefix.lib, "libsfftw.so"))
 
-            elif self.spec.variants["fft"].value == "mkl" and "^mkl" in spec:
+            elif (
+                self.spec.variants["fft"].value == "mkl"
+                and spec["fftw-api"].name in INTEL_MATH_LIBRARIES
+            ):
                 mklroot = env["MKLROOT"]
                 self.config["@LAINC@"] += " -I{0}".format(join_path(mklroot, "include/fftw"))
                 libfftw2x_cdft = join_path(
diff --git a/var/spack/repos/builtin/packages/hpctoolkit/package.py b/var/spack/repos/builtin/packages/hpctoolkit/package.py
index 1e342f326ad916..8d58956508a1d9 100644
--- a/var/spack/repos/builtin/packages/hpctoolkit/package.py
+++ b/var/spack/repos/builtin/packages/hpctoolkit/package.py
@@ -27,6 +27,8 @@ class Hpctoolkit(AutotoolsPackage):
     test_requires_compiler = True
 
     version("develop", branch="develop")
+    version("2023.08.stable", branch="release/2023.08")
+    version("2023.08.1", tag="2023.08.1", commit="753a72affd584a5e72fe153d1e8c47a394a3886e")
     version("2023.03.stable", branch="release/2023.03")
     version("2023.03.01", commit="9e0daf2ad169f6c7f6c60408475b3c2f71baebbf")
     version("2022.10.01", commit="e8a5cc87e8f5ddfd14338459a4106f8e0d162c83")
@@ -134,7 +136,7 @@ class Hpctoolkit(AutotoolsPackage):
     depends_on("xerces-c transcoder=iconv")
     depends_on("xz+pic libs=static", type="link")
     depends_on("yaml-cpp@0.7.0: +shared", when="@2022.10:")
-    depends_on("zlib+shared")
+    depends_on("zlib-api+shared")
 
     depends_on("cuda", when="+cuda")
     depends_on("oneapi-level-zero", when="+level_zero")
@@ -164,7 +166,7 @@ class Hpctoolkit(AutotoolsPackage):
     conflicts("%gcc@:4", when="@:2020", msg="hpctoolkit requires gnu gcc 5.x or later")
 
     conflicts("^binutils@2.35:2.35.1", msg="avoid binutils 2.35 and 2.35.1 (spews errors)")
-    conflicts("xz@5.2.7:5.2.8", msg="avoid xz 5.2.7:5.2.8 (broken symbol versions)")
+    conflicts("^xz@5.2.7:5.2.8", msg="avoid xz 5.2.7:5.2.8 (broken symbol versions)")
 
     conflicts("+cray", when="@2022.10.01", msg="hpcprof-mpi is not available in 2022.10.01")
     conflicts("+mpi", when="@2022.10.01", msg="hpcprof-mpi is not available in 2022.10.01")
@@ -204,7 +206,7 @@ def configure_args(self):
             "--with-libunwind=%s" % spec["libunwind"].prefix,
             "--with-xerces=%s" % spec["xerces-c"].prefix,
             "--with-lzma=%s" % spec["xz"].prefix,
-            "--with-zlib=%s" % spec["zlib"].prefix,
+            "--with-zlib=%s" % spec["zlib-api"].prefix,
         ]
 
         if spec.satisfies("@2022.10:"):
diff --git a/var/spack/repos/builtin/packages/hpcviewer/package.py b/var/spack/repos/builtin/packages/hpcviewer/package.py
index 670a28642769a7..414cf47019a013 100644
--- a/var/spack/repos/builtin/packages/hpcviewer/package.py
+++ b/var/spack/repos/builtin/packages/hpcviewer/package.py
@@ -47,6 +47,8 @@ class Hpcviewer(Package):
     maintainers("mwkrentel")
 
     darwin_sha = {
+        ("2023.07", "aarch64"): "6e3146fc3c6d778a256938a3589818ad3ac6496415f9fe27a012b6c1e7fbe766",
+        ("2023.07", "x86_64"): "0711a71d44e0323ec4a274983e63f07d13d09a41ead08427d273808326565cc9",
         ("2023.05", "aarch64"): "b34e1ebc021e91c7260cc91a888e966a81913691de04c5e972da613d0dc34294",
         ("2023.05", "x86_64"): "689c2c18f70d53a8e1f27527f65d30c61b6f70db98f63378a97f236926ef1ac5",
         ("2023.04", "aarch64"): "85fc1c8823e2ef442666d60e98674a55315771e57205a0d2cef739d39fea699f",
@@ -67,6 +69,9 @@ class Hpcviewer(Package):
     }
 
     viewer_sha = {
+        ("2023.07", "aarch64"): "641c151ed0bc5d85db40187eb39ba4bcb7a4fdeeb07d5b4d00ed6a6d457f59b4",
+        ("2023.07", "ppc64le"): "e76558377b5e64d8a07f6232468c8098d5aba32c2a6210c58bef26acd3ce8c9b",
+        ("2023.07", "x86_64"): "06db75b1aab80f1142058716ca295bb43956a2b315bd7f385ec4c3a74ade0cbb",
         ("2023.05", "aarch64"): "901b58b73890180b1cb7572d91c1b6cc205a5d3d50927c50d05d2b05554918c6",
         ("2023.05", "ppc64le"): "d948e4777aea3a0c06300aedd4ce04e28f97b3ac306f78d672a5f692152bbdef",
         ("2023.05", "x86_64"): "8c51df8b958ec600c9b7547461d7e9abb0e07a048d4031f58efd47df7ec79091",
@@ -178,6 +183,8 @@ class Hpcviewer(Package):
 
     system = platform.system().lower()
     machine = platform.machine().lower()
+    if machine == "arm64":
+        machine = "aarch64"
 
     # Versions for MacOSX / Darwin
     if system == "darwin":
@@ -193,7 +200,7 @@ class Hpcviewer(Package):
                     key[0],
                     url=viewer_url(*key),
                     sha256=viewer_sha[key],
-                    deprecated=(key[0] <= "2020.01"),
+                    deprecated=(key[0] <= "2020.99"),
                 )
 
                 # Current versions include the viewer and trace viewer in
diff --git a/var/spack/repos/builtin/packages/hpx-kokkos/package.py b/var/spack/repos/builtin/packages/hpx-kokkos/package.py
index 24b01f9da8e92f..27e88238294c6d 100644
--- a/var/spack/repos/builtin/packages/hpx-kokkos/package.py
+++ b/var/spack/repos/builtin/packages/hpx-kokkos/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import llnl.util.filesystem as fs
+
 from spack.package import *
 
 
@@ -11,9 +13,11 @@ class HpxKokkos(CMakePackage, CudaPackage, ROCmPackage):
 
     homepage = "https://github.com/STEllAR-GROUP/hpx-kokkos"
     url = "https://github.com/STEllAR-GROUP/hpx-kokkos/archive/0.0.0.tar.gz"
+    git = "https://github.com/STEllAR-GROUP/hpx-kokkos.git"
     maintainers("G-071", "msimberg")
 
-    version("master", git="https://github.com/STEllAR-GROUP/hpx-kokkos.git", branch="master")
+    version("master", branch="master")
+    version("0.4.0", sha256="dafef55521cf4bf7ab28ebad546ea1d3fb83fac3a9932e292db4ab3666cd833f")
     version("0.3.0", sha256="83c1d11dab95552ad0abdae767c71f757811d7b51d82bd231653dc942e89a45d")
     version("0.2.0", sha256="289b711cea26afe80be002fc521234c9194cd0e8f69863f3b08b654674dbe5d5")
     version("0.1.0", sha256="24edb817d0969f4aea1b68eab4984c2ea9a58f4760a9b8395e20f85b178f0850")
@@ -26,6 +30,14 @@ class HpxKokkos(CMakePackage, CudaPackage, ROCmPackage):
         description="Use the specified C++ standard when building.",
     )
 
+    future_types_map = {"polling": "event", "callback": "callback"}
+    variant(
+        "future_type",
+        default="polling",
+        values=future_types_map.keys(),
+        description="Integration type for GPU futures",
+    )
+
     depends_on("cmake@3.19:", type="build")
 
     depends_on("hpx")
@@ -42,7 +54,7 @@ class HpxKokkos(CMakePackage, CudaPackage, ROCmPackage):
 
     for cxxstd in cxxstds:
         depends_on("hpx cxxstd={0}".format(cxxstd), when="cxxstd={0}".format(cxxstd))
-        depends_on("kokkos std={0}".format(cxxstd), when="cxxstd={0}".format(cxxstd))
+        depends_on("kokkos cxxstd={0}".format(cxxstd), when="cxxstd={0}".format(cxxstd))
 
     # HPXKokkos explicitly supports CUDA and ROCm. Other GPU backends can be
     # used but without support in HPXKokkos. Other CPU backends, except Serial,
@@ -52,3 +64,29 @@ class HpxKokkos(CMakePackage, CudaPackage, ROCmPackage):
 
     depends_on("hpx +rocm", when="+rocm")
     depends_on("kokkos +rocm", when="+rocm")
+
+    def cmake_args(self):
+        spec, args = self.spec, []
+
+        args += [
+            self.define(
+                "HPX_KOKKOS_CUDA_FUTURE_TYPE",
+                self.future_types_map[spec.variants["future_type"].value],
+            ),
+            self.define("HPX_KOKKOS_ENABLE_TESTS", self.run_tests),
+            self.define("HPX_KOKKOS_ENABLE_BENCHMARKS", self.run_tests),
+        ]
+
+        if "+rocm" in self.spec:
+            args += [self.define("CMAKE_CXX_COMPILER", self.spec["hip"].hipcc)]
+
+        return args
+
+    build_directory = "spack-build"
+
+    def check(self):
+        if self.run_tests:
+            with fs.working_dir(self.build_directory):
+                cmake("--build", ".", "--target", "tests")
+                cmake("--build", ".", "--target", "benchmarks")
+                ctest("--output-on-failure")
diff --git a/var/spack/repos/builtin/packages/hpx/package.py b/var/spack/repos/builtin/packages/hpx/package.py
index f28360c2f604c9..5c0d390e590441 100644
--- a/var/spack/repos/builtin/packages/hpx/package.py
+++ b/var/spack/repos/builtin/packages/hpx/package.py
@@ -21,7 +21,8 @@ class Hpx(CMakePackage, CudaPackage, ROCmPackage):
     tags = ["e4s"]
 
     version("master", branch="master")
-    version("stable", tag="stable")
+    version("stable", tag="stable", commit="103a7b8e3719a0db948d1abde29de0ff91e070be")
+    version("1.9.1", sha256="1adae9d408388a723277290ddb33c699aa9ea72defadf3f12d4acc913a0ff22d")
     version("1.9.0", sha256="2a8dca78172fbb15eae5a5e9facf26ab021c845f9c09e61b1912e6cf9e72915a")
     version("1.8.1", sha256="2fc4c10f55e2e6bcdc6f6ff950e26c6d8e218e138fdbd885ee71ccf5c5549054")
     version("1.8.0", sha256="93f147ab7cf0ab4161f37680ea720d3baeb86540a95382f2fb591645b2a9b135")
@@ -167,15 +168,15 @@ class Hpx(CMakePackage, CudaPackage, ROCmPackage):
 
     # Certain Asio headers don't compile with nvcc from 1.17.0 onwards with
     # C++17. Starting with CUDA 11.3 they compile again.
-    conflicts("asio@1.17.0:", when="+cuda cxxstd=17 ^cuda@:11.2")
+    conflicts("^asio@1.17.0:", when="+cuda cxxstd=17 ^cuda@:11.2")
 
     # Starting from ROCm 5.0.0 hipcc miscompiles asio 1.17.0 and newer
-    conflicts("asio@1.17.0:", when="+rocm ^hip@5:")
+    conflicts("^asio@1.17.0:", when="+rocm ^hip@5:")
 
     # Boost and HIP don't work together in certain versions:
     # https://github.com/boostorg/config/issues/392. Boost 1.78.0 and HPX 1.8.0
     # both include a fix.
-    conflicts("boost@:1.77.0", when="@:1.7 +rocm")
+    conflicts("^boost@:1.77.0", when="@:1.7 +rocm")
 
     # libstdc++ has a broken valarray in some versions that clang/hipcc refuses
     # to compile:
@@ -195,8 +196,13 @@ class Hpx(CMakePackage, CudaPackage, ROCmPackage):
     # https://github.com/spack/spack/pull/17654
     # https://github.com/STEllAR-GROUP/hpx/issues/4829
     depends_on("boost+context", when="+generic_coroutines")
-    _msg_generic_coroutines = "This platform requires +generic_coroutines"
-    conflicts("~generic_coroutines", when="platform=darwin", msg=_msg_generic_coroutines)
+
+    _msg_generic_coroutines_platform = "This platform requires +generic_coroutines"
+    conflicts("~generic_coroutines", when="platform=darwin", msg=_msg_generic_coroutines_platform)
+
+    _msg_generic_coroutines_target = "This target requires +generic_coroutines"
+    conflicts("~generic_coroutines", when="target=aarch64:", msg=_msg_generic_coroutines_target)
+    conflicts("~generic_coroutines", when="target=arm:", msg=_msg_generic_coroutines_target)
 
     # Patches APEX
     patch("git_external.patch", when="@1.3.0 instrumentation=apex")
diff --git a/var/spack/repos/builtin/packages/hsa-rocr-dev/0002-Remove-explicit-RPATH-again.patch b/var/spack/repos/builtin/packages/hsa-rocr-dev/0002-Remove-explicit-RPATH-again.patch
index 58fd1e8cd94900..7d3150e40c3f29 100644
--- a/var/spack/repos/builtin/packages/hsa-rocr-dev/0002-Remove-explicit-RPATH-again.patch
+++ b/var/spack/repos/builtin/packages/hsa-rocr-dev/0002-Remove-explicit-RPATH-again.patch
@@ -1,17 +1,17 @@
-From b5a49e6de81e7a6cba86694ee5ba2486cd999976 Mon Sep 17 00:00:00 2001
-From: Harmen Stoppels 
-Date: Fri, 28 Aug 2020 18:26:54 +0200
-Subject: [PATCH] Remove explicit RPATH again
+From fb6bc54d50ec511118557bfad7f1b892adcc1a1d Mon Sep 17 00:00:00 2001
+From: Renjith Ravindran 
+Date: Tue, 10 Oct 2023 01:15:08 +0000
+Subject: [PATCH] Updating patch for the latest code
 
 ---
  src/CMakeLists.txt | 3 ---
  1 file changed, 3 deletions(-)
 
 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
-index 9de7842..66c6880 100644
+index 8fb02b1..b40c972 100644
 --- a/src/CMakeLists.txt
 +++ b/src/CMakeLists.txt
-@@ -134,9 +134,6 @@ target_include_directories( ${CORE_RUNTIME_TARGET}
+@@ -122,9 +122,6 @@ target_include_directories( ${CORE_RUNTIME_TARGET}
    ${CMAKE_CURRENT_SOURCE_DIR}
    ${CMAKE_CURRENT_SOURCE_DIR}/libamdhsacode )
  
@@ -19,8 +19,8 @@ index 9de7842..66c6880 100644
 -set_property(TARGET ${CORE_RUNTIME_TARGET} PROPERTY INSTALL_RPATH "$ORIGIN;$ORIGIN/../../lib;$ORIGIN/../../lib64;$ORIGIN/../lib64" )
 -
  ## ------------------------- Linux Compiler and Linker options -------------------------
- set ( HSA_CXX_FLAGS ${HSA_COMMON_CXX_FLAGS} -Werror -fexceptions -fno-rtti -fvisibility=hidden -Wno-error=missing-braces -Wno-error=sign-compare -Wno-sign-compare -Wno-write-strings -Wno-conversion-null -fno-math-errno -fno-threadsafe-statics -fmerge-all-constants -fms-extensions -Wno-error=comment -Wno-comment -Wno-error=pointer-arith -Wno-pointer-arith -Wno-error=unused-variable -Wno-error=unused-function )
+ set ( HSA_CXX_FLAGS ${HSA_COMMON_CXX_FLAGS} -fexceptions -fno-rtti -fvisibility=hidden -Wno-error=missing-braces -Wno-error=sign-compare -Wno-sign-compare -Wno-write-strings -Wno-conversion-null -fno-math-errno -fno-threadsafe-statics -fmerge-all-constants -fms-extensions -Wno-error=comment -Wno-comment -Wno-error=pointer-arith -Wno-pointer-arith -Wno-error=unused-variable -Wno-error=unused-function )
  
 -- 
-2.25.1
+2.31.1
 
diff --git a/var/spack/repos/builtin/packages/hsa-rocr-dev/package.py b/var/spack/repos/builtin/packages/hsa-rocr-dev/package.py
index 04d22e2ff8bf97..d6e42a83fcd640 100644
--- a/var/spack/repos/builtin/packages/hsa-rocr-dev/package.py
+++ b/var/spack/repos/builtin/packages/hsa-rocr-dev/package.py
@@ -17,14 +17,17 @@ class HsaRocrDev(CMakePackage):
 
     homepage = "https://github.com/RadeonOpenCompute/ROCR-Runtime"
     git = "https://github.com/RadeonOpenCompute/ROCR-Runtime.git"
-    url = "https://github.com/RadeonOpenCompute/ROCR-Runtime/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/ROCR-Runtime/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath", "haampie")
     libraries = ["libhsa-runtime64"]
 
     version("master", branch="master")
-
+    version("5.6.1", sha256="4de9a57c2092edf9398d671c8a2c60626eb7daf358caf710da70d9c105490221")
+    version("5.6.0", sha256="30875d440df9d8481ffb24d87755eae20a0efc1114849a72619ea954f1e9206c")
+    version("5.5.1", sha256="53d84ad5ba5086ed4ad67ad892c52c0e4eba8ddfa85c2dd341bf825f4d5fe4ee")
+    version("5.5.0", sha256="8dbc776b56f93ddaa2ca38bf3b88299b8091de7c1b3f2e481064896cf6808e6c")
     version("5.4.3", sha256="a600eed848d47a7578c60da7e64eb92f29bbce2ec67932b251eafd4c2974cb67")
     version("5.4.0", sha256="476cd18500cc227d01f6b44c00c7adc8574eb8234b6b4daefc219650183fa090")
     version("5.3.3", sha256="aca88d90f169f35bd65ce3366b8670c7cdbe3abc0a2056eab805d0192cfd7130")
@@ -105,12 +108,6 @@ class HsaRocrDev(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
     variant("shared", default=True, description="Build shared or static library")
     variant("image", default=True, description="build with or without image support")
 
@@ -148,6 +145,10 @@ class HsaRocrDev(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
     ]:
         depends_on("hsakmt-roct@" + ver, when="@" + ver)
@@ -156,10 +157,12 @@ class HsaRocrDev(CMakePackage):
         depends_on(
             "rocm-device-libs@" + ver, when="@{0} ^llvm-amdgpu ~rocm-device-libs".format(ver)
         )
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
 
     # Both 3.5.0 and 3.7.0 force INSTALL_RPATH in different ways
     patch("0001-Do-not-set-an-explicit-rpath-by-default-since-packag.patch", when="@3.5.0")
-    patch("0002-Remove-explicit-RPATH-again.patch", when="@3.7.0:")
+    patch("0002-Remove-explicit-RPATH-again.patch", when="@3.7.0:5.6")
 
     root_cmakelists_dir = "src"
 
@@ -199,4 +202,6 @@ def cmake_args(self):
 
             args.append(self.define("BITCODE_DIR", bitcode_dir))
 
+        if self.spec.satisfies("@5.6:"):
+            args.append("-DCMAKE_INSTALL_LIBDIR=lib")
         return args
diff --git a/var/spack/repos/builtin/packages/hsakmt-roct/package.py b/var/spack/repos/builtin/packages/hsakmt-roct/package.py
index 4d594406c7d7b0..444f7adcc1adb3 100644
--- a/var/spack/repos/builtin/packages/hsakmt-roct/package.py
+++ b/var/spack/repos/builtin/packages/hsakmt-roct/package.py
@@ -16,13 +16,16 @@ class HsakmtRoct(CMakePackage):
 
     homepage = "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface"
     git = "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface.git"
-    url = "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
-    maintainers("srekolam", "arjun-raj-kuppala", "renjithravindrankannath")
+    maintainers("srekolam", "renjithravindrankannath")
 
     version("master", branch="master")
-
+    version("5.6.1", sha256="d60b355bfd21a08e0e36270fd56f98d052c3c6edca47da887fa32bf32759c29b")
+    version("5.6.0", sha256="cd009c5c09f664f046c428ba9843582ab468f7b88d560747eb949d8d7f8c5567")
+    version("5.5.1", sha256="4ffde3fc1f91f24cdbf09263fd8e012a3995ad10854f4c1d866beab7b9f36bf4")
+    version("5.5.0", sha256="2b11fd8937c2b06cd4ddea2c3699fbf3d1651892c4c5957d38553b993dd9af18")
     version("5.4.3", sha256="3799abbe7177fbff3b304e2a363e2b39e8864f8650ae569b2b88b9291f9a710c")
     version("5.4.0", sha256="690a78a6e67ae2b3f518dbc4a1e267237d6a342e1063b31eef297f4a04d780f8")
     version("5.3.3", sha256="b5350de915997ed48072b37a21c2c44438028255f6cc147c25a196ad383c52e7")
@@ -103,24 +106,24 @@ class HsakmtRoct(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
     variant("shared", default=True, description="Build shared or static library")
 
     depends_on("pkgconfig", type="build", when="@4.5.0:")
     depends_on("cmake@3:", type="build")
     depends_on("numactl")
     depends_on("libdrm", when="@4.5.0:")
-    depends_on("llvm-amdgpu", type="test", when="@5.3.0:")
+
+    for ver in ["5.3.0", "5.4.0", "5.4.3"]:
+        depends_on("llvm-amdgpu@" + ver, type="test", when="@" + ver)
+
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+        depends_on("llvm-amdgpu@" + ver, type="test", when="@" + ver)
 
     # See https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/issues/72
     # and https://github.com/spack/spack/issues/28398
     patch("0001-Remove-compiler-support-libraries-and-libudev-as-req.patch", when="@4.5.0:5.2")
-    patch("0002-Remove-compiler-support-libraries-and-libudev-as-req-5.3.patch", when="@5.3.0:")
+    patch("0002-Remove-compiler-support-libraries-and-libudev-as-req-5.3.patch", when="@5.3.0:5.4")
 
     @property
     def install_targets(self):
@@ -130,7 +133,14 @@ def install_targets(self):
             return ["install"]
 
     def cmake_args(self):
-        return [self.define_from_variant("BUILD_SHARED_LIBS", "shared")]
+        args = []
+        if self.spec.satisfies("@:5.4.3"):
+            args.append(self.define_from_variant("BUILD_SHARED_LIBS", "shared"))
+        else:
+            args.append(self.define("BUILD_SHARED_LIBS", False))
+        if self.spec.satisfies("@5.4.3:"):
+            args.append("-DCMAKE_INSTALL_LIBDIR=lib")
+        return args
 
     @run_after("install")
     @on_package_attributes(run_tests=True)
@@ -148,7 +158,7 @@ def check_install(self):
                     self.spec["numactl"].prefix,
                     self.spec["pkgconfig"].prefix,
                     self.spec["llvm-amdgpu"].prefix,
-                    self.spec["zlib"].prefix,
+                    self.spec["zlib-api"].prefix,
                     self.spec["ncurses"].prefix,
                 ]
             )
diff --git a/var/spack/repos/builtin/packages/htslib/package.py b/var/spack/repos/builtin/packages/htslib/package.py
index e97639abbb3645..25caf7080655a6 100644
--- a/var/spack/repos/builtin/packages/htslib/package.py
+++ b/var/spack/repos/builtin/packages/htslib/package.py
@@ -42,7 +42,7 @@ class Htslib(AutotoolsPackage):
         description="use libdeflate for faster crc and deflate algorithms",
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2", when="@1.4:")
     depends_on("xz", when="@1.4:")
     depends_on("curl", when="@1.3:+libcurl")
diff --git a/var/spack/repos/builtin/packages/hugo/package.py b/var/spack/repos/builtin/packages/hugo/package.py
index 82f83b4cf624ee..964a69ba09fa5c 100644
--- a/var/spack/repos/builtin/packages/hugo/package.py
+++ b/var/spack/repos/builtin/packages/hugo/package.py
@@ -18,6 +18,7 @@ class Hugo(Package):
 
     maintainers("alecbcs")
 
+    version("0.118.2", sha256="915d7dcb44fba949c80858f9c2a55a11256162ba28a9067752f808cfe8faedaa")
     version("0.112.7", sha256="d706e52c74f0fb00000caf4e95b98e9d62c3536a134d5e26b433b1fa1e2a74aa")
     version("0.111.3", sha256="b6eeb13d9ed2e5d5c6895bae56480bf0fec24a564ad9d17c90ede14a7b240999")
     version("0.111.2", sha256="66500ae3a03cbf51a6ccf7404d01f42fdc454aa1eaea599c934860bbf0aa2fc5")
diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py
index d7fc90fdafdcfa..433d60a2ce7765 100644
--- a/var/spack/repos/builtin/packages/hypre/package.py
+++ b/var/spack/repos/builtin/packages/hypre/package.py
@@ -24,6 +24,7 @@ class Hypre(AutotoolsPackage, CudaPackage, ROCmPackage):
     test_requires_compiler = True
 
     version("develop", branch="master")
+    version("2.29.0", sha256="98b72115407a0e24dbaac70eccae0da3465f8f999318b2c9241631133f42d511")
     version("2.28.0", sha256="2eea68740cdbc0b49a5e428f06ad7af861d1e169ce6a12d2cf0aa2fc28c4a2ae")
     version("2.27.0", sha256="507a3d036bb1ac21a55685ae417d769dd02009bde7e09785d0ae7446b4ae1f98")
     version("2.26.0", sha256="c214084bddc61a06f3758d82947f7f831e76d7e3edeac2c78bb82d597686e05d")
@@ -75,6 +76,7 @@ class Hypre(AutotoolsPackage, CudaPackage, ROCmPackage):
     variant("gptune", default=False, description="Add the GPTune hookup code")
     variant("umpire", default=False, description="Enable Umpire support")
     variant("sycl", default=False, description="Enable SYCL support")
+    variant("magma", default=False, description="Enable MAGMA interface")
     variant("caliper", default=False, description="Enable Caliper support")
 
     # Patch to add gptune hookup codes
@@ -99,6 +101,7 @@ def patch(self):  # fix sequential compilation in 'src/seq_mv'
     depends_on("mpi", when="+mpi")
     depends_on("blas")
     depends_on("lapack")
+    depends_on("magma", when="+magma")
     depends_on("superlu-dist", when="+superlu-dist+mpi")
     depends_on("rocsparse", when="+rocm")
     depends_on("rocthrust", when="+rocm")
@@ -107,18 +110,23 @@ def patch(self):  # fix sequential compilation in 'src/seq_mv'
     depends_on("umpire", when="+umpire")
     depends_on("caliper", when="+caliper")
 
+    gpu_pkgs = ["magma", "umpire"]
     for sm_ in CudaPackage.cuda_arch_values:
-        depends_on(
-            "umpire+cuda cuda_arch={0}".format(sm_), when="+umpire+cuda cuda_arch={0}".format(sm_)
-        )
+        for pkg in gpu_pkgs:
+            depends_on(
+                "{0}+cuda cuda_arch={1}".format(pkg, sm_),
+                when="+{0}+cuda cuda_arch={1}".format(pkg, sm_),
+            )
+
     for gfx in ROCmPackage.amdgpu_targets:
-        depends_on(
-            "umpire+rocm amdgpu_target={0}".format(gfx),
-            when="+umpire+rocm amdgpu_target={0}".format(gfx),
-        )
+        for pkg in gpu_pkgs:
+            depends_on(
+                "{0}+rocm amdgpu_target={1}".format(pkg, gfx),
+                when="+{0}+rocm amdgpu_target={1}".format(pkg, gfx),
+            )
 
-    # Uses deprecated cuSPARSE functions/types (e.g. csrsv2Info_t).
-    depends_on("cuda@:11", when="+cuda")
+    # hypre@:2.28.0 uses deprecated cuSPARSE functions/types (e.g. csrsv2Info_t).
+    depends_on("cuda@:11", when="@:2.28.0+cuda")
 
     # Conflicts
     conflicts("+cuda", when="+int64")
@@ -149,6 +157,9 @@ def patch(self):  # fix sequential compilation in 'src/seq_mv'
     # Option added in v2.24.0
     conflicts("+sycl", when="@:2.23")
 
+    # Option added in v2.29.0
+    conflicts("+magma", when="@:2.28")
+
     configure_directory = "src"
 
     def url_for_version(self, version):
@@ -268,7 +279,7 @@ def configure_args(self):
             configure_args.extend(["--without-hip", "--disable-rocrand", "--disable-rocsparse"])
 
         if "+sycl" in spec:
-            configure_args.append("--with-scyl")
+            configure_args.append("--with-sycl")
             sycl_compatible_compilers = ["dpcpp", "icpx"]
             if not (os.path.basename(self.compiler.cxx) in sycl_compatible_compilers):
                 raise InstallError(
@@ -279,6 +290,11 @@ def configure_args(self):
         if "+unified-memory" in spec:
             configure_args.append("--enable-unified-memory")
 
+        if "+magma" in spec:
+            configure_args.append("--with-magma-include=%s" % spec["magma"].prefix.include)
+            configure_args.append("--with-magma-lib=%s" % spec["magma"].libs)
+            configure_args.append("--with-magma")
+
         configure_args.extend(self.enable_or_disable("fortran"))
 
         return configure_args
@@ -328,7 +344,7 @@ def install(self, spec, prefix):
 
     @run_after("install")
     def cache_test_sources(self):
-        self.cache_extra_test_sources(self.extra_install_tests)
+        cache_extra_test_sources(self, self.extra_install_tests)
 
     @property
     def _cached_tests_work_dir(self):
diff --git a/var/spack/repos/builtin/packages/icarus/package.py b/var/spack/repos/builtin/packages/icarus/package.py
index e6e39561cca82f..b290cd9fedbdfb 100644
--- a/var/spack/repos/builtin/packages/icarus/package.py
+++ b/var/spack/repos/builtin/packages/icarus/package.py
@@ -22,7 +22,7 @@ class Icarus(AutotoolsPackage):
     depends_on("flex", type="build")
     depends_on("gperf @3.0:", type="build")
     depends_on("readline @4.2:", type=("build", "link"))
-    depends_on("zlib", type=("build", "link"))
+    depends_on("zlib-api", type=("build", "link"))
 
     patch("fix-gcc-10.patch", when="@v10_3")
 
diff --git a/var/spack/repos/builtin/packages/icedtea/package.py b/var/spack/repos/builtin/packages/icedtea/package.py
index 04ed8ad572d960..f5a191e3440f32 100644
--- a/var/spack/repos/builtin/packages/icedtea/package.py
+++ b/var/spack/repos/builtin/packages/icedtea/package.py
@@ -69,7 +69,7 @@ class Icedtea(AutotoolsPackage):
     depends_on("libpng")
     depends_on("jpeg")
     depends_on("lcms")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("alsa-lib", when="platform=linux")
 
     provides("java@8", when="@3.4.0:3")
diff --git a/var/spack/repos/builtin/packages/id3lib/package.py b/var/spack/repos/builtin/packages/id3lib/package.py
index fb16fc044516b9..35395b80bb13a9 100644
--- a/var/spack/repos/builtin/packages/id3lib/package.py
+++ b/var/spack/repos/builtin/packages/id3lib/package.py
@@ -14,7 +14,7 @@ class Id3lib(AutotoolsPackage):
 
     version("3.8.3", sha256="2749cc3c0cd7280b299518b1ddf5a5bcfe2d1100614519b68702230e26c7d079")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # http://connie.slackware.com/~alien/slackbuilds/id3lib/build/id3lib-3.8.3_gcc4.diff
     # this is due to some changes in the c++ standard library headers
diff --git a/var/spack/repos/builtin/packages/idg/package.py b/var/spack/repos/builtin/packages/idg/package.py
new file mode 100644
index 00000000000000..92a23f01447c75
--- /dev/null
+++ b/var/spack/repos/builtin/packages/idg/package.py
@@ -0,0 +1,28 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Idg(CMakePackage):
+    """
+    Image Domain Gridding (IDG) is a fast method for convolutional resampling (gridding/degridding)
+    of radio astronomical data (visibilities). Direction-dependent effects (DDEs)
+    or A-tems can be applied in the gridding process.
+    """
+
+    homepage = "https://www.astron.nl/citt/IDG/"
+    git = "https://git.astron.nl/RD/idg.git"
+    url = "https://git.astron.nl/RD/idg/-/archive/1.2.0/idg-1.2.0.tar.gz"
+
+    maintainers("pelahi")
+
+    version("1.2.0", commit="ccf8951283c12547326800adae99440c70177449")
+    version("1.0.0", commit="3322756fb8b6e3bb1fe5293f3e07e40623ff8486")
+    version("0.8.1", commit="a09f3c85094c592f9304fff4c31e920c7592c3c3")
+
+    depends_on("boost")
+    depends_on("fftw-api@3")
+    depends_on("blas")
diff --git a/var/spack/repos/builtin/packages/igraph/package.py b/var/spack/repos/builtin/packages/igraph/package.py
index 5ee88ce0267a5f..62108600daf98d 100644
--- a/var/spack/repos/builtin/packages/igraph/package.py
+++ b/var/spack/repos/builtin/packages/igraph/package.py
@@ -6,12 +6,47 @@
 from spack.package import *
 
 
-class Igraph(AutotoolsPackage):
+class Igraph(CMakePackage, AutotoolsPackage):
     """igraph is a library for creating and manipulating graphs."""
 
     homepage = "https://igraph.org/"
     url = "https://github.com/igraph/igraph/releases/download/0.7.1/igraph-0.7.1.tar.gz"
 
+    version("0.10.6", sha256="99bf91ee90febeeb9a201f3e0c1d323c09214f0b5f37a4290dc3b63f52839d6d")
     version("0.7.1", sha256="d978030e27369bf698f3816ab70aa9141e9baf81c56cc4f55efbe5489b46b0df")
 
+    variant("shared", default=False, description="Enable shared build")
+
+    build_system(
+        conditional("cmake", when="@0.9:"), conditional("autotools", when="@:0.8"), default="cmake"
+    )
+
+    with when("build_system=cmake"):
+        depends_on("arpack-ng")
+        depends_on("blas")
+        depends_on("glpk+gmp@4.57:")
+        depends_on("gmp")
+        depends_on("lapack")
+
     depends_on("libxml2")
+
+    def cmake_args(self):
+        args = [
+            "-DIGRAPH_ENABLE_LTO=AUTO",
+            "-DIGRAPH_GLPK_SUPPORT=ON",
+            "-DIGRAPH_GRAPHML_SUPPORT=ON",
+            "-DIGRAPH_USE_INTERNAL_ARPACK=OFF",
+            "-DIGRAPH_USE_INTERNAL_BLAS=OFF",
+            "-DIGRAPH_USE_INTERNAL_GLPK=OFF",
+            "-DIGRAPH_USE_INTERNAL_GMP=OFF",
+            "-DIGRAPH_USE_INTERNAL_LAPACK=OFF",
+            "-DIGRAPH_USE_INTERNAL_PLFIT=ON",
+            "-DBLA_VENDOR=OpenBLAS",
+        ]
+
+        if "+shared" in self.spec:
+            args.append("-DBUILD_SHARED_LIBS=ON")
+        else:
+            args.append("-DBUILD_SHARED_LIBS=OFF")
+
+        return args
diff --git a/var/spack/repos/builtin/packages/intel-mkl/package.py b/var/spack/repos/builtin/packages/intel-mkl/package.py
index 7dd8ab41227aaa..c66235f382dae8 100644
--- a/var/spack/repos/builtin/packages/intel-mkl/package.py
+++ b/var/spack/repos/builtin/packages/intel-mkl/package.py
@@ -153,8 +153,7 @@ class IntelMkl(IntelPackage):
         multi=False,
     )
 
-    provides("blas")
-    provides("lapack")
+    provides("blas", "lapack")
     provides("lapack@3.9.0", when="@2020.4")
     provides("lapack@3.7.0", when="@11.3")
     provides("scalapack")
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-advisor/package.py b/var/spack/repos/builtin/packages/intel-oneapi-advisor/package.py
index 8bdf965b523fbb..fe2b7f34387c26 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-advisor/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-advisor/package.py
@@ -24,6 +24,12 @@ class IntelOneapiAdvisor(IntelOneApiPackage):
         "https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/advisor.html"
     )
 
+    version(
+        "2023.2.0",
+        url="https://registrationcenter-download.intel.com/akdlm//IRC_NAS/0b0e8bf2-30e4-4a26-b1ef-e369b0181b35/l_oneapi_advisor_p_2023.2.0.49489_offline.sh",
+        sha256="48ab7fa2b828a273d467c8f07efd64d6cf2fcdcfe0ff567bd1d1be7a5d5d8539",
+        expand=False,
+    )
     version(
         "2023.1.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a1ef7661-8921-4f19-ba40-2901231439f4/l_oneapi_advisor_p_2023.1.0.43480_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-ccl/package.py b/var/spack/repos/builtin/packages/intel-oneapi-ccl/package.py
index c1c90550cbf28e..2d300a55ff63ef 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-ccl/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-ccl/package.py
@@ -27,6 +27,12 @@ class IntelOneapiCcl(IntelOneApiLibraryPackage):
 
     depends_on("intel-oneapi-mpi")
 
+    version(
+        "2021.10.0",
+        url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/3230823d-f799-4d1f-8ef3-a17f086a7719/l_oneapi_ccl_p_2021.10.0.49084_offline.sh",
+        sha256="482b9a083c997df496309a40ef1293807fc0ce1c7c43ebe6be2acf568087544b",
+        expand=False,
+    )
     version(
         "2021.9.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/44e093fc-663b-4ae5-9c08-24a55211aca3/l_oneapi_ccl_p_2021.9.0.43543_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-compilers-classic/package.py b/var/spack/repos/builtin/packages/intel-oneapi-compilers-classic/package.py
index 1c29b84635d5ca..4fca5381e6f7ef 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-compilers-classic/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-compilers-classic/package.py
@@ -35,6 +35,7 @@ class IntelOneapiCompilersClassic(Package):
         "2021.7.1": "2022.2.1",
         "2021.8.0": "2023.0.0",
         "2021.9.0": "2023.1.0",
+        "2021.10.0": "2023.2.0",
     }.items():
         version(ver)
         depends_on("intel-oneapi-compilers@" + oneapi_ver, when="@" + ver, type="run")
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-compilers/package.py b/var/spack/repos/builtin/packages/intel-oneapi-compilers/package.py
index df36e993768e91..84b65a576181e0 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-compilers/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-compilers/package.py
@@ -3,11 +3,32 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import spack.compilers
 from spack.build_environment import dso_suffix
 from spack.package import *
 
 versions = [
+    {
+        "version": "2023.2.1",
+        "cpp": {
+            "url": "https://registrationcenter-download.intel.com/akdlm//IRC_NAS/ebf5d9aa-17a7-46a4-b5df-ace004227c0e/l_dpcpp-cpp-compiler_p_2023.2.1.8_offline.sh",
+            "sha256": "f5656b2f5bb5d904639e6ef1f90a2d2e760d2906e82ebc0dd387709738ca714b",
+        },
+        "ftn": {
+            "url": "https://registrationcenter-download.intel.com/akdlm//IRC_NAS/0d65c8d4-f245-4756-80c4-6712b43cf835/l_fortran-compiler_p_2023.2.1.8_offline.sh",
+            "sha256": "d4e36abc014c184698fec318a127f15a696b5333b3b0282aba1968b351207185",
+        },
+    },
+    {
+        "version": "2023.2.0",
+        "cpp": {
+            "url": "https://registrationcenter-download.intel.com/akdlm/IRC_NAS/748687b0-5a22-467c-86c6-c312fa0206b2/l_dpcpp-cpp-compiler_p_2023.2.0.49256_offline.sh",
+            "sha256": "21497b2dd2bc874794c2321561af313082725f61e3101e05a050f98b7351e08f",
+        },
+        "ftn": {
+            "url": "https://registrationcenter-download.intel.com/akdlm/IRC_NAS/237236c4-434b-4576-96ac-020ceeb22619/l_fortran-compiler_p_2023.2.0.49254_offline.sh",
+            "sha256": "37c0ad6f0013512d98e385f8708ca29b23c45fddc9ec76069f1d93663668d511",
+        },
+    },
     {
         "version": "2023.1.0",
         "cpp": {
@@ -143,16 +164,13 @@ class IntelOneapiCompilers(IntelOneApiPackage):
 
     homepage = "https://software.intel.com/content/www/us/en/develop/tools/oneapi.html"
 
-    depends_on("patchelf", type="build")
+    # See https://github.com/spack/spack/issues/39252
+    depends_on("patchelf@:0.17", type="build")
 
     # TODO: effectively gcc is a direct dependency of intel-oneapi-compilers, but we
     # cannot express that properly. For now, add conflicts for non-gcc compilers
     # instead.
-    for __compiler in spack.compilers.supported_compilers():
-        if __compiler != "gcc":
-            conflicts(
-                "%{0}".format(__compiler), msg="intel-oneapi-compilers must be installed with %gcc"
-            )
+    requires("%gcc", msg="intel-oneapi-compilers must be installed with %gcc")
 
     for v in versions:
         version(v["version"], expand=False, **v["cpp"])
@@ -208,8 +226,7 @@ def install(self, spec, prefix):
     def inject_rpaths(self):
         # Sets rpath so the compilers can work without setting LD_LIBRARY_PATH.
         patchelf = which("patchelf")
-        patchelf.add_default_arg("--set-rpath")
-        patchelf.add_default_arg(":".join(self._ld_library_path()))
+        patchelf.add_default_arg("--set-rpath", ":".join(self._ld_library_path()))
         for pd in ["bin", "lib", join_path("compiler", "lib", "intel64_lin")]:
             for file in find(self.component_prefix.linux.join(pd), "*", recursive=False):
                 # Try to patch all files, patchelf will do nothing and fail if file
@@ -253,7 +270,10 @@ def extend_config_flags(self):
             llvm_flags.append("-Wno-unused-command-line-argument")
 
         self.write_config_file(
-            common_flags + llvm_flags, self.component_prefix.linux.bin, ["icx", "icpx", "ifx"]
+            common_flags + llvm_flags, self.component_prefix.linux.bin, ["icx", "icpx"]
+        )
+        self.write_config_file(
+            common_flags + classic_flags, self.component_prefix.linux.bin, ["ifx"]
         )
         self.write_config_file(
             common_flags + classic_flags,
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-dal/package.py b/var/spack/repos/builtin/packages/intel-oneapi-dal/package.py
index bff7da51097737..29984f04381c80 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-dal/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-dal/package.py
@@ -26,6 +26,12 @@ class IntelOneapiDal(IntelOneApiLibraryPackage):
         "https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onedal.html"
     )
 
+    version(
+        "2023.2.0",
+        url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/fa218373-4b06-451f-8f4c-66b7d14b8e8b/l_daal_oneapi_p_2023.2.0.49574_offline.sh",
+        sha256="643c6b5a9d06bc82b610257645cde116dc0473935a3b969850fa72c6cb952eaf",
+        expand=False,
+    )
     version(
         "2023.1.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/c209d29f-3d06-45fb-8f04-7b2f47b93a7c/l_daal_oneapi_p_2023.1.0.46349_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-dnn/package.py b/var/spack/repos/builtin/packages/intel-oneapi-dnn/package.py
index 16738b93b8eb0b..0c8a0aeb76f513 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-dnn/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-dnn/package.py
@@ -26,6 +26,12 @@ class IntelOneapiDnn(IntelOneApiLibraryPackage):
         "https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onednn.html"
     )
 
+    version(
+        "2023.2.0",
+        url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/2d218b97-0175-4f8c-8dba-b528cec24d55/l_onednn_p_2023.2.0.49517_offline.sh",
+        sha256="96bb92b1b072e1886151b2fc0e48f27a2dc378cd92bd3f428f5166b83ae41798",
+        expand=False,
+    )
     version(
         "2023.1.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/df0fd85e-f52a-437a-8d49-be12b560607c/l_onednn_p_2023.1.0.46343_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-dpct/package.py b/var/spack/repos/builtin/packages/intel-oneapi-dpct/package.py
index 1fad5dc6120fe5..3c4fa410f598bb 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-dpct/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-dpct/package.py
@@ -19,6 +19,12 @@ class IntelOneapiDpct(IntelOneApiPackage):
 
     homepage = "https://www.intel.com/content/www/us/en/developer/tools/oneapi/dpc-compatibility-tool.html#gs.2p8km6"
 
+    version(
+        "2023.2.0",
+        url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/764119eb-2959-4b51-bb3c-3cf581c16186/l_dpcpp-ct_p_2023.2.0.49333_offline.sh",
+        sha256="3b4ca40c23a5114d4c5591694e4b43960bb55329455e86d8d876c3d5a366159b",
+        expand=False,
+    )
     version(
         "2023.1.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a9d2cc2d-9c1c-4de3-b6da-94229bbe0bca/l_dpcpp-ct_p_2023.1.0.44450_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-dpl/package.py b/var/spack/repos/builtin/packages/intel-oneapi-dpl/package.py
index 336b351cbd9e64..05282f92f4c4b4 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-dpl/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-dpl/package.py
@@ -22,6 +22,12 @@ class IntelOneapiDpl(IntelOneApiLibraryPackage):
 
     homepage = "https://github.com/oneapi-src/oneDPL"
 
+    version(
+        "2022.2.0",
+        url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/44f88a97-7526-48f0-8515-9bf1356eb7bb/l_oneDPL_p_2022.2.0.49287_offline.sh",
+        sha256="5f75e5c4e924b833a5b5d7a8cb812469d524a3ca4bda68c8ac850484dc0afd23",
+        expand=False,
+    )
     version(
         "2022.1.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/64075e93-4134-4d18-8941-827b71b7d8b9/l_oneDPL_p_2022.1.0.43490_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-inspector/package.py b/var/spack/repos/builtin/packages/intel-oneapi-inspector/package.py
index 9a0eb5538ef833..0c1e0f79abaf31 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-inspector/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-inspector/package.py
@@ -24,6 +24,12 @@ class IntelOneapiInspector(IntelOneApiPackage):
 
     homepage = "https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/inspector.html"
 
+    version(
+        "2023.2.0",
+        url="https://registrationcenter-download.intel.com/akdlm//IRC_NAS/2a99eafd-5109-41a1-9762-aee0c7ecbeb7/l_inspector_oneapi_p_2023.2.0.49304_offline.sh",
+        sha256="36b2ca94e5a69b68cbf9cbfde0a8e1aa58d02fb03f4d81db69769c96c20d4130",
+        expand=False,
+    )
     version(
         "2023.1.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/5e922b71-d701-4a88-b447-eb88fcb630e2/l_inspector_oneapi_p_2023.1.0.43486_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-ipp/package.py b/var/spack/repos/builtin/packages/intel-oneapi-ipp/package.py
index 05ae9499ce3e47..c29fb423b21a2f 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-ipp/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-ipp/package.py
@@ -27,6 +27,12 @@ class IntelOneapiIpp(IntelOneApiLibraryPackage):
         "https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/ipp.html"
     )
 
+    version(
+        "2021.9.0",
+        url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/616a3fba-4ab6-4317-a17b-2be4b737fc37/l_ipp_oneapi_p_2021.9.0.49454_offline.sh",
+        sha256="2c6e03dea143b6e508f5ff5f2dffb03a9d64b980453575e4a028ecd2c6aebbfe",
+        expand=False,
+    )
     version(
         "2021.8.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/732392fa-41b3-4a92-935e-6a2b823162a7/l_ipp_oneapi_p_2021.8.0.46345_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-ippcp/package.py b/var/spack/repos/builtin/packages/intel-oneapi-ippcp/package.py
index 3ca2c66a2237ab..4d0d6fe3c20733 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-ippcp/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-ippcp/package.py
@@ -28,6 +28,12 @@ class IntelOneapiIppcp(IntelOneApiLibraryPackage):
         "https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/ipp.html"
     )
 
+    version(
+        "2021.8.0",
+        url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/f488397a-bd8f-449f-9127-04de8426aa35/l_ippcp_oneapi_p_2021.8.0.49493_offline.sh",
+        sha256="ac380d98dc9a12007f11537a1a57a848d4ccb251c4773608b088cf677e72c6d8",
+        expand=False,
+    )
     version(
         "2021.7.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/3697d9d0-907f-40d4-a2a7-7d83c45b72cb/l_ippcp_oneapi_p_2021.7.0.43492_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-itac/package.py b/var/spack/repos/builtin/packages/intel-oneapi-itac/package.py
index a233cd15e81a13..3b53af927348de 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-itac/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-itac/package.py
@@ -27,6 +27,12 @@ class IntelOneapiItac(IntelOneApiPackage):
 
     maintainers("rscohn2")
 
+    version(
+        "2021.10.0",
+        url="https://registrationcenter-download.intel.com/akdlm//IRC_NAS/226adf12-b7f6-407e-95a9-8e9ab76d7631/l_itac_oneapi_p_2021.10.0.14_offline.sh",
+        sha256="cfff2ee19c793b64074b5490a16acbe8c9767f41d391d7c71c0004fdcec501c7",
+        expand=False,
+    )
     version(
         "2021.8.0",
         url="https://registrationcenter-download.intel.com/akdlm/irc_nas/19129/l_itac_oneapi_p_2021.8.0.25341_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-mkl/package.py b/var/spack/repos/builtin/packages/intel-oneapi-mkl/package.py
index 9f4b7c6b98cca8..db3fdd6d7ea8c0 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-mkl/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-mkl/package.py
@@ -25,6 +25,12 @@ class IntelOneapiMkl(IntelOneApiLibraryPackage):
         "https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html"
     )
 
+    version(
+        "2023.2.0",
+        url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/adb8a02c-4ee7-4882-97d6-a524150da358/l_onemkl_p_2023.2.0.49497_offline.sh",
+        sha256="4a0d93da85a94d92e0ad35dc0fc3b3ab7f040bd55ad374c4d5ec81a57a2b872b",
+        expand=False,
+    )
     version(
         "2023.1.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/cd17b7fe-500e-4305-a89b-bd5b42bfd9f8/l_onemkl_p_2023.1.0.46342_offline.sh",
@@ -97,6 +103,14 @@ class IntelOneapiMkl(IntelOneApiLibraryPackage):
     variant(
         "cluster", default=False, description="Build with cluster support: scalapack, blacs, etc"
     )
+    variant(
+        "mpi_family",
+        default="none",
+        values=("none", "mpich", "openmpi"),
+        description="MPI family",
+        multi=False,
+    )
+
     variant(
         "threads",
         default="none",
@@ -112,8 +126,7 @@ class IntelOneapiMkl(IntelOneApiLibraryPackage):
     provides("fftw-api@3")
     provides("scalapack", when="+cluster")
     provides("mkl")
-    provides("lapack")
-    provides("blas")
+    provides("lapack", "blas")
 
     @property
     def component_dir(self):
@@ -135,21 +148,6 @@ def libs(self):
         else:
             return IntelOneApiStaticLibraryList(libs, system_libs)
 
-    def setup_run_environment(self, env):
-        super().setup_run_environment(env)
-
-        # Support RPATH injection to the library directories when the '-mkl' or '-qmkl'
-        # flag of the Intel compilers are used outside the Spack build environment. We
-        # should not try to take care of other compilers because the users have to
-        # provide the linker flags anyway and are expected to take care of the RPATHs
-        # flags too. We prefer the __INTEL_POST_CFLAGS/__INTEL_POST_FFLAGS flags over
-        # the PRE ones so that any other RPATHs provided by the users on the command
-        # line come before and take precedence over the ones we inject here.
-        for d in self._find_mkl_libs(self.spec.satisfies("+shared")).directories:
-            flag = "-Wl,-rpath,{0}".format(d)
-            env.append_path("__INTEL_POST_CFLAGS", flag, separator=" ")
-            env.append_path("__INTEL_POST_FFLAGS", flag, separator=" ")
-
     def setup_dependent_build_environment(self, env, dependent_spec):
         # Only if environment modifications are desired (default is +envmods)
         if self.spec.satisfies("+envmods"):
@@ -178,16 +176,25 @@ def _find_mkl_libs(self, shared):
         if self.spec.satisfies("+cluster"):
             if any(
                 self.spec.satisfies(m)
-                for m in ["^intel-oneapi-mpi", "^intel-mpi", "^mpich", "^cray-mpich"]
+                for m in [
+                    "^intel-oneapi-mpi",
+                    "^intel-mpi",
+                    "^mpich",
+                    "^cray-mpich",
+                    "mpi_family=mpich",
+                ]
             ):
                 libs.append(self._xlp64_lib("libmkl_blacs_intelmpi"))
-            elif self.spec.satisfies("^openmpi"):
+            elif any(
+                self.spec.satisfies(m) for m in ["^openmpi", "^hpcx-mpi", "mpi_family=openmpi"]
+            ):
                 libs.append(self._xlp64_lib("libmkl_blacs_openmpi"))
             else:
                 raise RuntimeError(
                     (
-                        "intel-oneapi-mpi +cluster requires one of "
-                        "^intel-oneapi-mpi, ^intel-mpi, ^mpich, or ^openmpi"
+                        "intel-oneapi-mkl +cluster requires one of ^intel-oneapi-mpi, "
+                        "^intel-mpi, ^mpich, ^cray-mpich, mpi_family=mpich, ^openmpi, "
+                        "^hpcx-mpi, or mpi_family=openmpi"
                     )
                 )
 
@@ -197,9 +204,14 @@ def _find_mkl_libs(self, shared):
         resolved_libs = find_libraries(libs, lib_path, shared=shared)
         # Add MPI libraries for cluster support. If MPI is not in the
         # spec, then MKL is externally installed and application must
-        # link with MPI libaries
-        if self.spec.satisfies("+cluster ^mpi"):
-            resolved_libs = resolved_libs + self.spec["mpi"].libs
+        # link with MPI libaries. If MPI is in spec, but there are no
+        # libraries, then the package (e.g. hpcx-mpi) relies on the
+        # compiler wrapper to add the libraries.
+        try:
+            if self.spec.satisfies("+cluster ^mpi"):
+                resolved_libs = resolved_libs + self.spec["mpi"].libs
+        except spack.error.NoLibrariesError:
+            pass
         return resolved_libs
 
     def _xlp64_lib(self, lib):
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-mpi/package.py b/var/spack/repos/builtin/packages/intel-oneapi-mpi/package.py
index 41a95ac21367e7..fab6c8ca1f466b 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-mpi/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-mpi/package.py
@@ -21,6 +21,12 @@ class IntelOneapiMpi(IntelOneApiLibraryPackage):
 
     homepage = "https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/mpi-library.html"
 
+    version(
+        "2021.10.0",
+        url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/4f5871da-0533-4f62-b563-905edfb2e9b7/l_mpi_oneapi_p_2021.10.0.49374_offline.sh",
+        sha256="ab2e97d87b139201a2e7dab9a61ac6e8927b7783b459358c4ad69a1b1c064f40",
+        expand=False,
+    )
     version(
         "2021.9.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/718d6f8f-2546-4b36-b97b-bc58d5482ebf/l_mpi_oneapi_p_2021.9.0.43482_offline.sh",
@@ -105,6 +111,13 @@ class IntelOneapiMpi(IntelOneApiLibraryPackage):
     def component_dir(self):
         return "mpi"
 
+    @property
+    def env_script_args(self):
+        if "+external-libfabric" in self.spec:
+            return ("-i_mpi_ofi_internal=0",)
+        else:
+            return ()
+
     def setup_dependent_package(self, module, dep_spec):
         if "+generic-names" in self.spec:
             self.spec.mpicc = join_path(self.component_prefix.bin, "mpicc")
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-tbb/package.py b/var/spack/repos/builtin/packages/intel-oneapi-tbb/package.py
index 4b08a1b9895f87..6e169cbd84fe56 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-tbb/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-tbb/package.py
@@ -22,6 +22,12 @@ class IntelOneapiTbb(IntelOneApiLibraryPackage):
         "https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onetbb.html"
     )
 
+    version(
+        "2021.10.0",
+        url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/c95cd995-586b-4688-b7e8-2d4485a1b5bf/l_tbb_oneapi_p_2021.10.0.49543_offline.sh",
+        sha256="a10d319e67b6904d6199f8294e970124a064d9948cf7e2b5ebab94499aadc6ca",
+        expand=False,
+    )
     version(
         "2021.9.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/7dcd261b-12fa-418a-b61b-b3dd4d597466/l_tbb_oneapi_p_2021.9.0.43484_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-oneapi-vtune/package.py b/var/spack/repos/builtin/packages/intel-oneapi-vtune/package.py
index 8f5805aaf13f12..c2ed3f164e42af 100644
--- a/var/spack/repos/builtin/packages/intel-oneapi-vtune/package.py
+++ b/var/spack/repos/builtin/packages/intel-oneapi-vtune/package.py
@@ -25,6 +25,12 @@ class IntelOneapiVtune(IntelOneApiPackage):
 
     homepage = "https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/vtune-profiler.html"
 
+    version(
+        "2023.2.0",
+        url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/dfae6f23-6c90-4b9f-80e2-fa2a5037fe36/l_oneapi_vtune_p_2023.2.0.49485_offline.sh",
+        sha256="482a727afe0ac6f81eff51503857c28fcb79ffdba76260399900f3397fd0adbd",
+        expand=False,
+    )
     version(
         "2023.1.0",
         url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/4466ed1b-5d4a-4b30-9146-1eabc336c647/l_oneapi_vtune_p_2023.1.0.44286_offline.sh",
diff --git a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py
index 84810bacfa3370..50e7021de85d41 100644
--- a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py
+++ b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py
@@ -536,8 +536,7 @@ class IntelParallelStudio(IntelPackage):
     provides("ipp", when="+ipp")
 
     provides("mkl", when="+mkl")
-    provides("blas", when="+mkl")
-    provides("lapack", when="+mkl")
+    provides("blas", "lapack", when="+mkl")
     provides("scalapack", when="+mkl")
 
     provides("fftw-api@3", when="+mkl@professional.2017:")
diff --git a/var/spack/repos/builtin/packages/intel-tbb/gcc_13-2021.patch b/var/spack/repos/builtin/packages/intel-tbb/gcc_13-2021-v2.patch
similarity index 92%
rename from var/spack/repos/builtin/packages/intel-tbb/gcc_13-2021.patch
rename to var/spack/repos/builtin/packages/intel-tbb/gcc_13-2021-v2.patch
index e1e1b1116bf6f8..d1e87cd7c4d5ac 100644
--- a/var/spack/repos/builtin/packages/intel-tbb/gcc_13-2021.patch
+++ b/var/spack/repos/builtin/packages/intel-tbb/gcc_13-2021-v2.patch
@@ -16,6 +16,8 @@ Signed-off-by: Sam James 
 
 diff --git a/test/common/utils_assert.h b/test/common/utils_assert.h
 index 1df8ae72acc49fe38dac4d9bed4e9f4f26affcf5..0123ab881e124a800a5ebf8507050148038747d5 100644
+--- a/test/common/utils_assert.h
++++ b/test/common/utils_assert.h
 @@ -20,6 +20,8 @@
  #include "config.h"
  #include "utils_report.h"
diff --git a/var/spack/repos/builtin/packages/intel-tbb/package.py b/var/spack/repos/builtin/packages/intel-tbb/package.py
index 45761c7a06fcc3..14da30b2d430fe 100644
--- a/var/spack/repos/builtin/packages/intel-tbb/package.py
+++ b/var/spack/repos/builtin/packages/intel-tbb/package.py
@@ -125,7 +125,7 @@ class IntelTbb(CMakePackage, MakefilePackage):
     patch("gcc_generic-pedantic-4.4.patch", level=1, when="@:2019.0")
 
     # Patch and conflicts for GCC 13 support (#1031).
-    patch("gcc_13-2021.patch", when="@2021.1:")
+    patch("gcc_13-2021-v2.patch", when="@2021.1:")
     conflicts("%gcc@13", when="@:2021.3")
 
     # Patch cmakeConfig.cmake.in to find the libraries where we install them.
diff --git a/var/spack/repos/builtin/packages/intel-xed/package.py b/var/spack/repos/builtin/packages/intel-xed/package.py
index 63897b4184ee16..555d4154a220fd 100644
--- a/var/spack/repos/builtin/packages/intel-xed/package.py
+++ b/var/spack/repos/builtin/packages/intel-xed/package.py
@@ -21,13 +21,14 @@ class IntelXed(Package):
 
     # Current versions now have actual releases and tags.
     version("main", branch="main")
-    version("2023.06.07", tag="v2023.06.07")
-    version("2023.04.16", tag="v2023.04.16")
-    version("2022.10.11", tag="v2022.10.11")
-    version("2022.08.11", tag="v2022.08.11")
-    version("2022.04.17", tag="v2022.04.17")
-    version("12.0.1", tag="12.0.1")
-    version("11.2.0", tag="11.2.0")
+    version("2023.07.09", tag="v2023.07.09", commit="539a6a349cf7538a182ed3ee1f48bb9317eb185f")
+    version("2023.06.07", tag="v2023.06.07", commit="4dc77137f651def2ece4ac0416607b215c18e6e4")
+    version("2023.04.16", tag="v2023.04.16", commit="a3055cd0209f5c63c88e280bbff9579b1e2942e2")
+    version("2022.10.11", tag="v2022.10.11", commit="9fc12ab6c0ba7a9eaadb20135369b4b4107fa670")
+    version("2022.08.11", tag="v2022.08.11", commit="1ce1036aa4ab280f9a498136b37421ab390e42db")
+    version("2022.04.17", tag="v2022.04.17", commit="ef19f00de14a9c2c253c1c9b1119e1617280e3f2")
+    version("12.0.1", tag="12.0.1", commit="5976632eeaaaad7890c2109d0cfaf4012eaca3b8")
+    version("11.2.0", tag="11.2.0", commit="40125558530137444b4ee6fd26b445bfa105b543")
 
     # The old 2019.03.01 version (before there were tags).
     version("10.2019.03", commit="b7231de4c808db821d64f4018d15412640c34113", deprecated=True)
@@ -42,7 +43,14 @@ class IntelXed(Package):
         name="mbuild", placement=mdir, git=mbuild_git, tag="v2022.07.28", when="@2022.07:9999"
     )
 
-    resource(name="mbuild", placement=mdir, git=mbuild_git, tag="v2022.04.17", when="@:2022.06")
+    resource(
+        name="mbuild",
+        placement=mdir,
+        git=mbuild_git,
+        tag="v2022.04.17",
+        commit="ef19f00de14a9c2c253c1c9b1119e1617280e3f2",
+        when="@:2022.06",
+    )
 
     variant("debug", default=False, description="Enable debug symbols")
     variant("pic", default=False, description="Compile with position independent code.")
diff --git a/var/spack/repos/builtin/packages/intel/detection_test.yaml b/var/spack/repos/builtin/packages/intel/detection_test.yaml
new file mode 100644
index 00000000000000..076bfeaabac3bd
--- /dev/null
+++ b/var/spack/repos/builtin/packages/intel/detection_test.yaml
@@ -0,0 +1,19 @@
+paths:
+  - layout:
+      - executables:
+        - "bin/intel64/icc"
+        script: |
+          echo "icc (ICC) 18.0.5 20180823"
+          echo "Copyright (C) 1985-2018 Intel Corporation.  All rights reserved."
+      - executables:
+        - "bin/intel64/icpc"
+        script: |
+          echo "icpc (ICC) 18.0.5 20180823"
+          echo "Copyright (C) 1985-2018 Intel Corporation.  All rights reserved."
+      - executables:
+        - "bin/intel64/ifort"
+        script: |
+          echo "ifort (IFORT) 18.0.5 20180823"
+          echo "Copyright (C) 1985-2018 Intel Corporation.  All rights reserved."
+    results:
+      - spec: 'intel@18.0.5'
diff --git a/var/spack/repos/builtin/packages/interproscan/package.py b/var/spack/repos/builtin/packages/interproscan/package.py
index 4bd5956f1409c6..82380135a76feb 100644
--- a/var/spack/repos/builtin/packages/interproscan/package.py
+++ b/var/spack/repos/builtin/packages/interproscan/package.py
@@ -10,12 +10,16 @@ class Interproscan(Package):
     """InterProScan is the software package that allows sequences
     (protein and nucleic) to be scanned against InterPro's signatures.
     Signatures are predictive models, provided by several different
-    databases, that make up the InterPro consortium."""
+    databases, that make up the InterPro consortium. By default this
+    does not download all the databases you likely want, you'll either
+    want to build with +databases or download the data archive for the
+    version you installed to a separate location and modify interproscan.properties"""
 
     homepage = "https://www.ebi.ac.uk/interpro/interproscan.html"
     url = "https://github.com/ebi-pf-team/interproscan/archive/5.36-75.0.tar.gz"
     maintainers("snehring")
 
+    version("5.63-95.0", sha256="3d7babd09e64da3d7104c58f1e5104a298d69425e3210952331bc3f1ddf89ca6")
     version("5.61-93.0", sha256="70aca3b14983733fe5119b6978cb707156d006d7f737aa60ce6c9addd6c288e4")
     version("5.56-89.0", sha256="75e6a8f86ca17356a2f77f75b07d6d8fb7b397c9575f6e9716b64983e490b230")
     version("5.38-76.0", sha256="cb191ff8eee275689b789167a57b368ea5c06bbcd36b4de23e8bbbbdc0fc7434")
@@ -26,6 +30,41 @@ class Interproscan(Package):
         url="ftp://ftp.ebi.ac.uk/pub/software/unix/iprscan/4/RELEASE/4.8/iprscan_v4.8.tar.gz",
     )
 
+    resource(
+        when="@5.63-95.0 +databases",
+        name="databases",
+        url="https://ftp.ebi.ac.uk/pub/databases/interpro/iprscan/5/5.63-95.0/alt/interproscan-data-5.63-95.0.tar.gz",
+        sha256="6048eabe2eeaa4630b7a6a0b34d8c5a1724b0d22bba318c04c43777368e16cc4",
+    )
+
+    resource(
+        when="@5.61-93.0 +databases",
+        name="databases",
+        url="https://ftp.ebi.ac.uk/pub/databases/interpro/iprscan/5/5.61-93.0/alt/interproscan-data-5.61-93.0.tar.gz",
+        sha256="064aa4b4c3b2e27b457298359087878e48fc785bff801c95691f090d1b83867d",
+    )
+
+    resource(
+        when="5.56-89.0 +databases",
+        name="databases",
+        url="https://ftp.ebi.ac.uk/pub/databases/interpro/iprscan/5/5.56-89.0/alt/interproscan-data-5.56-89.0.tar.gz",
+        sha256="49cd0c69711f9469f3b68857f4581b23ff12765ca2b12893d18e5a9a5cd8032d",
+    )
+
+    resource(
+        when="5.38-76.0 +databases",
+        name="databases",
+        url="https://ftp.ebi.ac.uk/pub/databases/interpro/iprscan/5/5.38-76.0/alt/interproscan-data-5.38-76.0.tar.gz",
+        sha256="e05e15d701037504f92ecf849c20317e70df28e78ff1945826b3c1e16d9b9cce",
+    )
+
+    resource(
+        when="5.36-75.0 +databases",
+        name="databases",
+        url="https://ftp.ebi.ac.uk/pub/databases/interpro/iprscan/5/5.36-75.0/alt/interproscan-data-5.36-75.0.tar.gz",
+        sha256="e9b1e6f2d1c20d06661a31a08c973bc8ddf039a4cf1e45ec4443200375e5d6a4",
+    )
+
     resource(
         when="@:4.8",
         name="binaries",
@@ -33,6 +72,12 @@ class Interproscan(Package):
         sha256="551610a4682b112522f3ded5268f76ba9a47399a72e726fafb17cc938a50e7ee",
     )
 
+    variant(
+        "databases",
+        default=False,
+        description="Fetch and include databases in the install. Greatly increases install size.",
+    )
+
     depends_on("java@8.0:8.9", type=("build", "run"), when="@5:5.36-99.0")
     depends_on("java@11", type=("build", "run"), when="@5.37-76.0:")
     depends_on("maven", type="build", when="@5:")
@@ -63,6 +108,10 @@ def install(self, spec, prefix):
         target = join_path("core", "jms-implementation", "target", "interproscan-5-dist")
         install_tree(target, prefix)
 
+        if spec.satisfies("+databases"):
+            remove_directory_contents(prefix.data)
+            install_tree(f"interproscan-{self.spec.version}/data", prefix.data)
+
         # link the main shell script into the PATH
         symlink(join_path(prefix, "interproscan.sh"), join_path(prefix.bin, "interproscan.sh"))
 
diff --git a/var/spack/repos/builtin/packages/ior/package.py b/var/spack/repos/builtin/packages/ior/package.py
index 43238d3016168f..bad21c959ce475 100644
--- a/var/spack/repos/builtin/packages/ior/package.py
+++ b/var/spack/repos/builtin/packages/ior/package.py
@@ -26,6 +26,7 @@ class Ior(AutotoolsPackage):
 
     variant("hdf5", default=False, description="support IO with HDF5 backend")
     variant("ncmpi", default=False, description="support IO with NCMPI backend")
+    variant("lustre", default=False, description="support configurable Lustre striping values")
 
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
@@ -34,6 +35,7 @@ class Ior(AutotoolsPackage):
     depends_on("mpi")
     depends_on("hdf5+mpi", when="+hdf5")
     depends_on("parallel-netcdf", when="+ncmpi")
+    depends_on("lustre", when="+lustre")
 
     # The build for 3.2.0 fails if hdf5 is enabled
     # See https://github.com/hpc/ior/pull/124
@@ -43,6 +45,14 @@ class Ior(AutotoolsPackage):
         when="@3.2.0 +hdf5",
     )
 
+    # Needs patch to make Lustre variant work
+    # See https://github.com/hpc/ior/issues/353
+    patch(
+        "https://github.com/glennklockwood/ior/commit/e49476be64d4100c2da662ea415f327348b3d11d.patch?full_index=1",
+        sha256="ee3527023ef70ea9aee2e6208f8be7126d5a48f26c587deed3d6238b4f848a06",
+        when="+lustre",
+    )
+
     @run_before("autoreconf")
     def bootstrap(self):
         Executable("./bootstrap")()
@@ -64,4 +74,9 @@ def configure_args(self):
         else:
             config_args.append("--without-ncmpi")
 
+        if "+lustre" in spec:
+            config_args.append("--with-lustre")
+        else:
+            config_args.append("--without-lustre")
+
         return config_args
diff --git a/var/spack/repos/builtin/packages/ip/package.py b/var/spack/repos/builtin/packages/ip/package.py
index ebd53089741435..86c9f9592d818a 100644
--- a/var/spack/repos/builtin/packages/ip/package.py
+++ b/var/spack/repos/builtin/packages/ip/package.py
@@ -13,13 +13,16 @@ class Ip(CMakePackage):
 
     homepage = "https://noaa-emc.github.io/NCEPLIBS-ip"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-ip/archive/refs/tags/v3.3.3.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-ip"
 
-    maintainers("t-brown", "AlexanderRichert-NOAA", "edwardhartnett", "Hang-Lei-NOAA")
+    maintainers("AlexanderRichert-NOAA", "edwardhartnett", "Hang-Lei-NOAA")
 
+    version("develop", branch="develop")
     version("4.4.0", sha256="858d9201ce0bc4d16b83581ef94a4a0262f498ed1ea1b0535de2e575da7a8b8c")
     version("4.3.0", sha256="799308a868dea889d2527d96a0405af7b376869581410fe4cff681205e9212b4")
     # Note that versions 4.0-4.2 contain constants_mod module, and should not be used when
     # also compiling with packages containing Fortran modules of the same name, namely, FMS.
+    version("4.2.0", sha256="9b9f47106822044ff224c6dfd9f140c146dffc833904f2a0c5db7b5d8932e39e")
     version("4.1.0", sha256="b83ca037d9a5ad3eb0fb1acfe665c38b51e01f6bd73ce9fb8bb2a14f5f63cdbe")
     version("4.0.0", sha256="a2ef0cc4e4012f9cb0389fab6097407f4c623eb49772d96eb80c44f804aa86b8")
     version(
@@ -33,15 +36,28 @@ class Ip(CMakePackage):
     variant("shared", default=False, description="Build shared library", when="@4.1:")
     variant(
         "precision",
-        default=["4", "d"],
-        values=["4", "d"],
+        default=("4", "d"),
+        values=("4", "d"),
         multi=True,
         description="Set precision (_4/_d library versions)",
-        when="@4.1:",
+        when="@4.1",
+    )
+    variant(
+        "precision",
+        default=("4", "d"),
+        values=("4", "d", "8"),
+        multi=True,
+        description="Set precision (_4/_d/_8 library versions)",
+        when="@4.2:",
     )
 
+    conflicts("+shared ~pic")
+
     depends_on("sp")
     depends_on("sp@:2.3.3", when="@:4.0")
+    depends_on("sp precision=4", when="precision=4")
+    depends_on("sp precision=d", when="precision=d")
+    depends_on("sp precision=8", when="precision=8")
 
     def cmake_args(self):
         args = [
@@ -50,15 +66,17 @@ def cmake_args(self):
         ]
 
         if self.spec.satisfies("@4:"):
-            args.append(self.define("BUILD_TESTING", "NO"))
+            args.append(self.define("BUILD_TESTING", self.run_tests))
         else:
             args.append(self.define("ENABLE_TESTS", "NO"))
 
         if self.spec.satisfies("@4.1:"):
             args.append(self.define_from_variant("BUILD_SHARED_LIBS", "shared"))
-            for prec in ["4", "d"]:
-                if not self.spec.satisfies("precision=" + prec):
-                    args += ["-DBUILD_%s:BOOL=OFF" % prec.upper()]
+            args.append(self.define("BUILD_4", self.spec.satisfies("precision=4")))
+            args.append(self.define("BUILD_D", self.spec.satisfies("precision=d")))
+
+        if self.spec.satisfies("@4.2:"):
+            args.append(self.define("BUILD_8", self.spec.satisfies("precision=8")))
 
         return args
 
@@ -66,7 +84,7 @@ def setup_run_environment(self, env):
         suffixes = (
             self.spec.variants["precision"].value
             if self.spec.satisfies("@4.1:")
-            else ["4", "8", "d"]
+            else ("4", "8", "d")
         )
         shared = False if self.spec.satisfies("@:4.0") else self.spec.satisfies("+shared")
         for suffix in suffixes:
@@ -75,3 +93,7 @@ def setup_run_environment(self, env):
             )
             env.set("IP_LIB" + suffix, lib[0])
             env.set("IP_INC" + suffix, join_path(self.prefix, "include_" + suffix))
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/ipm/package.py b/var/spack/repos/builtin/packages/ipm/package.py
new file mode 100644
index 00000000000000..654707a96abdf2
--- /dev/null
+++ b/var/spack/repos/builtin/packages/ipm/package.py
@@ -0,0 +1,103 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+from spack.util.executable import Executable
+
+
+class Ipm(AutotoolsPackage):
+    """IPM is a portable profiling infrastructure for parallel codes.
+    It provides a low-overhead profile of application performance
+    and resource utilization in a parallel program. Communication,
+    computation, and IO are the primary focus."""
+
+    homepage = "https://github.com/nerscadmin/IPM"
+    git = "https://github.com/nerscadmin/IPM.git"
+
+    maintainers("Christoph-TU")
+
+    version("master", branch="master", preferred=True)
+    version("2.0.6", tag="2.0.6", commit="b008141ee16d39b33e20bffde615564afa107575")
+
+    variant("papi", default=False, description="Enable PAPI")
+    variant("cuda", default=False, description="Enable CUDA")
+    variant("libunwind", default=False, description="Enable libunwind")
+
+    variant(
+        "papi_multiplexing", default=False, when="+papi", description="Enable PAPI multiplexing"
+    )
+    variant(
+        "coll_details",
+        default=False,
+        description="Enable detailed monitoring of collective operations (experimental)",
+    )
+    variant("posixio", default=False, description="Enable POSIXIO")
+    variant("pmon", default=False, description="Enable power monitoring module")
+    variant("parser", default=False, description="Add dependencies for running ipm_parse")
+
+    depends_on("autoconf", type="build")
+    depends_on("automake", type="build")
+    depends_on("libtool", type="build")
+    depends_on("m4", type="build")
+
+    depends_on("mpi")
+    depends_on("papi", when="+papi")
+    depends_on("cuda", when="+cuda")
+    depends_on("libunwind", when="+libunwind")
+
+    # These are required when running the perl script ipm_parse,
+    # which is used to create reports from the generated xml file
+    depends_on("perl", type="run", when="+parser")
+    depends_on("ploticus", type="run", when="+parser")
+
+    # 2COMPLEX and 2DOUBLE_COMPLEX are non-standard types and lead
+    # to compile errors when building with coll_details
+    patch("remove_MPI_2COMPLEX_and_MPI_2DOUBLE_COMPLEX.patch", when="+coll_details")
+
+    def patch(self):
+        filter_file(r"#!/usr/bin/perl", "#!/usr/bin/env perl", "bin/ipm_parse")
+
+    def setup_build_environment(self, env):
+        spec = self.spec
+        env.set("MPICC", spec["mpi"].mpicc)
+        env.set("MPIFC", spec["mpi"].mpifc)
+        env.set("MPICXX", spec["mpi"].mpicxx)
+        env.set("MPIF77", spec["mpi"].mpif77)
+
+    def autoreconf(self, spec, prefix):
+        script = Executable(join_path(self.stage.source_path, "bootstrap.sh"))
+        script()
+
+    def configure_args(self):
+        args = []
+        spec = self.spec
+        if "+papi" in spec:
+            args.append("--with-papi={0}".format(spec["papi"].prefix))
+
+        if "+cuda" in spec:
+            args.append("--with-cudapath={0}".format(spec["cuda"].prefix))
+
+        if "+libunwind" in spec:
+            args.append("--with-libunwind={0}".format(spec["libunwind"].prefix))
+
+        if "+papi_multiplexing" in spec:
+            args.append("--enable-papi-multiplexing")
+
+        if "+posixio" in spec:
+            args.append("--enable-posixio")
+
+        if "+pmon" in spec:
+            args.append("--enable-pmon")
+
+        if "+coll_details" in spec:
+            args.append("--enable-coll-details")
+
+        args.extend(
+            [
+                "CFLAGS={0}".format(self.compiler.cc_pic_flag),
+                "CXXFLAGS={0}".format(self.compiler.cxx_pic_flag),
+            ]
+        )
+        return args
diff --git a/var/spack/repos/builtin/packages/ipm/remove_MPI_2COMPLEX_and_MPI_2DOUBLE_COMPLEX.patch b/var/spack/repos/builtin/packages/ipm/remove_MPI_2COMPLEX_and_MPI_2DOUBLE_COMPLEX.patch
new file mode 100644
index 00000000000000..2958e95a1c86b9
--- /dev/null
+++ b/var/spack/repos/builtin/packages/ipm/remove_MPI_2COMPLEX_and_MPI_2DOUBLE_COMPLEX.patch
@@ -0,0 +1,37 @@
+diff --git a/include/mod_mpi.h b/include/mod_mpi.h
+index 135a558..00b5382 100755
+--- a/include/mod_mpi.h
++++ b/include/mod_mpi.h
+@@ -316,8 +316,6 @@ extern char* ipm_mpi_op[MAXNUM_MPI_OPS];
+ #define IPM_MPI_2INTEGER                34
+ #define IPM_MPI_2REAL                   35
+ #define IPM_MPI_2DOUBLE_PRECISION       36
+-#define IPM_MPI_2COMPLEX                37
+-#define IPM_MPI_2DOUBLE_COMPLEX         38
+ 
+ extern char* ipm_mpi_type[MAXNUM_MPI_TYPES];
+ 
+@@ -380,8 +378,6 @@ extern char* ipm_mpi_type[MAXNUM_MPI_TYPES];
+     else if( mpitype==MPI_2INTEGER )          ipmtype=IPM_MPI_2INTEGER;	\
+     else if( mpitype==MPI_2REAL )             ipmtype=IPM_MPI_2REAL;		\
+     else if( mpitype==MPI_2DOUBLE_PRECISION ) ipmtype=IPM_MPI_2DOUBLE_PRECISION; \
+-    else if( mpitype==MPI_2COMPLEX )          ipmtype=IPM_MPI_2COMPLEX;	\
+-    else if( mpitype==MPI_2DOUBLE_COMPLEX )   ipmtype=IPM_MPI_2DOUBLE_COMPLEX;	\
+     else ipmtype=0;							\
+   }
+ 
+diff --git a/src/mod_mpi.c b/src/mod_mpi.c
+index 00ca4ab..d91e853 100755
+--- a/src/mod_mpi.c
++++ b/src/mod_mpi.c
+@@ -94,8 +94,6 @@ int mod_mpi_init(ipm_mod_t* mod, int flags)
+   ipm_mpi_type[IPM_MPI_2INTEGER] = "MPI_2INTEGER";
+   ipm_mpi_type[IPM_MPI_2REAL]    = "MPI_2REAL";
+   ipm_mpi_type[IPM_MPI_2DOUBLE_PRECISION] = "MPI_2DOUBLE_PRECISION";
+-  ipm_mpi_type[IPM_MPI_2COMPLEX]  = "MPI_2COMPLEX";
+-  ipm_mpi_type[IPM_MPI_2DOUBLE_COMPLEX] = "MPI_2DOUBLE_COMPLEX";
+ 
+   mod->state    = STATE_ACTIVE;
+   return IPM_OK;
+
+
diff --git a/var/spack/repos/builtin/packages/iq-tree/package.py b/var/spack/repos/builtin/packages/iq-tree/package.py
index 8c548da2e1ae11..7226dce4bc045c 100644
--- a/var/spack/repos/builtin/packages/iq-tree/package.py
+++ b/var/spack/repos/builtin/packages/iq-tree/package.py
@@ -14,9 +14,18 @@ class IqTree(CMakePackage):
     git = "https://github.com/iqtree/iqtree2.git"
     url = "https://github.com/Cibiv/IQ-TREE/archive/v1.6.12.tar.gz"
 
-    version("2.2.2.7", tag="v2.2.2.7", submodules=True)
-    version("2.1.3", tag="v2.1.3", submodules=True)
-    version("2.0.6", tag="v2.0.6", submodules=True)
+    version(
+        "2.2.2.7",
+        tag="v2.2.2.7",
+        commit="bd3468c7af6572ea29002dfdba377804f8f56c26",
+        submodules=True,
+    )
+    version(
+        "2.1.3", tag="v2.1.3", commit="3d31be9e56b05ffbc5f8488fc8285597b433c99f", submodules=True
+    )
+    version(
+        "2.0.6", tag="v2.0.6", commit="219e88407ac915a209a29808a81084bf0d5f1a84", submodules=True
+    )
     version("1.6.12", sha256="9614092de7a157de82c9cc402b19cc8bfa0cb0ffc93b91817875c2b4bb46a284")
 
     variant("openmp", default=True, description="Enable OpenMP support.")
@@ -29,7 +38,7 @@ class IqTree(CMakePackage):
 
     depends_on("boost+container+math+exception")
     depends_on("eigen")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("mpi", when="+mpi")
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/iqtree2/package.py b/var/spack/repos/builtin/packages/iqtree2/package.py
index 67a14b8f584666..c84cb6e8e6299d 100644
--- a/var/spack/repos/builtin/packages/iqtree2/package.py
+++ b/var/spack/repos/builtin/packages/iqtree2/package.py
@@ -18,4 +18,4 @@ class Iqtree2(CMakePackage):
 
     depends_on("boost", type="link")
     depends_on("eigen", type="link")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
diff --git a/var/spack/repos/builtin/packages/ispc/package.py b/var/spack/repos/builtin/packages/ispc/package.py
index 68b7ecf3824440..8bef2ce2f584c1 100644
--- a/var/spack/repos/builtin/packages/ispc/package.py
+++ b/var/spack/repos/builtin/packages/ispc/package.py
@@ -25,6 +25,7 @@ class Ispc(CMakePackage):
     executables = ["^ispc$"]
 
     version("main", branch="main")
+    version("1.20.0", sha256="8bd30ded7f96859451ead1cecf6f58ac8e937288fe0e5b98c56f6eba4be370b4")
     version("1.19.0", sha256="c1aeae4bdfb28004a6949394ea1b3daa3fdf12f646e17fcc0614861077dc8b6a")
     version("1.18.1", sha256="fee76d42fc0129f81489b7c2b9143e22a44c281940693c1c13cf1e3dd2ab207f")
     version("1.18.0", sha256="ecf1fc6ad5e39242e555b8e0ac539489939a9e475722eaa9da5caa4651cecf05")
@@ -40,8 +41,10 @@ class Ispc(CMakePackage):
     depends_on("bison", type="build")
     depends_on("flex", type="build")
     depends_on("ncurses", type="link")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
+    depends_on("tbb", type="link", when="platform=linux @1.20:")
     depends_on("llvm+clang")
+    depends_on("llvm libcxx=none", when="platform=darwin")
     depends_on("llvm@13:15", when="@1.19:")
     depends_on("llvm@11.0:14.0", when="@1.18")
     depends_on("llvm@11:14", when="@1.17")
@@ -49,6 +52,8 @@ class Ispc(CMakePackage):
     depends_on("llvm@11:", when="@1.16")
     depends_on("llvm@10:11", when="@1.15.0:1.15")
     depends_on("llvm@10.0:10", when="@1.13:1.14")
+    depends_on("llvm targets=arm,aarch64", when="target=arm:")
+    depends_on("llvm targets=arm,aarch64", when="target=aarch64:")
 
     patch(
         "don-t-assume-that-ncurses-zlib-are-system-libraries.patch",
@@ -62,9 +67,15 @@ class Ispc(CMakePackage):
         sha256="d3ccf547d3ba59779fd375e10417a436318f2200d160febb9f830a26f0daefdc",
     )
 
+    # Fix library lookup for NCurses in CMake
+    patch(
+        "https://patch-diff.githubusercontent.com/raw/ispc/ispc/pull/2638.patch?full_index=1",
+        when="@1.18:1.20",
+        sha256="3f7dae8d4a683fca2a6157bbcb7cbe9692ff2094b0f4afaf29be121c02b0b3ad",
+    )
+
     def setup_build_environment(self, env):
         if self.spec.satisfies("@1.18.0:"):
-            env.append_flags("LDFLAGS", "-lcurses")
             env.append_flags("LDFLAGS", "-lz")
 
     def patch(self):
@@ -76,14 +87,25 @@ def patch(self):
             filter_file("bit 32 64", "bit 64", "cmake/GenerateBuiltins.cmake")
 
     def cmake_args(self):
+        spec = self.spec
         args = []
-        args.append("-DARM_ENABLED=FALSE")
         args.append("-DISPC_NO_DUMPS=ON")  # otherwise, LLVM needs patching
+        args.append("-DCURSES_NEED_NCURSES=TRUE")
         args.append("-DISPC_INCLUDE_EXAMPLES=OFF")
         args.append("-DISPC_INCLUDE_TESTS=OFF")
         args.append("-DISPC_INCLUDE_UTILS=OFF")
+        if spec.satisfies("target=x86_64:") or spec.satisfies("target=x86:"):
+            args.append("-DARM_ENABLED=OFF")
+        elif spec.satisfies("target=aarch64:"):
+            args.append("-DARM_ENABLED=ON")
         return args
 
+    @run_after("install")
+    def check_install(self):
+        with working_dir(self.stage.source_path):
+            ispc = Executable(join_path(self.prefix, "bin", "ispc"))
+            ispc("--version")
+
     @classmethod
     def determine_version(cls, exe):
         output = Executable(exe)("--version", output=str, error=str)
diff --git a/var/spack/repos/builtin/packages/itk/package.py b/var/spack/repos/builtin/packages/itk/package.py
index 875683149ae250..0a956f3dfdb30a 100644
--- a/var/spack/repos/builtin/packages/itk/package.py
+++ b/var/spack/repos/builtin/packages/itk/package.py
@@ -58,15 +58,20 @@ class Itk(CMakePackage):
     depends_on("expat")
     depends_on("fftw-api")
     depends_on("googletest")
-    depends_on("hdf5+cxx")
+    depends_on("hdf5+cxx+hl")
     depends_on("jpeg")
     depends_on("libpng")
     depends_on("libtiff")
-    depends_on("mpi")
-    depends_on("zlib")
+    depends_on("zlib-api")
+
+    patch(
+        "https://github.com/InsightSoftwareConsortium/ITK/commit/9a719a0d2f5f489eeb9351b0ef913c3693147a4f.patch?full_index=1",
+        sha256="ec1f7fa71f2b7f05d9632c6b0321e7d436fff86fca92c60c12839b13ea79bd70",
+        when="@5.2.0:5.3.0",
+    )
 
     def cmake_args(self):
-        use_mkl = "^mkl" in self.spec
+        use_mkl = self.spec["fftw-api"].name in INTEL_MATH_LIBRARIES
         args = [
             self.define("BUILD_SHARED_LIBS", True),
             self.define("ITK_USE_SYSTEM_LIBRARIES", True),
diff --git a/var/spack/repos/builtin/packages/javafx/package.py b/var/spack/repos/builtin/packages/javafx/package.py
index f37429a4abe082..ceceb9f1c43f2f 100644
--- a/var/spack/repos/builtin/packages/javafx/package.py
+++ b/var/spack/repos/builtin/packages/javafx/package.py
@@ -20,7 +20,7 @@
             ),
         },
         "darwin": {
-            "aarch64": (
+            "arm64": (
                 "https://download2.gluonhq.com/openjfx/20.0.1/openjfx-20.0.1_osx-aarch64_bin-sdk.zip",
                 "baebdbbe283c17df62fc4c0bdc2bde4415f2253f99ba41437f9336e2272c255e",
             ),
diff --git a/var/spack/repos/builtin/packages/jedi-cmake/package.py b/var/spack/repos/builtin/packages/jedi-cmake/package.py
index b00aa529f777cf..84759fb50611ba 100644
--- a/var/spack/repos/builtin/packages/jedi-cmake/package.py
+++ b/var/spack/repos/builtin/packages/jedi-cmake/package.py
@@ -11,17 +11,13 @@ class JediCmake(CMakePackage):
 
     homepage = "https://github.com/JCSDA/jedi-cmake"
     git = "https://github.com/JCSDA/jedi-cmake.git"
-    # url = "https://github.com/JCSDA/jedi-cmake/archive/refs/tags/1.4.0.tar.gz"
 
     maintainers("climbfuji")
 
     version("master", branch="master", no_cache=True)
     version("develop", branch="develop", no_cache=True)
     version(
-        "1.4.0",
-        commit="36fc99bdff5d3d8835480b37a3dcc75e5f8da256",
-        preferred=True,
-        submodules=True,
+        "1.4.0", commit="36fc99bdff5d3d8835480b37a3dcc75e5f8da256", preferred=True, submodules=True
     )
     version("1.3.0", commit="729a9b2ec97a7e93cbc58213493f28ca11f08754")
 
diff --git a/var/spack/repos/builtin/packages/jhpcn-df/package.py b/var/spack/repos/builtin/packages/jhpcn-df/package.py
index ebe14cead5e62b..6854ced077e33a 100644
--- a/var/spack/repos/builtin/packages/jhpcn-df/package.py
+++ b/var/spack/repos/builtin/packages/jhpcn-df/package.py
@@ -26,7 +26,7 @@ class JhpcnDf(CMakePackage):
     variant("lz4", default=False, description="Enable lz4")
     variant("fortran", default=False, description="Enable Fortran Interface")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("lz4@:1.7", when="+lz4", type="link")
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/jose/package.py b/var/spack/repos/builtin/packages/jose/package.py
index 3b7e2c1609e36f..47f46787336fac 100644
--- a/var/spack/repos/builtin/packages/jose/package.py
+++ b/var/spack/repos/builtin/packages/jose/package.py
@@ -18,5 +18,5 @@ class Jose(AutotoolsPackage):
 
     depends_on("pkgconfig", type="build")
     depends_on("jansson@2.10:")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("openssl@1.0.2:")
diff --git a/var/spack/repos/builtin/packages/jsoncpp/package.py b/var/spack/repos/builtin/packages/jsoncpp/package.py
index b5eff5150d7e75..bd20e2b22e1a23 100644
--- a/var/spack/repos/builtin/packages/jsoncpp/package.py
+++ b/var/spack/repos/builtin/packages/jsoncpp/package.py
@@ -6,7 +6,7 @@
 from spack.package import *
 
 
-class Jsoncpp(CMakePackage):
+class Jsoncpp(CMakePackage, MesonPackage):
     """JsonCpp is a C++ library that allows manipulating JSON values,
     including serialization and deserialization to and from strings.
     It can also preserve existing comment in unserialization/serialization
@@ -33,22 +33,25 @@ class Jsoncpp(CMakePackage):
     version("1.7.4", sha256="10dcd0677e80727e572a1e462193e51a5fde3e023b99e144b2ee1a469835f769")
     version("1.7.3", sha256="1cfcad14054039ba97c22531888796cb9369e6353f257aacaad34fda956ada53")
 
-    variant(
-        "build_type",
-        default="RelWithDebInfo",
-        description="The build type to build",
-        values=("Debug", "Release", "RelWithDebInfo", "MinSizeRel", "Coverage"),
-    )
-
+    # From 1.9.3 onwards CMAKE_CXX_STANDARD is finally set to 11.
     variant(
         "cxxstd",
         default="default",
-        values=("default", "98", "11", "14", "17"),
+        values=("default", conditional("98", when="@:1.8"), "11", "14", "17"),
         multi=False,
         description="Use the specified C++ standard when building.",
+        when="@:1.9.2 build_system=cmake",
     )
 
-    depends_on("cmake@3.1:", type="build")
+    build_system("cmake", conditional("meson", when="@1.9.2:"), default="cmake")
+
+    with when("build_system=cmake"):
+        depends_on("cmake@3.1:", type="build")
+        depends_on("cmake@3.9:", when="@1.9:", type="build")
+
+    with when("build_system=meson"):
+        depends_on("meson@0.49.0:", type="build")
+
     depends_on("python", type="test")
 
     # Ref: https://github.com/open-source-parsers/jsoncpp/pull/1023
@@ -62,13 +65,20 @@ def patch(self):
             "src/lib_json/json_value.cpp",
         )
 
+
+class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder):
     def cmake_args(self):
-        args = ["-DBUILD_SHARED_LIBS=ON"]
-        cxxstd = self.spec.variants["cxxstd"].value
-        if cxxstd != "default":
-            args.append("-DCMAKE_CXX_STANDARD={0}".format(cxxstd))
-        if self.run_tests:
-            args.append("-DJSONCPP_WITH_TESTS=ON")
-        else:
-            args.append("-DJSONCPP_WITH_TESTS=OFF")
+        args = [
+            self.define("BUILD_SHARED_LIBS", True),
+            self.define("JSONCPP_WITH_TESTS", self.pkg.run_tests),
+        ]
+        if "cxxstd" in self.spec.variants:
+            cxxstd = self.spec.variants["cxxstd"].value
+            if cxxstd != "default":
+                args.append(self.define("CMAKE_CXX_STANDARD", cxxstd))
         return args
+
+
+class MesonBuilder(spack.build_systems.meson.MesonBuilder):
+    def meson_args(self):
+        return ["-Dtests={}".format("true" if self.pkg.run_tests else "false")]
diff --git a/var/spack/repos/builtin/packages/julia/package.py b/var/spack/repos/builtin/packages/julia/package.py
index 67006f870158ad..4115f148726d13 100644
--- a/var/spack/repos/builtin/packages/julia/package.py
+++ b/var/spack/repos/builtin/packages/julia/package.py
@@ -26,6 +26,7 @@ class Julia(MakefilePackage):
     maintainers("vchuravy", "haampie", "giordano")
 
     version("master", branch="master")
+    version("1.9.2", sha256="015438875d591372b80b09d01ba899657a6517b7c72ed41222298fef9d4ad86b")
     version("1.9.0", sha256="48f4c8a7d5f33d0bc6ce24226df20ab49e385c2d0c3767ec8dfdb449602095b2")
     version("1.8.5", sha256="d31026cc6b275d14abce26fd9fd5b4552ac9d2ce8bde4291e494468af5743031")
     version("1.8.4", sha256="b7b8ee64fb947db8d61104f231e1b25342fe330d29e0d2273f93c264f32c5333")
@@ -123,7 +124,7 @@ class Julia(MakefilePackage):
         when="^llvm@12.0.1",
         patches=patch(
             "https://github.com/JuliaLang/llvm-project/compare/fed41342a82f5a3a9201819a82bf7a48313e296b...980d2f60a8524c5546397db9e8bbb7d6ea56c1b7.patch",
-            sha256="10cb42f80c2eaad3e9c87cb818b6676f1be26737bdf972c77392d71707386aa4",
+            sha256="37f2f6193e1205ea49b9a56100a70b038b64abf402115f263c6132cdf0df80c3",
         ),
     )
     depends_on(
@@ -131,7 +132,7 @@ class Julia(MakefilePackage):
         when="^llvm@13.0.1",
         patches=patch(
             "https://github.com/JuliaLang/llvm-project/compare/75e33f71c2dae584b13a7d1186ae0a038ba98838...2f4460bd46aa80d4fe0d80c3dabcb10379e8d61b.patch",
-            sha256="45f72c59ae5cf45461e9cd8b224ca49b739d885c79b3786026433c6c22f83b5f",
+            sha256="d9e7f0befeddddcba40eaed3895c4f4734980432b156c39d7a251bc44abb13ca",
         ),
     )
     depends_on(
@@ -139,7 +140,7 @@ class Julia(MakefilePackage):
         when="^llvm@14.0.6",
         patches=patch(
             "https://github.com/JuliaLang/llvm-project/compare/f28c006a5895fc0e329fe15fead81e37457cb1d1...381043941d2c7a5157a011510b6d0386c171aae7.diff",
-            sha256="f3def26930832532bbcd861d41b31ae03db993bc2b3510f89ef831a30bd3e099",
+            sha256="f3fd1803459bdaac0e26d0f3b1874b0e3f97e9411a9e98043d36f788ab4fd00e",
         ),
     )
 
@@ -165,6 +166,7 @@ class Julia(MakefilePackage):
     depends_on("patchelf@0.13:", type="build")
     depends_on("perl", type="build")
     depends_on("libwhich", type="build")
+    depends_on("python", type="build")
 
     depends_on("blas")  # note: for now openblas is fixed...
     depends_on("curl tls=mbedtls +nghttp2 +libssh2")
@@ -184,7 +186,8 @@ class Julia(MakefilePackage):
     depends_on("suite-sparse +pic")
     depends_on("unwind")
     depends_on("utf8proc")
-    depends_on("zlib +shared +pic +optimize")
+    depends_on("zlib-api")
+    depends_on("zlib +shared +pic +optimize", when="^zlib")
 
     # Patches for julia
     patch("julia-1.6-system-libwhich-and-p7zip-symlink.patch", when="@1.6.0:1.6")
@@ -251,7 +254,6 @@ def setup_build_environment(self, env):
             "pcre2",
             "suite-sparse",
             "utf8proc",
-            "zlib",
         ]
         if "+openlibm" in self.spec:
             pkgs.append("openlibm")
@@ -260,6 +262,8 @@ def setup_build_environment(self, env):
         for pkg in pkgs:
             for dir in self.spec[pkg].libs.directories:
                 env.prepend_path(linker_var, dir)
+        for dir in self.spec["zlib-api"].libs.directories:
+            env.prepend_path(linker_var, dir)
 
     def edit(self, spec, prefix):
         # TODO: use a search query for blas / lapack?
@@ -315,6 +319,8 @@ def edit(self, spec, prefix):
             "JULIA_PRECOMPILE:={0}".format("1" if spec.variants["precompile"].value else "0"),
             # we want to use `patchelf --add-rpath` instead of `patchelf --set-rpath`
             "override PATCHELF_SET_RPATH_ARG:=--add-rpath",  # @1.9:
+            # Otherwise, Julia tries to download and build ittapi
+            "USE_INTEL_JITEVENTS:=0",  # @1.9:
         ]
 
         options.append("USEGCC:={}".format("1" if "%gcc" in spec else "0"))
diff --git a/var/spack/repos/builtin/packages/justbuild/package.py b/var/spack/repos/builtin/packages/justbuild/package.py
index d32dd0d99cd223..06a350821fbcba 100644
--- a/var/spack/repos/builtin/packages/justbuild/package.py
+++ b/var/spack/repos/builtin/packages/justbuild/package.py
@@ -22,8 +22,12 @@ class Justbuild(Package):
     maintainers("asartori86")
 
     version("master", branch="master")
-    version("1.1.2", tag="v1.1.2")
-    version("1.0.0", tag="v1.0.0")
+    version("1.2.2", tag="v1.2.2", commit="e1ee04684c34ae30ac3c91b6753e99a81a9dc51c")
+    version("1.2.1", tag="v1.2.1", commit="959cd90083d0c783389cd09e187c98322c16469f")
+    version("1.1.4", tag="v1.1.4", commit="32e96afd159f2158ca129fd00bf02c273d8e1e48")
+    version("1.1.3", tag="v1.1.3", commit="3aed5d450aec38be18edec822ac2efac6d49a938")
+    version("1.1.2", tag="v1.1.2", commit="67b486e2ce6ab657a98b2212a9b6f68935d07a29")
+    version("1.0.0", tag="v1.0.0", commit="c29b671f798e82ba26b5f54ebc9e24c7dcfb8166")
 
     depends_on("python@3:", type=("build", "run"))
     depends_on("wget", type=("build", "run"))
@@ -32,6 +36,10 @@ class Justbuild(Package):
 
     def setup_build_environment(self, env):
         ar = which("ar")
+        if self.spec.version < Version("1.2.1"):
+            family = ', "COMPILER_FAMILY":"unknown"'
+        else:
+            family = ', "TOOLCHAIN_CONFIG": {"FAMILY": "unknown"}'
         if self.spec.satisfies("%gcc@10:"):
             gcc = which("gcc")
             gpp = which("g++")
@@ -41,7 +49,7 @@ def setup_build_environment(self, env):
                 + '  "CC":"{0}"'.format(gcc.path)
                 + ', "CXX":"{0}"'.format(gpp.path)
                 + ', "AR":"{0}"'.format(ar.path)
-                + ', "COMPILER_FAMILY":"unknown"'
+                + family
                 + ', "ENV":{'
                 + '    "PATH":"{0}"'.format(os.environ["PATH"])
                 + "   }"
diff --git a/var/spack/repos/builtin/packages/k8/package.py b/var/spack/repos/builtin/packages/k8/package.py
index 81a756f8577262..9ca18bf6784006 100644
--- a/var/spack/repos/builtin/packages/k8/package.py
+++ b/var/spack/repos/builtin/packages/k8/package.py
@@ -17,7 +17,7 @@ class K8(Package):
 
     version("0.2.4", sha256="da8a99c7f1ce7f0cb23ff07ce10510e770686b906d5431442a5439743c0b3c47")
 
-    depends_on("zlib", type="run")
+    depends_on("zlib-api", type="run")
 
     def install(self, spec, prefix):
         if sys.platform == "darwin":
diff --git a/var/spack/repos/builtin/packages/kafka/package.py b/var/spack/repos/builtin/packages/kafka/package.py
index 5f05fbf52b170e..228bd8e0a3f194 100644
--- a/var/spack/repos/builtin/packages/kafka/package.py
+++ b/var/spack/repos/builtin/packages/kafka/package.py
@@ -18,6 +18,15 @@ class Kafka(Package):
     list_url = "https://www-eu.apache.org/dist/kafka/"
     list_depth = 1
 
+    version(
+        "2.13-3.5.1", sha256="f7b74d544023f2c0ec52a179de59975cb64e34ea03650d829328b407b560e4da"
+    )
+    version(
+        "2.13-3.5.0", sha256="2a3a468e069be570f16d612a8196c2f7a9207b7eac865be934d20ce1252eb2f8"
+    )
+    version(
+        "2.13-3.4.1", sha256="a76f17a52b8f2cd31de11571ff366a714820b6e4e02893b0159d09db0edaf998"
+    )
     version(
         "2.13-2.4.0", sha256="c1c5246c7075459687b3160b713a001f5cd1cc563b9a3db189868d2f22aa9110"
     )
diff --git a/var/spack/repos/builtin/packages/kallisto/package.py b/var/spack/repos/builtin/packages/kallisto/package.py
index 85e4740f93e5e3..2e8cc2da93404e 100644
--- a/var/spack/repos/builtin/packages/kallisto/package.py
+++ b/var/spack/repos/builtin/packages/kallisto/package.py
@@ -20,7 +20,7 @@ class Kallisto(CMakePackage):
     # HDF5 support is optional beginning with version 0.46.2.
     variant("hdf5", when="@0.46.2:", default=False, description="Build with HDF5 support")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("hdf5", when="@:0.43")
     depends_on("hdf5", when="+hdf5")
 
diff --git a/var/spack/repos/builtin/packages/kassiopeia/package.py b/var/spack/repos/builtin/packages/kassiopeia/package.py
index a09be3bb8142d1..004d47756841c1 100644
--- a/var/spack/repos/builtin/packages/kassiopeia/package.py
+++ b/var/spack/repos/builtin/packages/kassiopeia/package.py
@@ -41,7 +41,7 @@ class Kassiopeia(CMakePackage):
     variant("boost", default=False, description="Build Boost dependent modules")
 
     depends_on("cmake@3.13:", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("root@6.0.0:", when="+root")
     depends_on("vtk@6.1:", when="+vtk")
     depends_on("mpi", when="+mpi")
diff --git a/var/spack/repos/builtin/packages/kcov/package.py b/var/spack/repos/builtin/packages/kcov/package.py
index 86ca3950cc1883..2c7a4009372c35 100644
--- a/var/spack/repos/builtin/packages/kcov/package.py
+++ b/var/spack/repos/builtin/packages/kcov/package.py
@@ -17,7 +17,7 @@ class Kcov(CMakePackage):
     version("38", sha256="b37af60d81a9b1e3b140f9473bdcb7975af12040feb24cc666f9bb2bb0be68b4")
 
     depends_on("cmake@2.8.4:", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("curl")
     depends_on("elfutils", when="platform=linux")
     depends_on("binutils +libiberty", when="platform=linux", type="link")
diff --git a/var/spack/repos/builtin/packages/kealib/package.py b/var/spack/repos/builtin/packages/kealib/package.py
index d4b17a7fb92538..3d9bdd7629c6b2 100644
--- a/var/spack/repos/builtin/packages/kealib/package.py
+++ b/var/spack/repos/builtin/packages/kealib/package.py
@@ -24,12 +24,14 @@ class Kealib(CMakePackage):
     """
 
     homepage = "http://www.kealib.org/"
-    url = "https://github.com/ubarsc/kealib/releases/download/kealib-1.5.0/kealib-1.5.0.tar.gz"
+    url = "https://github.com/ubarsc/kealib/releases/download/kealib-1.5.2/kealib-1.5.2.tar.gz"
     git = "https://github.com/ubarsc/kealib"
 
     maintainers("gillins", "neilflood", "petebunting")
 
     version("develop", git=git)
+    version("1.5.2", sha256="c4e17c472761a39e45184b5fa687395b319ac75430e0f6584dbf4cec6e335572")
+    version("1.5.1", sha256="06cd547b1e40394b9539beaf6982bd249e8ee93d6150295e9cd9161d00829657")
     version("1.5.0", sha256="d19a0fb051019f87fe413bda76472bf4fff8fca52ede92e0ffd983caeafd05b8")
     version("1.4.15", sha256="40f2573c00f005f93c1fa88f1f13bfbd485cbc7a9b3f1c706931e69bff17dae4")
     version("1.4.12", sha256="0b100e36b3e25e57487aa197d7be47f22e1b30afb16a57fdaa5f877696ec321e")
@@ -39,8 +41,9 @@ class Kealib(CMakePackage):
     version("1.4.8", sha256="0f24d8478865abcb17865c8f49c0370095726c529b8ac373ffae018ad3d40a02")
     version("1.4.7", sha256="ec38751b3b555d3a26f0c7445f2d2cd9d7c3a3502237519a206a50cb58df56ec")
 
-    depends_on("cmake@2.8.10:", type="build")
-    depends_on("hdf5+cxx+hl")
+    depends_on("cmake@3.5:", type="build")
+    depends_on("hdf5+cxx+hl", when="@:1.5.1")
+    depends_on("hdf5+cxx", when="@1.5.2:")
 
     patch("cmake.patch", when="@1.4.7")
 
diff --git a/var/spack/repos/builtin/packages/keepassxc/package.py b/var/spack/repos/builtin/packages/keepassxc/package.py
index baf67627c84f60..2ecc7d894dba9c 100644
--- a/var/spack/repos/builtin/packages/keepassxc/package.py
+++ b/var/spack/repos/builtin/packages/keepassxc/package.py
@@ -35,7 +35,7 @@ class Keepassxc(CMakePackage):
     # The following libraries are required:
     depends_on("qt+dbus~framework@5.2:")
     depends_on("libgcrypt@1.6:", type="link")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("minizip", when="+autotype")
     depends_on("libmicrohttpd", type="link")
     depends_on("libsodium@1.0.12:", type="link")
diff --git a/var/spack/repos/builtin/packages/kitty/package.py b/var/spack/repos/builtin/packages/kitty/package.py
index 47e068a474283b..ee21b7ecc733cf 100644
--- a/var/spack/repos/builtin/packages/kitty/package.py
+++ b/var/spack/repos/builtin/packages/kitty/package.py
@@ -33,7 +33,7 @@ class Kitty(Package):
     depends_on("python@3.5:", type=("build", "run"))
     depends_on("harfbuzz@1.5.0:")
     depends_on("libxkbcommon@0.5:")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libpng")
     depends_on("gl", type=("build", "link", "run"))
     depends_on("pkgconfig", type="build")
diff --git a/var/spack/repos/builtin/packages/kmergenie/package.py b/var/spack/repos/builtin/packages/kmergenie/package.py
index 96e3e42eff2724..f7ed2aaab12dcf 100644
--- a/var/spack/repos/builtin/packages/kmergenie/package.py
+++ b/var/spack/repos/builtin/packages/kmergenie/package.py
@@ -18,7 +18,7 @@ class Kmergenie(MakefilePackage):
     depends_on("python", type=("build", "run"))
     depends_on("py-setuptools", type=("build", "run"))
     depends_on("r", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         install_tree(self.stage.source_path, prefix.bin)
diff --git a/var/spack/repos/builtin/packages/knem/package.py b/var/spack/repos/builtin/packages/knem/package.py
index 85f1c8d1d6259b..3e1bcd925c8b3b 100644
--- a/var/spack/repos/builtin/packages/knem/package.py
+++ b/var/spack/repos/builtin/packages/knem/package.py
@@ -32,8 +32,8 @@ class Knem(AutotoolsPackage):
     variant("hwloc", default=True, description="Enable hwloc in the user-space tools")
 
     patch(
-        "https://gitlab.inria.fr/knem/knem/-/commit/5c8cb902d6040df58cdc4e4e4c10d1f1426c3525.patch",
-        sha256="78885a02d6f031a793db6a7190549f8d64c8606b353051d65f8e3f802b801902",
+        "https://gitlab.inria.fr/knem/knem/-/commit/5c8cb902d6040df58cdc4e4e4c10d1f1426c3525.diff",
+        sha256="a422277f02247bde680d4a3c8ccb8c05498a79109ba1ade4a037bedd6efe3c79",
         when="@1.1.4",
     )
 
@@ -49,12 +49,7 @@ class Knem(AutotoolsPackage):
     # Ideally, we should list all non-Linux-based platforms here:
     conflicts("platform=darwin")
 
-    # All compilers except for gcc are in conflict:
-    for __compiler in spack.compilers.supported_compilers():
-        if __compiler != "gcc":
-            conflicts(
-                "%{0}".format(__compiler), msg="Linux kernel module must be compiled with gcc"
-            )
+    requires("%gcc", msg="Linux kernel module must be compiled with gcc")
 
     @run_before("build")
     def override_kernel_compiler(self):
diff --git a/var/spack/repos/builtin/packages/kokkos-kernels/package.py b/var/spack/repos/builtin/packages/kokkos-kernels/package.py
index 404ea0520d6253..9f1cb88c4d72d9 100644
--- a/var/spack/repos/builtin/packages/kokkos-kernels/package.py
+++ b/var/spack/repos/builtin/packages/kokkos-kernels/package.py
@@ -70,12 +70,37 @@ class KokkosKernels(CMakePackage, CudaPackage):
         depends_on("kokkos+%s" % backend.lower(), when="+%s" % backend.lower())
 
     space_etis = {
-        "execspace_cuda": ("auto", "", "cuda"),
-        "execspace_openmp": ("auto", "", "openmp"),
-        "execspace_threads": ("auto", "", "threads"),
-        "execspace_serial": ("auto", "", "serial"),
-        "memspace_cudauvmspace": ("auto", "", "cuda"),
-        "memspace_cudaspace": ("auto", "", "cuda"),
+        "execspace_cuda": (
+            "auto",
+            "Whether to pre instantiate kernels for the execution space Kokkos::Cuda",
+            "cuda",
+        ),
+        "execspace_openmp": (
+            "auto",
+            "Whether to pre instantiate kernels for the execution space "
+            "Kokkos::Experimental::OpenMPTarget",
+            "openmp",
+        ),
+        "execspace_threads": (
+            "auto",
+            "Whether to build kernels for the execution space Kokkos::Threads",
+            "threads",
+        ),
+        "execspace_serial": (
+            "auto",
+            "Whether to build kernels for the execution space Kokkos::Serial",
+            "serial",
+        ),
+        "memspace_cudauvmspace": (
+            "auto",
+            "Whether to pre instantiate kernels for the memory space Kokkos::CudaUVMSpace",
+            "cuda",
+        ),
+        "memspace_cudaspace": (
+            "auto",
+            "Whether to pre instantiate kernels for the memory space Kokkos::CudaSpace",
+            "cuda",
+        ),
     }
     for eti in space_etis:
         deflt, descr, backend_required = space_etis[eti]
@@ -97,7 +122,7 @@ class KokkosKernels(CMakePackage, CudaPackage):
     }
     for eti in numeric_etis:
         deflt, cmake_name, vals = numeric_etis[eti]
-        variant(eti, default=deflt, values=vals, multi=True)
+        variant(eti, default=deflt, description=eti, values=vals, multi=True)
 
     tpls = {
         # variant name   #deflt   #spack name   #root var name #docstring
diff --git a/var/spack/repos/builtin/packages/kokkos-nvcc-wrapper/package.py b/var/spack/repos/builtin/packages/kokkos-nvcc-wrapper/package.py
index 463f3132f41769..1ecc52340c5b4e 100644
--- a/var/spack/repos/builtin/packages/kokkos-nvcc-wrapper/package.py
+++ b/var/spack/repos/builtin/packages/kokkos-nvcc-wrapper/package.py
@@ -19,6 +19,7 @@ class KokkosNvccWrapper(Package):
 
     maintainers("Rombur")
 
+    version("4.1.00", sha256="cf725ea34ba766fdaf29c884cfe2daacfdc6dc2d6af84042d1c78d0f16866275")
     version("4.0.01", sha256="bb942de8afdd519fd6d5d3974706bfc22b6585a62dd565c12e53bdb82cd154f0")
     version("4.0.00", sha256="1829a423883d4b44223c7c3a53d3c51671145aad57d7d23e6a1a4bebf710dcf6")
     version("3.7.02", sha256="5024979f06bc8da2fb696252a66297f3e0e67098595a0cc7345312b3b4aa0f54")
diff --git a/var/spack/repos/builtin/packages/kokkos/package.py b/var/spack/repos/builtin/packages/kokkos/package.py
index 0c99b1170af2ba..5eb7ce5cca0ac3 100644
--- a/var/spack/repos/builtin/packages/kokkos/package.py
+++ b/var/spack/repos/builtin/packages/kokkos/package.py
@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os.path
 
-from llnl.util import tty
+from llnl.util import lang, tty
 
 from spack.package import *
 
@@ -25,6 +25,7 @@ class Kokkos(CMakePackage, CudaPackage, ROCmPackage):
 
     version("master", branch="master")
     version("develop", branch="develop")
+    version("4.1.00", sha256="cf725ea34ba766fdaf29c884cfe2daacfdc6dc2d6af84042d1c78d0f16866275")
     version("4.0.01", sha256="bb942de8afdd519fd6d5d3974706bfc22b6585a62dd565c12e53bdb82cd154f0")
     version("4.0.00", sha256="1829a423883d4b44223c7c3a53d3c51671145aad57d7d23e6a1a4bebf710dcf6")
     version("3.7.02", sha256="5024979f06bc8da2fb696252a66297f3e0e67098595a0cc7345312b3b4aa0f54")
@@ -138,6 +139,17 @@ class Kokkos(CMakePackage, CudaPackage, ROCmPackage):
     cuda_arches = spack_cuda_arch_map.values()
     conflicts("+cuda", when="cuda_arch=none")
 
+    # Kokkos support only one cuda_arch at a time
+    variant(
+        "cuda_arch",
+        description="CUDA architecture",
+        values=("none",) + CudaPackage.cuda_arch_values,
+        default="none",
+        multi=False,
+        sticky=True,
+        when="+cuda",
+    )
+
     amdgpu_arch_map = {
         "gfx900": "vega900",
         "gfx906": "vega906",
@@ -174,22 +186,14 @@ class Kokkos(CMakePackage, CudaPackage, ROCmPackage):
         description="Intel GPU architecture",
     )
 
-    devices_values = list(devices_variants.keys())
-    for dev in devices_variants:
-        dflt, desc = devices_variants[dev]
+    for dev, (dflt, desc) in devices_variants.items():
         variant(dev, default=dflt, description=desc)
     conflicts("+cuda", when="+rocm", msg="CUDA and ROCm are not compatible in Kokkos.")
 
-    options_values = list(options_variants.keys())
-    for opt in options_values:
-        if "cuda" in opt:
-            conflicts("+%s" % opt, when="~cuda", msg="Must enable CUDA to use %s" % opt)
-        dflt, desc = options_variants[opt]
-        variant(opt, default=dflt, description=desc)
+    for opt, (dflt, desc) in options_variants.items():
+        variant(opt, default=dflt, description=desc, when=("+cuda" if "cuda" in opt else None))
 
-    tpls_values = list(tpls_variants.keys())
-    for tpl in tpls_values:
-        dflt, desc = tpls_variants[tpl]
+    for tpl, (dflt, desc) in tpls_variants.items():
         variant(tpl, default=dflt, description=desc)
         depends_on(tpl, when="+%s" % tpl)
 
@@ -199,29 +203,30 @@ class Kokkos(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("kokkos-nvcc-wrapper@master", when="@master+wrapper")
     conflicts("+wrapper", when="~cuda")
 
-    stds = ["11", "14", "17", "20"]
-    # TODO: This should be named cxxstd for consistency with other packages
-    variant("std", default="17", values=stds, multi=False)
+    cxxstds = ["11", "14", "17", "20"]
+    variant("cxxstd", default="17", values=cxxstds, multi=False, description="C++ standard")
     variant("pic", default=False, description="Build position independent code")
 
-    conflicts("std=11", when="@3.7:")
-    conflicts("std=14", when="@4.0:")
+    conflicts("cxxstd=11", when="@3.7:")
+    conflicts("cxxstd=14", when="@4.0:")
 
-    conflicts("+cuda", when="std=17 ^cuda@:10")
-    conflicts("+cuda", when="std=20 ^cuda@:11")
+    conflicts("+cuda", when="cxxstd=17 ^cuda@:10")
+    conflicts("+cuda", when="cxxstd=20 ^cuda@:11")
 
     # SYCL and OpenMPTarget require C++17 or higher
-    for stdver in stds[: stds.index("17")]:
-        conflicts("+sycl", when="std={0}".format(stdver), msg="SYCL requires C++17 or higher")
+    for cxxstdver in cxxstds[: cxxstds.index("17")]:
+        conflicts(
+            "+sycl", when="cxxstd={0}".format(cxxstdver), msg="SYCL requires C++17 or higher"
+        )
         conflicts(
             "+openmptarget",
-            when="std={0}".format(stdver),
+            when="cxxstd={0}".format(cxxstdver),
             msg="OpenMPTarget requires C++17 or higher",
         )
 
     # HPX should use the same C++ standard
-    for std in stds:
-        depends_on("hpx cxxstd={0}".format(std), when="+hpx std={0}".format(std))
+    for cxxstd in cxxstds:
+        depends_on("hpx cxxstd={0}".format(cxxstd), when="+hpx cxxstd={0}".format(cxxstd))
 
     # HPX version constraints
     depends_on("hpx@:1.6", when="@:3.5 +hpx")
@@ -262,7 +267,7 @@ def append_args(self, cmake_prefix, cmake_options, spack_options):
             optname = "Kokkos_%s_%s" % (cmake_prefix, opt.upper())
             # Explicitly enable or disable
             option = self.define_from_variant(optname, variant_name)
-            if option not in spack_options:
+            if option:
                 spack_options.append(option)
 
     def setup_dependent_package(self, module, dependent_spec):
@@ -282,17 +287,22 @@ def cmake_args(self):
 
         options = [
             from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
-            from_variant("CMAKE_CXX_STANDARD", "std"),
+            from_variant("CMAKE_CXX_STANDARD", "cxxstd"),
             from_variant("BUILD_SHARED_LIBS", "shared"),
         ]
 
         spack_microarches = []
         if "+cuda" in spec:
-            # this is a list
-            for cuda_arch in spec.variants["cuda_arch"].value:
-                if not cuda_arch == "none":
-                    kokkos_arch_name = self.spack_cuda_arch_map[cuda_arch]
-                    spack_microarches.append(kokkos_arch_name)
+            if isinstance(spec.variants["cuda_arch"].value, str):
+                cuda_arch = spec.variants["cuda_arch"].value
+            else:
+                if len(spec.variants["cuda_arch"].value) > 1:
+                    msg = "Kokkos supports only one cuda_arch at a time."
+                    raise InstallError(msg)
+                cuda_arch = spec.variants["cuda_arch"].value[0]
+            if cuda_arch != "none":
+                kokkos_arch_name = self.spack_cuda_arch_map[cuda_arch]
+                spack_microarches.append(kokkos_arch_name)
 
         kokkos_microarch_name = self.get_microarch(spec.target)
         if kokkos_microarch_name:
@@ -314,11 +324,11 @@ def cmake_args(self):
         for arch in spack_microarches:
             options.append(self.define("Kokkos_ARCH_" + arch.upper(), True))
 
-        self.append_args("ENABLE", self.devices_values, options)
-        self.append_args("ENABLE", self.options_values, options)
-        self.append_args("ENABLE", self.tpls_values, options)
+        self.append_args("ENABLE", self.devices_variants.keys(), options)
+        self.append_args("ENABLE", self.options_variants.keys(), options)
+        self.append_args("ENABLE", self.tpls_variants.keys(), options)
 
-        for tpl in self.tpls_values:
+        for tpl in self.tpls_variants:
             if spec.variants[tpl].value:
                 options.append(self.define(tpl + "_DIR", spec[tpl].prefix))
 
@@ -329,7 +339,11 @@ def cmake_args(self):
                 self.define("CMAKE_CXX_COMPILER", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
             )
 
-        return options
+        if self.spec.satisfies("%oneapi") or self.spec.satisfies("%intel"):
+            options.append(self.define("CMAKE_CXX_FLAGS", "-fp-model=precise"))
+
+        # Remove duplicate options
+        return lang.dedupe(options)
 
     test_script_relative_path = join_path("scripts", "spack_test")
 
diff --git a/var/spack/repos/builtin/packages/krakenuniq/package.py b/var/spack/repos/builtin/packages/krakenuniq/package.py
index e092c783f15543..44171008af4018 100644
--- a/var/spack/repos/builtin/packages/krakenuniq/package.py
+++ b/var/spack/repos/builtin/packages/krakenuniq/package.py
@@ -26,7 +26,7 @@ class Krakenuniq(Package):
     variant("jellyfish", default=False, description="Install jellyfish v1.1.")
 
     depends_on("bzip2")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("wget", when="+jellyfish")
 
     def install(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/krb5/package.py b/var/spack/repos/builtin/packages/krb5/package.py
index 92c88948af5955..84dc616066e01d 100644
--- a/var/spack/repos/builtin/packages/krb5/package.py
+++ b/var/spack/repos/builtin/packages/krb5/package.py
@@ -35,6 +35,7 @@ class Krb5(AutotoolsPackage):
     depends_on("openssl")
     depends_on("gettext")
     depends_on("findutils", type="build")
+    depends_on("pkgconfig", type="build", when="^openssl~shared")
 
     variant(
         "shared", default=True, description="install shared libraries if True, static if false"
@@ -81,7 +82,9 @@ def configure_args(self):
             args.append("CFLAGS=-fcommon")
 
         if self.spec["openssl"].satisfies("~shared"):
-            args.append("LDFLAGS=-L%s -lz" % self.spec["zlib"].prefix.lib)
+            pkgconf = which("pkg-config")
+            ssllibs = pkgconf("--static", "--libs", "openssl", output=str)
+            args.append(f"LDFLAGS={ssllibs}")
 
         return args
 
diff --git a/var/spack/repos/builtin/packages/kripke/package.py b/var/spack/repos/builtin/packages/kripke/package.py
index 274289208ae96a..36768b0ff22a90 100644
--- a/var/spack/repos/builtin/packages/kripke/package.py
+++ b/var/spack/repos/builtin/packages/kripke/package.py
@@ -19,23 +19,42 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage):
     maintainers("vsrana01")
 
     version("develop", branch="develop", submodules=False)
-    version("1.2.4", submodules=False, tag="v1.2.4")
-    version("1.2.3", submodules=True, tag="v1.2.3")
-    version("1.2.2", submodules=True, tag="v1.2.2-CORAL2")
-    version("1.2.1", submodules=True, tag="v1.2.1-CORAL2")
-    version("1.2.0", submodules=True, tag="v1.2.0-CORAL2")
+    version(
+        "1.2.4", submodules=False, tag="v1.2.4", commit="d85c6bc462f17a2382b11ba363059febc487f771"
+    )
+    version(
+        "1.2.3", submodules=True, tag="v1.2.3", commit="66046d8cd51f5bcf8666fd8c810322e253c4ce0e"
+    )
+    version(
+        "1.2.2",
+        submodules=True,
+        tag="v1.2.2-CORAL2",
+        commit="a12bce71e751f8f999009aa2fd0839b908b118a4",
+    )
+    version(
+        "1.2.1",
+        submodules=True,
+        tag="v1.2.1-CORAL2",
+        commit="c36453301ddd684118bb0fb426cfa62764d42398",
+    )
+    version(
+        "1.2.0",
+        submodules=True,
+        tag="v1.2.0-CORAL2",
+        commit="67e4b0a2f092009d61f44b5122111d388a3bec2a",
+    )
 
     variant("mpi", default=True, description="Build with MPI.")
     variant("openmp", default=False, description="Build with OpenMP enabled.")
     variant("caliper", default=False, description="Build with Caliper support enabled.")
 
     depends_on("mpi", when="+mpi")
-    depends_on("blt")
-    depends_on("cmake")
+    depends_on("blt", type="build")
     depends_on("caliper", when="+caliper")
     depends_on("chai~examples+raja")
     depends_on("raja~exercises~examples")
     depends_on("umpire~examples")
+    conflicts("^blt@:0.3.6", when="+rocm")
 
     def cmake_args(self):
         spec = self.spec
diff --git a/var/spack/repos/builtin/packages/kvtree/package.py b/var/spack/repos/builtin/packages/kvtree/package.py
index 4b14714b78464c..8177e76c8f117d 100644
--- a/var/spack/repos/builtin/packages/kvtree/package.py
+++ b/var/spack/repos/builtin/packages/kvtree/package.py
@@ -26,7 +26,7 @@ class Kvtree(CMakePackage):
     version("1.0.3", sha256="c742cdb1241ef4cb13767019204d5350a3c4383384bed9fb66680b93ff44b0d4")
     version("1.0.2", sha256="56fb5b747758c24a907a8380e8748d296900d94de9547bc15f6b427ac4ae2ec4")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     variant("mpi", default=True, description="Build with MPI message packing")
     depends_on("mpi", when="+mpi")
diff --git a/var/spack/repos/builtin/packages/lammps/package.py b/var/spack/repos/builtin/packages/lammps/package.py
index 80faba00454d50..b2d3d111334b8e 100644
--- a/var/spack/repos/builtin/packages/lammps/package.py
+++ b/var/spack/repos/builtin/packages/lammps/package.py
@@ -23,39 +23,72 @@ class Lammps(CMakePackage, CudaPackage, ROCmPackage):
     maintainers("rbberger")
 
     # rules for new versions and deprecation
-    # * new stable versions should be marked as preferred=True
+    # * new stable versions should be added to stable_versions set
     # * a stable version that has updates and any of its outdated update releases should be
     #   marked deprecated=True
     # * patch releases older than a stable release should be marked deprecated=True
     version("develop", branch="develop")
-    version("20230615", sha256="8470ed7b26ccd3728f4b44a7f1c520f1af23a648af685fd30b42b840fdfae2ff")
-    version("20230328", sha256="14f5a5c37e4b46466e90d8b35476800e66acee74999f7358f4c12dfe662bfd99")
-    version("20230208", sha256="60221242145da4479e5b207d9a0eed90af4168d7a297b4dc8c0e7f2b3215602e")
-    version("20221222", sha256="75372ee7ef982767fc4ed4dc95e20ddca8247419adeb0c1276c40e43d1eab955")
-    version("20221103", sha256="d28517b84b157d4e46a1a64ed787b4662d8f2f5ade3f5a04bb0caed068f32f7e")
-    version("20220915", sha256="392b8d35fc7919b0efaf8b389259a9b795a817e58e75e380467c63d03a0dab75")
-    version("20220803", sha256="f37cb0b35c1682ffceae5826aadce47278aa7003099a1655fcea43acd7d37926")
+    version(
+        "20230802",
+        sha256="48dc8b0b0583689e80ea2052275acbc3e3fce89707ac557e120db5564257f7df",
+        preferred=True,
+    )
+    version(
+        "20230615",
+        sha256="8470ed7b26ccd3728f4b44a7f1c520f1af23a648af685fd30b42b840fdfae2ff",
+        deprecated=True,
+    )
+    version(
+        "20230328",
+        sha256="14f5a5c37e4b46466e90d8b35476800e66acee74999f7358f4c12dfe662bfd99",
+        deprecated=True,
+    )
+    version(
+        "20230208",
+        sha256="60221242145da4479e5b207d9a0eed90af4168d7a297b4dc8c0e7f2b3215602e",
+        deprecated=True,
+    )
+    version(
+        "20221222",
+        sha256="75372ee7ef982767fc4ed4dc95e20ddca8247419adeb0c1276c40e43d1eab955",
+        deprecated=True,
+    )
+    version(
+        "20221103",
+        sha256="d28517b84b157d4e46a1a64ed787b4662d8f2f5ade3f5a04bb0caed068f32f7e",
+        deprecated=True,
+    )
+    version(
+        "20220915",
+        sha256="392b8d35fc7919b0efaf8b389259a9b795a817e58e75e380467c63d03a0dab75",
+        deprecated=True,
+    )
+    version(
+        "20220803",
+        sha256="f37cb0b35c1682ffceae5826aadce47278aa7003099a1655fcea43acd7d37926",
+        deprecated=True,
+    )
+    version(
+        "20220623.4", sha256="42541b4dbd0d339d16ddb377e76d192bc3d1d5712fdf9e2cdc838fc980d0a0cf"
+    )
     version(
         "20220623.3",
         sha256="8a276a01b50d37eecfe6eb36f420f354cde51936d20aca7944dea60d3c098c89",
-        preferred=True,
+        deprecated=True,
     )
     version(
         "20220623.2",
         sha256="8a560213e83919623525c4a7c4b5f0eda35cdf3b0c0e6548fd891379e04ca9e6",
-        preferred=True,
         deprecated=True,
     )
     version(
         "20220623.1",
         sha256="58e3b2b984f8935bb0db5631e143be2826c45ffd48844f7c394f36624a3e17a2",
-        preferred=True,
         deprecated=True,
     )
     version(
         "20220623",
         sha256="d27ede095c9f00cd13a26f967a723d07cf8f4df65c700ed73573577bc173d5ce",
-        preferred=True,
         deprecated=True,
     )
     version(
@@ -94,26 +127,21 @@ class Lammps(CMakePackage, CudaPackage, ROCmPackage):
         deprecated=True,
     )
     version(
-        "20210929.3",
-        sha256="e4c274f0dc5fdedc43f2b365156653d1105197a116ff2bafe893523cdb22532e",
-        preferred=True,
+        "20210929.3", sha256="e4c274f0dc5fdedc43f2b365156653d1105197a116ff2bafe893523cdb22532e"
     )
     version(
         "20210929.2",
         sha256="9318ca816cde16a9a4174bf22a1966f5f2155cb32c0ad5a6757633276411fb36",
-        preferred=True,
         deprecated=True,
     )
     version(
         "20210929.1",
         sha256="5000b422c9c245b92df63507de5aa2ea4af345ea1f00180167aaa084b711c27c",
-        preferred=True,
         deprecated=True,
     )
     version(
         "20210929",
         sha256="2dff656cb21fd9a6d46c818741c99d400cfb1b12102604844663b655fb2f893d",
-        preferred=True,
         deprecated=True,
     )
     version(
@@ -166,11 +194,7 @@ class Lammps(CMakePackage, CudaPackage, ROCmPackage):
         sha256="2c5ba2c7935ad559ca94ee826e8727e65b49ef4582eb856534fffba70e44101a",
         deprecated=True,
     )
-    version(
-        "20201029",
-        sha256="759705e16c1fedd6aa6e07d028cc0c78d73c76b76736668420946a74050c3726",
-        preferred=True,
-    )
+    version("20201029", sha256="759705e16c1fedd6aa6e07d028cc0c78d73c76b76736668420946a74050c3726")
     version(
         "20200721",
         sha256="845bfeddb7b667799a1a5dbc166b397d714c3d2720316604a979d3465b4190a9",
@@ -186,11 +210,7 @@ class Lammps(CMakePackage, CudaPackage, ROCmPackage):
         sha256="c49d77fd602d28ebd8cf10f7359b9fc4d14668c72039028ed7792453d416de73",
         deprecated=True,
     )
-    version(
-        "20200303",
-        sha256="a1a2e3e763ef5baecea258732518d75775639db26e60af1634ab385ed89224d1",
-        preferred=True,
-    )
+    version("20200303", sha256="a1a2e3e763ef5baecea258732518d75775639db26e60af1634ab385ed89224d1")
     version(
         "20200227",
         sha256="1aabcf38bc72285797c710b648e906151a912c36b634a9c88ac383aacf85516e",
@@ -322,6 +342,21 @@ class Lammps(CMakePackage, CudaPackage, ROCmPackage):
         deprecated=True,
     )
 
+    stable_versions = {
+        "20230802",
+        "20220623.4",
+        "20220623.3",
+        "20220623.2",
+        "20220623.1",
+        "20220623",
+        "20210929.3",
+        "20210929.2",
+        "20210929.1",
+        "20210929",
+        "20201029",
+        "20200303",
+    }
+
     def url_for_version(self, version):
         split_ver = str(version).split(".")
         vdate = dt.datetime.strptime(split_ver[0], "%Y%m%d")
@@ -330,14 +365,10 @@ def url_for_version(self, version):
         else:
             update = "_update{0}".format(split_ver[1])
 
-        is_stable = (
-            version in self.versions
-            and "preferred" in self.versions[version]
-            and self.versions[version]["preferred"]
-        )
-
         return "https://github.com/lammps/lammps/archive/{0}_{1}{2}.tar.gz".format(
-            "stable" if is_stable else "patch", vdate.strftime("%d%b%Y").lstrip("0"), update
+            "stable" if str(version) in Lammps.stable_versions else "patch",
+            vdate.strftime("%d%b%Y").lstrip("0"),
+            update,
         )
 
     # List of supported optional packages
@@ -654,15 +685,20 @@ def url_for_version(self, version):
 
     # Older LAMMPS does not compile with Kokkos 4.x
     conflicts(
-        "^kokkos @4:",
-        when="@:20230615",
-        msg="LAMMPS is incompatible with Kokkos 4.x until @20230615",
+        "^kokkos@4:",
+        when="@:20230802",
+        msg="LAMMPS is incompatible with Kokkos 4.x until @20230802",
     )
 
     patch("lib.patch", when="@20170901")
     patch("660.patch", when="@20170922")
     patch("gtest_fix.patch", when="@:20210310 %aocc@3.2.0")
-    patch("intel-aocc.patch", when="@20220324:20221103 +intel %aocc")
+
+    # This patch merged to LAMMPS trunk at 20221222 and backported to
+    # stable version 20220623.4. We still patch all other affected
+    # versions here
+    patch("intel-aocc.patch", when="@20220324:20220623.3,20220803:20221103 +intel %aocc")
+
     patch(
         "https://github.com/lammps/lammps/commit/562300996285fdec4ef74542383276898555af06.patch?full_index=1",
         sha256="e6f1b62bbfdc79d632f4cea98019202d0dd25aa4ae61a70df1164cb4f290df79",
@@ -755,7 +791,7 @@ def cmake_args(self):
             # FFTW libraries are available and enable them by default.
             if "^fftw" in spec or "^cray-fftw" in spec or "^amdfftw" in spec:
                 args.append(self.define("FFT", "FFTW3"))
-            elif "^mkl" in spec:
+            elif spec["fftw-api"].name in INTEL_MATH_LIBRARIES:
                 args.append(self.define("FFT", "MKL"))
             elif "^armpl-gcc" in spec or "^acfl" in spec:
                 args.append(self.define("FFT", "FFTW3"))
diff --git a/var/spack/repos/builtin/packages/landsfcutil/package.py b/var/spack/repos/builtin/packages/landsfcutil/package.py
index 7b7ba2e578d90c..64752d3e1ea6ac 100644
--- a/var/spack/repos/builtin/packages/landsfcutil/package.py
+++ b/var/spack/repos/builtin/packages/landsfcutil/package.py
@@ -14,11 +14,21 @@ class Landsfcutil(CMakePackage):
 
     homepage = "https://github.com/NOAA-EMC/NCEPLIBS-landsfcutil"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-landsfcutil/archive/refs/tags/v2.4.1.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-landsfcutil"
 
     maintainers("edwardhartnett", "AlexanderRichert-NOAA", "Hang-Lei-NOAA")
 
+    version("develop", branch="develop")
     version("2.4.1", sha256="831c5005a480eabe9a8542b4deec838c2650f6966863ea2711cc0cc5db51ca14")
 
+    variant("pfunit", default=False, description="Enable pFunit testing")
+
+    depends_on("pfunit", when="+pfunit")
+
+    def cmake_args(self):
+        args = [self.define("ENABLE_TESTS", self.run_tests)]
+        return args
+
     def setup_run_environment(self, env):
         for suffix in ("4", "d"):
             lib = find_libraries(
@@ -33,3 +43,7 @@ def flag_handler(self, name, flags):
             if name == "fflags":
                 flags.append("-Free")
         return (None, None, flags)
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/lapackpp/package.py b/var/spack/repos/builtin/packages/lapackpp/package.py
index 071fe28cffbab5..a69e4e7bdffe16 100644
--- a/var/spack/repos/builtin/packages/lapackpp/package.py
+++ b/var/spack/repos/builtin/packages/lapackpp/package.py
@@ -11,6 +11,8 @@
 _versions = [
     # LAPACK++,     BLAS++
     ["master", "master"],
+    ["2023.08.25", "2023.08.25"],
+    ["2023.06.00", "2023.06.00"],
     ["2022.07.00", "2022.07.00"],
     ["2022.05.00", "2022.05.00"],
     ["2020.10.00", "2020.10.00"],
@@ -31,6 +33,12 @@ class Lapackpp(CMakePackage, CudaPackage, ROCmPackage):
     maintainers("teonnik", "Sely85", "G-Ragghianti", "mgates3")
 
     version("master", branch="master")
+    version(
+        "2023.08.25", sha256="9effdd616a4a183a9b37c2ad33c85ddd3d6071b183e8c35e02243fbaa7333d4d"
+    )
+    version(
+        "2023.06.00", sha256="93df8392c859071120e00239feb96a0e060c0bb5176ee3a4f03eb9777c4edead"
+    )
     version(
         "2022.07.00", sha256="11e59efcc7ea0764a2bfc0e0f7b1abf73cee2943c1df11a19601780641a9aa18"
     )
@@ -51,6 +59,7 @@ class Lapackpp(CMakePackage, CudaPackage, ROCmPackage):
     )
 
     variant("shared", default=True, description="Build shared library")
+    variant("sycl", default=False, description="Build support for the SYCL backend")
 
     # Match each LAPACK++ version to a specific BLAS++ version
     for lpp_ver, bpp_ver in _versions:
@@ -58,6 +67,8 @@ class Lapackpp(CMakePackage, CudaPackage, ROCmPackage):
 
     depends_on("blaspp ~cuda", when="~cuda")
     depends_on("blaspp +cuda", when="+cuda")
+    depends_on("blaspp ~sycl", when="~sycl")
+    depends_on("blaspp +sycl", when="+sycl")
     depends_on("blaspp ~rocm", when="~rocm")
     for val in ROCmPackage.amdgpu_targets:
         depends_on("blaspp +rocm amdgpu_target=%s" % val, when="amdgpu_target=%s" % val)
@@ -66,8 +77,15 @@ class Lapackpp(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("lapack")
     depends_on("rocblas", when="+rocm")
     depends_on("rocsolver", when="+rocm")
+    depends_on("intel-oneapi-mkl threads=openmp", when="+sycl")
+
+    backend_msg = "LAPACK++ supports only one GPU backend at a time"
+    conflicts("+rocm", when="+cuda", msg=backend_msg)
+    conflicts("+rocm", when="+sycl", msg=backend_msg)
+    conflicts("+cuda", when="+sycl", msg=backend_msg)
+    conflicts("+sycl", when="@:2023.06.00", msg="+sycl requires LAPACK++ version 2023.08.25")
 
-    conflicts("+rocm", when="+cuda", msg="LAPACK++ can only support one GPU backend at a time")
+    requires("%oneapi", when="+sycl", msg="lapackpp+sycl must be compiled with %oneapi")
 
     def cmake_args(self):
         spec = self.spec
@@ -78,6 +96,8 @@ def cmake_args(self):
                 backend = "cuda"
             if "+rocm" in spec:
                 backend = "hip"
+            if "+sycl" in spec:
+                backend = "sycl"
 
         args = [
             "-DBUILD_SHARED_LIBS=%s" % ("+shared" in spec),
@@ -93,8 +113,8 @@ def cmake_args(self):
 
     def check(self):
         # If the tester fails to build, ensure that the check() fails.
-        if os.path.isfile(join_path(self.build_directory, "test", "tester")):
-            with working_dir(self.build_directory):
+        if os.path.isfile(join_path(self.builder.build_directory, "test", "tester")):
+            with working_dir(self.builder.build_directory):
                 make("check")
         else:
             raise Exception("The tester was not built!")
diff --git a/var/spack/repos/builtin/packages/last/package.py b/var/spack/repos/builtin/packages/last/package.py
index 8e1b953e301f03..a59da3ef5382be 100644
--- a/var/spack/repos/builtin/packages/last/package.py
+++ b/var/spack/repos/builtin/packages/last/package.py
@@ -16,10 +16,15 @@ class Last(MakefilePackage):
     git = "https://gitlab.com/mcfrith/last.git"
     maintainers("snehring")
 
+    version("1499", commit="2cc68d3ba8ae5ca46ceeb69302aef18b9db04f46")
     version("1282", commit="4368be912f4759e52b549940276f1adf087f489a")
     version("869", sha256="6371a6282bc1bb02a5e5013cc463625f2ce3e7746ff2ea0bdf9fe6b15605a67c")
 
+    depends_on("zlib-api")
+
     def edit(self, spec, prefix):
+        if not spec.satisfies("target=x86_64:"):
+            filter_file("-msse4", "", "makefile")
         files = ["mat-doc.sh", "mat-inc.sh", "seed-doc.sh", "seed-inc.sh"]
         if spec.satisfies("@1282:"):
             files.append("gc-inc.sh")
diff --git a/var/spack/repos/builtin/packages/lbann/package.py b/var/spack/repos/builtin/packages/lbann/package.py
index 54a0c690f2868d..faae710921e563 100644
--- a/var/spack/repos/builtin/packages/lbann/package.py
+++ b/var/spack/repos/builtin/packages/lbann/package.py
@@ -140,11 +140,25 @@ class Lbann(CachedCMakePackage, CudaPackage, ROCmPackage):
     variant("onednn", default=False, description="Support for OneDNN")
     variant("onnx", default=False, description="Support for exporting models into ONNX format")
     variant("nvshmem", default=False, description="Support for NVSHMEM")
-    variant("python", default=True, description="Support for Python extensions (e.g. Data Reader)")
-    variant("pfe", default=True, description="Python Frontend for generating and launching models")
+    variant(
+        "python",
+        default=True,
+        sticky=True,
+        description="Support for Python extensions (e.g. Data Reader)",
+    )
+    variant(
+        "pfe",
+        default=True,
+        sticky=True,
+        description="Python Frontend for generating and launching models",
+    )
     variant("boost", default=False, description="Enable callbacks that use Boost libraries")
     variant("asan", default=False, description="Build with support for address-sanitizer")
     variant("unit_tests", default=False, description="Support for unit testing")
+    variant("caliper", default=False, description="Support for instrumentation with caliper")
+    variant(
+        "shared", default=True, sticky=True, description="Enables the build of shared libraries"
+    )
 
     # LBANN benefits from high performance linkers, but passing these in as command
     # line options forces the linker flags to unnecessarily propagate to all
@@ -168,9 +182,7 @@ class Lbann(CachedCMakePackage, CudaPackage, ROCmPackage):
     conflicts("~python", when="@0.91:0.101")
     conflicts("~pfe", when="@0.91:0.101")
 
-    for comp in spack.compilers.supported_compilers():
-        if comp != "clang":
-            conflicts("+lld", when="%" + comp)
+    requires("%clang", when="+lld")
 
     conflicts("+lld", when="+gold")
     conflicts("+gold", when="platform=darwin", msg="gold does not work on Darwin")
@@ -232,6 +244,8 @@ class Lbann(CachedCMakePackage, CudaPackage, ROCmPackage):
     depends_on("dihydrogen@:0.0,0.2:", when="@:0.90,0.102: +dihydrogen")
     conflicts("~dihydrogen", when="+distconv")
 
+    depends_on("hdf5+mpi", when="+distconv")
+
     for arch in CudaPackage.cuda_arch_values:
         depends_on("hydrogen cuda_arch=%s" % arch, when="+cuda cuda_arch=%s" % arch)
         depends_on("aluminum cuda_arch=%s" % arch, when="+al +cuda cuda_arch=%s" % arch)
@@ -257,6 +271,7 @@ class Lbann(CachedCMakePackage, CudaPackage, ROCmPackage):
     depends_on("hwloc@1.11.0:1.11", when="@0.95:0.101 +hwloc")
     depends_on("hwloc +cuda +nvml", when="+cuda")
     depends_on("hwloc@2.3.0:", when="+rocm")
+    depends_on("hiptt", when="+rocm")
 
     depends_on("half", when="+half")
 
@@ -299,7 +314,7 @@ class Lbann(CachedCMakePackage, CudaPackage, ROCmPackage):
     depends_on("py-protobuf+cpp@3.10.0:", type=("build", "run"), when="@:0.90,0.99: +pfe")
 
     depends_on("protobuf+shared@3.10.0:", when="@:0.90,0.99:")
-    depends_on("zlib", when="protobuf@3.11.0:")
+    depends_on("zlib-api", when="protobuf@3.11.0:")
 
     # using cereal@1.3.1 and above requires changing the
     # find_package call to lowercase, so stick with :1.3.0
@@ -316,6 +331,8 @@ class Lbann(CachedCMakePackage, CudaPackage, ROCmPackage):
     depends_on("spdlog", when="@:0.90,0.102:")
     depends_on("zstr")
 
+    depends_on("caliper+adiak+mpi", when="+caliper")
+
     generator("ninja")
 
     def setup_build_environment(self, env):
@@ -331,12 +348,16 @@ def _get_sys_type(self, spec):
             sys_type = env["SYS_TYPE"]
         return sys_type
 
+    @property
+    def libs(self):
+        shared = True if "+shared" in self.spec else False
+        return find_libraries("liblbann", root=self.prefix, shared=shared, recursive=True)
+
     @property
     def cache_name(self):
         hostname = socket.gethostname()
-        if "SYS_TYPE" in env:
-            # Get a hostname that has no node identifier
-            hostname = hostname.rstrip("1234567890")
+        # Get a hostname that has no node identifier
+        hostname = hostname.rstrip("1234567890")
         return "LBANN_{0}_{1}-{2}-{3}@{4}.cmake".format(
             hostname,
             self.spec.version,
@@ -349,6 +370,7 @@ def initconfig_compiler_entries(self):
         spec = self.spec
         entries = super().initconfig_compiler_entries()
         entries.append(cmake_cache_string("CMAKE_CXX_STANDARD", "17"))
+        entries.append(cmake_cache_option("BUILD_SHARED_LIBS", "+shared" in spec))
         if not spec.satisfies("^cmake@3.23.0"):
             # There is a bug with using Ninja generator in this version
             # of CMake
@@ -364,6 +386,17 @@ def initconfig_compiler_entries(self):
             entries.append(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", "-fuse-ld=gold"))
             entries.append(cmake_cache_string("CMAKE_SHARED_LINKER_FLAGS", "-fuse-ld=gold"))
 
+        # Set the generator in the cached config
+        if self.spec.satisfies("generator=make"):
+            entries.append(cmake_cache_string("CMAKE_GENERATOR", "Unix Makefiles"))
+        if self.spec.satisfies("generator=ninja"):
+            entries.append(cmake_cache_string("CMAKE_GENERATOR", "Ninja"))
+            entries.append(
+                cmake_cache_string(
+                    "CMAKE_MAKE_PROGRAM", "{0}/ninja".format(spec["ninja"].prefix.bin)
+                )
+            )
+
         return entries
 
     def initconfig_hardware_entries(self):
@@ -411,6 +444,7 @@ def initconfig_package_entries(self):
             ("LBANN_WITH_ALUMINUM", "al"),
             ("LBANN_WITH_ADDRESS_SANITIZER", "asan"),
             ("LBANN_WITH_BOOST", "boost"),
+            ("LBANN_WITH_CALIPER", "caliper"),
             ("LBANN_WITH_CONDUIT", "conduit"),
             ("LBANN_WITH_NVSHMEM", "nvshmem"),
             ("LBANN_WITH_FFT", "fft"),
diff --git a/var/spack/repos/builtin/packages/lcio/package.py b/var/spack/repos/builtin/packages/lcio/package.py
index 244cb3ac4be752..8f02063af4c760 100644
--- a/var/spack/repos/builtin/packages/lcio/package.py
+++ b/var/spack/repos/builtin/packages/lcio/package.py
@@ -16,7 +16,7 @@ class Lcio(CMakePackage):
 
     tags = ["hep"]
 
-    maintainers("gaede", "vvolkl")
+    maintainers("gaede", "vvolkl", "jmcarcell")
 
     version("master", branch="master")
     version("2.20", sha256="5ef92c9ef04ce468ffb48be0ec6010377a400b064e352cb50f9f4c9599e7e990")
@@ -107,6 +107,7 @@ def setup_run_environment(self, env):
         env.prepend_path("PYTHONPATH", self.prefix.python)
         # needed for the python bindings to find "Exceptions.h"
         env.prepend_path("CPATH", self.prefix)
+        env.prepend_path("LD_LIBRARY_PATH", self.spec["lcio"].libs.directories[0])
 
     @run_after("install")
     def install_source(self):
diff --git a/var/spack/repos/builtin/packages/lcms/package.py b/var/spack/repos/builtin/packages/lcms/package.py
index ed6907618a93ce..4620a208a550f1 100644
--- a/var/spack/repos/builtin/packages/lcms/package.py
+++ b/var/spack/repos/builtin/packages/lcms/package.py
@@ -25,7 +25,7 @@ def url_for_version(self, version):
 
     depends_on("jpeg")
     depends_on("libtiff")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     @property
     def libs(self):
diff --git a/var/spack/repos/builtin/packages/lcov/package.py b/var/spack/repos/builtin/packages/lcov/package.py
index 808601c1089735..dc7d3aa1b637c3 100644
--- a/var/spack/repos/builtin/packages/lcov/package.py
+++ b/var/spack/repos/builtin/packages/lcov/package.py
@@ -20,5 +20,12 @@ class Lcov(MakefilePackage):
     version("1.15", sha256="c1cda2fa33bec9aa2c2c73c87226cfe97de0831887176b45ee523c5e30f8053a")
     version("1.14", sha256="14995699187440e0ae4da57fe3a64adc0a3c5cf14feab971f8db38fb7d8f071a")
 
+    depends_on("perl")
+
     def install(self, spec, prefix):
-        make("DESTDIR=", "PREFIX=%s" % prefix, "install")
+        make(
+            "LCOV_PERL_PATH=%s" % self.spec["perl"].command.path,
+            "DESTDIR=",
+            "PREFIX=%s" % prefix,
+            "install",
+        )
diff --git a/var/spack/repos/builtin/packages/ldak/package.py b/var/spack/repos/builtin/packages/ldak/package.py
index 941b04955be480..d074d90ea6830b 100644
--- a/var/spack/repos/builtin/packages/ldak/package.py
+++ b/var/spack/repos/builtin/packages/ldak/package.py
@@ -12,31 +12,68 @@ class Ldak(Package):
     homepage = "https://dougspeed.com/ldak/"
     url = "https://dougspeed.com/wp-content/uploads/source.zip"
 
-    version("5.1", sha256="ae3eb8c2ef31af210e138336fd6edcd0e3a26ea9bae89fd6c0c6ea33e3a1517e")
+    maintainers("snehring")
 
-    variant("mkl", default=False, description="Use MKL")
+    version("5.2", sha256="ba3de4eb4f2d664b3c2a54bef2eb66d1a498ac423179e97a5795d010161b1805")
+    version(
+        "5.1",
+        sha256="ae3eb8c2ef31af210e138336fd6edcd0e3a26ea9bae89fd6c0c6ea33e3a1517e",
+        deprecated=True,
+    )
 
-    depends_on("zlib")
+    variant("glpk", default=False, description="Use glpk instead of vendored qsopt")
+
+    depends_on("zlib-api")
     depends_on("blas")
     depends_on("lapack")
-    depends_on("mkl", when="+mkl")
-
-    for t in ["aarch64", "arm", "ppc", "ppc64", "ppc64le", "ppcle", "sparc", "sparc64", "x86"]:
-        conflicts("target={0}:".format(t), msg="libspot is available linux x86_64 only")
-
-    def setup_build_environment(self, env):
-        env.append_flags("LDLIBS", "-lm")
-        env.append_flags("LDLIBS", "-lz")
-        libs = (self.spec["lapack"].libs + self.spec["blas"].libs).ld_flags
-        env.append_flags("LDLIBS", libs)
-        if self.spec.platform == "darwin":
-            env.append_flags("LDLIBS", "libqsopt.mac.a")
+    depends_on("openblas threads=openmp", when="^openblas")
+    depends_on("intel-mkl threads=openmp", when="^intel-mkl")
+    depends_on("intel-oneapi-mkl threads=openmp", when="^intel-oneapi-mkl")
+    depends_on("glpk", when="+glpk")
+
+    requires("target=x86_64:", when="~glpk", msg="bundled qsopt is only for x86_64")
+    requires(
+        "^openblas",
+        *[f"^{intel_pkg}" for intel_pkg in INTEL_MATH_LIBRARIES],
+        policy="one_of",
+        msg="Only mkl or openblas are supported for blas/lapack with ldak",
+    )
+    conflicts("platform=cray", when="~glpk", msg="bundled qsopt only for linux or mac")
+
+    phases = ["build", "install"]
+
+    def build(self, spec, prefix):
+        libs = [
+            "-lm",
+            (self.spec["lapack"].libs + self.spec["blas"].libs).link_flags,
+            self.spec["zlib-api"].libs.link_flags,
+        ]
+        includes = [
+            (self.spec["lapack"].headers + self.spec["blas"].headers).include_flags,
+            self.spec["zlib-api"].headers.include_flags,
+        ]
+
+        if self.spec.satisfies("~glpk"):
+            if self.spec.satisfies("platform=darwin"):
+                libs.append("libqsopt.mac.a")
+            else:
+                libs.append("libqsopt.linux.a")
         else:
-            env.append_flags("LDLIBS", "libqsopt.linux.a")
+            includes.append(self.spec["glpk"].headers.include_flags)
+            libs.append(self.spec["glpk"].libs.link_flags)
+        if self.spec.satisfies("^mkl"):
+            filter_file("#define MKL.*", "#define MKL 1", "ldak.c")
+        if self.spec.satisfies("^openblas"):
+            filter_file("#define MKL.*", "#define MKL 2", "ldak.c")
+            filter_file("#if MKL==2", "#if MKL==2\n#include \n", "ldak.c")
+        if self.spec.satisfies("+glpk"):
+            filter_file("#define MET.*", "#define MET 1", "ldak.c")
+            filter_file('#include"glpk.h"', "#include", "ldak.c")
+            filter_file(r"weights\[", "tally3[", "weightfuns.c")
+        cc = Executable(spack_cc)
+        args = ["ldak.c", self.compiler.openmp_flag, "-o", "ldak"] + includes + libs
+        cc(*args)
 
     def install(self, spec, prefix):
-        if self.spec.satisfies("~mkl"):
-            filter_file("#define MKL.*", "#define MKL 0", "ldak.c")
-        make("ldak")
         mkdirp(prefix.bin)
         install("ldak", prefix.bin.ldak)
diff --git a/var/spack/repos/builtin/packages/ldc-bootstrap/package.py b/var/spack/repos/builtin/packages/ldc-bootstrap/package.py
index e41ad9ae246ec1..68e0e36c9ba849 100644
--- a/var/spack/repos/builtin/packages/ldc-bootstrap/package.py
+++ b/var/spack/repos/builtin/packages/ldc-bootstrap/package.py
@@ -25,7 +25,7 @@ class LdcBootstrap(CMakePackage):
     version("0.17.4", sha256="48428afde380415640f3db4e38529345f3c8485b1913717995547f907534c1c3")
 
     depends_on("llvm@3.7:")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libconfig")
     depends_on("curl")
     depends_on("libedit")
diff --git a/var/spack/repos/builtin/packages/ldc/package.py b/var/spack/repos/builtin/packages/ldc/package.py
index 9d843221e7acff..af76fcc5194e9c 100644
--- a/var/spack/repos/builtin/packages/ldc/package.py
+++ b/var/spack/repos/builtin/packages/ldc/package.py
@@ -24,7 +24,7 @@ class Ldc(CMakePackage):
     variant("shared", default=True, description="Build runtime and tooling as shared libs")
 
     depends_on("llvm@3.9:")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libconfig")
     depends_on("curl")
     depends_on("libedit")
diff --git a/var/spack/repos/builtin/packages/legion/package.py b/var/spack/repos/builtin/packages/legion/package.py
index 6f422d8e0ee17b..65b703f10ed9f7 100644
--- a/var/spack/repos/builtin/packages/legion/package.py
+++ b/var/spack/repos/builtin/packages/legion/package.py
@@ -27,15 +27,16 @@ class Legion(CMakePackage, ROCmPackage):
 
     maintainers("pmccormick", "streichler", "elliottslaughter")
     tags = ["e4s"]
-    version("23.03.0", tag="legion-23.03.0")
-    version("22.12.0", tag="legion-22.12.0")
-    version("22.09.0", tag="legion-22.09.0")
-    version("22.06.0", tag="legion-22.06.0")
-    version("22.03.0", tag="legion-22.03.0")
-    version("21.12.0", tag="legion-21.12.0")
-    version("21.09.0", tag="legion-21.09.0")
-    version("21.06.0", tag="legion-21.06.0")
-    version("21.03.0", tag="legion-21.03.0")
+    version("23.06.0", tag="legion-23.06.0", commit="7b5ff2fb9974511c28aec8d97b942f26105b5f6d")
+    version("23.03.0", tag="legion-23.03.0", commit="12f6051c9d75229d00ac0b31d6be1ff2014f7e6a")
+    version("22.12.0", tag="legion-22.12.0", commit="9ed6f4d6b579c4f17e0298462e89548a4f0ed6e5")
+    version("22.09.0", tag="legion-22.09.0", commit="5b6e013ad74fa6b4c5a24cbb329c676b924550a9")
+    version("22.06.0", tag="legion-22.06.0", commit="f721be968fb969339334b07a3175a0400700eced")
+    version("22.03.0", tag="legion-22.03.0", commit="bf6ce4560c99397da4a5cf61a306b521ec7069d0")
+    version("21.12.0", tag="legion-21.12.0", commit="e1443112edaa574804b3b9d2a24803e937b127fd")
+    version("21.09.0", tag="legion-21.09.0", commit="5a991b714cf55c3eaa513c7a18abb436d86a0a90")
+    version("21.06.0", tag="legion-21.06.0", commit="30e00fa6016527c4cf60025a461fb7865f8def6b")
+    version("21.03.0", tag="legion-21.03.0", commit="0cf9ddd60c227c219c8973ed0580ddc5887c9fb2")
     version("stable", branch="stable")
     version("master", branch="master")
     version("cr", branch="control_replication")
@@ -46,17 +47,18 @@ class Legion(CMakePackage, ROCmPackage):
     # use for general (not legion development) use cases.
     depends_on("mpi", when="network=mpi")
     depends_on("mpi", when="network=gasnet")  # MPI is required to build gasnet (needs mpicc).
+    depends_on("ucx", when="network=ucx")
     depends_on("ucx", when="conduit=ucx")
     depends_on("mpi", when="conduit=mpi")
-    depends_on("cuda@10.0:11.9", when="+cuda_unsupported_compiler")
-    depends_on("cuda@10.0:11.9", when="+cuda")
+    depends_on("cuda@10.0:11.9", when="+cuda_unsupported_compiler @:23.03.0")
+    depends_on("cuda@10.0:11.9", when="+cuda @:23.03.0")
+    depends_on("cuda@10.0:12.2", when="+cuda_unsupported_compiler @23.06.0:")
+    depends_on("cuda@10.0:12.2", when="+cuda @23.06.0:")
     depends_on("hdf5", when="+hdf5")
     depends_on("hwloc", when="+hwloc")
 
     # cuda-centric
-    # reminder for arch numbers to names: 60=pascal, 70=volta, 75=turing, 80=ampere
-    # TODO: we could use a map here to clean up and use naming vs. numbers.
-    cuda_arch_list = ("60", "70", "75", "80")
+    cuda_arch_list = CudaPackage.cuda_arch_values
     for nvarch in cuda_arch_list:
         depends_on(
             "kokkos@3.3.01:+cuda+cuda_lambda+wrapper cuda_arch={0}".format(nvarch),
@@ -97,18 +99,19 @@ class Legion(CMakePackage, ROCmPackage):
 
     depends_on("kokkos@3.3.01:+rocm", when="+kokkos+rocm")
 
-    depends_on("python@3", when="+python")
+    # https://github.com/StanfordLegion/legion/#dependencies
+    depends_on("python@3.8:", when="+python")
+    depends_on("py-cffi", when="+python")
+    depends_on("py-numpy", when="+python")
     depends_on("papi", when="+papi")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
 
     # A C++ standard variant to work-around some odd behaviors with apple-clang
     # but this might be helpful for other use cases down the road.  Legion's
     # current development policy is C++11 or greater so we capture that aspect
     # here.
     cpp_stds = ["11", "14", "17", "20"]
-    variant("cxxstd", default="11", values=cpp_stds, multi=False)
-
-    # TODO: Need a AMD/HIP variant to match support landing in 21.03.0.
+    variant("cxxstd", default="11", description="C++ standard", values=cpp_stds, multi=False)
 
     # Network transport layer: the underlying data transport API should be used for
     # distributed data movement.  For Legion, gasnet is the currently the most
@@ -118,7 +121,7 @@ class Legion(CMakePackage, ROCmPackage):
     variant(
         "network",
         default="none",
-        values=("gasnet", "mpi", "none"),
+        values=("gasnet", "mpi", "ucx", "none"),
         description="The network communications/transport layer to use.",
         multi=False,
     )
@@ -148,42 +151,26 @@ def validate_gasnet_root(value):
         else:
             return True
 
-    variant(
-        "gasnet_root",
-        default="none",
-        values=validate_gasnet_root,
-        description="Path to a pre-installed version of GASNet (prefix directory).",
-        multi=False,
-    )
-    conflicts("gasnet_root", when="network=mpi")
-
-    variant(
-        "conduit",
-        default="none",
-        values=("aries", "ibv", "udp", "mpi", "ucx", "none"),
-        description="The gasnet conduit(s) to enable.",
-        multi=False,
-    )
-
-    conflicts(
-        "conduit=none",
-        when="network=gasnet",
-        msg="a conduit must be selected when 'network=gasnet'",
-    )
-
-    gasnet_conduits = ("aries", "ibv", "udp", "mpi", "ucx")
-    for c in gasnet_conduits:
-        conflict_str = "conduit=%s" % c
-        conflicts(
-            conflict_str, when="network=mpi", msg="conduit attribute requires 'network=gasnet'."
+    with when("network=gasnet"):
+        variant(
+            "gasnet_root",
+            default="none",
+            values=validate_gasnet_root,
+            description="Path to a pre-installed version of GASNet (prefix directory).",
+            multi=False,
+        )
+        variant(
+            "conduit",
+            default="none",
+            values=("none", "aries", "ibv", "udp", "mpi", "ucx", "ofi-slingshot11"),
+            description="The GASNet conduit(s) to enable.",
+            sticky=True,
+            multi=False,
         )
         conflicts(
-            conflict_str, when="network=none", msg="conduit attribute requires 'network=gasnet'."
+            "conduit=none", msg="the 'conduit' variant must be set to a value other than 'none'"
         )
-
-    variant("gasnet_debug", default=False, description="Build gasnet with debugging enabled.")
-    conflicts("+gasnet_debug", when="network=mpi")
-    conflicts("+gasnet_debug", when="network=none")
+        variant("gasnet_debug", default=False, description="Build gasnet with debugging enabled.")
 
     variant("shared", default=False, description="Build shared libraries.")
 
@@ -197,12 +184,6 @@ def validate_gasnet_root(value):
         description="Enable runtime privildge checks in Legion accessors.",
     )
 
-    variant(
-        "enable_tls",
-        default=False,
-        description="Enable thread-local-storage of the Legion context.",
-    )
-
     variant(
         "output_level",
         default="warning",
@@ -236,6 +217,7 @@ def validate_gasnet_root(value):
     conflicts("+cuda_hijack", when="~cuda")
 
     variant("fortran", default=False, description="Enable Fortran bindings.")
+    requires("+bindings", when="+fortran")
 
     variant("hdf5", default=False, description="Enable support for HDF5.")
 
@@ -258,6 +240,8 @@ def validate_gasnet_root(value):
     variant("papi", default=False, description="Enable PAPI performance measurements.")
 
     variant("python", default=False, description="Enable Python support.")
+    requires("+bindings", when="+python")
+    requires("+shared", when="+python")
 
     variant("zlib", default=True, description="Enable zlib support.")
 
@@ -277,11 +261,17 @@ def validate_gasnet_root(value):
         default=512,
         description="Maximum number of fields allowed in a logical region.",
     )
+    variant(
+        "max_num_nodes",
+        values=int,
+        default=1024,
+        description="Maximum number of nodes supported by Legion.",
+    )
 
     def setup_build_environment(self, build_env):
         spec = self.spec
         if "+rocm" in spec:
-            build_env.set("HIP_PATH", spec["hip"].prefix)
+            build_env.set("HIP_PATH", "{0}/hip".format(spec["hip"].prefix))
 
     def cmake_args(self):
         spec = self.spec
@@ -300,17 +290,21 @@ def cmake_args(self):
                 options.append("-DLegion_EMBED_GASNet_LOCALSRC=%s" % gasnet_dir)
 
             gasnet_conduit = spec.variants["conduit"].value
-            options.append("-DGASNet_CONDUIT=%s" % gasnet_conduit)
+
+            if "-" in gasnet_conduit:
+                gasnet_conduit, gasnet_system = gasnet_conduit.split("-")
+                options.append("-DGASNet_CONDUIT=%s" % gasnet_conduit)
+                options.append("-DGASNet_SYSTEM=%s" % gasnet_system)
+            else:
+                options.append("-DGASNet_CONDUIT=%s" % gasnet_conduit)
 
             if "+gasnet_debug" in spec:
                 options.append("-DLegion_EMBED_GASNet_CONFIGURE_ARGS=--enable-debug")
         elif "network=mpi" in spec:
             options.append("-DLegion_NETWORKS=mpi")
-            if spec.variants["gasnet_root"].value != "none":
-                raise InstallError("'gasnet_root' is only valid when 'network=gasnet'.")
+        elif "network=ucx" in spec:
+            options.append("-DLegion_NETWORKS=ucx")
         else:
-            if spec.variants["gasnet_root"].value != "none":
-                raise InstallError("'gasnet_root' is only valid when 'network=gasnet'.")
             options.append("-DLegion_EMBED_GASNet=OFF")
 
         if "+shared" in spec:
@@ -324,9 +318,6 @@ def cmake_args(self):
         if "+privilege_checks" in spec:
             # default is off.
             options.append("-DLegion_PRIVILEGE_CHECKS=ON")
-        if "+enable_tls" in spec:
-            # default is off.
-            options.append("-DLegion_ENABLE_TLS=ON")
         if "output_level" in spec:
             level = str.upper(spec.variants["output_level"].value)
             options.append("-DLegion_OUTPUT_LEVEL=%s" % level)
@@ -403,7 +394,6 @@ def cmake_args(self):
             # default is off.
             options.append("-DLegion_BUILD_BINDINGS=ON")
             options.append("-DLegion_REDOP_COMPLEX=ON")  # required for bindings
-            options.append("-DLegion_USE_Fortran=ON")
 
         if spec.variants["build_type"].value == "Debug":
             cmake_cxx_flags.extend(["-DDEBUG_REALM", "-DDEBUG_LEGION", "-ggdb"])
@@ -423,6 +413,17 @@ def cmake_args(self):
             maxfields = maxfields << 1
         options.append("-DLegion_MAX_FIELDS=%d" % maxfields)
 
+        maxnodes = int(spec.variants["max_num_nodes"].value)
+        if maxnodes <= 0:
+            maxnodes = 1024
+        # make sure maxnodes is a power of two.  if not,
+        # find the next largest power of two and use that...
+        if maxnodes & (maxnodes - 1) != 0:
+            while maxnodes & maxnodes - 1:
+                maxnodes = maxnodes & maxnodes - 1
+            maxnodes = maxnodes << 1
+        options.append("-DLegion_MAX_NUM_NODES=%d" % maxnodes)
+
         # This disables Legion's CMake build system's logic for targeting the native
         # CPU architecture in favor of Spack-provided compiler flags
         options.append("-DBUILD_MARCH:STRING=")
diff --git a/var/spack/repos/builtin/packages/leptonica/package.py b/var/spack/repos/builtin/packages/leptonica/package.py
index 5a74572ca150ec..ffe6f6b0931718 100644
--- a/var/spack/repos/builtin/packages/leptonica/package.py
+++ b/var/spack/repos/builtin/packages/leptonica/package.py
@@ -23,7 +23,7 @@ class Leptonica(CMakePackage):
     depends_on("jpeg")
     depends_on("libpng")
     depends_on("libtiff")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libwebp+libwebpmux+libwebpdemux")
     depends_on("openjpeg")
 
diff --git a/var/spack/repos/builtin/packages/less/package.py b/var/spack/repos/builtin/packages/less/package.py
index f8cdef2e6a3106..e395157a0d7c2b 100644
--- a/var/spack/repos/builtin/packages/less/package.py
+++ b/var/spack/repos/builtin/packages/less/package.py
@@ -15,6 +15,9 @@ class Less(AutotoolsPackage):
     url = "https://www.greenwoodsoftware.com/less/less-551.zip"
     list_url = "https://www.greenwoodsoftware.com/less/download.html"
 
+    depends_on("ncurses")
+
+    version("643", sha256="3bb417c4b909dfcb0adafc371ab87f0b22e8b15f463ec299d156c495fc9aa196")
     version("590", sha256="69056021c365b16504cf5bd3864436a5e50cb2f98b76cd68b99b457064139375")
     version("551", sha256="2630db16ef188e88b513b3cc24daa9a798c45643cc7da06e549c9c00cfd84244")
     version("530", sha256="8c1652ba88a726314aa2616d1c896ca8fe9a30253a5a67bc21d444e79a6c6bc3")
diff --git a/var/spack/repos/builtin/packages/lfortran/package.py b/var/spack/repos/builtin/packages/lfortran/package.py
index 6b9b42216d08f1..0f574f0c857f3d 100644
--- a/var/spack/repos/builtin/packages/lfortran/package.py
+++ b/var/spack/repos/builtin/packages/lfortran/package.py
@@ -24,7 +24,7 @@ class Lfortran(CMakePackage):
     depends_on("python@3:", type="build", when="@main")
     depends_on("cmake", type="build")
     depends_on("llvm@11:15", type=("build", "run"), when="+llvm")
-    depends_on("zlib", type="build")
+    depends_on("zlib-api")
     depends_on("re2c", type="build", when="@main")
     depends_on("bison@:3.4", type="build", when="@main")
     depends_on("binutils@2.38:", type="build", when="platform=linux")
diff --git a/var/spack/repos/builtin/packages/lftp/package.py b/var/spack/repos/builtin/packages/lftp/package.py
index 6c8df1ad0929e2..3e1e2969307994 100644
--- a/var/spack/repos/builtin/packages/lftp/package.py
+++ b/var/spack/repos/builtin/packages/lftp/package.py
@@ -25,14 +25,14 @@ class Lftp(AutotoolsPackage):
     depends_on("ncurses")
     depends_on("openssl")
     depends_on("readline")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def configure_args(self):
         args = [
             "--with-expat={0}".format(self.spec["expat"].prefix),
             "--with-openssl={0}".format(self.spec["openssl"].prefix),
             "--with-readline={0}".format(self.spec["readline"].prefix),
-            "--with-zlib={0}".format(self.spec["zlib"].prefix),
+            "--with-zlib={0}".format(self.spec["zlib-api"].prefix),
             "--disable-dependency-tracking",
         ]
         if self.spec["iconv"].name == "libc":
diff --git a/var/spack/repos/builtin/packages/lhapdf/package.py b/var/spack/repos/builtin/packages/lhapdf/package.py
index a58107b8a200bf..41c3887f89fdba 100644
--- a/var/spack/repos/builtin/packages/lhapdf/package.py
+++ b/var/spack/repos/builtin/packages/lhapdf/package.py
@@ -41,6 +41,15 @@ class Lhapdf(AutotoolsPackage):
     depends_on("py-setuptools", type="build", when="+python")
     depends_on("gettext", type="build", when="+python")
 
+    def setup_build_environment(self, env):
+        # Add -lintl if provided by gettext, otherwise libintl is provided by the system's glibc:
+        if (
+            self.spec.satisfies("+python")
+            and "gettext" in self.spec
+            and "intl" in self.spec["gettext"].libs.names
+        ):
+            env.append_flags("LDFLAGS", "-L" + self.spec["gettext"].prefix.lib)
+
     def configure_args(self):
         args = ["FCFLAGS=-O3", "CFLAGS=-O3", "CXXFLAGS=-O3"]
 
diff --git a/var/spack/repos/builtin/packages/libarchive/package.py b/var/spack/repos/builtin/packages/libarchive/package.py
index c86bf2f4d2f128..b8ab7600d454d3 100644
--- a/var/spack/repos/builtin/packages/libarchive/package.py
+++ b/var/spack/repos/builtin/packages/libarchive/package.py
@@ -16,6 +16,8 @@ class Libarchive(AutotoolsPackage):
 
     maintainers("haampie")
 
+    version("3.7.1", sha256="5d24e40819768f74daf846b99837fc53a3a9dcdf3ce1c2003fe0596db850f0f0")
+    version("3.7.0", sha256="d937886a14b48c4287c4d343644feb294a14b31b7926ba9a4f1777123ce7c2cc")
     version("3.6.2", sha256="ba6d02f15ba04aba9c23fd5f236bb234eab9d5209e95d1c4df85c44d5f19b9b3")
 
     # Deprecated versions
@@ -85,7 +87,7 @@ class Libarchive(AutotoolsPackage):
     )
     variant(
         "crypto",
-        default="mbedtls",
+        default="openssl",
         values=("mbedtls", "nettle", "openssl"),
         description="What crypto library to use for mtree and xar hashes",
     )
@@ -100,7 +102,7 @@ class Libarchive(AutotoolsPackage):
     depends_on("lz4", when="compression=lz4")
     depends_on("lzo", when="compression=lzo2")
     depends_on("xz", when="compression=lzma")
-    depends_on("zlib", when="compression=zlib")
+    depends_on("zlib-api", when="compression=zlib")
     depends_on("zstd", when="compression=zstd")
 
     depends_on("nettle", when="crypto=nettle")
diff --git a/var/spack/repos/builtin/packages/libassuan/package.py b/var/spack/repos/builtin/packages/libassuan/package.py
index 72ba599c465056..02c15940eb2062 100644
--- a/var/spack/repos/builtin/packages/libassuan/package.py
+++ b/var/spack/repos/builtin/packages/libassuan/package.py
@@ -17,6 +17,7 @@ class Libassuan(AutotoolsPackage):
 
     maintainers("alalazo")
 
+    version("2.5.6", sha256="e9fd27218d5394904e4e39788f9b1742711c3e6b41689a31aa3380bd5aa4f426")
     version("2.5.5", sha256="8e8c2fcc982f9ca67dcbb1d95e2dc746b1739a4668bc20b3a3c5be632edb34e4")
     version("2.5.4", sha256="c080ee96b3bd519edd696cfcebdecf19a3952189178db9887be713ccbcb5fbf0")
     version("2.5.3", sha256="91bcb0403866b4e7c4bc1cc52ed4c364a9b5414b3994f718c70303f7f765e702")
diff --git a/var/spack/repos/builtin/packages/libbson/package.py b/var/spack/repos/builtin/packages/libbson/package.py
index f3382fe7128bc1..082c50a6acd258 100644
--- a/var/spack/repos/builtin/packages/libbson/package.py
+++ b/var/spack/repos/builtin/packages/libbson/package.py
@@ -15,6 +15,7 @@ class Libbson(Package):
 
     maintainers("michaelkuhn")
 
+    version("1.24.4", sha256="2f4a3e8943bfe3b8672c2053f88cf74acc8494dc98a45445f727901eee141544")
     version("1.23.4", sha256="209406c91fcf7c63aa633179a0a6b1b36ba237fb77e0470fd81f7299a408e334")
     version("1.23.3", sha256="c8f951d4f965d455f37ae2e10b72914736fc0f25c4ffc14afc3cbadd1a574ef6")
     version("1.23.2", sha256="123c358827eea07cd76a31c40281bb1c81b6744f6587c96d0cf217be8b1234e3")
diff --git a/var/spack/repos/builtin/packages/libcatalyst/package.py b/var/spack/repos/builtin/packages/libcatalyst/package.py
index ed7aa445783730..9000ca137fc158 100644
--- a/var/spack/repos/builtin/packages/libcatalyst/package.py
+++ b/var/spack/repos/builtin/packages/libcatalyst/package.py
@@ -3,6 +3,10 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import subprocess
+
+import llnl.util.tty as tty
+
 from spack.package import *
 
 
@@ -14,25 +18,47 @@ class Libcatalyst(CMakePackage):
     git = "https://gitlab.kitware.com/paraview/catalyst.git"
     url = "https://gitlab.kitware.com/api/v4/projects/paraview%2Fcatalyst/packages/generic/catalyst/v2.0.0/catalyst-v2.0.0.tar.gz"
 
-    maintainers("mathstuf")
-
-    version("2.0.0-rc3", sha256="8862bd0a4d0be2176b4272f9affda1ea4e5092087acbb99a2fe2621c33834e05")
-
-    # master as of 2021-05-12
-    version("0.20210512", commit="8456ccd6015142b5a7705f79471361d4f5644fa7")
+    maintainers("mathstuf", "ayenpure")
+    version("master", branch="master")
+    version("2.0.0-rc4", sha256="cb491e4ccd344156cc2494f65b9f38885598c16d12e1016c36e2ee0bc3640863")
 
     variant("mpi", default=False, description="Enable MPI support")
+    variant("conduit", default=False, description="Use external Conduit for Catalyst")
 
     depends_on("mpi", when="+mpi")
-
-    # TODO: catalyst doesn't support an external conduit
-    # depends_on('conduit')
+    depends_on("conduit", when="+conduit")
 
     def cmake_args(self):
         """Populate cmake arguments for libcatalyst."""
         args = [
             "-DCATALYST_BUILD_TESTING=OFF",
             self.define_from_variant("CATALYST_USE_MPI", "mpi"),
+            self.define_from_variant("CATALYST_WITH_EXTERNAL_CONDUIT", "conduit"),
         ]
 
         return args
+
+    def setup_run_environment(self, env):
+        spec = self.spec
+        if spec.satisfies("+conduit"):
+            env.prepend_path("CMAKE_PREFIX_PATH", spec["conduit"].prefix)
+
+    @on_package_attributes(run_tests=True)
+    @run_after("install")
+    def build_test(self):
+        testdir = "smoke_test_build"
+        cmakeExampleDir = join_path(self.stage.source_path, "examples")
+        cmake_args = [
+            cmakeExampleDir,
+            "-DBUILD_SHARED_LIBS=ON",
+            self.define("CMAKE_PREFIX_PATH", self.prefix),
+        ]
+        cmake = which(self.spec["cmake"].prefix.bin.cmake)
+
+        with working_dir(testdir, create=True):
+            cmake(*cmake_args)
+            cmake(*(["--build", "."]))
+            tty.info("Running Catalyst test")
+
+            res = subprocess.run(["adaptor0/adaptor0_test", "catalyst"])
+            assert res.returncode == 0
diff --git a/var/spack/repos/builtin/packages/libceed/package.py b/var/spack/repos/builtin/packages/libceed/package.py
index 830a827daa4a36..e74468646b824c 100644
--- a/var/spack/repos/builtin/packages/libceed/package.py
+++ b/var/spack/repos/builtin/packages/libceed/package.py
@@ -15,18 +15,19 @@ class Libceed(MakefilePackage, CudaPackage, ROCmPackage):
     maintainers("jedbrown", "v-dobrev", "tzanio", "jeremylt")
 
     version("develop", branch="main")
-    version("0.11.0", tag="v0.11.0")
-    version("0.10.1", tag="v0.10.1")
-    version("0.9", tag="v0.9.0")
-    version("0.8", tag="v0.8")
-    version("0.7", tag="v0.7")
+    version("0.12.0", tag="v0.12.0", commit="60ef3feef7f5137af55ea7336903743d94ee71a8")
+    version("0.11.0", tag="v0.11.0", commit="8ec64e9ae9d5df169dba8c8ee61d8ec8907b8f80")
+    version("0.10.1", tag="v0.10.1", commit="74532b27052d94e943eb8bc76257fbd710103614")
+    version("0.9", tag="v0.9.0", commit="d66340f5aae79e564186ab7514a1cd08b3a1b06b")
+    version("0.8", tag="v0.8", commit="e8f234590eddcce2220edb1d6e979af7a3c35f82")
+    version("0.7", tag="v0.7", commit="0bc92be5158efcbeb80d3d59240233bf5b2f748c")
     version(
         "0.6", commit="c7f533e01e2f3f6720fbf37aac2af2ffed225f60"
     )  # tag v0.6 + small portability fixes
-    version("0.5", tag="v0.5")
-    version("0.4", tag="v0.4")
-    version("0.2", tag="v0.2")
-    version("0.1", tag="v0.1")
+    version("0.5", tag="v0.5", commit="12804ff7ea2ac608ae5494437379e4f626cf5cb7")
+    version("0.4", tag="v0.4", commit="40b9dad77dea06a1608fa8b93a0d8b9c993ee43d")
+    version("0.2", tag="v0.2", commit="113004cb41757b819325a4b3a8a7dfcea5156531")
+    version("0.1", tag="v0.1", commit="74e0540e2478136394f75869675056eb6aba67cc")
 
     variant("occa", default=False, description="Enable OCCA backends")
     variant("debug", default=False, description="Enable debug build")
diff --git a/var/spack/repos/builtin/packages/libcuml/package.py b/var/spack/repos/builtin/packages/libcuml/package.py
index 11f772644ef477..67af9b46f1ca2d 100644
--- a/var/spack/repos/builtin/packages/libcuml/package.py
+++ b/var/spack/repos/builtin/packages/libcuml/package.py
@@ -17,7 +17,7 @@ class Libcuml(CMakePackage):
     version("0.15.0", sha256="5c9c656ae4eaa94a426e07d7385fd5ea0e5dc7abff806af2941aee10d4ca99c7")
 
     depends_on("cmake@3.14:", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libcudf@0.8:")
     depends_on("cuda@9.2:")
     depends_on("blas")
diff --git a/var/spack/repos/builtin/packages/libdap4/package.py b/var/spack/repos/builtin/packages/libdap4/package.py
index a8e2a229b53786..2603abab8c03a9 100644
--- a/var/spack/repos/builtin/packages/libdap4/package.py
+++ b/var/spack/repos/builtin/packages/libdap4/package.py
@@ -38,4 +38,8 @@ def configure_args(self):
         # need, so grab this path manually:
         libxml2_include = self.spec["libxml2"].prefix.include
         args = ["CPPFLAGS=-I{0}".format(libxml2_include)]
+        if self.spec.satisfies("^libxml2 ~shared"):
+            xml2_config = which("xml2-config")
+            xml2_flags = xml2_config("--libs", output=str).strip()
+            args.append(f"LIBS={xml2_flags}")
         return args
diff --git a/var/spack/repos/builtin/packages/libdeflate/package.py b/var/spack/repos/builtin/packages/libdeflate/package.py
index 0a3c1e11c735bb..91d823f55a929e 100644
--- a/var/spack/repos/builtin/packages/libdeflate/package.py
+++ b/var/spack/repos/builtin/packages/libdeflate/package.py
@@ -3,10 +3,11 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+from spack.build_systems import cmake, makefile
 from spack.package import *
 
 
-class Libdeflate(MakefilePackage):
+class Libdeflate(MakefilePackage, CMakePackage):
     """Heavily optimized library for DEFLATE/zlib/gzip compression and decompression"""
 
     homepage = "https://github.com/ebiggers/libdeflate"
@@ -14,11 +15,29 @@ class Libdeflate(MakefilePackage):
 
     maintainers("dorton21")
 
+    version("1.18", sha256="225d982bcaf553221c76726358d2ea139bb34913180b20823c782cede060affd")
+    version("1.14", sha256="89e7df898c37c3427b0f39aadcf733731321a278771d20fc553f92da8d4808ac")
     version("1.10", sha256="5c1f75c285cd87202226f4de49985dcb75732f527eefba2b3ddd70a8865f2533")
     version("1.7", sha256="a5e6a0a9ab69f40f0f59332106532ca76918977a974e7004977a9498e3f11350")
 
-    depends_on("zlib")
+    build_system(
+        conditional("makefile", when="@:1.14"),
+        conditional("cmake", when="@1.15:"),
+        default="cmake",
+    )
+
+    depends_on("zlib-api")
     depends_on("gzip")
 
-    def patch(self):
-        filter_file(r"\/usr\/local", self.prefix, "Makefile")
+    with when("build_system=cmake"):
+        depends_on("cmake@3.7:", type="build")
+
+
+class MakefileBuilder(makefile.MakefileBuilder):
+    def install(self, pkg, spec, prefix):
+        with working_dir(self.build_directory):
+            make("install", f"PREFIX={prefix}")
+
+
+class CMakeBuilder(cmake.CMakeBuilder):
+    pass
diff --git a/var/spack/repos/builtin/packages/libdwarf/package.py b/var/spack/repos/builtin/packages/libdwarf/package.py
index 3da923e73a9660..93659acd0cd5bb 100644
--- a/var/spack/repos/builtin/packages/libdwarf/package.py
+++ b/var/spack/repos/builtin/packages/libdwarf/package.py
@@ -35,7 +35,7 @@ class Libdwarf(Package):
     version("20130126", sha256="c23c847935f8612f4fcdcfa0b3311f1553dcbd95bb683d3d5e030440201192fe")
     depends_on("elfutils@0.163", when="@20160507", type="link")
     depends_on("elf", type="link")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     parallel = False
 
@@ -63,7 +63,7 @@ def install(self, spec, prefix):
             configure("--prefix=" + prefix, "--enable-shared", *extra_config_args)
             filter_file(
                 r"^dwfzlib\s*=\s*-lz",
-                "dwfzlib=-L{0} -lz".format(self.spec["zlib"].prefix.lib),
+                "dwfzlib=-L{0} -lz".format(self.spec["zlib-api"].prefix.lib),
                 "Makefile",
             )
             make()
@@ -94,7 +94,7 @@ def install(self, spec, prefix):
             configure("--prefix=" + prefix)
             filter_file(
                 r"^dwfzlib\s*=\s*-lz",
-                "dwfzlib=-L{0} -lz".format(self.spec["zlib"].prefix.lib),
+                "dwfzlib=-L{0} -lz".format(self.spec["zlib-api"].prefix.lib),
                 "Makefile",
             )
 
diff --git a/var/spack/repos/builtin/packages/libelf/package.py b/var/spack/repos/builtin/packages/libelf/package.py
index f2b61b3747bccc..015978d5731e21 100644
--- a/var/spack/repos/builtin/packages/libelf/package.py
+++ b/var/spack/repos/builtin/packages/libelf/package.py
@@ -51,3 +51,10 @@ def configure_args(self):
 
     def install(self, spec, prefix):
         make("install", parallel=False)
+
+    def flag_handler(self, name, flags):
+        if name == "cflags":
+            if self.spec.satisfies("%clang@16:"):
+                flags.append("-Wno-error=implicit-int")
+                flags.append("-Wno-error=implicit-function-declaration")
+        return (flags, None, None)
diff --git a/var/spack/repos/builtin/packages/libevent/package.py b/var/spack/repos/builtin/packages/libevent/package.py
index c0ece0386c5a5a..7969fb63d58d23 100644
--- a/var/spack/repos/builtin/packages/libevent/package.py
+++ b/var/spack/repos/builtin/packages/libevent/package.py
@@ -41,6 +41,10 @@ class Libevent(AutotoolsPackage):
     depends_on("openssl@:1.0", when="@:2.0+openssl")
     depends_on("openssl", when="+openssl")
 
+    depends_on("autoconf", type="build")
+    depends_on("automake", type="build")
+    depends_on("libtool", type="build")
+
     def url_for_version(self, version):
         if version >= Version("2.0.22"):
             url = "https://github.com/libevent/libevent/releases/download/release-{0}-stable/libevent-{0}-stable.tar.gz"
@@ -54,6 +58,9 @@ def libs(self):
         libs = find_libraries("libevent", root=self.prefix, shared=True, recursive=True)
         return LibraryList(libs)
 
+    def autoreconf(self, spec, prefix):
+        Executable("./autogen.sh")()
+
     def configure_args(self):
         spec = self.spec
         configure_args = []
diff --git a/var/spack/repos/builtin/packages/libfabric/package.py b/var/spack/repos/builtin/packages/libfabric/package.py
index 0f48b16f9a0aa6..fe0c9c6a40e70d 100644
--- a/var/spack/repos/builtin/packages/libfabric/package.py
+++ b/var/spack/repos/builtin/packages/libfabric/package.py
@@ -22,6 +22,8 @@ class Libfabric(AutotoolsPackage):
     executables = ["^fi_info$"]
 
     version("main", branch="main")
+    version("1.19.0", sha256="f14c764be9103e80c46223bde66e530e5954cb28b3835b57c8e728479603ef9e")
+    version("1.18.2", sha256="64d7837853ca84d2a413fdd96534b6a81e6e777cc13866e28cf86cd0ccf1b93e")
     version("1.18.1", sha256="4615ae1e22009e59c72ae03c20adbdbd4a3dce95aeefbc86cc2bf1acc81c9e38")
     version("1.18.0", sha256="912fb7c7b3cf2a91140520962b004a1c5d2f39184adbbd98ae5919b0178afd43")
     version("1.17.1", sha256="8b372ddb3f46784c53fdad50a701a6eb0e661239aee45a42169afbedf3644035")
@@ -153,6 +155,18 @@ def setup_build_environment(self, env):
         if self.run_tests:
             env.prepend_path("PATH", self.prefix.bin)
 
+    # To enable this package add it to the LD_LIBRARY_PATH
+    def setup_run_environment(self, env):
+        libfabric_home = self.spec["libfabric"].prefix
+        env.prepend_path("LD_LIBRARY_PATH", libfabric_home.lib)
+        env.prepend_path("LD_LIBRARY_PATH", libfabric_home.lib64)
+
+    # To enable this package add it to the LD_LIBRARY_PATH
+    def setup_dependent_run_environment(self, env, dependent_spec):
+        libfabric_home = self.spec["libfabric"].prefix
+        env.prepend_path("LD_LIBRARY_PATH", libfabric_home.lib)
+        env.prepend_path("LD_LIBRARY_PATH", libfabric_home.lib64)
+
     @when("@main")
     def autoreconf(self, spec, prefix):
         bash = which("bash")
diff --git a/var/spack/repos/builtin/packages/libffi/package.py b/var/spack/repos/builtin/packages/libffi/package.py
index 2526b615b71ef1..c67bd82ddd2f52 100644
--- a/var/spack/repos/builtin/packages/libffi/package.py
+++ b/var/spack/repos/builtin/packages/libffi/package.py
@@ -40,7 +40,11 @@ def headers(self):
 
     def flag_handler(self, name, flags):
         if name == "cflags":
-            if self.spec.satisfies("%oneapi@2023:") or self.spec.satisfies("%arm@23.04:"):
+            if (
+                self.spec.satisfies("%oneapi@2023:")
+                or self.spec.satisfies("%arm@23.04:")
+                or self.spec.satisfies("%clang@16:")
+            ):
                 flags.append("-Wno-error=implicit-function-declaration")
         return (flags, None, None)
 
diff --git a/var/spack/repos/builtin/packages/libfirefly/package.py b/var/spack/repos/builtin/packages/libfirefly/package.py
new file mode 100644
index 00000000000000..a4fc6b00a03876
--- /dev/null
+++ b/var/spack/repos/builtin/packages/libfirefly/package.py
@@ -0,0 +1,29 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Libfirefly(CMakePackage):
+    """A standalone C++ Library for vectors calculations"""
+
+    homepage = "https://libfirefly.tbhaxor.com"
+    url = "https://github.com/tbhaxor/firefly/archive/refs/tags/v2.1.0.tar.gz"
+    git = "https://github.com/tbhaxor/firefly.git"
+
+    maintainers("tbhaxor")
+
+    version("master", branch="master")
+    version("2.1.0", sha256="4de4b216c73199a1826de7a0d45205b401603315347d7947d8b5950d3e6b893d")
+
+    variant(
+        "double-precision",
+        description="Enables double type instead of float when enabled",
+        default=True,
+    )
+
+    def cmake_args(self):
+        args = [self.define_from_variant("Firefly_ENABLE_DOUBLE_PRECISION", "double-precision")]
+        return args
diff --git a/var/spack/repos/builtin/packages/libfms/package.py b/var/spack/repos/builtin/packages/libfms/package.py
index 81eb5ce7d352ac..8c806968b9d6c3 100644
--- a/var/spack/repos/builtin/packages/libfms/package.py
+++ b/var/spack/repos/builtin/packages/libfms/package.py
@@ -17,7 +17,7 @@ class Libfms(CMakePackage):
     maintainers("v-dobrev", "tzanio", "cwsmith")
 
     version("develop", branch="master")
-    version("0.2.0", tag="v0.2")
+    version("0.2.0", tag="v0.2", commit="a66cb96711cc404c411f1bf07ca8db09b6f894eb")
 
     variant("conduit", default=True, description="Build with Conduit I/O support")
     variant("shared", default=True, description="Build shared libraries")
diff --git a/var/spack/repos/builtin/packages/libfontenc/package.py b/var/spack/repos/builtin/packages/libfontenc/package.py
index 60f3e5f7e79cb0..58eecb3496ea79 100644
--- a/var/spack/repos/builtin/packages/libfontenc/package.py
+++ b/var/spack/repos/builtin/packages/libfontenc/package.py
@@ -15,7 +15,7 @@ class Libfontenc(AutotoolsPackage, XorgPackage):
     version("1.1.7", sha256="5e5f210329823f08f97bfe9fd5b4105070c789bc5aef88ce01d86d8203d4aa9f")
     version("1.1.3", sha256="6fba26760ca8d5045f2b52ddf641c12cedc19ee30939c6478162b7db8b6220fb")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     depends_on("xproto")
     depends_on("pkgconfig", type="build")
diff --git a/var/spack/repos/builtin/packages/libfuse/package.py b/var/spack/repos/builtin/packages/libfuse/package.py
index b5907b95b3eb4f..7180fa8181a9f1 100644
--- a/var/spack/repos/builtin/packages/libfuse/package.py
+++ b/var/spack/repos/builtin/packages/libfuse/package.py
@@ -16,6 +16,9 @@ class Libfuse(MesonPackage):
     homepage = "https://github.com/libfuse/libfuse"
     url = "https://github.com/libfuse/libfuse/releases/download/fuse-2.9.9/fuse-2.9.9.tar.gz"
 
+    keep_werror = "all"
+
+    version("3.16.2", sha256="1bc306be1a1f4f6c8965fbdd79c9ccca021fdc4b277d501483a711cbd7dbcd6c")
     version("3.11.0", sha256="25a00226d2d449c15b2f08467d6d5ebbb2a428260c4ab773721c32adbc6da072")
     version("3.10.5", sha256="e73f75e58da59a0e333d337c105093c496c0fd7356ef3a5a540f560697c9c4e6")
     version("3.10.4", sha256="bfcb2520fd83db29e9fefd57d3abd5285f38ad484739aeee8e03fbec9b2d984a")
@@ -65,7 +68,7 @@ def url_for_version(self, version):
     conflicts("platform=darwin", msg="libfuse does not support OS-X, use macfuse instead")
 
     # Drops the install script which does system configuration
-    patch("0001-Do-not-run-install-script.patch", when="@3: ~system_install")
+    patch("0001-Do-not-run-install-script.patch", when="@3:3.11 ~system_install")
     patch(
         "https://src.fedoraproject.org/rpms/fuse3/raw/0519b7bf17c4dd1b31ee704d49f8ed94aa5ba6ab/f/fuse3-gcc11.patch",
         sha256="3ad6719d2393b46615b5787e71778917a7a6aaa189ba3c3e0fc16d110a8414ec",
@@ -115,6 +118,9 @@ def meson_args(self):
         if "~system_install" in self.spec:
             # Fix meson's setup if meson does not have the host system's udev package:
             args.append("-Dudevrulesdir={0}".format(self.prefix.etc.rules.d))
+
+            if self.spec.satisfies("@3.12:"):
+                args.append("-Dinitscriptdir=")
         else:
             # Likewise, but with +system_install, it may install to /lib/udev/rules.d:
             args.append("-Dudevrulesdir={0}".format("/lib/udev/rules.d"))
diff --git a/var/spack/repos/builtin/packages/libgeotiff/package.py b/var/spack/repos/builtin/packages/libgeotiff/package.py
index 9e597cd66001d7..f1024d7b5cf736 100644
--- a/var/spack/repos/builtin/packages/libgeotiff/package.py
+++ b/var/spack/repos/builtin/packages/libgeotiff/package.py
@@ -17,6 +17,8 @@ class Libgeotiff(AutotoolsPackage):
 
     maintainers("adamjstewart")
 
+    version("1.7.1", sha256="05ab1347aaa471fc97347d8d4269ff0c00f30fa666d956baba37948ec87e55d6")
+    version("1.7.0", sha256="fc304d8839ca5947cfbeb63adb9d1aa47acef38fc6d6689e622926e672a99a7e")
     version("1.6.0", sha256="9311017e5284cffb86f2c7b7a9df1fb5ebcdc61c30468fb2e6bca36e4272ebca")
     version("1.5.1", sha256="f9e99733c170d11052f562bcd2c7cb4de53ed405f7acdde4f16195cd3ead612c")
     version("1.5.0", sha256="1c0bef329c60f770ed128e8b273945100f1a4b5abd161ac61e93bc947b0624dd")
@@ -27,7 +29,7 @@ class Libgeotiff(AutotoolsPackage):
     variant("jpeg", default=True, description="Include jpeg support")
     variant("proj", default=True, description="Use PROJ.x library")
 
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("jpeg", when="+jpeg")
     depends_on("libtiff")
     depends_on("proj", when="+proj")
@@ -58,7 +60,7 @@ def configure_args(self):
         args = ["--with-libtiff={0}".format(spec["libtiff"].prefix)]
 
         if "+zlib" in spec:
-            args.append("--with-zlib={0}".format(spec["zlib"].prefix))
+            args.append("--with-zlib={0}".format(spec["zlib-api"].prefix))
         else:
             args.append("--with-zlib=no")
 
@@ -79,7 +81,7 @@ def configure_args(self):
                 if dep not in self.spec:
                     continue
                 ldflags.append(self.spec[dep].libs.search_flags)
-                libs.append(self.spec[dep].libs.link_flags) 
+                libs.append(self.spec[dep].libs.link_flags)
             if ldflags or libs:
                 args.append("LDFLAGS=%s" % " ".join(ldflags))
                 args.append("LIBS=%s" % " ".join(libs))
diff --git a/var/spack/repos/builtin/packages/libgit2/package.py b/var/spack/repos/builtin/packages/libgit2/package.py
index 6898d1d1e9b528..dd09fd8e1e843c 100644
--- a/var/spack/repos/builtin/packages/libgit2/package.py
+++ b/var/spack/repos/builtin/packages/libgit2/package.py
@@ -16,7 +16,13 @@ class Libgit2(CMakePackage):
     homepage = "https://libgit2.github.com/"
     url = "https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz"
 
+    version("1.7.0", sha256="d9d0f84a86bf98b73e68997f5c1543cc5067d0ca9c7a5acaba3e8d117ecefef3")
     version("1.6.4", sha256="d25866a4ee275a64f65be2d9a663680a5cf1ed87b7ee4c534997562c828e500d")
+    version("1.6.3", sha256="a8e2a09835eabb24ace2fd597a78af182e1e199a894e99a90e4c87c849fcd9c4")
+    version("1.6.2", sha256="d557fbf35557bb5df53cbf38ae0081edb4a36494ec0d19741fa673e509245f8f")
+    version("1.6.1", sha256="25f7d3e2e7f398477a6e18271c5a68510853d0fe826a6287811cebccf92e773f")
+    version("1.5.2", sha256="57638ac0e319078f56a7e17570be754515e5b1276d3750904b4214c92e8fa196")
+    version("1.5.1", sha256="7074f1e2697992b82402501182db254fe62d64877b12f6e4c64656516f4cde88")
     version("1.5.0", sha256="8de872a0f201b33d9522b817c92e14edb4efad18dae95cf156cf240b2efff93e")
     version("1.4.4", sha256="e9923e9916a32f54c661d55d79c28fa304cb23617639e68bff9f94d3e18f2d4b")
     version("1.4.3", sha256="f48b961e463a9e4e7e7e58b21a0fb5a9b2a1d24d9ba4d15870a0c9b8ad965163")
diff --git a/var/spack/repos/builtin/packages/libharu/package.py b/var/spack/repos/builtin/packages/libharu/package.py
index d0ec677c85d695..b575c51ed1fab3 100644
--- a/var/spack/repos/builtin/packages/libharu/package.py
+++ b/var/spack/repos/builtin/packages/libharu/package.py
@@ -26,7 +26,7 @@ class Libharu(AutotoolsPackage):
     depends_on("autoconf", type=("build"))
     depends_on("automake", type=("build"))
     depends_on("libpng")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def autoreconf(self, spec, prefix):
         """execute their autotools wrapper script"""
@@ -39,7 +39,7 @@ def configure_args(self):
         spec = self.spec
         args = []
 
-        args.append("--with-zlib={0}".format(spec["zlib"].prefix))
+        args.append("--with-zlib={0}".format(spec["zlib-api"].prefix))
         args.append("--with-png={0}".format(spec["libpng"].prefix))
 
         return args
diff --git a/var/spack/repos/builtin/packages/libhio/package.py b/var/spack/repos/builtin/packages/libhio/package.py
index 0c457172e63081..a394c701972f58 100644
--- a/var/spack/repos/builtin/packages/libhio/package.py
+++ b/var/spack/repos/builtin/packages/libhio/package.py
@@ -25,7 +25,7 @@ class Libhio(AutotoolsPackage):
     #
     version("master", branch="master")
     version("1.4.1.6", sha256="863e7274f9e32d97bd5d9e6745ad9449735bdc8bd5623f152a32be45e6f3a212")
-    version("1.4.1.5", sha256="af5cb2a799a8470ed95847c3b07ea3ad61f8f7d5a2b79c52a46ca784846e8962")
+    version("1.4.1.5", sha256="76f5a0eb306e9596cdef558586ed2ba47bf50a9cc7782beb1a8b37115c00b1ee")
     version("1.4.1.4", sha256="6998a424cff97be9a207032b3addd19f292d8ebda72043be92a8f942ae3b4da1")
     version("1.4.1.3", sha256="b6ad2354f1bc597e7e55fc989ff50944835d64149f4925c2f45df950919e4d08")
     version("1.4.1.2", sha256="87a0f9b72b7aa69485c40d9bd36f02af653b70e8eed3eb0b50eaeb02ff0a9049")
diff --git a/var/spack/repos/builtin/packages/libiberty/package.py b/var/spack/repos/builtin/packages/libiberty/package.py
index bb15334c3ff0c1..4c82320a186cb8 100644
--- a/var/spack/repos/builtin/packages/libiberty/package.py
+++ b/var/spack/repos/builtin/packages/libiberty/package.py
@@ -19,6 +19,7 @@ class Libiberty(AutotoolsPackage, GNUMirrorPackage):
     gnu_mirror_path = "binutils/binutils-2.31.1.tar.xz"
     maintainers("mwkrentel")
 
+    version("2.41", sha256="ae9a5789e23459e59606e6714723f2d3ffc31c03174191ef0d015bdf06007450")
     version("2.40", sha256="0f8a4c272d7f17f369ded10a4aca28b8e304828e95526da482b0ccc4dfc9d8e1")
     version("2.37", sha256="820d9724f020a3e69cb337893a0b63c2db161dadcb0e06fc11dc29eb1e84a32c")
     version("2.36.1", sha256="e81d9edf373f193af428a0f256674aea62a9d74dfe93f65192d4eae030b0f3b0")
diff --git a/var/spack/repos/builtin/packages/libid3tag/package.py b/var/spack/repos/builtin/packages/libid3tag/package.py
index 8da31bd278008f..e62e143922195d 100644
--- a/var/spack/repos/builtin/packages/libid3tag/package.py
+++ b/var/spack/repos/builtin/packages/libid3tag/package.py
@@ -17,7 +17,7 @@ class Libid3tag(AutotoolsPackage):
 
     version("0.15.1b", sha256="63da4f6e7997278f8a3fef4c6a372d342f705051d1eeb6a46a86b03610e26151")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("gperf")
 
     # source: https://git.archlinux.org/svntogit/packages.git/tree/trunk/10_utf16.diff?h=packages/libid3tag
diff --git a/var/spack/repos/builtin/packages/libjpeg-turbo/package.py b/var/spack/repos/builtin/packages/libjpeg-turbo/package.py
index 66f856f299e7b3..cf7b54b113e480 100644
--- a/var/spack/repos/builtin/packages/libjpeg-turbo/package.py
+++ b/var/spack/repos/builtin/packages/libjpeg-turbo/package.py
@@ -18,6 +18,8 @@ class LibjpegTurbo(CMakePackage, AutotoolsPackage):
     homepage = "https://libjpeg-turbo.org/"
     url = "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.3.tar.gz"
 
+    version("3.0.0", sha256="171dae5d73560bc94006a7c0c3281bd9bfde6a34f7e41e66f930a1a9162bd7df")
+    version("2.1.5.1", sha256="61846251941e5791005fb7face196eec24541fce04f12570c308557529e92c75")
     version("2.1.5", sha256="254f3642b04e309fee775123133c6464181addc150499561020312ec61c1bf7c")
     version("2.1.4", sha256="a78b05c0d8427a90eb5b4eb08af25309770c8379592bb0b8a863373128e6143f")
     version("2.1.3", sha256="dbda0c685942aa3ea908496592491e5ec8160d2cf1ec9d5fd5470e50768e7859")
diff --git a/var/spack/repos/builtin/packages/libjxl/package.py b/var/spack/repos/builtin/packages/libjxl/package.py
index 5ce1317824f1c7..9e1403bb5967ab 100644
--- a/var/spack/repos/builtin/packages/libjxl/package.py
+++ b/var/spack/repos/builtin/packages/libjxl/package.py
@@ -14,8 +14,12 @@ class Libjxl(CMakePackage):
     git = "https://github.com/libjxl/libjxl.git"
 
     version("main", branch="main", submodules=True)
-    version("0.7.0", tag="v0.7.0", submodules=True)
-    version("0.6.1", tag="v0.6.1", submodules=True)
+    version(
+        "0.7.0", tag="v0.7.0", commit="f95da131cf7c7ccd4da256356fde2fec1fa23bb5", submodules=True
+    )
+    version(
+        "0.6.1", tag="v0.6.1", commit="a205468bc5d3a353fb15dae2398a101dff52f2d3", submodules=True
+    )
 
     depends_on("cmake@3.10:", type="build")
     depends_on("brotli")
diff --git a/var/spack/repos/builtin/packages/libkml/package.py b/var/spack/repos/builtin/packages/libkml/package.py
index 19eadb883d7435..ab59d6d720b29a 100644
--- a/var/spack/repos/builtin/packages/libkml/package.py
+++ b/var/spack/repos/builtin/packages/libkml/package.py
@@ -32,7 +32,8 @@ class Libkml(CMakePackage):
     depends_on("expat@2.1.0:")
     depends_on("minizip@1.2.8:")
     depends_on("uriparser")
-    depends_on("zlib@1.2.8:")
+    depends_on("zlib-api")
+    conflicts("^zlib@:1.2.7")
     depends_on("googletest@1.7.0:", type="link")
     depends_on("swig", when="+java", type="build")
     depends_on("swig", when="+python", type="build")
diff --git a/var/spack/repos/builtin/packages/libksba/package.py b/var/spack/repos/builtin/packages/libksba/package.py
index 44f22fd6c00caa..5230bcb6a5bf42 100644
--- a/var/spack/repos/builtin/packages/libksba/package.py
+++ b/var/spack/repos/builtin/packages/libksba/package.py
@@ -17,6 +17,7 @@ class Libksba(AutotoolsPackage):
 
     maintainers("alalazo")
 
+    version("1.6.4", sha256="bbb43f032b9164d86c781ffe42213a83bf4f2fee91455edfa4654521b8b03b6b")
     version("1.6.3", sha256="3f72c68db30971ebbf14367527719423f0a4d5f8103fc9f4a1c01a9fa440de5c")
 
     # Deprecated over CVE-2022-3515 (https://gnupg.org/blog/20221017-pepe-left-the-ksba.html)
diff --git a/var/spack/repos/builtin/packages/liblbxutil/package.py b/var/spack/repos/builtin/packages/liblbxutil/package.py
index cad591781bb553..0be107cb184771 100644
--- a/var/spack/repos/builtin/packages/liblbxutil/package.py
+++ b/var/spack/repos/builtin/packages/liblbxutil/package.py
@@ -18,7 +18,7 @@ class Liblbxutil(AutotoolsPackage, XorgPackage):
     depends_on("xproto")
     depends_on("pkgconfig", type="build")
     depends_on("util-macros", type="build")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     # There is a bug in the library that causes the following messages:
     # undefined symbol: Xfree
diff --git a/var/spack/repos/builtin/packages/libluv/package.py b/var/spack/repos/builtin/packages/libluv/package.py
index abf42d47f08ee5..ff9a9db5e62d09 100644
--- a/var/spack/repos/builtin/packages/libluv/package.py
+++ b/var/spack/repos/builtin/packages/libluv/package.py
@@ -14,17 +14,26 @@ class Libluv(CMakePackage):
     homepage = "https://github.com/luvit/luv"
     url = "https://github.com/luvit/luv/releases/download/1.36.0-0/luv-1.36.0-0.tar.gz"
 
+    version("1.45.0-0", sha256="fa6c46fb09f88320afa7f88017efd7b0d2b3a0158c5ba5b6851340b0332a2b81")
     version("1.44.2-1", sha256="3eb5c7bc44f61fbc4148ea30e3221d410263e0ffa285672851fc19debf9e5c30")
+    version("1.44.2-0", sha256="30639f8e0fac7fb0c3a04b94a00f73c6d218c15765347ceb0998a6b72464b6cf")
     version("1.43.0-0", sha256="567a6f3dcdcf8a9b54ddc57ffef89d1e950d72832b85ee81c8c83a9d4e0e9de2")
     version("1.42.0-1", sha256="4b6fbaa89d2420edf6070ad9e522993e132bd7eb2540ff754c2b9f1497744db2")
     version("1.42.0-0", sha256="b5228a9d0eaacd9f862b6270c732d5c90773a28ce53b6d9e32a14050e7947f36")
     version("1.36.0-0", sha256="f2e7eb372574f25c6978c1dc74280d22efdcd7df2dda4a286c7fe7dceda26445")
 
+    # https://github.com/neovim/neovim/issues/25770
+    # up to 1.45 (included) dynamic library on macOS did not have the @rpath prefix, being not
+    # usable on this platform.
+    # from 1.46, by requiring a newer cmake version, CMP0042 is in place and it works correctly.
+    depends_on("cmake@3:", type="build")
+
     depends_on("lua-lang", type="link")
     depends_on("libuv", type="link")
 
     def cmake_args(self):
         args = [
+            self.define("CMAKE_POLICY_DEFAULT_CMP0042", "NEW"),
             "-DLUA_BUILD_TYPE=System",
             "-DBUILD_STATIC_LIBS=ON",
             "-DBUILD_SHARED_LIBS=ON",
diff --git a/var/spack/repos/builtin/packages/libmng/package.py b/var/spack/repos/builtin/packages/libmng/package.py
index 28a9cde69260ca..1b582b22f5c94f 100644
--- a/var/spack/repos/builtin/packages/libmng/package.py
+++ b/var/spack/repos/builtin/packages/libmng/package.py
@@ -18,7 +18,7 @@ class Libmng(CMakePackage):
     version("2.0.2", sha256="4908797bb3541fb5cd8fffbe0b1513ed163509f2a4d57a78b26a96f8d1dd05a2")
 
     depends_on("jpeg")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("lcms")
 
     def patch(self):
diff --git a/var/spack/repos/builtin/packages/libmolgrid/package.py b/var/spack/repos/builtin/packages/libmolgrid/package.py
index a486a2359ac6f0..20932dac869ecd 100644
--- a/var/spack/repos/builtin/packages/libmolgrid/package.py
+++ b/var/spack/repos/builtin/packages/libmolgrid/package.py
@@ -20,7 +20,7 @@ class Libmolgrid(CMakePackage):
     version("0.5.3", sha256="a9f7a62cdeb516bc62a06b324cdd33b095a787df175c6166d74a8d30b6916abb")
     version("0.5.2", sha256="e732d13a96c2f374d57a73999119bef700172d392c195c751214aa6ac6680c3a")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("boost +regex +test +program_options +system +filesystem +iostreams +python")
     depends_on("openbabel@3:~gui~cairo")
     depends_on("cuda@11")
@@ -36,5 +36,6 @@ def cmake_args(self):
         args = [
             "-DOPENBABEL3_INCLUDE_DIR=" + ob_incl,
             "-DOPENBABEL3_LIBRARIES=" + ob_libs,
+            f"-DPYTHON_EXECUTABLE={self.spec['python'].command.path}",
         ]
         return args
diff --git a/var/spack/repos/builtin/packages/libpng/package.py b/var/spack/repos/builtin/packages/libpng/package.py
index ba585addd36696..7187c99706d1d4 100644
--- a/var/spack/repos/builtin/packages/libpng/package.py
+++ b/var/spack/repos/builtin/packages/libpng/package.py
@@ -28,7 +28,7 @@ class Libpng(CMakePackage):
     version("1.5.30", sha256="7d76275fad2ede4b7d87c5fd46e6f488d2a16b5a69dc968ffa840ab39ba756ed")
     version("1.2.57", sha256="0f4620e11fa283fedafb474427c8e96bf149511a1804bdc47350963ae5cf54d8")
 
-    depends_on("zlib@1.0.4:")  # 1.2.5 or later recommended
+    depends_on("zlib-api")
 
     variant(
         "libs",
@@ -39,14 +39,22 @@ class Libpng(CMakePackage):
     )
     variant("pic", default=False, description="PIC")
 
+    @property
+    def libs(self):
+        # v1.2 does not have a version-less symlink
+        libraries = f"libpng{self.version.up_to(2).joined}"
+        shared = "libs=shared" in self.spec
+        return find_libraries(libraries, root=self.prefix, shared=shared, recursive=True)
+
+
 class CMakeBuilder(CMakeBuilder):
     def cmake_args(self):
         args = [
-            self.define("CMAKE_CXX_FLAGS", self.spec["zlib"].headers.include_flags),
-            self.define("ZLIB_ROOT", self.spec["zlib"].prefix),
+            self.define("CMAKE_CXX_FLAGS", self.spec["zlib-api"].headers.include_flags),
+            self.define("ZLIB_ROOT", self.spec["zlib-api"].prefix),
             self.define("PNG_SHARED", "shared" in self.spec.variants["libs"].value),
             self.define("PNG_STATIC", "static" in self.spec.variants["libs"].value),
-            self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic")
+            self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
         ]
         if self.spec.satisfies("platform=darwin target=aarch64:"):
             args.append("-DPNG_ARM_NEON=off")
diff --git a/var/spack/repos/builtin/packages/libpostal/package.py b/var/spack/repos/builtin/packages/libpostal/package.py
new file mode 100644
index 00000000000000..d04b67e51f3b43
--- /dev/null
+++ b/var/spack/repos/builtin/packages/libpostal/package.py
@@ -0,0 +1,44 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Libpostal(AutotoolsPackage):
+    """A C library for parsing/normalizing street addresses around the world.
+    Powered by statistical NLP and open geo data."""
+
+    homepage = "https://github.com/openvenues/libpostal"
+    url = "https://github.com/openvenues/libpostal/archive/refs/tags/v1.1.tar.gz"
+
+    maintainers("jgaeb")
+
+    version("1.1", sha256="8cc473a05126895f183f2578ca234428d8b58ab6fadf550deaacd3bd0ae46032")
+    version("1.0.0", sha256="3035af7e15b2894069753975d953fa15a86d968103913dbf8ce4b8aa26231644")
+    version("0.3.4", sha256="8b3b95660c5b5d4fe48045b9acb000d1a0eb19d58d0c2d2041e78d9a96d88716")
+    version("0.3.3", sha256="dc73de37d7f7b96f329fd213dcbac540f2ae92fbef9c079fd64fbc8daeb87b01")
+    version("0.3.2", sha256="9a1590eadf4ebe84979113b71059410413adf239b2999d22d11fe8778945f2c1")
+    version("0.3.1", sha256="68c51a5fdae41e1cac474742789ba5a46a38e307a0a2450cb2d3e33b4f17cf4d")
+    version("0.3", sha256="28c19e21bab13425a76aa65a8435f4b3909611056c2ff439c39b4e57b2a70150")
+
+    depends_on("autoconf", type="build")
+    depends_on("automake", type="build")
+    depends_on("libtool", type="build")
+    depends_on("m4", type="build")
+    depends_on("curl", type="build")
+    depends_on("pkgconfig", type="build")
+
+    def autoreconf(self, spec, prefix):
+        which("sh")("bootstrap.sh")
+
+    def configure_args(self):
+        args = ["--datadir={0}".format(self.prefix.share)]
+
+        # Check if the target is Apple's ARM-based M1 chip.
+        arch = self.spec.architecture
+        if arch.platform == "darwin" and arch.target == "m1":
+            args.append("--disable-sse2")
+
+        return args
diff --git a/var/spack/repos/builtin/packages/libpressio-sperr/package.py b/var/spack/repos/builtin/packages/libpressio-sperr/package.py
index cd636f2fc9c070..35576e33a7a08a 100644
--- a/var/spack/repos/builtin/packages/libpressio-sperr/package.py
+++ b/var/spack/repos/builtin/packages/libpressio-sperr/package.py
@@ -10,17 +10,19 @@ class LibpressioSperr(CMakePackage):
     """A LibPressio plugin for Sperr"""
 
     homepage = "https://github.com/robertu94/libpressio-sperr"
-    url = "https://github.com/robertu94/libpressio-sperr/archive/refs/tags/0.0.1.tar.gz"
+    url = "https://github.com/robertu94/libpressio-sperr/archive/refs/tags/0.0.4.tar.gz"
     git = homepage
 
     maintainers("robertu94")
 
-    depends_on("libpressio@0.88.0:", when="@0.0.3:")
-    depends_on("libpressio@:0.88.0", when="@:0.0.2")
-    depends_on("sperr")
-    depends_on("pkgconfig", type="build")
-
     version("master", branch="master")
+    version("0.0.4", sha256="97f2879460b1a28ed8ebf0c300c1cf7ceeb2c7aa7b8a1307ed19bf8cce0b7941")
     version("0.0.3", sha256="e0d1fd083419aaaa243cbf780b7de17aeb96533000071088aa21ec238d358ecc")
     version("0.0.2", sha256="61995d687f9e7e798e17ec7238d19d917890dc0ff5dec18293b840c4d6f8c115")
     version("0.0.1", sha256="e2c164822708624b97654046b42abff704594cba6537d6d0646d485bdf2d03ca")
+
+    depends_on("libpressio@0.88.0:", when="@0.0.3:")
+    depends_on("libpressio@:0.88.0", when="@:0.0.2")
+    depends_on("sperr@:0.6.2", when="@:0.0.3")
+    depends_on("sperr@0.7.1:", when="@0.0.4:")
+    depends_on("pkgconfig", type="build")
diff --git a/var/spack/repos/builtin/packages/libpressio/package.py b/var/spack/repos/builtin/packages/libpressio/package.py
index 05f0f25d238630..2c2bfa48e3da3d 100644
--- a/var/spack/repos/builtin/packages/libpressio/package.py
+++ b/var/spack/repos/builtin/packages/libpressio/package.py
@@ -344,6 +344,8 @@ def cmake_args(self):
             args.append("-DLIBPRESSIO_HAS_QoZ=ON")
         if "+cusz" in self.spec:
             args.append("-DLIBPRESSIO_HAS_CUSZ=ON")
+        if self.spec.satisfies("+cusz +cuda"):
+            args.append("-DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined")
         if "+core" in self.spec:
             args.append("-DLIBPRESSIO_BUILD_MODE=FULL")
         else:
diff --git a/var/spack/repos/builtin/packages/libproxy/package.py b/var/spack/repos/builtin/packages/libproxy/package.py
index 5d3945675227b0..65915bb1f55549 100644
--- a/var/spack/repos/builtin/packages/libproxy/package.py
+++ b/var/spack/repos/builtin/packages/libproxy/package.py
@@ -22,7 +22,7 @@ class Libproxy(CMakePackage):
     variant("perl", default=False, description="Enable Perl bindings")
     variant("python", default=False, description="Enable Python bindings", when="@0.4.16:")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("perl", type=("build", "run"), when="+perl")
 
     extends("python@:3.8", when="+python")
diff --git a/var/spack/repos/builtin/packages/librdkafka/package.py b/var/spack/repos/builtin/packages/librdkafka/package.py
index d2a6e8977b47f4..b097d2513da017 100644
--- a/var/spack/repos/builtin/packages/librdkafka/package.py
+++ b/var/spack/repos/builtin/packages/librdkafka/package.py
@@ -13,6 +13,7 @@ class Librdkafka(AutotoolsPackage):
     homepage = "https://github.com/edenhill/librdkafka"
     url = "https://github.com/edenhill/librdkafka/archive/v1.5.0.tar.gz"
 
+    version("2.2.0", sha256="af9a820cbecbc64115629471df7c7cecd40403b6c34bfdbb9223152677a47226")
     version("2.1.1", sha256="7be1fc37ab10ebdc037d5c5a9b35b48931edafffae054b488faaff99e60e0108")
     version("2.1.0", sha256="d8e76c4b1cde99e283a19868feaaff5778aa5c6f35790036c5ef44bc5b5187aa")
     version("2.0.2", sha256="f321bcb1e015a34114c83cf1aa7b99ee260236aab096b85c003170c90a47ca9d")
diff --git a/var/spack/repos/builtin/packages/libreproc/package.py b/var/spack/repos/builtin/packages/libreproc/package.py
index d39eada3150279..3417e4dcf85b43 100644
--- a/var/spack/repos/builtin/packages/libreproc/package.py
+++ b/var/spack/repos/builtin/packages/libreproc/package.py
@@ -20,8 +20,7 @@ class Libreproc(CMakePackage):
     variant("shared", default=True, description="Build shared libraries")
 
     depends_on("cmake@3.14:", type="build")
-    depends_on("zlib+shared", type="link", when="+shared")
-    depends_on("zlib~shared", type="link", when="~shared")
+    depends_on("zlib-api", type="link")
 
     def cmake_args(self):
         return [
diff --git a/var/spack/repos/builtin/packages/libressl/package.py b/var/spack/repos/builtin/packages/libressl/package.py
index d436df8abf82df..0fca30e3539129 100644
--- a/var/spack/repos/builtin/packages/libressl/package.py
+++ b/var/spack/repos/builtin/packages/libressl/package.py
@@ -16,14 +16,15 @@ class Libressl(AutotoolsPackage):
 
     maintainers("eschnett")
 
+    version("3.7.2", sha256="b06aa538fefc9c6b33c4db4931a09a5f52d9d2357219afcbff7d93fe12ebf6f7")
+    version("3.6.3", sha256="87b1bbe36e9eec8d0ae5f04c83d36b2c5b0e581784c7eb0817025ed29eadea37")
     version("3.6.1", sha256="acfac61316e93b919c28d62d53037ca734de85c46b4d703f19fd8395cf006774")
 
     variant("shared", default=True, description="Build shared libraries")
     variant("static", default=False, description="Build static libraries")
 
     def configure_args(self):
-        args = [
-            "--enable-shared" if "+shared" in spec else "--disable-shared",
-            "--enable-static" if "+static" in spec else "--disable-static",
-        ]
+        args = []
+        args.extend(self.enable_or_disable("shared"))
+        args.extend(self.enable_or_disable("static"))
         return args
diff --git a/var/spack/repos/builtin/packages/librom/package.py b/var/spack/repos/builtin/packages/librom/package.py
index 444a74b3bf7b55..94f28aa318ac2e 100644
--- a/var/spack/repos/builtin/packages/librom/package.py
+++ b/var/spack/repos/builtin/packages/librom/package.py
@@ -17,7 +17,7 @@ class Librom(AutotoolsPackage):
 
     depends_on("lapack")
     depends_on("mpi")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libszip")
     depends_on("hdf5")
     depends_on("perl")
@@ -34,7 +34,7 @@ def configure_args(self):
         args = [
             "--with-lapack={0}".format(spec["lapack"].prefix),
             "--with-lapack-libs={0}".format(spec["lapack"].libs.ld_flags),
-            "--with-zlib={0}".format(spec["zlib"].prefix),
+            "--with-zlib={0}".format(spec["zlib-api"].prefix),
             "--with-szlib={0}".format(spec["libszip"].prefix),
             "--with-hdf5={0}".format(spec["hdf5"].prefix),
             "--with-MPICC={0}".format(spec["mpi"].mpicc),
diff --git a/var/spack/repos/builtin/packages/librsb/package.py b/var/spack/repos/builtin/packages/librsb/package.py
index bf42228fd061f9..2d5b9f0a16ce40 100644
--- a/var/spack/repos/builtin/packages/librsb/package.py
+++ b/var/spack/repos/builtin/packages/librsb/package.py
@@ -21,7 +21,7 @@ class Librsb(AutotoolsPackage):
     version("1.2.0.9", sha256="f421f5d572461601120933e3c1cfee2ca69e6ecc92cbb11baa4e86bdedd3d9fa")
     version("1.2.0.8", sha256="8bebd19a1866d80ade13eabfdd0f07ae7e8a485c0b975b5d15f531ac204d80cb")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("googletest", type="build", when="+googletest")
     conflicts("%apple-clang")
     # conflicts('%clang')
@@ -46,8 +46,8 @@ def configure_args(self):
             "--enable-openmp",
             "--with-zlib",
             "--enable-fortran-module-install",
-            "CPPFLAGS={0}".format(self.spec["zlib"].headers.include_flags),
-            "LDFLAGS={0}".format(self.spec["zlib"].libs.search_flags),
+            "CPPFLAGS={0}".format(self.spec["zlib-api"].headers.include_flags),
+            "LDFLAGS={0}".format(self.spec["zlib-api"].libs.search_flags),
         ]
         if "+asan" in self.spec:
             args.append("CFLAGS=-O0 -ggdb -fsanitize=address -fno-omit-frame-pointer")
diff --git a/var/spack/repos/builtin/packages/librsvg/package.py b/var/spack/repos/builtin/packages/librsvg/package.py
index 1eb493061d1972..f338a5d83fb159 100644
--- a/var/spack/repos/builtin/packages/librsvg/package.py
+++ b/var/spack/repos/builtin/packages/librsvg/package.py
@@ -12,6 +12,7 @@ class Librsvg(AutotoolsPackage):
     homepage = "https://wiki.gnome.org/Projects/LibRsvg"
     url = "https://download.gnome.org/sources/librsvg/2.44/librsvg-2.44.14.tar.xz"
 
+    version("2.56.2", sha256="3ec3c4d8f73e0ba4b9130026969e8371c092b734298d36e2fdb3eb4afcec1200")
     version("2.51.0", sha256="89d32e38445025e1b1d9af3dd9d3aeb9f6fce527aeecbecf38b369b34c80c038")
     version("2.50.2", sha256="6211f271ce4cd44a7318190d36712e9cea384a933d3e3570004edeb210a056d3")
     version("2.50.0", sha256="b3fadba240f09b9c9898ab20cb7311467243e607cf8f928b7c5f842474ee3df4")
@@ -22,15 +23,29 @@ class Librsvg(AutotoolsPackage):
 
     depends_on("gobject-introspection", type="build")
     depends_on("pkgconfig", type="build")
-    depends_on("rust", type="build", when="@2.41:")
+    # rust minimal version from NEWS file
+    depends_on("rust@1.65:", when="@2.56.1:", type="build")
+    # upper bound because "Unaligned references to packed fields are a hard
+    # error" starting from 1.69
+    depends_on("rust@1.40:1.68", when="@2.50:2.51", type="build")
+    depends_on("rust", when="@2.41:", type="build")
     depends_on("gtk-doc", type="build", when="+doc")
-    depends_on("cairo+gobject")
-    depends_on("gdk-pixbuf")
-    depends_on("glib")
-    depends_on("libcroco")
-    depends_on("pango")
+
+    # requirements according to `configure` file
+    depends_on("cairo@1.16:+gobject", when="@2.50:")
+    depends_on("cairo@1.15.12:+gobject", when="@2.44.14:")
+    depends_on("cairo@1.2.0:+gobject")
+    depends_on("libcroco@0.6.1:", when="@:2.44.14")
+    depends_on("gdk-pixbuf@2.20:")
+    depends_on("glib@2.50:", when="@2.50:")
+    depends_on("glib@2.48:", when="@2.44.14:")
+    depends_on("glib@2.12:")
+    depends_on("harfbuzz@2:", when="@2.50:")
+    depends_on("libxml2@2.9:")
+    depends_on("pango@1.46:", when="@2.51:")
+    depends_on("pango@1.38:")
+
     depends_on("libffi")
-    depends_on("libxml2")
     depends_on("shared-mime-info")
 
     def url_for_version(self, version):
diff --git a/var/spack/repos/builtin/packages/libsolv/package.py b/var/spack/repos/builtin/packages/libsolv/package.py
index e3b5d61e038e6f..f58e7463a13e12 100644
--- a/var/spack/repos/builtin/packages/libsolv/package.py
+++ b/var/spack/repos/builtin/packages/libsolv/package.py
@@ -20,8 +20,7 @@ class Libsolv(CMakePackage):
     variant("conda", default=False, description="Include solv/conda.h")
 
     depends_on("expat", type="link")
-    depends_on("zlib+shared", type="link", when="+shared")
-    depends_on("zlib~shared", type="link", when="~shared")
+    depends_on("zlib-api", type="link")
 
     def cmake_args(self):
         return [
diff --git a/var/spack/repos/builtin/packages/libssh/package.py b/var/spack/repos/builtin/packages/libssh/package.py
index e5b1207db4b13e..e239f3fb3ca3bd 100644
--- a/var/spack/repos/builtin/packages/libssh/package.py
+++ b/var/spack/repos/builtin/packages/libssh/package.py
@@ -19,7 +19,7 @@ class Libssh(CMakePackage):
     variant("gssapi", default=True, description="Build with gssapi support")
     depends_on("openssl@:1.0", when="@:0.7")
     depends_on("openssl")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("krb5", when="+gssapi")
 
     def url_for_version(self, version):
diff --git a/var/spack/repos/builtin/packages/libssh2/package.py b/var/spack/repos/builtin/packages/libssh2/package.py
index 49619345e6e820..34b4127301a295 100644
--- a/var/spack/repos/builtin/packages/libssh2/package.py
+++ b/var/spack/repos/builtin/packages/libssh2/package.py
@@ -23,7 +23,12 @@ class Libssh2(AutotoolsPackage, CMakePackage):
 
     build_system("autotools", "cmake", default="autotools")
 
-    variant("crypto", default="openssl", values=("openssl", conditional("mbedtls", when="@1.8:")))
+    variant(
+        "crypto",
+        default="openssl",
+        description="The backend to use for cryptography",
+        values=("openssl", conditional("mbedtls", when="@1.8:")),
+    )
     variant("shared", default=True, description="Build shared libraries")
 
     with when("build_system=cmake"):
@@ -36,7 +41,7 @@ class Libssh2(AutotoolsPackage, CMakePackage):
         depends_on("openssl@:1", when="@:1.9")
 
     depends_on("mbedtls@:2 +pic", when="crypto=mbedtls")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("xz")
 
     # libssh2 adds its own deps in the pc file even when doing shared linking,
diff --git a/var/spack/repos/builtin/packages/libtheora/exit-prior-to-running-configure.patch b/var/spack/repos/builtin/packages/libtheora/exit-prior-to-running-configure.patch
deleted file mode 100644
index 99992c39c28dc1..00000000000000
--- a/var/spack/repos/builtin/packages/libtheora/exit-prior-to-running-configure.patch
+++ /dev/null
@@ -1,23 +0,0 @@
-From 0060fd48c12a59a080974ca3754bf0eab9ab6d35 Mon Sep 17 00:00:00 2001
-From: Howard Pritchard 
-Date: Tue, 24 Nov 2020 15:14:41 -0700
-Subject: [PATCH] exit prior to running configure
-
-Signed-off-by: Howard Pritchard 
-
-diff --git a/autogen.sh b/autogen.sh
-index bbca69dc..4de1e783 100755
---- a/autogen.sh
-+++ b/autogen.sh
-@@ -112,6 +112,8 @@ if test -z "$*"; then
-         echo "to pass any to it, please specify them on the $0 command line."
- fi
- 
-+exit 0
-+
- echo "Generating configuration files for $package, please wait...."
- 
- echo "  $ACLOCAL $ACLOCAL_FLAGS"
--- 
-2.18.2
-
diff --git a/var/spack/repos/builtin/packages/libtheora/package.py b/var/spack/repos/builtin/packages/libtheora/package.py
index 6386da3497d85f..6ec88aa91ccf24 100644
--- a/var/spack/repos/builtin/packages/libtheora/package.py
+++ b/var/spack/repos/builtin/packages/libtheora/package.py
@@ -17,7 +17,10 @@ class Libtheora(AutotoolsPackage, MSBuildPackage):
 
     homepage = "https://www.theora.org"
     url = "http://downloads.xiph.org/releases/theora/libtheora-1.1.1.tar.xz"
+    git = "https://gitlab.xiph.org/xiph/theora.git"
 
+    version("master", branch="master")
+    version("stable", branch="theora-1.1")
     version("1.1.1", sha256="f36da409947aa2b3dcc6af0a8c2e3144bc19db2ed547d64e9171c59c66561c61")
     version("1.1.0", sha256="3d7b4fb1c115f1a530afd430eed2e8861fa57c8b179ec2d5a5d8f1cd0c7a4268")
 
@@ -43,12 +46,23 @@ class Libtheora(AutotoolsPackage, MSBuildPackage):
         "msbuild", "autotools", default="autotools" if sys.platform != "win32" else "msbuild"
     )
 
-    patch("exit-prior-to-running-configure.patch", when="@1.1.1")
     patch("fix_encoding.patch", when="@1.1:")
     patch(
-        "https://gitlab.xiph.org/xiph/theora/-/commit/7288b539c52e99168488dc3a343845c9365617c8.patch",
-        sha256="8b1f256fa6bfb4ce1355c5be1104e8cfe695c8484d8ea19db06c006880a02298",
-        when="^libpng@1.6:",
+        "https://gitlab.xiph.org/xiph/theora/-/commit/7288b539c52e99168488dc3a343845c9365617c8.diff",
+        sha256="e01ef71a1c19783a0b323b90a625e5c360ddb7ee03d2b6c201f1519f1704ea11",
+        when="@:1.1.1 ^libpng@1.6:",
+    )
+    # add -no-undefined
+    patch(
+        "https://gitlab.xiph.org/xiph/theora/-/commit/391ab0e99f2ad730231dbe5fc1154b990087f17d.diff",
+        sha256="d9bb5a9573819a27b3a925b1b66c33b36d9bca11b05d8aef88566eb6c8700690",
+        when="@:1.1.1",
+    )
+    # link theoraenc to theoradec
+    patch(
+        "https://gitlab.xiph.org/xiph/theora/-/commit/133b951b60fd845eabbc38bf7acd998bb9be75fc.diff",
+        sha256="e01511aff0130a40c889868d3713a56458744f39d1bb5ad98c8058da50233aa7",
+        when="@:1.1.1",
     )
     patch("libtheora-inc-external-ogg.patch", when="platform=windows")
 
@@ -62,10 +76,9 @@ def configure_args(self):
 
     def autoreconf(self, pkg, spec, prefix):
         sh = which("sh")
-        if self.spec.satisfies("target=aarch64:"):
-            sh("./autogen.sh", "prefix={0}".format(prefix), "--build=arm-linux")
-        else:
-            sh("./autogen.sh", "prefix={0}".format(prefix))
+        # arguments are passed on to configure, let it just print its version
+        # and exit, so that configure can run in the configure build phase
+        sh("./autogen.sh", "-V")
 
 
 class MSBuildBuilder(MSBuildBuilder):
diff --git a/var/spack/repos/builtin/packages/libtiff/package.py b/var/spack/repos/builtin/packages/libtiff/package.py
index 395bb4c1b4569d..5977520d4ec136 100644
--- a/var/spack/repos/builtin/packages/libtiff/package.py
+++ b/var/spack/repos/builtin/packages/libtiff/package.py
@@ -38,6 +38,7 @@ class Libtiff(CMakePackage, AutotoolsPackage):
 
     maintainers("adamjstewart")
 
+    version("4.5.1", sha256="d7f38b6788e4a8f5da7940c5ac9424f494d8a79eba53d555f4a507167dca5e2b")
     version("4.5.0", sha256="c7a1d9296649233979fa3eacffef3fa024d73d05d589cb622727b5b08c423464")
     version("4.4.0", sha256="917223b37538959aca3b790d2d73aa6e626b688e02dcda272aec24c2f498abed")
     version("4.3.0", sha256="0e46e5acb087ce7d1ac53cf4f56a09b221537fc86dfc5daaad1c2e89e1b37ac8")
@@ -86,8 +87,8 @@ class Libtiff(CMakePackage, AutotoolsPackage):
         depends_on("cmake@2.8.9:", when="@4.0.6:4.0.9", type="build")
         depends_on("cmake@3:", when="@4.0.5", type="build")
 
-    depends_on("zlib", when="+zlib")
-    depends_on("zlib", when="+pixarlog")
+    depends_on("zlib-api", when="+zlib")
+    depends_on("zlib-api", when="+pixarlog")
     depends_on("jpeg@5:", when="+jpeg")
     depends_on("jbigkit", when="+jbig")
     depends_on("lerc", when="+lerc")
diff --git a/var/spack/repos/builtin/packages/libtirpc/macos-1.3.3.patch b/var/spack/repos/builtin/packages/libtirpc/macos-1.3.3.patch
new file mode 100644
index 00000000000000..8e6f1efe421f78
--- /dev/null
+++ b/var/spack/repos/builtin/packages/libtirpc/macos-1.3.3.patch
@@ -0,0 +1,105 @@
+diff --git a/src/Makefile.in b/src/Makefile.in
+--- a/src/Makefile.in
++++ b/src/Makefile.in
+@@ -90,7 +90,7 @@
+ build_triplet = @build@
+ host_triplet = @host@
+ @AUTHDES_TRUE@am__append_1 = auth_des.c  authdes_prot.c  des_crypt.c  des_impl.c  des_soft.c  svc_auth_des.c
+-@SYMVERS_TRUE@am__append_2 = -Wl,--version-script=$(srcdir)/libtirpc.map
++@SYMVERS_TRUE@am__append_2 = -Wl
+ @GSS_TRUE@am__append_3 = auth_gss.c authgss_prot.c svc_auth_gss.c \
+ @GSS_TRUE@			   rpc_gss_utils.c
+ 
+diff --git a/src/getpeereid.c b/src/getpeereid.c
+--- a/src/getpeereid.c
++++ b/src/getpeereid.c
+@@ -24,7 +24,7 @@
+  * SUCH DAMAGE.
+  */
+ 
+-
++#ifndef __APPLE__
+ #include 
+ #include 
+ #include 
+@@ -49,3 +49,4 @@
+ 	*egid = uc.gid;
+ 	return (0);
+  }
++#endif
+diff --git a/src/rpc_com.h b/src/rpc_com.h
+--- a/src/rpc_com.h
++++ b/src/rpc_com.h
+@@ -42,6 +42,11 @@
+ 
+ #include 
+ 
++#ifdef __APPLE__
++    #define SOL_IPV6 IPPROTO_IPV6
++    #define SOL_IP IPPROTO_IP
++#endif
++
+ #ifdef __cplusplus
+ extern "C" {
+ #endif
+diff --git a/src/svc_dg.c b/src/svc_dg.c
+--- a/src/svc_dg.c
++++ b/src/svc_dg.c
+@@ -37,6 +37,7 @@
+  *
+  * Does some caching in the hopes of achieving execute-at-most-once semantics.
+  */
++#define __APPLE_USE_RFC_3542
+ #include 
+ #include 
+ #include 
+diff --git a/src/svc_raw.c b/src/svc_raw.c
+--- a/src/svc_raw.c
++++ b/src/svc_raw.c
+@@ -43,6 +43,7 @@
+ #include 
+ #include 
+ #include 
++#include 
+ 
+ #ifndef UDPMSGSIZE
+ #define	UDPMSGSIZE 8800
+diff --git a/src/xdr_float.c b/src/xdr_float.c
+--- a/src/xdr_float.c
++++ b/src/xdr_float.c
+@@ -83,7 +83,11 @@
+ };
+ #else
+ 
++#ifndef __APPLE__
+ #include 
++#else
++#include 
++#endif
+ #define IEEEFP
+ 
+ #endif /* vax */
+diff --git a/tirpc/reentrant.h b/tirpc/reentrant.h
+--- a/tirpc/reentrant.h
++++ b/tirpc/reentrant.h
+@@ -36,7 +36,7 @@
+  * These definitions are only guaranteed to be valid on Linux. 
+  */
+ 
+-#if defined(__linux__)
++#if defined(__linux__) || defined(__APPLE__)
+ 
+ #include 
+ 
+diff --git a/tirpc/rpc/rpcent.h b/tirpc/rpc/rpcent.h
+--- a/tirpc/rpc/rpcent.h
++++ b/tirpc/rpc/rpcent.h
+@@ -50,7 +50,7 @@
+ 
+ /* These are defined in /usr/include/rpc/netdb.h, unless we are using
+    the C library without RPC support. */
+-#if defined(__UCLIBC__) && !defined(__UCLIBC_HAS_RPC__) || !defined(__GLIBC__)
++#if defined(__UCLIBC__) && !defined(__UCLIBC_HAS_RPC__) || !defined(__GLIBC__) && !defined(__APPLE__)
+ struct rpcent {
+ 	char	*r_name;	/* name of server for this rpc program */
+ 	char	**r_aliases;	/* alias list */
diff --git a/var/spack/repos/builtin/packages/libtirpc/package.py b/var/spack/repos/builtin/packages/libtirpc/package.py
index e32755d02776d7..cd11256d2f77b5 100644
--- a/var/spack/repos/builtin/packages/libtirpc/package.py
+++ b/var/spack/repos/builtin/packages/libtirpc/package.py
@@ -12,6 +12,7 @@ class Libtirpc(AutotoolsPackage):
     homepage = "https://sourceforge.net/projects/libtirpc/"
     url = "https://sourceforge.net/projects/libtirpc/files/libtirpc/1.1.4/libtirpc-1.1.4.tar.bz2/download"
 
+    version("1.3.3", sha256="6474e98851d9f6f33871957ddee9714fdcd9d8a5ee9abb5a98d63ea2e60e12f3")
     version("1.2.6", sha256="4278e9a5181d5af9cd7885322fdecebc444f9a3da87c526e7d47f7a12a37d1cc")
     version("1.1.4", sha256="2ca529f02292e10c158562295a1ffd95d2ce8af97820e3534fe1b0e3aec7561d")
 
@@ -23,10 +24,17 @@ class Libtirpc(AutotoolsPackage):
 
     # Remove -pipe flag to compiler in Makefiles when using nvhpc
     patch("libtirpc-remove-pipe-flag-for-nvhpc.patch", when="%nvhpc")
-
-    # FIXME: build error on macOS
-    # auth_none.c:81:9: error: unknown type name 'mutex_t'
-    conflicts("platform=darwin", msg="Does not build on macOS")
+    # Allow to build on macOS
+    # - Remove versioning linker flags and include
+    # - Include missing / apple specific headers
+    # - Add apple pre-processor guards to guard / ignore some sections
+    # Taken from:
+    # https://github.com/unfs3/unfs3/pull/25#issuecomment-1631198490
+    patch("macos-1.3.3.patch", when="@1.3.3 platform=darwin")
+
+    # Only the latest version is known to build on macOS. Previous versions fail
+    # with auth_none.c:81:9: error: unknown type name 'mutex_t'
+    conflicts("platform=darwin", when="@:1.3.2", msg="Does not build on macOS")
 
     @property
     def headers(self):
@@ -45,4 +53,9 @@ def configure_args(self):
         if spec.satisfies("~gssapi"):
             args.append("--disable-gssapi")
 
+        # See discussion in
+        # https://github.com/unfs3/unfs3/pull/25#issuecomment-1631198490
+        if spec.satisfies("@1.3.3 platform=darwin"):
+            args.append("--disable-gssapi")
+
         return args
diff --git a/var/spack/repos/builtin/packages/libunwind/package.py b/var/spack/repos/builtin/packages/libunwind/package.py
index 39f806486bf52a..b9115455a2a03e 100644
--- a/var/spack/repos/builtin/packages/libunwind/package.py
+++ b/var/spack/repos/builtin/packages/libunwind/package.py
@@ -85,7 +85,7 @@ class Libunwind(AutotoolsPackage):
     depends_on("m4", type="build", when=reconf_versions)
 
     depends_on("xz", type="link", when="+xz")
-    depends_on("zlib", type="link", when="+zlib")
+    depends_on("zlib-api", type="link", when="+zlib")
 
     conflicts("platform=darwin", msg="Non-GNU libunwind needs ELF libraries Darwin does not have")
 
diff --git a/var/spack/repos/builtin/packages/libuv/package.py b/var/spack/repos/builtin/packages/libuv/package.py
index 8845270bb15a10..5c838db449a850 100644
--- a/var/spack/repos/builtin/packages/libuv/package.py
+++ b/var/spack/repos/builtin/packages/libuv/package.py
@@ -10,7 +10,12 @@ class Libuv(AutotoolsPackage):
 
     homepage = "https://libuv.org"
     url = "https://dist.libuv.org/dist/v1.44.1/libuv-v1.44.1-dist.tar.gz"
+    list_url = "https://dist.libuv.org/dist"
+    list_depth = 1
 
+    version("1.46.0", sha256="94f101111ef3209340d7f09c2aa150ddb4feabd2f9d87d47d9f5bded835b8094")
+    version("1.45.0", sha256="3793d8c0d6fa587721d010d0555b7e82443fd4e8b3c91e529eb6607592f52b87")
+    version("1.44.2", sha256="8ff28f6ac0d6d2a31d2eeca36aff3d7806706c7d3f5971f5ee013ddb0bdd2e9e")
     version("1.44.1", sha256="b7293cefb470e17774dcf5d62c4c969636172726155b55ceef5092b7554863cc")
     version("1.44.0", sha256="6c52494401cfe8d08fb4ec245882f0bd4b1572b5a8e79d6c418b855422a1a27d")
     version("1.43.0", sha256="90d72bb7ae18de2519d0cac70eb89c319351146b90cd3f91303a492707e693a4")
diff --git a/var/spack/repos/builtin/packages/libvorbis/package.py b/var/spack/repos/builtin/packages/libvorbis/package.py
index 9f4fc58fc99757..cfd0420ca36bd8 100644
--- a/var/spack/repos/builtin/packages/libvorbis/package.py
+++ b/var/spack/repos/builtin/packages/libvorbis/package.py
@@ -22,5 +22,8 @@ class Libvorbis(AutotoolsPackage):
 
     depends_on("pkgconfig", type="build")
 
+    def patch(self):
+        filter_file(r"-force_cpusubtype_ALL", "", "configure", string=True)
+
     # `make check` crashes when run in parallel
     parallel = False
diff --git a/var/spack/repos/builtin/packages/libwebsockets/package.py b/var/spack/repos/builtin/packages/libwebsockets/package.py
index 06828304557623..db15138ff6c17e 100644
--- a/var/spack/repos/builtin/packages/libwebsockets/package.py
+++ b/var/spack/repos/builtin/packages/libwebsockets/package.py
@@ -19,5 +19,5 @@ class Libwebsockets(CMakePackage):
     version("2.0.3", sha256="cf0e91b564c879ab98844385c98e7c9e298cbb969dbc251a3f18a47feb94342c")
     version("1.7.9", sha256="86a5105881ea2cb206f8795483d294e9509055decf60436bcc1e746262416438")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("openssl")
diff --git a/var/spack/repos/builtin/packages/libwmf/package.py b/var/spack/repos/builtin/packages/libwmf/package.py
index 1bd0751708892e..040bf9dc9514eb 100644
--- a/var/spack/repos/builtin/packages/libwmf/package.py
+++ b/var/spack/repos/builtin/packages/libwmf/package.py
@@ -28,7 +28,7 @@ class Libwmf(AutotoolsPackage):
     depends_on("libxml2")
     depends_on("libpng")
     depends_on("libjpeg")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def configure_args(self):
         args = ["--disable-static"]
diff --git a/var/spack/repos/builtin/packages/libxc/package.py b/var/spack/repos/builtin/packages/libxc/package.py
index 46853d75a0dd14..1652ae772aeeba 100644
--- a/var/spack/repos/builtin/packages/libxc/package.py
+++ b/var/spack/repos/builtin/packages/libxc/package.py
@@ -13,6 +13,9 @@ class Libxc(AutotoolsPackage, CudaPackage):
     homepage = "https://tddft.org/programs/libxc/"
     url = "https://gitlab.com/libxc/libxc/-/archive/6.1.0/libxc-6.1.0.tar.gz"
 
+    version("6.2.2", sha256="3b0523924579cf494cafc6fea92945257f35692b004217d3dfd3ea7ca780e8dc")
+    version("6.2.1", sha256="b5f3b4514db6bc4ccda1da90ac6176ea1f82e12241cc66427c58cbc4a5197b9b")
+    version("6.2.0", sha256="3d25878782b5f94e7e4d41bd6de27f98983584cd0be0c65e69a9ada986b56b4d")
     version("6.1.0", sha256="f593745fa47ebfb9ddc467aaafdc2fa1275f0d7250c692ce9761389a90dd8eaf")
     version("6.0.0", sha256="0c774e8e195dd92800b9adf3df5f5721e29acfe9af4b191a9937c7de4f9aa9f6")
     version("5.2.3", sha256="7b7a96d8eeb472c7b8cca7ac38eae27e0a8113ef44dae5359b0eb12592b4bcf2")
@@ -30,6 +33,8 @@ class Libxc(AutotoolsPackage, CudaPackage):
     version("2.2.1", sha256="ade61c1fa4ed238edd56408fd8ee6c2e305a3d5753e160017e2a71817c98fd00")
 
     variant("shared", default=True, description="Build shared libraries")
+    variant("kxc", default=False, when="@5:", description="Build with third derivatives")
+    variant("lxc", default=False, when="@5:", description="Build with fourth derivatives")
 
     conflicts("+shared +cuda", msg="Only ~shared supported with +cuda")
     conflicts("+cuda", when="@:4", msg="CUDA support only in libxc 5.0.0 and above")
@@ -118,6 +123,10 @@ def configure_args(self):
         args = []
         args += self.enable_or_disable("shared")
         args += self.enable_or_disable("cuda")
+        if "+kxc" in self.spec:
+            args.append("--enable-kxc")
+        if "+lxc" in self.spec:
+            args.append("--enable-lxc")
         return args
 
     @run_after("configure")
diff --git a/var/spack/repos/builtin/packages/libxcrypt/package.py b/var/spack/repos/builtin/packages/libxcrypt/package.py
index dd85dcd9b04f0d..428720add35372 100644
--- a/var/spack/repos/builtin/packages/libxcrypt/package.py
+++ b/var/spack/repos/builtin/packages/libxcrypt/package.py
@@ -30,7 +30,12 @@ def url_for_version(self, version):
     version("4.4.16", sha256="a98f65b8baffa2b5ba68ee53c10c0a328166ef4116bce3baece190c8ce01f375")
     version("4.4.15", sha256="8bcdef03bc65f9dbda742e56820435b6f13eea59fb903765141c6467f4655e5a")
 
-    variant("obsolete_api", default=False, when="@4.4.30:")
+    variant(
+        "obsolete_api",
+        default=False,
+        description="Enable all compatibility interfaces",
+        when="@4.4.30:",
+    )
 
     patch("truncating-conversion.patch", when="@4.4.30")
 
diff --git a/var/spack/repos/builtin/packages/libxkbcommon/package.py b/var/spack/repos/builtin/packages/libxkbcommon/package.py
index 772220a4a5e1cf..2ab85c99b37812 100644
--- a/var/spack/repos/builtin/packages/libxkbcommon/package.py
+++ b/var/spack/repos/builtin/packages/libxkbcommon/package.py
@@ -20,6 +20,8 @@ class Libxkbcommon(MesonPackage, AutotoolsPackage):
         conditional("meson", when="@0.9:"), conditional("autotools", when="@:0.8"), default="meson"
     )
 
+    version("1.5.0", sha256="560f11c4bbbca10f495f3ef7d3a6aa4ca62b4f8fb0b52e7d459d18a26e46e017")
+    version("1.4.1", sha256="943c07a1e2198026d8102b17270a1f406e4d3d6bbc4ae105b9e1b82d7d136b39")
     version("1.4.0", sha256="106cec5263f9100a7e79b5f7220f889bc78e7d7ffc55d2b6fdb1efefb8024031")
     version(
         "0.8.2",
@@ -39,6 +41,9 @@ class Libxkbcommon(MesonPackage, AutotoolsPackage):
 
     variant("wayland", default=False, description="Enable Wayland support")
 
+    depends_on("meson@0.41:", type="build", when="@0.9:")
+    depends_on("meson@0.49:", type="build", when="@1.0:")
+    depends_on("meson@0.51:", type="build", when="@1.5:")
     depends_on("pkgconfig@0.9.0:", type="build")
     depends_on("bison", type="build")
     depends_on("util-macros")
@@ -49,6 +54,7 @@ class Libxkbcommon(MesonPackage, AutotoolsPackage):
     depends_on("wayland@1.2.0:", when="+wayland")
     depends_on("wayland-protocols@1.7:", when="+wayland")
 
+
 class MesonBuilder(spack.build_systems.meson.MesonBuilder):
     def meson_args(self):
         args = [
@@ -64,7 +70,8 @@ def setup_build_environment(self, env):
             deps = ["zlib", "xz", "iconv"]
             ldflags = [self.spec[lib].libs.search_flags for lib in deps]
             libs = [self.spec[lib].libs.link_flags for lib in deps]
-            env.set("LDFLAGS", " ".join(ldflags+libs))
+            env.set("LDFLAGS", " ".join(ldflags + libs))
+
 
 class AutotoolsBuilder(spack.build_systems.autotools.AutotoolsBuilder):
     def configure_args(self):
diff --git a/var/spack/repos/builtin/packages/libxml2/package.py b/var/spack/repos/builtin/packages/libxml2/package.py
index 31c4deff3a2b5b..dd598cdbc37187 100644
--- a/var/spack/repos/builtin/packages/libxml2/package.py
+++ b/var/spack/repos/builtin/packages/libxml2/package.py
@@ -17,6 +17,8 @@ class Libxml2(AutotoolsPackage, NMakePackage):
     url = "https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.13.tar.xz"
     list_url = "https://gitlab.gnome.org/GNOME/libxml2/-/releases"
 
+    maintainers("AlexanderRichert-NOAA")
+
     def url_for_version(self, version):
         if version >= Version("2.9.13"):
             url = "https://download.gnome.org/sources/libxml2/{0}/libxml2-{1}.tar.xz"
@@ -39,13 +41,15 @@ def url_for_version(self, version):
 
     variant("python", default=False, description="Enable Python support")
     variant("shared", default=True, description="Build shared library")
-    variant("pic", default=False, description="Enable position-independent code (PIC)")
+    variant("pic", default=True, description="Enable position-independent code (PIC)")
+
+    conflicts("~pic+shared")
 
     depends_on("pkgconfig@0.9.0:", type="build", when="build_system=autotools")
     # conditional on non Windows, but rather than specify for each platform
     # specify for non Windows builder, which has equivalent effect
     depends_on("iconv", when="build_system=autotools")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("xz")
 
     # avoid cycle dependency for concretizer
@@ -70,12 +74,18 @@ def url_for_version(self, version):
     # Use NAN/INFINITY if available to avoid SIGFPE
     # See https://gitlab.gnome.org/GNOME/libxml2/-/merge_requests/186
     patch(
-        "https://gitlab.gnome.org/GNOME/libxml2/-/commit/c9925454fd384a17c8c03d358c6778a552e9287b.patch",
-        sha256="3e06d42596b105839648070a5921157fe284b932289ffdbfa304ddc3457e5637",
+        "https://gitlab.gnome.org/GNOME/libxml2/-/commit/c9925454fd384a17c8c03d358c6778a552e9287b.diff",
+        sha256="5dc43fed02b443d2563a502a52caafe39477c06fc30b70f786d5ed3eb5aea88d",
         when="@2.9.11:2.9.14",
     )
     build_system(conditional("nmake", when="platform=windows"), "autotools", default="autotools")
 
+    def flag_handler(self, name, flags):
+        if name == "cflags" and self.spec.satisfies("+pic"):
+            flags.append(self.compiler.cc_pic_flag)
+            flags.append("-DPIC")
+        return (flags, None, None)
+
     @property
     def command(self):
         return Executable(self.prefix.bin.join("xml2-config"))
@@ -238,6 +248,10 @@ def configure_args(self):
         else:
             args.append("--without-python")
 
+        args.extend(self.enable_or_disable("shared"))
+        # PIC setting is taken care of above by self.flag_handler()
+        args.append("--without-pic")
+
         return args
 
 
@@ -260,8 +274,9 @@ def configure(self, pkg, spec, prefix):
                 "iconv=no",
                 "zlib=yes",
                 "lzma=yes",
-                "lib=%s" % ";".join((spec["zlib"].prefix.lib, spec["xz"].prefix.lib)),
-                "include=%s" % ";".join((spec["zlib"].prefix.include, spec["xz"].prefix.include)),
+                "lib=%s" % ";".join((spec["zlib-api"].prefix.lib, spec["xz"].prefix.lib)),
+                "include=%s"
+                % ";".join((spec["zlib-api"].prefix.include, spec["xz"].prefix.include)),
             ]
             if "+python" in spec:
                 opts.append("python=yes")
diff --git a/var/spack/repos/builtin/packages/libxpm/package.py b/var/spack/repos/builtin/packages/libxpm/package.py
index bbdad61aa2791d..a0dfa0617b3802 100644
--- a/var/spack/repos/builtin/packages/libxpm/package.py
+++ b/var/spack/repos/builtin/packages/libxpm/package.py
@@ -30,3 +30,13 @@ def flag_handler(self, name, flags):
         if name == "ldflags" and "intl" in self.spec["gettext"].libs.names:
             flags.append("-lintl")
         return env_flags(name, flags)
+
+    def configure_args(self):
+        args = []
+        if self.spec.satisfies("^gettext ~shared"):
+            libs = self.spec["gettext"].libs.search_flags
+            libs += " " + self.spec["gettext"].libs.link_flags
+            libs += " " + self.spec["iconv"].libs.search_flags
+            libs += " " + self.spec["iconv"].libs.link_flags
+            args.append(f"LIBS={libs}")
+        return args
diff --git a/var/spack/repos/builtin/packages/libxslt/package.py b/var/spack/repos/builtin/packages/libxslt/package.py
index ed59366a80e4c6..0680d47dea444c 100644
--- a/var/spack/repos/builtin/packages/libxslt/package.py
+++ b/var/spack/repos/builtin/packages/libxslt/package.py
@@ -30,7 +30,7 @@ class Libxslt(AutotoolsPackage):
     depends_on("libxml2")
     depends_on("libxml2+python", when="+python")
     depends_on("xz")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libgcrypt", when="+crypto")
 
     depends_on("python+shared", when="+python")
diff --git a/var/spack/repos/builtin/packages/libxsmm/package.py b/var/spack/repos/builtin/packages/libxsmm/package.py
index 4e853c41cf31f8..4de81dace46dd2 100644
--- a/var/spack/repos/builtin/packages/libxsmm/package.py
+++ b/var/spack/repos/builtin/packages/libxsmm/package.py
@@ -82,6 +82,9 @@ class Libxsmm(MakefilePackage):
     # ().
     depends_on("binutils+ld+gas@2.33:", type="build", when="@:1.17")
 
+    # Intel Architecture or compatible CPU required
+    requires("target=x86_64:")
+
     @property
     def libs(self):
         result = find_libraries(["libxsmm", "libxsmmf"], root=self.prefix, recursive=True)
diff --git a/var/spack/repos/builtin/packages/libyogrt/package.py b/var/spack/repos/builtin/packages/libyogrt/package.py
index 79d164e3267f45..34a6a97fcc5a14 100644
--- a/var/spack/repos/builtin/packages/libyogrt/package.py
+++ b/var/spack/repos/builtin/packages/libyogrt/package.py
@@ -51,7 +51,7 @@ class Libyogrt(AutotoolsPackage):
 
     conflicts("scheduler=lsf", when="@:1.22")
 
-    variant("static", default="False", description="build static library")
+    variant("static", default=False, description="build static library")
 
     def url_for_version(self, version):
         if version < Version("1.21"):
@@ -61,6 +61,12 @@ def url_for_version(self, version):
                 version
             )
 
+    def flag_handler(self, name, flags):
+        if name == "cflags":
+            if self.spec.satisfies("%oneapi"):
+                flags.append("-Wno-error=implicit-function-declaration")
+        return (flags, None, None)
+
     def configure_args(self):
         args = []
 
diff --git a/var/spack/repos/builtin/packages/libzip/package.py b/var/spack/repos/builtin/packages/libzip/package.py
index 93bbdce5ea7248..f475541864173f 100644
--- a/var/spack/repos/builtin/packages/libzip/package.py
+++ b/var/spack/repos/builtin/packages/libzip/package.py
@@ -16,7 +16,7 @@ class Libzip(AutotoolsPackage):
     version("1.3.2", sha256="ab4c34eb6c3a08b678cd0f2450a6c57a13e9618b1ba34ee45d00eb5327316457")
     version("1.2.0", sha256="6cf9840e427db96ebf3936665430bab204c9ebbd0120c326459077ed9c907d9f")
 
-    depends_on("zlib@1.1.2:")
+    depends_on("zlib-api")
 
     @property
     def headers(self):
diff --git a/var/spack/repos/builtin/packages/libzmq/package.py b/var/spack/repos/builtin/packages/libzmq/package.py
index 15349324da0746..f8adbfc37b15db 100644
--- a/var/spack/repos/builtin/packages/libzmq/package.py
+++ b/var/spack/repos/builtin/packages/libzmq/package.py
@@ -12,12 +12,13 @@ class Libzmq(AutotoolsPackage):
     """The ZMQ networking/concurrency library and core API"""
 
     homepage = "https://zguide.zeromq.org/"
-    url = "https://github.com/zeromq/libzmq/releases/download/v4.3.2/zeromq-4.3.2.tar.gz"
+    url = "https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz"
     git = "https://github.com/zeromq/libzmq.git"
 
     maintainers("dennisklein")
 
     version("master", branch="master")
+    version("4.3.5", sha256="6653ef5910f17954861fe72332e68b03ca6e4d9c7160eb3a8de5a5a913bfab43")
     version("4.3.4", sha256="c593001a89f5a85dd2ddf564805deb860e02471171b3f204944857336295c3e5")
     version("4.3.3", sha256="9d9285db37ae942ed0780c016da87060497877af45094ff9e1a1ca736e3875a2")
     version("4.3.2", sha256="ebd7b5c830d6428956b67a0454a7f8cbed1de74b3b01e5c33c5378e22740f763")
@@ -72,21 +73,21 @@ class Libzmq(AutotoolsPackage):
     patch(
         "https://github.com/zeromq/libzmq/commit/92b2c38a2c51a1942a380c7ee08147f7b1ca6845.patch?full_index=1",
         sha256="310b8aa57a8ea77b7ac74debb3bf928cbafdef5e7ca35beaac5d9c61c7edd239",
-        when="@4.3.3:4.3.4 %gcc@11:",
+        when="@4.3.3:4.3.4",
     )
 
     # Fix build issues with gcc-12
     patch(
         "https://github.com/zeromq/libzmq/pull/4334.patch?full_index=1",
         sha256="edca864cba914481a5c97d2e975ba64ca1d2fbfc0044e9a78c48f1f7b2bedb6f",
-        when="@4.3.4 %gcc@12:",
+        when="@4.3.4",
     )
 
     # Fix static assertion failure with gcc-13
     patch(
         "https://github.com/zeromq/libzmq/commit/438d5d88392baffa6c2c5e0737d9de19d6686f0d.patch?full_index=1",
         sha256="e15a8bfe8131f3e648fd79f3c1c931f99cd896b2733a7df1760f5b4354a0687c",
-        when="@4.3.3:4.3.4 %gcc@13:",
+        when="@4.3.3:4.3.4",
     )
 
     def url_for_version(self, version):
@@ -104,14 +105,16 @@ def autoreconf(self, spec, prefix):
     def configure_args(self):
         config_args = []
 
+        config_args.extend(self.with_or_without("docs"))
         config_args.extend(self.enable_or_disable("drafts"))
         config_args.extend(self.enable_or_disable("libbsd"))
+        config_args.extend(self.with_or_without("libsodium"))
         config_args.extend(self.enable_or_disable("libunwind"))
-
-        if "+libsodium" in self.spec:
-            config_args.append("--with-libsodium=" + self.spec["libsodium"].prefix)
-        if "~docs" in self.spec:
-            config_args.append("--without-docs")
+        # the package won't compile with newer compilers because warnings
+        # are converted to errors. Hence, disable such conversion.
+        # this option was only added in version 4.2.3.
+        if self.spec.version >= Version("4.2.3"):
+            config_args.append("--disable-Werror")
         if "clang" in self.compiler.cc:
             config_args.append("CFLAGS=-Wno-gnu")
             config_args.append("CXXFLAGS=-Wno-gnu")
diff --git a/var/spack/repos/builtin/packages/liggghts/cpp-17.patch b/var/spack/repos/builtin/packages/liggghts/cpp-17.patch
new file mode 100644
index 00000000000000..73c4bffdd0f0de
--- /dev/null
+++ b/var/spack/repos/builtin/packages/liggghts/cpp-17.patch
@@ -0,0 +1,75 @@
+diff --git a/src/math_vector.h b/src/math_vector.h
+index 2b8704af..79c0cedd 100644
+--- a/src/math_vector.h
++++ b/src/math_vector.h
+@@ -94,7 +94,7 @@ inline void vec_neg(vector &dest) {                                // -a
+   dest[2] = -dest[2]; }
+ 
+ inline void vec_norm(vector &dest) {                                 // a/|a|
+-  register double f = sqrt(vec_dot(dest, dest));
++  double f = sqrt(vec_dot(dest, dest));
+   dest[0] /= f;
+   dest[1] /= f;
+   dest[2] /= f; }
+@@ -222,7 +222,7 @@ inline void form_subtr(shape &dest, form &src) {                // m_a-m_b
+   dest[3] -= src[3]; dest[4] -= src[4]; dest[5] -= src[5]; }
+ 
+ inline int form_inv(form &m_inv, form &m) {                        // m^-1
+-  register double det = form_det(m);
++  double det = form_det(m);
+   if (fzero(det)) return 0;
+   m_inv[0] = (m[1]*m[2]-m[3]*m[3])/det;
+   m_inv[1] = (m[0]*m[2]-m[4]*m[4])/det;
+@@ -377,7 +377,7 @@ inline void form4_unit(form4 &dest) {
+   dest[0] = dest[1] = dest[2] = dest[3] = 1.0; }
+ 
+ inline double form4_det(form4 &m) {
+-  register double f = m[6]*m[7]-m[5]*m[8];
++  double f = m[6]*m[7]-m[5]*m[8];
+   return m[0]*(
+       m[1]*(m[2]*m[3]-m[4]*m[4])+
+       m[5]*(2.0*m[4]*m[7]-m[2]*m[5])-m[3]*m[7]*m[7])+f*f+
+@@ -387,7 +387,7 @@ inline double form4_det(form4 &m) {
+         m[9]*(m[4]*m[4]-m[2]*m[3])); }
+ 
+ inline int form4_inv(form4 &m_inv, form4 &m) {
+-  register double det = form4_det(m);
++  double det = form4_det(m);
+   if (fzero(det)) return 0;
+   m_inv[0] = (m[1]*(m[2]*m[3]-m[4]*m[4])+
+       m[5]*(2.0*m[4]*m[7]-m[2]*m[5])-m[3]*m[7]*m[7])/det;
+diff --git a/src/pair.cpp b/src/pair.cpp
+index c0889f72..8c212715 100644
+--- a/src/pair.cpp
++++ b/src/pair.cpp
+@@ -566,7 +566,7 @@ void Pair::init_tables_disp(double cut_lj_global)
+     }
+     
+     rsq = rsq_lookup.f;
+-    register double x2 = g2*rsq, a2 = 1.0/x2;
++    double x2 = g2*rsq, a2 = 1.0/x2;
+     x2 = a2*exp(-x2);
+ 
+     rdisptable[i] = rsq_lookup.f;
+@@ -612,7 +612,7 @@ void Pair::init_tables_disp(double cut_lj_global)
+   if (rsq_lookup.f < (cut_lj_globalsq = cut_lj_global * cut_lj_global)) {
+     rsq_lookup.f = cut_lj_globalsq;
+     
+-    register double x2 = g2*rsq, a2 = 1.0/x2;
++    double x2 = g2*rsq, a2 = 1.0/x2;
+     x2 = a2*exp(-x2);
+     f_tmp = g8*(((6.0*a2+6.0)*a2+3.0)*a2+1.0)*x2*rsq;
+     e_tmp = g6*((a2+1.0)*a2+0.5)*x2;
+diff --git a/src/utils.h b/src/utils.h
+index fab00e9b..5a122627 100644
+--- a/src/utils.h
++++ b/src/utils.h
+@@ -67,7 +67,7 @@ namespace Utils {
+ 
+   inline std::string int_to_string(int a)
+   {
+-    return static_cast< std::ostringstream & >(( std::ostringstream() << std::dec << a ) ).str();
++    return static_cast< std::ostringstream & >(( std::ostringstream().flush() << std::dec << a ) ).str();
+   }
+ 
+   inline std::string double_to_string(double dbl)
diff --git a/var/spack/repos/builtin/packages/liggghts/makefile-llvm-based-compiler.patch b/var/spack/repos/builtin/packages/liggghts/makefile-llvm-based-compiler.patch
new file mode 100644
index 00000000000000..a5c26300a34392
--- /dev/null
+++ b/var/spack/repos/builtin/packages/liggghts/makefile-llvm-based-compiler.patch
@@ -0,0 +1,21 @@
+diff --git a/src/MAKE/Makefile.auto b/src/MAKE/Makefile.auto
+index 239f886..8f42e73 100644
+--- a/src/MAKE/Makefile.auto
++++ b/src/MAKE/Makefile.auto
+@@ -816,12 +816,14 @@ ifeq ($(USE_VTK), "ON")
+             endif
+         endif
+     endif
+-    open_bracket := (
+-    close_bracket := )
++    open_bracket := ("
++    close_bracket := ")
++    message := message
+     space :=
+     space +=
+     VTK_TMP := $(subst $(open_bracket),$(space),$(VTK_TMP))
+     VTK_TMP := $(subst $(close_bracket),$(space),$(VTK_TMP))
++    VTK_TMP := $(subst $(message),$(space),$(VTK_TMP))
+     VTK_MAJOR_VERSION := $(patsubst "%",%,$(word $(words $(VTK_TMP)),$(VTK_TMP)))
+     ifeq ($(AUTO_DEBUG),1)
+         $(shell $(ECHO) "#vtk_major_version: $(VTK_MAJOR_VERSION)" >> $(AUTO_LOG_FILE))
diff --git a/var/spack/repos/builtin/packages/liggghts/makefile.patch b/var/spack/repos/builtin/packages/liggghts/makefile.patch
new file mode 100644
index 00000000000000..370e4b8dad21f0
--- /dev/null
+++ b/var/spack/repos/builtin/packages/liggghts/makefile.patch
@@ -0,0 +1,240 @@
+diff --git a/src/MAKE/Makefile.auto b/src/MAKE/Makefile.auto
+index dde9e72..239f886 100644
+--- a/src/MAKE/Makefile.auto
++++ b/src/MAKE/Makefile.auto
+@@ -440,12 +440,12 @@ ifeq ($(USE_MPI), "ON")
+         TMP_INC = -I$(MPI_INC)
+     endif
+     # We assume that the compiler supports #pragma message
+-    TMP := $(shell $(ECHO) '\#include  \n \#if defined(MPICH) \n \#pragma message "MPICH" \n \#elif defined(OPEN_MPI) \n \#pragma message "OpenMPI" \n \#else \n \#pragma message "Unknown" \n \#endif' > $(TMPFILE) && $(MPICXX) $(OPT_LVL) $(PROF_FLAG) $(TMP_INC) -xc++ -E $(TMPFILE) 2> /dev/null | grep pragma | grep -m 1 message || echo -1)
++    TMP := $(shell $(ECHO) '#include  \n #if defined(MPICH) \n #pragma message "MPICH" \n #elif defined(OPEN_MPI) \n #pragma message "OpenMPI" \n #else \n #pragma message "Unknown" \n #endif' > $(TMPFILE) && $(MPICXX) $(OPT_LVL) $(PROF_FLAG) $(TMP_INC) -xc++ -E $(TMPFILE) 2> /dev/null | grep pragma | grep -m 1 message || echo -1)
+     # See if compilation has worked out
+     ifeq ($(TMP), -1)
+         # Maybe it failed because of the optimization as -Og is not known
+         ifeq ($(USE_DEBUG), "ON")
+-            TMP := $(shell $(ECHO) '\#include  \n \#if defined(MPICH) \n \#pragma message "MPICH" \n \#elif defined(OPEN_MPI) \n \#pragma message "OpenMPI" \n \#else \n \#pragma message "Unknown" \n \#endif' > $(TMPFILE) && $(MPICXX) -O0 -g $(PROF_FLAG) $(TMP_INC) -xc++ -E $(TMPFILE) 2> /dev/null | grep pragma | grep -m 1 message || echo -1)
++            TMP := $(shell $(ECHO) '#include  \n #if defined(MPICH) \n #pragma message "MPICH" \n #elif defined(OPEN_MPI) \n #pragma message "OpenMPI" \n #else \n #pragma message "Unknown" \n #endif' > $(TMPFILE) && $(MPICXX) -O0 -g $(PROF_FLAG) $(TMP_INC) -xc++ -E $(TMPFILE) 2> /dev/null | grep pragma | grep -m 1 message || echo -1)
+             ifeq ($(TMP), -1)
+                 $(error 'Could not compile a simple MPI example (testing with -Og and -O0). Test was done with MPI_INC="$(TMP_INC)" and MPICXX="$(MPICXX)"')
+             else
+@@ -566,7 +566,7 @@ else
+         $(shell $(ECHO) "#Compiling with mpi stubs" >> $(AUTO_LOG_FILE))
+         $(shell $(ECHO) "#Command: $(CXX) $(EXTRA_INC) $(EXTRA_LIB) $(EXTRA_ADDLIBS) -xc++ $(LDFLAGS) $(CCFLAGS) -o /dev/null $(TMPFILE)")
+     endif
+-    TMP := $(shell $(ECHO) '\#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_INC) $(EXTRA_LIB) $(EXTRA_ADDLIBS) -xc++ $(LDFLAGS) $(CCFLAGS) -o /dev/null $(TMPFILE) 2>> $(AUTO_LOG_FILE) && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_INC) $(EXTRA_LIB) $(EXTRA_ADDLIBS) -xc++ $(LDFLAGS) $(CCFLAGS) -o /dev/null $(TMPFILE) 2>> $(AUTO_LOG_FILE) && echo 0 || echo -1)
+     ifeq ($(TMP), -1)
+         $(error 'Could not compile a simple c++ example. Please make sure that you have run "make stubs" before compiling LIGGGHTS itself. Test was done with CXX=$(CXX), EXTRA_INC=$(EXTRA_INC), EXTRA_LIB=$(EXTRA_LIB) and EXTRA_ADDLIBS=$(EXTRA_ADDLIBS).')
+     endif
+@@ -595,7 +595,7 @@ endif
+ HAVE_MATH_SPECIAL_FUNCS = 0
+ # For c++17 this is included without any further defines
+ ifeq ($(CXXVERSION),17)
+-    TMP := $(shell $(ECHO) '\#include  \n int main(){ std::beta(1,1); }' > $(TMPFILE) && $(CXX) $(EXTRA_INC) $(EXTRA_LIB) $(EXTRA_ADDLIBS) -xc++ $(LDFLAGS) $(CCFLAGS) -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include  \n int main(){ std::beta(1,1); }' > $(TMPFILE) && $(CXX) $(EXTRA_INC) $(EXTRA_LIB) $(EXTRA_ADDLIBS) -xc++ $(LDFLAGS) $(CCFLAGS) -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+     ifeq ($(TMP),0)
+         HAVE_MATH_SPECIAL_FUNCS = 1
+     endif
+@@ -604,14 +604,14 @@ ifeq ($(CXXVERSION),17)
+ else
+     # For c++11 we need to check if ISO 29124:2010 is supported
+     ifeq ($(CXXVERSION),11)
+-        TMP := $(shell $(ECHO) '\#define __STDCPP_WANT_MATH_SPEC_FUNCS__ 1 \n \#include  \n \#if !defined(__STDCPP_MATH_SPEC_FUNCS__) || __STDCPP_MATH_SPEC_FUNCS__ < 201003L \n \#error "STOP" \n \#endif \n int main(){ std::beta(1,1); }' > $(TMPFILE) && $(CXX) $(EXTRA_INC) $(EXTRA_LIB) $(EXTRA_ADDLIBS) -xc++ $(LDFLAGS) $(CCFLAGS) -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++        TMP := $(shell $(ECHO) '#define __STDCPP_WANT_MATH_SPEC_FUNCS__ 1 \n #include  \n #if !defined(__STDCPP_MATH_SPEC_FUNCS__) || __STDCPP_MATH_SPEC_FUNCS__ < 201003L \n #error "STOP" \n #endif \n int main(){ std::beta(1,1); }' > $(TMPFILE) && $(CXX) $(EXTRA_INC) $(EXTRA_LIB) $(EXTRA_ADDLIBS) -xc++ $(LDFLAGS) $(CCFLAGS) -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+         ifeq ($(TMP),0)
+             HAVE_MATH_SPECIAL_FUNCS = 1
+         endif
+     endif
+ endif
+ ifeq ($(HAVE_MATH_SPECIAL_FUNCS),0)
+-    TMP := $(shell $(ECHO) '\#include  \n int main(){ std::tr1::beta(1,1); }' > $(TMPFILE) && $(CXX) $(EXTRA_INC) $(EXTRA_LIB) $(EXTRA_ADDLIBS) -xc++ $(LDFLAGS) $(CCFLAGS) -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include  \n int main(){ std::tr1::beta(1,1); }' > $(TMPFILE) && $(CXX) $(EXTRA_INC) $(EXTRA_LIB) $(EXTRA_ADDLIBS) -xc++ $(LDFLAGS) $(CCFLAGS) -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+     HAVE_TR1_CMATH = 0
+     ifeq ($(TMP),0)
+         HAVE_TR1_CMATH = 1
+@@ -729,7 +729,7 @@ ifeq ($(USE_VTK), "ON")
+         $(shell $(ECHO) "#vtk major version detection" >> $(AUTO_LOG_FILE))
+     endif
+     # note we assume here that our compiler supports #pragma message
+-    VTK_TMP := $(shell $(ECHO) '\#include  \n \#define XSTR(x) STR(x) \n \#define STR(x) \#x \n \#pragma message XSTR(VTK_MAJOR_VERSION)' > $(TMPFILE) && $(CXX) -Wno-deprecated -E $(VTK_INC) -xc++ $(TMPFILE) 2>> $(AUTO_LOG_FILE) | tee -a $(AUTO_LOG_FILE) | grep "pragma" | grep "message" || echo -1)
++    VTK_TMP := $(shell $(ECHO) '#include  \n #define XSTR(x) STR(x) \n #define STR(x) #x \n #pragma message XSTR(VTK_MAJOR_VERSION)' > $(TMPFILE) && $(CXX) -Wno-deprecated -E $(VTK_INC) -xc++ $(TMPFILE) 2>> $(AUTO_LOG_FILE) | tee -a $(AUTO_LOG_FILE) | grep "pragma" | grep "message" || echo -1)
+     ifeq ($(AUTO_DEBUG),1)
+         $(shell $(ECHO) "#vtk major version detection result: $(VTK_TMP)" >> $(AUTO_LOG_FILE))
+     endif
+@@ -744,7 +744,7 @@ ifeq ($(USE_VTK), "ON")
+         ifeq ($(VTK_INC),-I)
+             VTK_INC =
+         endif
+-        VTK_TMP := $(shell $(ECHO) '\#include  \n \#define XSTR(x) STR(x) \n \#define STR(x) \#x \n \#pragma message XSTR(VTK_MAJOR_VERSION)' > $(TMPFILE) && $(CXX) -Wno-deprecated -E $(VTK_INC) -xc++ $(TMPFILE) 2>> $(AUTO_LOG_FILE) | tee -a $(AUTO_LOG_FILE) | grep "pragma" | grep "message" || echo -1)
++        VTK_TMP := $(shell $(ECHO) '#include  \n #define XSTR(x) STR(x) \n #define STR(x) #x \n #pragma message XSTR(VTK_MAJOR_VERSION)' > $(TMPFILE) && $(CXX) -Wno-deprecated -E $(VTK_INC) -xc++ $(TMPFILE) 2>> $(AUTO_LOG_FILE) | tee -a $(AUTO_LOG_FILE) | grep "pragma" | grep "message" || echo -1)
+         ifeq ($(AUTO_DEBUG),1)
+             $(shell $(ECHO) "#vtk major version detection result (lib): $(VTK_TMP)" >> $(AUTO_LOG_FILE))
+         endif
+@@ -797,7 +797,7 @@ ifeq ($(USE_VTK), "ON")
+                 # At this stage we now have VTK downloaded. Next we need to compile it
+                 $(info VTK has been downloaded and will be compiled now. This can take several minutes.)
+                 OBJDIR := $(PWD)
+-                TMP := $(shell $(ECHO) '\#!/bin/bash \n cd "$(OBJDIR)/$(LIB_PATH)/vtk" \n mkdir -p build \n cd src \n git checkout $(VTK_VERSION_TAG) &>> $(AUTO_LOG_FILE) \n cd ../build \n cmake -DBUILD_TESTING:BOOL=OFF -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_INSTALL_PREFIX=../install -DModule_vtkIOMPIParallel:BOOL=ON -DVTK_Group_MPI:BOOL=ON -DVTK_Group_Rendering:BOOL=OFF -DVTK_RENDERING_BACKEND:STRING=None -DVTK_USE_X:BOOL=OFF -DModule_vtkIOMPIImage:BOOL=ON -DModule_vtkParallelMPI:BOOL=ON ../src &>> $(AUTO_LOG_FILE) \n make &>> $(AUTO_LOG_FILE) \n make install &>> $(AUTO_LOG_FILE)' > $(TMPFILE))
++                TMP := $(shell $(ECHO) '#!/bin/bash \n cd "$(OBJDIR)/$(LIB_PATH)/vtk" \n mkdir -p build \n cd src \n git checkout $(VTK_VERSION_TAG) &>> $(AUTO_LOG_FILE) \n cd ../build \n cmake -DBUILD_TESTING:BOOL=OFF -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_INSTALL_PREFIX=../install -DModule_vtkIOMPIParallel:BOOL=ON -DVTK_Group_MPI:BOOL=ON -DVTK_Group_Rendering:BOOL=OFF -DVTK_RENDERING_BACKEND:STRING=None -DVTK_USE_X:BOOL=OFF -DModule_vtkIOMPIImage:BOOL=ON -DModule_vtkParallelMPI:BOOL=ON ../src &>> $(AUTO_LOG_FILE) \n make &>> $(AUTO_LOG_FILE) \n make install &>> $(AUTO_LOG_FILE)' > $(TMPFILE))
+                 TMP := $(shell bash $(TMPFILE) && echo 0 || echo -1)
+                 ifeq ($(TMP), -1)
+                     $(error 'Compilation of vtk failed. Please install it manually')
+@@ -807,7 +807,7 @@ ifeq ($(USE_VTK), "ON")
+                 ifeq ($(VTK_INC),-I)
+                     VTK_INC =
+                 endif
+-                VTK_TMP := $(shell $(ECHO) '\#include  \n \#define XSTR(x) STR(x) \n \#define STR(x) \#x \n \#pragma message XSTR(VTK_MAJOR_VERSION)' > $(TMPFILE) && $(CXX) -Wno-deprecated -E $(VTK_INC) -xc++ $(TMPFILE) 2>> $(AUTO_LOG_FILE) | tee -a $(AUTO_LOG_FILE) | grep "pragma" | grep "message" || echo -1)
++                VTK_TMP := $(shell $(ECHO) '#include  \n #define XSTR(x) STR(x) \n #define STR(x) #x \n #pragma message XSTR(VTK_MAJOR_VERSION)' > $(TMPFILE) && $(CXX) -Wno-deprecated -E $(VTK_INC) -xc++ $(TMPFILE) 2>> $(AUTO_LOG_FILE) | tee -a $(AUTO_LOG_FILE) | grep "pragma" | grep "message" || echo -1)
+                 ifeq ($(AUTO_DEBUG),1)
+                     $(shell $(ECHO) "#vtk major version detection result (lib): $(VTK_TMP)" >> $(AUTO_LOG_FILE))
+                 endif
+@@ -826,7 +826,7 @@ ifeq ($(USE_VTK), "ON")
+     ifeq ($(AUTO_DEBUG),1)
+         $(shell $(ECHO) "#vtk_major_version: $(VTK_MAJOR_VERSION)" >> $(AUTO_LOG_FILE))
+     endif
+-    VTK_TMP := $(shell $(ECHO) '\#include  \n \#define XSTR(x) STR(x) \n \#define STR(x) \#x \n \#pragma message XSTR(VTK_MINOR_VERSION)' > $(TMPFILE) && $(CXX) -Wno-deprecated -E $(VTK_INC) -xc++ $(TMPFILE) 2>> $(AUTO_LOG_FILE) | tee -a $(AUTO_LOG_FILE) | grep "pragma" | grep "message" || echo -1)
++    VTK_TMP := $(shell $(ECHO) '#include  \n #define XSTR(x) STR(x) \n #define STR(x) #x \n #pragma message XSTR(VTK_MINOR_VERSION)' > $(TMPFILE) && $(CXX) -Wno-deprecated -E $(VTK_INC) -xc++ $(TMPFILE) 2>> $(AUTO_LOG_FILE) | tee -a $(AUTO_LOG_FILE) | grep "pragma" | grep "message" || echo -1)
+     ifeq ($(VTK_TMP), -1)
+         $(error Could not obtain VTK_MINOR_VERSION)
+     endif
+@@ -885,7 +885,7 @@ ifeq ($(USE_VTK), "ON")
+                 VTK_LIB =
+             endif
+         endif
+-        TMP := $(shell $(ECHO) '\#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) -lvtkCommon$(VTK_APPENDIX_5) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++        TMP := $(shell $(ECHO) '#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) -lvtkCommon$(VTK_APPENDIX_5) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+         ifeq ($(TMP), -1)
+             ifeq ($(VTK_LIB_SET), 0)
+                 VTK_LIB := -L$(dir $(shell find $(VTK_BASE_PATH)/lib* -name 'libvtkCommon.so' | tail -n 1))
+@@ -893,7 +893,7 @@ ifeq ($(USE_VTK), "ON")
+                     VTK_LIB =
+                 endif
+             endif
+-            TMP := $(shell $(ECHO) '\#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) -lvtkCommon $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++            TMP := $(shell $(ECHO) '#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) -lvtkCommon $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+             ifeq ($(TMP), -1)
+                 $(error 'Could not determine suitable appendix of VTK library with VTK_INC="$(VTK_INC)", VTK_LIB="$(VTK_LIB)" and VTK_APPENDIX="$(VTK_APPENDIX)"')
+             else
+@@ -924,7 +924,7 @@ ifeq ($(USE_VTK), "ON")
+             $(shell $(ECHO) "#vtk_lib: $(VTK_LIB)" >> $(AUTO_LOG_FILE))
+             $(shell $(ECHO) "#appendix command: $(CXX) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) -lvtksys$(VTK_APPENDIX) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE)" >> $(AUTO_LOG_FILE))
+         endif
+-        TMP := $(shell $(ECHO) '\#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) -lvtksys$(VTK_APPENDIX) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2>> $(AUTO_LOG_FILE) && echo 0 || echo -1)
++        TMP := $(shell $(ECHO) '#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) -lvtksys$(VTK_APPENDIX) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2>> $(AUTO_LOG_FILE) && echo 0 || echo -1)
+         ifeq ($(TMP), -1)
+             ifeq ($(AUTO_DEBUG),1)
+                 $(shell $(ECHO) "#attempting without appendix" >> $(AUTO_LOG_FILE))
+@@ -935,7 +935,7 @@ ifeq ($(USE_VTK), "ON")
+                     VTK_LIB =
+                 endif
+             endif
+-            TMP := $(shell $(ECHO) '\#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) -lvtksys $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2>> $(AUTO_LOG_FILE) && echo 0 || echo -1)
++            TMP := $(shell $(ECHO) '#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) -lvtksys $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2>> $(AUTO_LOG_FILE) && echo 0 || echo -1)
+             ifeq ($(TMP), -1)
+                 $(error 'Could not determine suitable appendix of VTK library with VTK_INC="$(VTK_INC)", VTK_LIB="$(VTK_LIB)" and VTK_APPENDIX="$(VTK_APPENDIX)"')
+             else
+@@ -1025,9 +1025,9 @@ ifeq ($(USE_VTK), "ON")
+         $(shell $(ECHO) "#vtk_addlibs: $(VTK_ADDLIBS)" >> $(AUTO_LOG_FILE))
+         $(shell $(ECHO) "#vtk_rpath: $(VTK_RPATH)" >> $(AUTO_LOG_FILE))
+         $(shell $(ECHO) "#vtk compile test:" >> $(AUTO_LOG_FILE))
+-        TMP := $(shell $(ECHO) "\#include  \n int main(){}" > $(TMPFILE) && $(CXX) $(VTK_RPATH) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) $(VTK_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) &>> $(AUTO_LOG_FILE))
++        TMP := $(shell $(ECHO) "#include  \n int main(){}" > $(TMPFILE) && $(CXX) $(VTK_RPATH) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) $(VTK_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) &>> $(AUTO_LOG_FILE))
+     endif
+-    TMP := $(shell $(ECHO) '\#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(VTK_RPATH) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) $(VTK_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(VTK_RPATH) $(EXTRA_LIB) $(VTK_LIB) $(VTK_INC) $(EXTRA_ADDLIBS) $(VTK_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+     ifeq ($(TMP), -1)
+         $(error 'Could not compile VTK example with VTK_INC="$(VTK_INC)", VTK_LIB="$(VTK_LIB)" and VTK_ADDLIBS="$(VTK_ADDLIBS)"')
+     endif
+@@ -1057,7 +1057,7 @@ ifeq ($(USE_SUPERQUADRICS), "ON")
+     ifeq ($(REQUIRE_BOOST),1)
+         BOOST_INC ?= $(BOOST_INC_USR)
+         # Include test
+-        TMP := $(shell $(ECHO) '\#include "boost/math/special_functions/beta.hpp" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(BOOST_INC) $(EXTRA_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++        TMP := $(shell $(ECHO) '#include "boost/math/special_functions/beta.hpp" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(BOOST_INC) $(EXTRA_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+         ifeq ($(TMP), -1)
+             $(error 'Could not compile boost example with BOOST_INC="$(BOOST_INC)" as boost/math/special_functions/beta.hpp could not be found')
+         endif
+@@ -1082,7 +1082,7 @@ ifeq ($(USE_JPG), "ON")
+         $(shell $(ECHO) "#JPG_ADDLIBS: $(JPG_ADDLIBS)" >> $(AUTO_LOG_FILE))
+         $(shell $(ECHO) "jpg compile test:" >> $(AUTO_LOG_FILE))
+     endif
+-    TMP := $(shell $(ECHO) '\#include  \n \#include  \n \#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(JPG_INC) $(EXTRA_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2>> $(AUTO_LOG_FILE) && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include  \n #include  \n #include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(JPG_INC) $(EXTRA_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2>> $(AUTO_LOG_FILE) && echo 0 || echo -1)
+     ifeq ($(TMP), -1)
+         $(error 'Could not compile jpg example with JPG_INC="$(JPG_INC)"')
+     endif
+@@ -1090,7 +1090,7 @@ ifeq ($(USE_JPG), "ON")
+         $(shell $(ECHO) "jpg link test:" >> $(AUTO_LOG_FILE))
+     endif
+     # Linking test
+-    TMP := $(shell $(ECHO) '\#include  \n \#include  \n \#include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(JPG_LIB) $(JPG_INC) $(EXTRA_ADDLIBS) $(JPG_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2>> $(AUTO_LOG_FILE) && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include  \n #include  \n #include  \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(JPG_LIB) $(JPG_INC) $(EXTRA_ADDLIBS) $(JPG_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2>> $(AUTO_LOG_FILE) && echo 0 || echo -1)
+     ifeq ($(TMP), -1)
+         $(error 'Could not compile and link jpg example with JPG_INC="$(JPG_INC)", JPG_LIB="$(JPG_LIB)" and JPG_ADDLIBS="$(JPG_ADDLIBS)"')
+     endif
+@@ -1119,7 +1119,7 @@ ifeq ($(USE_CONVEX), "ON")
+     CONVEX_ADDLIBS += -lccd
+     # Test settings
+     # Link test
+-    TMP := $(shell $(ECHO) '\#include "ccd/ccd.h" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(CONVEX_LIB) $(CONVEX_INC) $(EXTRA_ADDLIBS) $(CONVEX_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include "ccd/ccd.h" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(CONVEX_LIB) $(CONVEX_INC) $(EXTRA_ADDLIBS) $(CONVEX_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+     # Automatic download and compilation if AUTODOWNLOAD_CONVEX is set
+     ifeq ($(TMP), -1)
+         ifeq ($(AUTOINSTALL_CONVEX), "ON")
+@@ -1168,7 +1168,7 @@ ifeq ($(USE_CONVEX), "ON")
+             endif
+             # At this stage we now have libccd downloaded. Next we need to compile it
+             OBJDIR := $(PWD)
+-            TMP := $(shell $(ECHO) '\#!/bin/bash \n cd "$(OBJDIR)/$(LIB_PATH)/libccd/src" \n make PREFIX="$(PWD)/../../" USE_DOUBLE=yes &> /dev/null' > $(TMPFILE))
++            TMP := $(shell $(ECHO) '#!/bin/bash \n cd "$(OBJDIR)/$(LIB_PATH)/libccd/src" \n make PREFIX="$(PWD)/../../" USE_DOUBLE=yes &> /dev/null' > $(TMPFILE))
+             TMP := $(shell bash $(TMPFILE) && echo 0 || echo -1)
+             ifeq ($(TMP), -1)
+                 $(error 'Compilation of libccd failed. Please install it manually')
+@@ -1178,12 +1178,12 @@ ifeq ($(USE_CONVEX), "ON")
+         endif
+     endif
+     # Include test
+-    TMP := $(shell $(ECHO) '\#include "ccd/ccd.h" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(CONVEX_INC) $(EXTRA_ADDLIBS) $(CCFLAGS) -xc++ -E $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include "ccd/ccd.h" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(CONVEX_INC) $(EXTRA_ADDLIBS) $(CCFLAGS) -xc++ -E $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+     ifeq ($(TMP), -1)
+         $(error 'Could not compile Convex (libccd) example with CONVEX_INC="$(CONVEX_INC)"')
+     endif
+     # Link test
+-    TMP := $(shell $(ECHO) '\#include "ccd/ccd.h" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(CONVEX_LIB) $(CONVEX_INC) $(EXTRA_ADDLIBS) $(CONVEX_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include "ccd/ccd.h" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(CONVEX_LIB) $(CONVEX_INC) $(EXTRA_ADDLIBS) $(CONVEX_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+     ifeq ($(TMP), -1)
+         $(error 'Could not compile and link Convex (libccd) example with CONVEX_INC="$(CONVEX_INC)", CONVEX_LIB="$(CONVEX_LIB)" and CONVEX_ADDLIBS="$(CONVEX_ADDLIBS)"')
+     endif
+@@ -1210,7 +1210,7 @@ ifeq ($(USE_MFEM), "ON")
+     MFEM_LIB ?= -L$(LIB_PATH)/mfem
+     MFEM_ADDLIBS += -lmfem
+     # Link test
+-    TMP := $(shell $(ECHO) '\#include "mfem.hpp" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(MFEM_INC) $(MFEM_LIB) $(EXTRA_ADDLIBS) $(MFEM_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include "mfem.hpp" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(MFEM_INC) $(MFEM_LIB) $(EXTRA_ADDLIBS) $(MFEM_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+     ifeq ($(TMP), -1)
+         ifeq ($(AUTOINSTALL_MFEM), "ON")
+             $(info 'Could not compile MFEM example. As AUTOINSTALL_MFEM is set to "ON". MFEM will now be automatically downloaded to ../lib/mfem')
+@@ -1257,7 +1257,7 @@ ifeq ($(USE_MFEM), "ON")
+             # At this stage we now have MFEM downloaded. Next we need to compile it
+             TMP := $(shell ls $(LIB_PATH)/mfem/libmfem.a && echo 0 || echo -1)
+             ifeq ($(TMP), -1)
+-                TMP := $(shell $(ECHO) '\#!/bin/bash \n cd $(LIB_PATH)/mfem \n make config \n make all -j 4' > $(TMPFILE))
++                TMP := $(shell $(ECHO) '#!/bin/bash \n cd $(LIB_PATH)/mfem \n make config \n make all -j 4' > $(TMPFILE))
+                 TMP := $(shell bash $(TMPFILE) && echo 0 || echo -1)
+                 ifeq ($(TMP), -1)
+                     $(error 'Compilation of MFEM failed. Please install it manually')
+@@ -1270,12 +1270,12 @@ ifeq ($(USE_MFEM), "ON")
+ 
+ 
+     # Include test
+-    TMP := $(shell $(ECHO) '\#include "mfem.hpp" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(MFEM_INC) $(EXTRA_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include "mfem.hpp" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(MFEM_INC) $(EXTRA_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+     ifeq ($(TMP), -1)
+         $(error 'Could not compile MFEM example with MFEM_INC="$(MFEM_INC)"')
+     endif
+     # Link test
+-    TMP := $(shell $(ECHO) '\#include "mfem.hpp" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(MFEM_INC) $(MFEM_LIB) $(EXTRA_ADDLIBS) $(MFEM_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
++    TMP := $(shell $(ECHO) '#include "mfem.hpp" \n int main(){}' > $(TMPFILE) && $(CXX) $(EXTRA_LIB) $(MFEM_INC) $(MFEM_LIB) $(EXTRA_ADDLIBS) $(MFEM_ADDLIBS) $(CCFLAGS) -xc++ -o /dev/null $(TMPFILE) 2> /dev/null && echo 0 || echo -1)
+     ifeq ($(TMP), -1)
+         $(error 'Could not compile and link MFEM example. Test was done with MFEM_INC="$(MFEM_INC)", MFEM_LIB="$(MFEM_LIB)" and MFEM_ADDLIBS="$(MFEM_ADDLIBS)"')
+     endif
diff --git a/var/spack/repos/builtin/packages/liggghts/package.py b/var/spack/repos/builtin/packages/liggghts/package.py
index 8fe4b74852e787..dac43ff3655115 100644
--- a/var/spack/repos/builtin/packages/liggghts/package.py
+++ b/var/spack/repos/builtin/packages/liggghts/package.py
@@ -16,6 +16,8 @@ class Liggghts(MakefilePackage):
     url = "https://github.com/CFDEMproject/LIGGGHTS-PUBLIC/archive/3.8.0.tar.gz"
     git = "ssh://git@github.com/CFDEMproject/LIGGGHTS-PUBLIC.git"
 
+    maintainers("SofiaXu")
+
     version("3.8.0", sha256="9cb2e6596f584463ac2f80e3ff7b9588b7e3638c44324635b6329df87b90ab03")
 
     variant("mpi", default=True, description="Enable MPI support")
@@ -27,8 +29,14 @@ class Liggghts(MakefilePackage):
     depends_on("vtk@6.1.0:8.2.0")
     depends_on("mpi", when="+mpi")
     depends_on("jpeg", when="+jpeg")
-    depends_on("zlib", when="+gzip")
-
+    depends_on("zlib-api", when="+gzip")
+    # patch for makefile test code
+    patch("makefile.patch")
+    # patch for clang and oneapi
+    patch("makefile-llvm-based-compiler.patch", when="%clang")
+    patch("makefile-llvm-based-compiler.patch", when="%oneapi")
+    # C++17 support
+    patch("cpp-17.patch")
     build_directory = "src"
     build_targets = ["auto"]
 
@@ -55,9 +63,9 @@ def edit(self, spec, prefix):
 
         if "+mpi" in spec:
             mpi = spec["mpi"]
-            makefile.filter(r"^#(MPICXX_USER=).*", r"\1{0}".format(mpi.mpicxx))
-            makefile.filter(r"^#(MPI_INC_USER=).*", r"\1{0}".format(mpi.prefix.include))
-            makefile.filter(r"^#(MPI_LIB_USER=).*", r"\1{0}".format(mpi.prefix.lib))
+            makefile.filter(r"^#(MPICXX_USR=).*", r"\1{0}".format(mpi.mpicxx))
+            makefile.filter(r"^#(MPI_INC_USR=).*", r"\1{0}".format(mpi.prefix.include))
+            makefile.filter(r"^#(MPI_LIB_USR=).*", r"\1{0}".format(mpi.prefix.lib))
         else:
             makefile.filter(r"^(USE_MPI = ).*", r'\1"OFF"')
             # Set path to C++ compiler.
@@ -70,8 +78,8 @@ def edit(self, spec, prefix):
         if "+jpeg" in spec:
             jpeg = spec["jpeg"]
             makefile.filter(r"^(USE_JPG = ).*", r'\1"ON"')
-            makefile.filter(r"^#(JPG_INC_USER=-I).*", r"\1{0}".format(jpeg.prefix.include))
-            makefile.filter(r"^#(JPG_LIB_USER=-L).*", r"\1{0}".format(jpeg.prefix.lib))
+            makefile.filter(r"^#(JPG_INC_USR=-I).*", r"\1{0}".format(jpeg.prefix.include))
+            makefile.filter(r"^#(JPG_LIB_USR=-L).*", r"\1{0}".format(jpeg.prefix.lib))
 
         if "+gzip" in spec:
             makefile.filter(r"^(USE_GZIP = ).*", r'\1"ON"')
diff --git a/var/spack/repos/builtin/packages/likwid/package.py b/var/spack/repos/builtin/packages/likwid/package.py
index 4398dd330f28d5..f7ebb21048c4c4 100644
--- a/var/spack/repos/builtin/packages/likwid/package.py
+++ b/var/spack/repos/builtin/packages/likwid/package.py
@@ -188,12 +188,18 @@ def install(self, spec, prefix):
                 "HWLOC_INCLUDE_DIR = {0}".format(spec["hwloc"].prefix.include),
                 "config.mk",
             )
-            filter_file(
-                "^#HWLOC_LIB_DIR.*",
-                "HWLOC_LIB_DIR = {0}".format(spec["hwloc"].prefix.lib),
-                "config.mk",
-            )
-            filter_file("^#HWLOC_LIB_NAME.*", "HWLOC_LIB_NAME = hwloc", "config.mk")
+            ll = spec["hwloc"].libs
+            if len(ll.directories) > 0 and len(ll.names) > 0:
+                filter_file(
+                    "^#HWLOC_LIB_DIR.*",
+                    "HWLOC_LIB_DIR = {0}".format(ll.directories[0]),
+                    "config.mk",
+                )
+                filter_file(
+                    "^#HWLOC_LIB_NAME.*", "HWLOC_LIB_NAME = {0}".format(ll.names[0]), "config.mk"
+                )
+            else:
+                raise InstallError("Failed to find library path and/or name of hwloc dependency")
 
         # https://github.com/RRZE-HPC/likwid/issues/287
         if self.spec.satisfies("@:5.0.2 %gcc@10:"):
diff --git a/var/spack/repos/builtin/packages/linaro-forge/package.py b/var/spack/repos/builtin/packages/linaro-forge/package.py
new file mode 100644
index 00000000000000..e67613fef1f621
--- /dev/null
+++ b/var/spack/repos/builtin/packages/linaro-forge/package.py
@@ -0,0 +1,143 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+import platform
+import subprocess
+
+from spack.package import *
+
+
+class LinaroForge(Package):
+    """Build reliable and optimized code for the right results on multiple
+    Server and HPC architectures, from the latest compilers and C++ standards
+    to Intel, 64-bit Arm, AMD, OpenPOWER and Nvidia GPU hardware. Linaro Forge
+    combines Linaro DDT, the leading debugger for time-saving high performance
+    application debugging, Linaro MAP, the trusted performance profiler for
+    invaluable optimization advice across native and Python HPC codes, and
+    Linaro Performance Reports for advanced reporting capabilities."""
+
+    homepage = "https://www.linaroforge.com"
+    maintainers("kenche-linaro")
+
+    if platform.machine() in ["aarch64", "arm64"]:
+        version(
+            "23.0.4", sha256="a19e6b247badaa52f78815761f71fb95a565024b7f79bdfb2f602f18b47a881c"
+        )
+        version(
+            "23.0.3", sha256="a7e23ef2a187f8e2d6a6692cafb931c9bb614abf58e45ea9c2287191c4c44f02"
+        )
+        version(
+            "23.0.2", sha256="698fda8f7cc05a06909e5dcc50b9956f94135d7b12e84ffb21999a5b45c70c74"
+        )
+        version(
+            "23.0.1", sha256="552e4a3f408ed4eb5f1bfbb83c94530ee8733579c56c3e98050c0ad2d43eb433"
+        )
+        version("23.0", sha256="7ae20bb27d539751d1776d1e09a65dcce821fc6a75f924675439f791261783fb")
+        version(
+            "22.1.4", sha256="4e2af481a37b4c99dba0de6fac75ac945316955fc4170d06e321530adea7ac9f"
+        )
+        version(
+            "21.1.3", sha256="4a4ff7372aad5a31fc9e18b7b6c493691ab37d8d44a3158584e62d1ab82b0eeb"
+        )
+    elif platform.machine() == "ppc64le":
+        version(
+            "23.0.4", sha256="927c1ba733cf63027243060586b196f8262e545d898712044c359a6af6fc5795"
+        )
+        version(
+            "23.0.3", sha256="5ff9770f4bc4a2df4bac8a2544a9d6bad9fba2556420fa2e659e5c21e741caf7"
+        )
+        version(
+            "23.0.2", sha256="181b157bdfc8609b49addf63023f920ebb609dbc9a126e9dc26605188b756ff0"
+        )
+        version(
+            "23.0.1", sha256="08cffef2195ea96872d56e827f320eed40aaa82fd3b62d4c661a598fb2fb3a47"
+        )
+        version("23.0", sha256="0962c7e0da0f450cf6daffe1156e1f59e02c9f643df458ec8458527afcde5b4d")
+        version(
+            "22.1.3", sha256="6479c3a4ae6ce6648c37594eb0266161f06d9f89010fca9299855848661dda49"
+        )
+        version(
+            "22.0.4", sha256="f4cb5bcbaa67f9209299fe4653186a2829760b8b16a2883913aa43766375b04c"
+        )
+        version(
+            "21.1.3", sha256="eecbc5686d60994c5468b2d7cd37bebe5d9ac0ba37bd1f98fbfc69b071db541e"
+        )
+    elif platform.machine() == "x86_64":
+        version(
+            "23.0.4", sha256="41a81840a273ea9a232efb4f031149867c5eff7a6381d787e18195f1171caac4"
+        )
+        version(
+            "23.0.3", sha256="f2a010b94838f174f057cd89d12d03a89ca946163536eab178dd1ec877cdc27f"
+        )
+        version(
+            "23.0.2", sha256="565f0c073c6c8cbb06c062ca414e3f6ff8c6ca6797b03d247b030a9fbc55a5b1"
+        )
+        version(
+            "23.0.1", sha256="1d681891c0c725363f0f45584c9b79e669d5c9782158453b7d24b4b865d72755"
+        )
+        version("23.0", sha256="f4ab12289c992dd07cb1a15dd985ef4713d1f9c0cf362ec5e9c995cca9b1cf81")
+        version(
+            "22.1.3", sha256="4f8a8b1df6ad712e89c82eedf4bd85b93b57b3c8d5b37d13480ff058fa8f4467"
+        )
+        version(
+            "22.0.4", sha256="a2c8c1da38b9684d7c4656a98b3fc42777b03fd474cd0bf969324804f47587e5"
+        )
+        version(
+            "21.1.3", sha256="03dc82f1d075deb6f08d1e3e6592dc9b630d406c08a1316d89c436b5874f3407"
+        )
+
+    variant(
+        "probe",
+        default=False,
+        description='Detect available PMU counters via "forge-probe" during install',
+    )
+
+    variant("accept-eula", default=False, description="Accept the EULA")
+
+    # forge-probe executes with "/usr/bin/env python"
+    depends_on("python@2.7:", type="build", when="+probe")
+
+    # Licensing
+    license_required = True
+    license_comment = "#"
+    license_files = ["licences/Licence"]
+    license_vars = [
+        "ALLINEA_LICENSE_DIR",
+        "ALLINEA_LICENCE_DIR",
+        "ALLINEA_LICENSE_FILE",
+        "ALLINEA_LICENCE_FILE",
+    ]
+    license_url = "https://docs.linaroforge.com/latest/html/licenceserver/index.html"
+
+    def url_for_version(self, version):
+        pre = "arm" if version < Version("23.0") else "linaro"
+        return f"https://downloads.linaroforge.com/{version}/{pre}-forge-{version}-linux-{platform.machine()}.tar"
+
+    @run_before("install")
+    def abort_without_eula_acceptance(self):
+        install_example = "spack install linaro-forge +accept-eula"
+        license_terms_path = os.path.join(self.stage.source_path, "license_terms")
+        if not self.spec.variants["accept-eula"].value:
+            raise InstallError(
+                "\n\n\nNOTE:\nUse +accept-eula "
+                + "during installation "
+                + "to accept the license terms in:\n"
+                + "  {0}\n".format(os.path.join(license_terms_path, "license_agreement.txt"))
+                + "  {0}\n\n".format(os.path.join(license_terms_path, "supplementary_terms.txt"))
+                + "Example: '{0}'\n".format(install_example)
+            )
+
+    def install(self, spec, prefix):
+        subprocess.call(["./textinstall.sh", "--accept-license", prefix])
+        if spec.satisfies("+probe"):
+            probe = join_path(prefix, "bin", "forge-probe")
+            subprocess.call([probe, "--install", "global"])
+
+    def setup_run_environment(self, env):
+        # Only PATH is needed for Forge.
+        # Adding lib to LD_LIBRARY_PATH can cause conflicts with Forge's internal libs.
+        env.clear()
+        env.prepend_path("PATH", join_path(self.prefix, "bin"))
diff --git a/var/spack/repos/builtin/packages/linux-headers/package.py b/var/spack/repos/builtin/packages/linux-headers/package.py
index bbbf0f0381e8c4..8e1d995695b069 100644
--- a/var/spack/repos/builtin/packages/linux-headers/package.py
+++ b/var/spack/repos/builtin/packages/linux-headers/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import os
+
 from spack.package import *
 
 
@@ -14,9 +16,14 @@ class LinuxHeaders(Package):
     list_url = "https://www.kernel.org/pub/linux/kernel"
     list_depth = 2
 
+    version("6.5.2", sha256="2027e14057d568ad3ddc100dadf4c8853a49b031270478a61d88f6011572650f")
     version("6.2.8", sha256="fed0ad87d42f83a70ce019ff2800bc30a855e672e72bf6d54a014d98d344f665")
     version("4.9.10", sha256="bd6e05476fd8d9ea4945e11598d87bc97806bbc8d03556abbaaf809707661525")
 
+    def url_for_version(self, version):
+        url = "https://www.kernel.org/pub/linux/kernel/v{0}.x/linux-{1}.tar.xz"
+        return url.format(version.up_to(1), version)
+
     def setup_build_environment(self, env):
         # This variable is used in the Makefile. If it is defined on the
         # system, it can break the build if there is no build recipe for
@@ -24,4 +31,12 @@ def setup_build_environment(self, env):
         env.unset("ARCH")
 
     def install(self, spec, prefix):
-        make("headers_install", "INSTALL_HDR_PATH={0}".format(prefix))
+        make("headers")
+
+        src, dst = join_path("usr", "include"), join_path(prefix, "include")
+
+        # This copy_tree + ignore is really poor API design, but the it avoids
+        # running make install_headers which depends on rsync.
+        copy_tree(
+            src, dst, ignore=lambda f: os.path.isfile(join_path(src, f)) and not f.endswith(".h")
+        )
diff --git a/var/spack/repos/builtin/packages/llvm-amdgpu/001-Add-i1-mul-patterns.patch b/var/spack/repos/builtin/packages/llvm-amdgpu/001-Add-i1-mul-patterns.patch
new file mode 100644
index 00000000000000..f93fcb99dbf458
--- /dev/null
+++ b/var/spack/repos/builtin/packages/llvm-amdgpu/001-Add-i1-mul-patterns.patch
@@ -0,0 +1,2842 @@
+From a0f3d7f45075a3e9545c0c9fa25a9f8fc840cdd7 Mon Sep 17 00:00:00 2001
+From: Renjith Ravindran 
+Date: Mon, 25 Sep 2023 18:38:17 +0000
+Subject: [PATCH] i1 muls can sometimes happen after SCEV. They resulted in
+ ISel failures because we were missing the patterns for them.
+
+---
+ llvm/lib/Target/AMDGPU/SIInstructions.td |   10 +
+ llvm/test/CodeGen/AMDGPU/mul.ll          | 2676 ++++++++++++++++++++--
+ 2 files changed, 2544 insertions(+), 142 deletions(-)
+
+diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
+index 03b2160a1..3bf4e42de 100644
+--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
++++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
+@@ -2372,6 +2372,11 @@ def : GCNPat <
+   (S_AND_B64 $src0, $src1)
+ >;
+ 
++def : GCNPat <
++  (i1 (mul i1:$src0, i1:$src1)),
++  (S_AND_B64 $src0, $src1)
++>;
++
+ def : GCNPat <
+   (i1 (or i1:$src0, i1:$src1)),
+   (S_OR_B64 $src0, $src1)
+@@ -2411,6 +2416,11 @@ def : GCNPat <
+   (S_AND_B32 $src0, $src1)
+ >;
+ 
++def : GCNPat <
++  (i1 (mul i1:$src0, i1:$src1)),
++  (S_AND_B32 $src0, $src1)
++>;
++
+ def : GCNPat <
+   (i1 (or i1:$src0, i1:$src1)),
+   (S_OR_B32 $src0, $src1)
+diff --git a/llvm/test/CodeGen/AMDGPU/mul.ll b/llvm/test/CodeGen/AMDGPU/mul.ll
+index 85dd59a0c..a8973d845 100644
+--- a/llvm/test/CodeGen/AMDGPU/mul.ll
++++ b/llvm/test/CodeGen/AMDGPU/mul.ll
+@@ -1,20 +1,129 @@
+-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,SI,FUNC %s
+-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,VI,FUNC %s
+-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=FUNC,GFX9PLUS %s
+-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx1010 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=FUNC,GFX9PLUS %s
+-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=FUNC,GFX9PLUS %s
+-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=EG,FUNC %s
++; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
++; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=SI %s
++; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=VI %s
++; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
++; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx1010 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10 %s
++; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11 %s
++; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG %s
+ 
+ ; mul24 and mad24 are affected
+ 
+-; FUNC-LABEL: {{^}}test_mul_v2i32:
+-; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+-; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+-
+-; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+-; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+-
+ define amdgpu_kernel void @test_mul_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %in) {
++; SI-LABEL: test_mul_v2i32:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
++; SI-NEXT:    s_mov_b32 s7, 0xf000
++; SI-NEXT:    s_mov_b32 s6, -1
++; SI-NEXT:    s_mov_b32 s10, s6
++; SI-NEXT:    s_mov_b32 s11, s7
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mov_b32 s8, s2
++; SI-NEXT:    s_mov_b32 s9, s3
++; SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
++; SI-NEXT:    s_mov_b32 s4, s0
++; SI-NEXT:    s_mov_b32 s5, s1
++; SI-NEXT:    s_waitcnt vmcnt(0)
++; SI-NEXT:    v_mul_lo_u32 v1, v1, v3
++; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
++; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: test_mul_v2i32:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; VI-NEXT:    s_mov_b32 s7, 0xf000
++; VI-NEXT:    s_mov_b32 s6, -1
++; VI-NEXT:    s_mov_b32 s10, s6
++; VI-NEXT:    s_mov_b32 s11, s7
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mov_b32 s8, s2
++; VI-NEXT:    s_mov_b32 s9, s3
++; VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
++; VI-NEXT:    s_mov_b32 s4, s0
++; VI-NEXT:    s_mov_b32 s5, s1
++; VI-NEXT:    s_waitcnt vmcnt(0)
++; VI-NEXT:    v_mul_lo_u32 v1, v1, v3
++; VI-NEXT:    v_mul_lo_u32 v0, v0, v2
++; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: test_mul_v2i32:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_mov_b32 s10, s6
++; GFX9-NEXT:    s_mov_b32 s11, s7
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mov_b32 s8, s2
++; GFX9-NEXT:    s_mov_b32 s9, s3
++; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
++; GFX9-NEXT:    s_mov_b32 s4, s0
++; GFX9-NEXT:    s_mov_b32 s5, s1
++; GFX9-NEXT:    s_waitcnt vmcnt(0)
++; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v3
++; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v2
++; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: test_mul_v2i32:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s10, s6
++; GFX10-NEXT:    s_mov_b32 s11, s7
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mov_b32 s8, s2
++; GFX10-NEXT:    s_mov_b32 s9, s3
++; GFX10-NEXT:    s_mov_b32 s4, s0
++; GFX10-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
++; GFX10-NEXT:    s_mov_b32 s5, s1
++; GFX10-NEXT:    s_waitcnt vmcnt(0)
++; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v3
++; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v2
++; GFX10-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: test_mul_v2i32:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
++; GFX11-NEXT:    s_mov_b32 s6, -1
++; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s10, s6
++; GFX11-NEXT:    s_mov_b32 s11, s7
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mov_b32 s8, s2
++; GFX11-NEXT:    s_mov_b32 s9, s3
++; GFX11-NEXT:    s_mov_b32 s4, s0
++; GFX11-NEXT:    buffer_load_b128 v[0:3], off, s[8:11], 0
++; GFX11-NEXT:    s_mov_b32 s5, s1
++; GFX11-NEXT:    s_waitcnt vmcnt(0)
++; GFX11-NEXT:    v_mul_lo_u32 v1, v1, v3
++; GFX11-NEXT:    v_mul_lo_u32 v0, v0, v2
++; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[4:7], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: test_mul_v2i32:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    TEX 0 @6
++; EG-NEXT:    ALU 3, @9, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 6:
++; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
++; EG-NEXT:    ALU clause starting at 8:
++; EG-NEXT:     MOV * T0.X, KC0[2].Z,
++; EG-NEXT:    ALU clause starting at 9:
++; EG-NEXT:     MULLO_INT * T0.Y, T0.Y, T0.W,
++; EG-NEXT:     LSHR T1.X, KC0[2].Y, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, T0.X, T0.Z,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++entry:
+   %b_ptr = getelementptr <2 x i32>, ptr addrspace(1) %in, i32 1
+   %a = load <2 x i32>, ptr addrspace(1) %in
+   %b = load <2 x i32>, ptr addrspace(1) %b_ptr
+@@ -23,18 +132,142 @@ define amdgpu_kernel void @test_mul_v2i32(ptr addrspace(1) %out, ptr addrspace(1
+   ret void
+ }
+ 
+-; FUNC-LABEL: {{^}}v_mul_v4i32:
+-; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+-; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+-; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+-; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+-
+-; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+-; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+-; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+-; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+-
+ define amdgpu_kernel void @v_mul_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %in) {
++; SI-LABEL: v_mul_v4i32:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
++; SI-NEXT:    s_mov_b32 s7, 0xf000
++; SI-NEXT:    s_mov_b32 s6, -1
++; SI-NEXT:    s_mov_b32 s10, s6
++; SI-NEXT:    s_mov_b32 s11, s7
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mov_b32 s8, s2
++; SI-NEXT:    s_mov_b32 s9, s3
++; SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
++; SI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
++; SI-NEXT:    s_mov_b32 s4, s0
++; SI-NEXT:    s_mov_b32 s5, s1
++; SI-NEXT:    s_waitcnt vmcnt(0)
++; SI-NEXT:    v_mul_lo_u32 v3, v3, v7
++; SI-NEXT:    v_mul_lo_u32 v2, v2, v6
++; SI-NEXT:    v_mul_lo_u32 v1, v1, v5
++; SI-NEXT:    v_mul_lo_u32 v0, v0, v4
++; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: v_mul_v4i32:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; VI-NEXT:    s_mov_b32 s7, 0xf000
++; VI-NEXT:    s_mov_b32 s6, -1
++; VI-NEXT:    s_mov_b32 s10, s6
++; VI-NEXT:    s_mov_b32 s11, s7
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mov_b32 s8, s2
++; VI-NEXT:    s_mov_b32 s9, s3
++; VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
++; VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
++; VI-NEXT:    s_mov_b32 s4, s0
++; VI-NEXT:    s_mov_b32 s5, s1
++; VI-NEXT:    s_waitcnt vmcnt(0)
++; VI-NEXT:    v_mul_lo_u32 v3, v3, v7
++; VI-NEXT:    v_mul_lo_u32 v2, v2, v6
++; VI-NEXT:    v_mul_lo_u32 v1, v1, v5
++; VI-NEXT:    v_mul_lo_u32 v0, v0, v4
++; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: v_mul_v4i32:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_mov_b32 s10, s6
++; GFX9-NEXT:    s_mov_b32 s11, s7
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mov_b32 s8, s2
++; GFX9-NEXT:    s_mov_b32 s9, s3
++; GFX9-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
++; GFX9-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
++; GFX9-NEXT:    s_mov_b32 s4, s0
++; GFX9-NEXT:    s_mov_b32 s5, s1
++; GFX9-NEXT:    s_waitcnt vmcnt(0)
++; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v7
++; GFX9-NEXT:    v_mul_lo_u32 v2, v2, v6
++; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v5
++; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v4
++; GFX9-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: v_mul_v4i32:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s10, s6
++; GFX10-NEXT:    s_mov_b32 s11, s7
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mov_b32 s8, s2
++; GFX10-NEXT:    s_mov_b32 s9, s3
++; GFX10-NEXT:    s_clause 0x1
++; GFX10-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
++; GFX10-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
++; GFX10-NEXT:    s_mov_b32 s4, s0
++; GFX10-NEXT:    s_mov_b32 s5, s1
++; GFX10-NEXT:    s_waitcnt vmcnt(0)
++; GFX10-NEXT:    v_mul_lo_u32 v3, v3, v7
++; GFX10-NEXT:    v_mul_lo_u32 v2, v2, v6
++; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v5
++; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v4
++; GFX10-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: v_mul_v4i32:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
++; GFX11-NEXT:    s_mov_b32 s6, -1
++; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s10, s6
++; GFX11-NEXT:    s_mov_b32 s11, s7
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mov_b32 s8, s2
++; GFX11-NEXT:    s_mov_b32 s9, s3
++; GFX11-NEXT:    s_clause 0x1
++; GFX11-NEXT:    buffer_load_b128 v[0:3], off, s[8:11], 0
++; GFX11-NEXT:    buffer_load_b128 v[4:7], off, s[8:11], 0 offset:16
++; GFX11-NEXT:    s_mov_b32 s4, s0
++; GFX11-NEXT:    s_mov_b32 s5, s1
++; GFX11-NEXT:    s_waitcnt vmcnt(0)
++; GFX11-NEXT:    v_mul_lo_u32 v3, v3, v7
++; GFX11-NEXT:    v_mul_lo_u32 v2, v2, v6
++; GFX11-NEXT:    v_mul_lo_u32 v1, v1, v5
++; GFX11-NEXT:    v_mul_lo_u32 v0, v0, v4
++; GFX11-NEXT:    buffer_store_b128 v[0:3], off, s[4:7], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: v_mul_v4i32:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    TEX 1 @6
++; EG-NEXT:    ALU 5, @11, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T2.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 6:
++; EG-NEXT:     VTX_READ_128 T1.XYZW, T0.X, 16, #1
++; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
++; EG-NEXT:    ALU clause starting at 10:
++; EG-NEXT:     MOV * T0.X, KC0[2].Z,
++; EG-NEXT:    ALU clause starting at 11:
++; EG-NEXT:     MULLO_INT * T0.W, T0.W, T1.W,
++; EG-NEXT:     MULLO_INT * T0.Z, T0.Z, T1.Z,
++; EG-NEXT:     MULLO_INT * T0.Y, T0.Y, T1.Y,
++; EG-NEXT:     LSHR T2.X, KC0[2].Y, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, T0.X, T1.X,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++entry:
+   %b_ptr = getelementptr <4 x i32>, ptr addrspace(1) %in, i32 1
+   %a = load <4 x i32>, ptr addrspace(1) %in
+   %b = load <4 x i32>, ptr addrspace(1) %b_ptr
+@@ -43,24 +276,232 @@ define amdgpu_kernel void @v_mul_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %
+   ret void
+ }
+ 
+-; FUNC-LABEL: {{^}}s_trunc_i64_mul_to_i32:
+-; GCN: s_load_dword
+-; GCN: s_load_dword
+-; GCN: s_mul_i32
+-; GCN: buffer_store_dword
+ define amdgpu_kernel void @s_trunc_i64_mul_to_i32(ptr addrspace(1) %out, i64 %a, i64 %b) {
++; SI-LABEL: s_trunc_i64_mul_to_i32:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_load_dword s7, s[0:1], 0xd
++; SI-NEXT:    s_mov_b32 s3, 0xf000
++; SI-NEXT:    s_mov_b32 s2, -1
++; SI-NEXT:    s_mov_b32 s0, s4
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mul_i32 s4, s7, s6
++; SI-NEXT:    s_mov_b32 s1, s5
++; SI-NEXT:    v_mov_b32_e32 v0, s4
++; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: s_trunc_i64_mul_to_i32:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_load_dword s7, s[0:1], 0x34
++; VI-NEXT:    s_mov_b32 s3, 0xf000
++; VI-NEXT:    s_mov_b32 s2, -1
++; VI-NEXT:    s_mov_b32 s0, s4
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mul_i32 s4, s7, s6
++; VI-NEXT:    s_mov_b32 s1, s5
++; VI-NEXT:    v_mov_b32_e32 v0, s4
++; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: s_trunc_i64_mul_to_i32:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x34
++; GFX9-NEXT:    ; kill: killed $sgpr0_sgpr1
++; GFX9-NEXT:    s_mov_b32 s3, 0xf000
++; GFX9-NEXT:    s_mov_b32 s2, -1
++; GFX9-NEXT:    s_mov_b32 s0, s4
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mul_i32 s4, s7, s6
++; GFX9-NEXT:    s_mov_b32 s1, s5
++; GFX9-NEXT:    v_mov_b32_e32 v0, s4
++; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: s_trunc_i64_mul_to_i32:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_clause 0x1
++; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x34
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX10-NEXT:    s_mul_i32 s0, s2, s6
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    v_mov_b32_e32 v0, s0
++; GFX10-NEXT:    buffer_store_dword v0, off, s[4:7], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: s_trunc_i64_mul_to_i32:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_clause 0x1
++; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
++; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x34
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX11-NEXT:    s_mul_i32 s0, s0, s6
++; GFX11-NEXT:    s_mov_b32 s6, -1
++; GFX11-NEXT:    v_mov_b32_e32 v0, s0
++; GFX11-NEXT:    buffer_store_b32 v0, off, s[4:7], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: s_trunc_i64_mul_to_i32:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    ALU clause starting at 4:
++; EG-NEXT:     LSHR * T0.X, KC0[2].Y, literal.x,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++; EG-NEXT:     MULLO_INT * T1.X, KC0[3].Y, KC0[2].W,
++entry:
+   %mul = mul i64 %b, %a
+   %trunc = trunc i64 %mul to i32
+   store i32 %trunc, ptr addrspace(1) %out, align 8
+   ret void
+ }
+ 
+-; FUNC-LABEL: {{^}}v_trunc_i64_mul_to_i32:
+-; GCN: s_load_dword
+-; GCN: s_load_dword
+-; GCN: v_mul_lo_u32
+-; GCN: buffer_store_dword
+ define amdgpu_kernel void @v_trunc_i64_mul_to_i32(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) nounwind {
++; SI-LABEL: v_trunc_i64_mul_to_i32:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
++; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
++; SI-NEXT:    s_mov_b32 s3, 0xf000
++; SI-NEXT:    s_mov_b32 s2, -1
++; SI-NEXT:    s_mov_b32 s14, s2
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mov_b32 s12, s6
++; SI-NEXT:    s_mov_b32 s13, s7
++; SI-NEXT:    s_mov_b32 s15, s3
++; SI-NEXT:    s_mov_b32 s10, s2
++; SI-NEXT:    s_mov_b32 s11, s3
++; SI-NEXT:    buffer_load_dword v0, off, s[12:15], 0
++; SI-NEXT:    buffer_load_dword v1, off, s[8:11], 0
++; SI-NEXT:    s_mov_b32 s0, s4
++; SI-NEXT:    s_mov_b32 s1, s5
++; SI-NEXT:    s_waitcnt vmcnt(0)
++; SI-NEXT:    v_mul_lo_u32 v0, v1, v0
++; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: v_trunc_i64_mul_to_i32:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; VI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
++; VI-NEXT:    s_mov_b32 s3, 0xf000
++; VI-NEXT:    s_mov_b32 s2, -1
++; VI-NEXT:    s_mov_b32 s14, s2
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mov_b32 s12, s6
++; VI-NEXT:    s_mov_b32 s13, s7
++; VI-NEXT:    s_mov_b32 s15, s3
++; VI-NEXT:    s_mov_b32 s10, s2
++; VI-NEXT:    s_mov_b32 s11, s3
++; VI-NEXT:    buffer_load_dword v0, off, s[12:15], 0
++; VI-NEXT:    buffer_load_dword v1, off, s[8:11], 0
++; VI-NEXT:    s_mov_b32 s0, s4
++; VI-NEXT:    s_mov_b32 s1, s5
++; VI-NEXT:    s_waitcnt vmcnt(0)
++; VI-NEXT:    v_mul_lo_u32 v0, v1, v0
++; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: v_trunc_i64_mul_to_i32:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; GFX9-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
++; GFX9-NEXT:    s_mov_b32 s3, 0xf000
++; GFX9-NEXT:    s_mov_b32 s2, -1
++; GFX9-NEXT:    s_mov_b32 s14, s2
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mov_b32 s12, s6
++; GFX9-NEXT:    s_mov_b32 s13, s7
++; GFX9-NEXT:    s_mov_b32 s15, s3
++; GFX9-NEXT:    s_mov_b32 s10, s2
++; GFX9-NEXT:    s_mov_b32 s11, s3
++; GFX9-NEXT:    buffer_load_dword v0, off, s[12:15], 0
++; GFX9-NEXT:    buffer_load_dword v1, off, s[8:11], 0
++; GFX9-NEXT:    s_mov_b32 s0, s4
++; GFX9-NEXT:    s_mov_b32 s1, s5
++; GFX9-NEXT:    s_waitcnt vmcnt(0)
++; GFX9-NEXT:    v_mul_lo_u32 v0, v1, v0
++; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: v_trunc_i64_mul_to_i32:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_clause 0x1
++; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; GFX10-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
++; GFX10-NEXT:    s_mov_b32 s2, -1
++; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s14, s2
++; GFX10-NEXT:    s_mov_b32 s15, s3
++; GFX10-NEXT:    s_mov_b32 s10, s2
++; GFX10-NEXT:    s_mov_b32 s11, s3
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mov_b32 s12, s6
++; GFX10-NEXT:    s_mov_b32 s13, s7
++; GFX10-NEXT:    buffer_load_dword v0, off, s[12:15], 0
++; GFX10-NEXT:    buffer_load_dword v1, off, s[8:11], 0
++; GFX10-NEXT:    s_mov_b32 s0, s4
++; GFX10-NEXT:    s_mov_b32 s1, s5
++; GFX10-NEXT:    s_waitcnt vmcnt(0)
++; GFX10-NEXT:    v_mul_lo_u32 v0, v1, v0
++; GFX10-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: v_trunc_i64_mul_to_i32:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_clause 0x1
++; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
++; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
++; GFX11-NEXT:    s_mov_b32 s10, -1
++; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s14, s10
++; GFX11-NEXT:    s_mov_b32 s15, s11
++; GFX11-NEXT:    s_mov_b32 s2, s10
++; GFX11-NEXT:    s_mov_b32 s3, s11
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mov_b32 s12, s6
++; GFX11-NEXT:    s_mov_b32 s13, s7
++; GFX11-NEXT:    buffer_load_b32 v0, off, s[12:15], 0
++; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0
++; GFX11-NEXT:    s_mov_b32 s8, s4
++; GFX11-NEXT:    s_mov_b32 s9, s5
++; GFX11-NEXT:    s_waitcnt vmcnt(0)
++; GFX11-NEXT:    v_mul_lo_u32 v0, v1, v0
++; GFX11-NEXT:    buffer_store_b32 v0, off, s[8:11], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: v_trunc_i64_mul_to_i32:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 1, @10, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    TEX 1 @6
++; EG-NEXT:    ALU 2, @12, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T2.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 6:
++; EG-NEXT:     VTX_READ_32 T1.X, T1.X, 0, #1
++; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
++; EG-NEXT:    ALU clause starting at 10:
++; EG-NEXT:     MOV T0.X, KC0[2].Z,
++; EG-NEXT:     MOV * T1.X, KC0[2].W,
++; EG-NEXT:    ALU clause starting at 12:
++; EG-NEXT:     LSHR T2.X, KC0[2].Y, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, T1.X, T0.X,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++entry:
+   %a = load i64, ptr addrspace(1) %aptr, align 8
+   %b = load i64, ptr addrspace(1) %bptr, align 8
+   %mul = mul i64 %b, %a
+@@ -71,13 +512,93 @@ define amdgpu_kernel void @v_trunc_i64_mul_to_i32(ptr addrspace(1) %out, ptr add
+ 
+ ; This 64-bit multiply should just use MUL_HI and MUL_LO, since the top
+ ; 32-bits of both arguments are sign bits.
+-; FUNC-LABEL: {{^}}mul64_sext_c:
+-; EG-DAG: MULLO_INT
+-; EG-DAG: MULHI_INT
+-; SI-DAG: s_mulk_i32
+-; SI-DAG: v_mul_hi_i32
+-; VI: v_mad_i64_i32
++
+ define amdgpu_kernel void @mul64_sext_c(ptr addrspace(1) %out, i32 %in) {
++; SI-LABEL: mul64_sext_c:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
++; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
++; SI-NEXT:    v_mov_b32_e32 v0, 0x50
++; SI-NEXT:    s_mov_b32 s3, 0xf000
++; SI-NEXT:    s_mov_b32 s2, -1
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    v_mul_hi_i32 v1, s4, v0
++; SI-NEXT:    s_mulk_i32 s4, 0x50
++; SI-NEXT:    v_mov_b32_e32 v0, s4
++; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: mul64_sext_c:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
++; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
++; VI-NEXT:    v_mov_b32_e32 v0, 0x50
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    v_mad_i64_i32 v[0:1], s[2:3], s2, v0, 0
++; VI-NEXT:    s_mov_b32 s3, 0xf000
++; VI-NEXT:    s_mov_b32 s2, -1
++; VI-NEXT:    s_nop 2
++; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: mul64_sext_c:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
++; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mul_hi_i32 s0, s2, 0x50
++; GFX9-NEXT:    s_mulk_i32 s2, 0x50
++; GFX9-NEXT:    v_mov_b32_e32 v0, s2
++; GFX9-NEXT:    v_mov_b32_e32 v1, s0
++; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: mul64_sext_c:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_clause 0x1
++; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x2c
++; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
++; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mul_i32 s0, s2, 0x50
++; GFX10-NEXT:    s_mul_hi_i32 s1, s2, 0x50
++; GFX10-NEXT:    v_mov_b32_e32 v0, s0
++; GFX10-NEXT:    v_mov_b32_e32 v1, s1
++; GFX10-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: mul64_sext_c:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_clause 0x1
++; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
++; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mul_i32 s3, s2, 0x50
++; GFX11-NEXT:    s_mul_hi_i32 s2, s2, 0x50
++; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
++; GFX11-NEXT:    v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s2
++; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s2, -1
++; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: mul64_sext_c:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 4, @4, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    ALU clause starting at 4:
++; EG-NEXT:     MULHI_INT * T0.Y, KC0[2].Z, literal.x,
++; EG-NEXT:    80(1.121039e-43), 0(0.000000e+00)
++; EG-NEXT:     LSHR T1.X, KC0[2].Y, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, KC0[2].Z, literal.y,
++; EG-NEXT:    2(2.802597e-45), 80(1.121039e-43)
+ entry:
+   %0 = sext i32 %in to i64
+   %1 = mul i64 %0, 80
+@@ -85,14 +606,125 @@ entry:
+   ret void
+ }
+ 
+-; FUNC-LABEL: {{^}}v_mul64_sext_c:
+-; EG-DAG: MULLO_INT
+-; EG-DAG: MULHI_INT
+-; SI-DAG: v_mul_lo_u32
+-; SI-DAG: v_mul_hi_i32
+-; VI: v_mad_i64_i32
+-; GCN: s_endpgm
+ define amdgpu_kernel void @v_mul64_sext_c(ptr addrspace(1) %out, ptr addrspace(1) %in) {
++; SI-LABEL: v_mul64_sext_c:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
++; SI-NEXT:    s_mov_b32 s7, 0xf000
++; SI-NEXT:    s_mov_b32 s6, -1
++; SI-NEXT:    s_mov_b32 s10, s6
++; SI-NEXT:    s_mov_b32 s11, s7
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mov_b32 s8, s2
++; SI-NEXT:    s_mov_b32 s9, s3
++; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
++; SI-NEXT:    s_movk_i32 s2, 0x50
++; SI-NEXT:    s_mov_b32 s4, s0
++; SI-NEXT:    s_mov_b32 s5, s1
++; SI-NEXT:    s_waitcnt vmcnt(0)
++; SI-NEXT:    v_mul_hi_i32 v1, v0, s2
++; SI-NEXT:    v_mul_lo_u32 v0, v0, s2
++; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: v_mul64_sext_c:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; VI-NEXT:    s_mov_b32 s7, 0xf000
++; VI-NEXT:    s_mov_b32 s6, -1
++; VI-NEXT:    s_mov_b32 s10, s6
++; VI-NEXT:    s_mov_b32 s11, s7
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mov_b32 s8, s2
++; VI-NEXT:    s_mov_b32 s9, s3
++; VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
++; VI-NEXT:    s_movk_i32 s2, 0x50
++; VI-NEXT:    s_mov_b32 s4, s0
++; VI-NEXT:    s_mov_b32 s5, s1
++; VI-NEXT:    s_waitcnt vmcnt(0)
++; VI-NEXT:    v_mad_i64_i32 v[0:1], s[2:3], v0, s2, 0
++; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: v_mul64_sext_c:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_mov_b32 s10, s6
++; GFX9-NEXT:    s_mov_b32 s11, s7
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mov_b32 s8, s2
++; GFX9-NEXT:    s_mov_b32 s9, s3
++; GFX9-NEXT:    buffer_load_dword v0, off, s[8:11], 0
++; GFX9-NEXT:    s_movk_i32 s2, 0x50
++; GFX9-NEXT:    s_mov_b32 s4, s0
++; GFX9-NEXT:    s_mov_b32 s5, s1
++; GFX9-NEXT:    s_waitcnt vmcnt(0)
++; GFX9-NEXT:    v_mul_hi_i32 v1, v0, s2
++; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s2
++; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: v_mul64_sext_c:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s10, s6
++; GFX10-NEXT:    s_mov_b32 s11, s7
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mov_b32 s8, s2
++; GFX10-NEXT:    s_mov_b32 s9, s3
++; GFX10-NEXT:    s_mov_b32 s4, s0
++; GFX10-NEXT:    buffer_load_dword v0, off, s[8:11], 0
++; GFX10-NEXT:    s_mov_b32 s5, s1
++; GFX10-NEXT:    s_waitcnt vmcnt(0)
++; GFX10-NEXT:    v_mul_hi_i32 v1, 0x50, v0
++; GFX10-NEXT:    v_mul_lo_u32 v0, 0x50, v0
++; GFX10-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: v_mul64_sext_c:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
++; GFX11-NEXT:    s_mov_b32 s6, -1
++; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s10, s6
++; GFX11-NEXT:    s_mov_b32 s11, s7
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mov_b32 s8, s2
++; GFX11-NEXT:    s_mov_b32 s9, s3
++; GFX11-NEXT:    s_mov_b32 s4, s0
++; GFX11-NEXT:    buffer_load_b32 v0, off, s[8:11], 0
++; GFX11-NEXT:    s_mov_b32 s5, s1
++; GFX11-NEXT:    s_waitcnt vmcnt(0)
++; GFX11-NEXT:    v_mul_hi_i32 v1, 0x50, v0
++; GFX11-NEXT:    v_mul_lo_u32 v0, 0x50, v0
++; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[4:7], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: v_mul64_sext_c:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    TEX 0 @6
++; EG-NEXT:    ALU 4, @9, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 6:
++; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
++; EG-NEXT:    ALU clause starting at 8:
++; EG-NEXT:     MOV * T0.X, KC0[2].Z,
++; EG-NEXT:    ALU clause starting at 9:
++; EG-NEXT:     MULHI_INT * T0.Y, T0.X, literal.x,
++; EG-NEXT:    80(1.121039e-43), 0(0.000000e+00)
++; EG-NEXT:     LSHR T1.X, KC0[2].Y, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, T0.X, literal.y,
++; EG-NEXT:    2(2.802597e-45), 80(1.121039e-43)
++entry:
+   %val = load i32, ptr addrspace(1) %in, align 4
+   %ext = sext i32 %val to i64
+   %mul = mul i64 %ext, 80
+@@ -100,12 +732,122 @@ define amdgpu_kernel void @v_mul64_sext_c(ptr addrspace(1) %out, ptr addrspace(1
+   ret void
+ }
+ 
+-; FUNC-LABEL: {{^}}v_mul64_sext_inline_imm:
+-; SI-DAG: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, 9
+-; SI-DAG: v_mul_hi_i32 v{{[0-9]+}}, v{{[0-9]+}}, 9
+-; VI: v_mad_i64_i32 v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, 9, 0
+-; GCN: s_endpgm
+ define amdgpu_kernel void @v_mul64_sext_inline_imm(ptr addrspace(1) %out, ptr addrspace(1) %in) {
++; SI-LABEL: v_mul64_sext_inline_imm:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
++; SI-NEXT:    s_mov_b32 s7, 0xf000
++; SI-NEXT:    s_mov_b32 s6, -1
++; SI-NEXT:    s_mov_b32 s10, s6
++; SI-NEXT:    s_mov_b32 s11, s7
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mov_b32 s8, s2
++; SI-NEXT:    s_mov_b32 s9, s3
++; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
++; SI-NEXT:    s_mov_b32 s4, s0
++; SI-NEXT:    s_mov_b32 s5, s1
++; SI-NEXT:    s_waitcnt vmcnt(0)
++; SI-NEXT:    v_mul_hi_i32 v1, v0, 9
++; SI-NEXT:    v_mul_lo_u32 v0, v0, 9
++; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: v_mul64_sext_inline_imm:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; VI-NEXT:    s_mov_b32 s7, 0xf000
++; VI-NEXT:    s_mov_b32 s6, -1
++; VI-NEXT:    s_mov_b32 s10, s6
++; VI-NEXT:    s_mov_b32 s11, s7
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mov_b32 s8, s2
++; VI-NEXT:    s_mov_b32 s9, s3
++; VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
++; VI-NEXT:    s_mov_b32 s4, s0
++; VI-NEXT:    s_mov_b32 s5, s1
++; VI-NEXT:    s_waitcnt vmcnt(0)
++; VI-NEXT:    v_mad_i64_i32 v[0:1], s[2:3], v0, 9, 0
++; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: v_mul64_sext_inline_imm:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_mov_b32 s10, s6
++; GFX9-NEXT:    s_mov_b32 s11, s7
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mov_b32 s8, s2
++; GFX9-NEXT:    s_mov_b32 s9, s3
++; GFX9-NEXT:    buffer_load_dword v0, off, s[8:11], 0
++; GFX9-NEXT:    s_mov_b32 s4, s0
++; GFX9-NEXT:    s_mov_b32 s5, s1
++; GFX9-NEXT:    s_waitcnt vmcnt(0)
++; GFX9-NEXT:    v_mul_hi_i32 v1, v0, 9
++; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 9
++; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: v_mul64_sext_inline_imm:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s10, s6
++; GFX10-NEXT:    s_mov_b32 s11, s7
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mov_b32 s8, s2
++; GFX10-NEXT:    s_mov_b32 s9, s3
++; GFX10-NEXT:    s_mov_b32 s4, s0
++; GFX10-NEXT:    buffer_load_dword v0, off, s[8:11], 0
++; GFX10-NEXT:    s_mov_b32 s5, s1
++; GFX10-NEXT:    s_waitcnt vmcnt(0)
++; GFX10-NEXT:    v_mul_hi_i32 v1, v0, 9
++; GFX10-NEXT:    v_mul_lo_u32 v0, v0, 9
++; GFX10-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: v_mul64_sext_inline_imm:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
++; GFX11-NEXT:    s_mov_b32 s6, -1
++; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s10, s6
++; GFX11-NEXT:    s_mov_b32 s11, s7
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mov_b32 s8, s2
++; GFX11-NEXT:    s_mov_b32 s9, s3
++; GFX11-NEXT:    s_mov_b32 s4, s0
++; GFX11-NEXT:    buffer_load_b32 v0, off, s[8:11], 0
++; GFX11-NEXT:    s_mov_b32 s5, s1
++; GFX11-NEXT:    s_waitcnt vmcnt(0)
++; GFX11-NEXT:    v_mul_hi_i32 v1, v0, 9
++; GFX11-NEXT:    v_mul_lo_u32 v0, v0, 9
++; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[4:7], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: v_mul64_sext_inline_imm:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    TEX 0 @6
++; EG-NEXT:    ALU 4, @9, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 6:
++; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
++; EG-NEXT:    ALU clause starting at 8:
++; EG-NEXT:     MOV * T0.X, KC0[2].Z,
++; EG-NEXT:    ALU clause starting at 9:
++; EG-NEXT:     MULHI_INT * T0.Y, T0.X, literal.x,
++; EG-NEXT:    9(1.261169e-44), 0(0.000000e+00)
++; EG-NEXT:     LSHR T1.X, KC0[2].Y, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, T0.X, literal.y,
++; EG-NEXT:    2(2.802597e-45), 9(1.261169e-44)
++entry:
+   %val = load i32, ptr addrspace(1) %in, align 4
+   %ext = sext i32 %val to i64
+   %mul = mul i64 %ext, 9
+@@ -113,22 +855,202 @@ define amdgpu_kernel void @v_mul64_sext_inline_imm(ptr addrspace(1) %out, ptr ad
+   ret void
+ }
+ 
+-; FUNC-LABEL: {{^}}s_mul_i32:
+-; GCN: s_load_dword [[SRC0:s[0-9]+]],
+-; GCN: s_load_dword [[SRC1:s[0-9]+]],
+-; GCN: s_mul_i32 [[SRESULT:s[0-9]+]], [[SRC0]], [[SRC1]]
+-; GCN: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]]
+-; GCN: buffer_store_dword [[VRESULT]],
+-; GCN: s_endpgm
+ define amdgpu_kernel void @s_mul_i32(ptr addrspace(1) %out, [8 x i32], i32 %a, [8 x i32], i32 %b) nounwind {
++; SI-LABEL: s_mul_i32:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dword s4, s[0:1], 0x13
++; SI-NEXT:    s_load_dword s5, s[0:1], 0x1c
++; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
++; SI-NEXT:    s_mov_b32 s3, 0xf000
++; SI-NEXT:    s_mov_b32 s2, -1
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mul_i32 s4, s4, s5
++; SI-NEXT:    v_mov_b32_e32 v0, s4
++; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: s_mul_i32:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dword s4, s[0:1], 0x4c
++; VI-NEXT:    s_load_dword s5, s[0:1], 0x70
++; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
++; VI-NEXT:    s_mov_b32 s3, 0xf000
++; VI-NEXT:    s_mov_b32 s2, -1
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mul_i32 s4, s4, s5
++; VI-NEXT:    v_mov_b32_e32 v0, s4
++; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: s_mul_i32:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x4c
++; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x70
++; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mul_i32 s0, s2, s3
++; GFX9-NEXT:    v_mov_b32_e32 v0, s0
++; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: s_mul_i32:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_clause 0x2
++; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x4c
++; GFX10-NEXT:    s_load_dword s3, s[0:1], 0x70
++; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
++; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mul_i32 s0, s2, s3
++; GFX10-NEXT:    v_mov_b32_e32 v0, s0
++; GFX10-NEXT:    buffer_store_dword v0, off, s[4:7], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: s_mul_i32:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_clause 0x2
++; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x4c
++; GFX11-NEXT:    s_load_b32 s3, s[0:1], 0x70
++; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mul_i32 s2, s2, s3
++; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX11-NEXT:    v_mov_b32_e32 v0, s2
++; GFX11-NEXT:    s_mov_b32 s2, -1
++; GFX11-NEXT:    buffer_store_b32 v0, off, s[0:3], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: s_mul_i32:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 2, @4, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    ALU clause starting at 4:
++; EG-NEXT:     LSHR * T0.X, KC0[2].Y, literal.x,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++; EG-NEXT:     MULLO_INT * T1.X, KC0[4].Z, KC0[6].W,
++entry:
+   %mul = mul i32 %a, %b
+   store i32 %mul, ptr addrspace(1) %out, align 4
+   ret void
+ }
+ 
+-; FUNC-LABEL: {{^}}v_mul_i32:
+-; GCN: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+ define amdgpu_kernel void @v_mul_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) {
++; SI-LABEL: v_mul_i32:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
++; SI-NEXT:    s_mov_b32 s7, 0xf000
++; SI-NEXT:    s_mov_b32 s6, -1
++; SI-NEXT:    s_mov_b32 s10, s6
++; SI-NEXT:    s_mov_b32 s11, s7
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mov_b32 s8, s2
++; SI-NEXT:    s_mov_b32 s9, s3
++; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
++; SI-NEXT:    s_mov_b32 s4, s0
++; SI-NEXT:    s_mov_b32 s5, s1
++; SI-NEXT:    s_waitcnt vmcnt(0)
++; SI-NEXT:    v_mul_lo_u32 v0, v0, v1
++; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: v_mul_i32:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; VI-NEXT:    s_mov_b32 s7, 0xf000
++; VI-NEXT:    s_mov_b32 s6, -1
++; VI-NEXT:    s_mov_b32 s10, s6
++; VI-NEXT:    s_mov_b32 s11, s7
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mov_b32 s8, s2
++; VI-NEXT:    s_mov_b32 s9, s3
++; VI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
++; VI-NEXT:    s_mov_b32 s4, s0
++; VI-NEXT:    s_mov_b32 s5, s1
++; VI-NEXT:    s_waitcnt vmcnt(0)
++; VI-NEXT:    v_mul_lo_u32 v0, v0, v1
++; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: v_mul_i32:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_mov_b32 s10, s6
++; GFX9-NEXT:    s_mov_b32 s11, s7
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mov_b32 s8, s2
++; GFX9-NEXT:    s_mov_b32 s9, s3
++; GFX9-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
++; GFX9-NEXT:    s_mov_b32 s4, s0
++; GFX9-NEXT:    s_mov_b32 s5, s1
++; GFX9-NEXT:    s_waitcnt vmcnt(0)
++; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
++; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: v_mul_i32:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s10, s6
++; GFX10-NEXT:    s_mov_b32 s11, s7
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mov_b32 s8, s2
++; GFX10-NEXT:    s_mov_b32 s9, s3
++; GFX10-NEXT:    s_mov_b32 s4, s0
++; GFX10-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
++; GFX10-NEXT:    s_mov_b32 s5, s1
++; GFX10-NEXT:    s_waitcnt vmcnt(0)
++; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
++; GFX10-NEXT:    buffer_store_dword v0, off, s[4:7], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: v_mul_i32:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
++; GFX11-NEXT:    s_mov_b32 s6, -1
++; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s10, s6
++; GFX11-NEXT:    s_mov_b32 s11, s7
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mov_b32 s8, s2
++; GFX11-NEXT:    s_mov_b32 s9, s3
++; GFX11-NEXT:    s_mov_b32 s4, s0
++; GFX11-NEXT:    buffer_load_b64 v[0:1], off, s[8:11], 0
++; GFX11-NEXT:    s_mov_b32 s5, s1
++; GFX11-NEXT:    s_waitcnt vmcnt(0)
++; GFX11-NEXT:    v_mul_lo_u32 v0, v0, v1
++; GFX11-NEXT:    buffer_store_b32 v0, off, s[4:7], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: v_mul_i32:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    TEX 0 @6
++; EG-NEXT:    ALU 2, @9, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 6:
++; EG-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
++; EG-NEXT:    ALU clause starting at 8:
++; EG-NEXT:     MOV * T0.X, KC0[2].Z,
++; EG-NEXT:    ALU clause starting at 9:
++; EG-NEXT:     LSHR T1.X, KC0[2].Y, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, T0.X, T0.Y,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++entry:
+   %b_ptr = getelementptr i32, ptr addrspace(1) %in, i32 1
+   %a = load i32, ptr addrspace(1) %in
+   %b = load i32, ptr addrspace(1) %b_ptr
+@@ -137,6 +1059,298 @@ define amdgpu_kernel void @v_mul_i32(ptr addrspace(1) %out, ptr addrspace(1) %in
+   ret void
+ }
+ 
++define amdgpu_kernel void @s_mul_i1(ptr addrspace(1) %out, [8 x i32], i1 %a, [8 x i32], i1 %b) nounwind {
++; SI-LABEL: s_mul_i1:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dword s2, s[0:1], 0x13
++; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
++; SI-NEXT:    s_load_dword s3, s[0:1], 0x1c
++; SI-NEXT:    s_mov_b32 s7, 0xf000
++; SI-NEXT:    s_mov_b32 s6, -1
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_bitcmp1_b32 s2, 0
++; SI-NEXT:    s_cselect_b64 s[0:1], -1, 0
++; SI-NEXT:    s_bitcmp1_b32 s3, 0
++; SI-NEXT:    s_cselect_b64 s[2:3], -1, 0
++; SI-NEXT:    s_and_b64 s[0:1], s[0:1], s[2:3]
++; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
++; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: s_mul_i1:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dword s2, s[0:1], 0x4c
++; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
++; VI-NEXT:    s_load_dword s3, s[0:1], 0x70
++; VI-NEXT:    s_mov_b32 s7, 0xf000
++; VI-NEXT:    s_mov_b32 s6, -1
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_bitcmp1_b32 s2, 0
++; VI-NEXT:    s_cselect_b64 s[0:1], -1, 0
++; VI-NEXT:    s_bitcmp1_b32 s3, 0
++; VI-NEXT:    s_cselect_b64 s[2:3], -1, 0
++; VI-NEXT:    s_and_b64 s[0:1], s[0:1], s[2:3]
++; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
++; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: s_mul_i1:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x4c
++; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
++; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x70
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_bitcmp1_b32 s2, 0
++; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
++; GFX9-NEXT:    s_bitcmp1_b32 s3, 0
++; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
++; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], s[2:3]
++; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
++; GFX9-NEXT:    buffer_store_byte v0, off, s[4:7], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: s_mul_i1:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_clause 0x2
++; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x4c
++; GFX10-NEXT:    s_load_dword s3, s[0:1], 0x70
++; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
++; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_bitcmp1_b32 s2, 0
++; GFX10-NEXT:    s_cselect_b32 s0, -1, 0
++; GFX10-NEXT:    s_bitcmp1_b32 s3, 0
++; GFX10-NEXT:    s_cselect_b32 s1, -1, 0
++; GFX10-NEXT:    s_and_b32 s0, s0, s1
++; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
++; GFX10-NEXT:    buffer_store_byte v0, off, s[4:7], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: s_mul_i1:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_clause 0x2
++; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x4c
++; GFX11-NEXT:    s_load_b32 s3, s[0:1], 0x70
++; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_bitcmp1_b32 s2, 0
++; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
++; GFX11-NEXT:    s_bitcmp1_b32 s3, 0
++; GFX11-NEXT:    s_cselect_b32 s3, -1, 0
++; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
++; GFX11-NEXT:    s_and_b32 s2, s2, s3
++; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s2
++; GFX11-NEXT:    s_mov_b32 s2, -1
++; GFX11-NEXT:    buffer_store_b8 v0, off, s[0:3], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: s_mul_i1:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 0, @10, KC0[], KC1[]
++; EG-NEXT:    TEX 1 @6
++; EG-NEXT:    ALU 12, @11, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 6:
++; EG-NEXT:     VTX_READ_8 T1.X, T0.X, 72, #3
++; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 108, #3
++; EG-NEXT:    ALU clause starting at 10:
++; EG-NEXT:     MOV * T0.X, 0.0,
++; EG-NEXT:    ALU clause starting at 11:
++; EG-NEXT:     AND_INT T0.W, KC0[2].Y, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, T1.X, T0.X,
++; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
++; EG-NEXT:     AND_INT T1.W, PS, 1,
++; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
++; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
++; EG-NEXT:     LSHL T0.X, PV.W, PS,
++; EG-NEXT:     LSHL * T0.W, literal.x, PS,
++; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
++; EG-NEXT:     MOV T0.Y, 0.0,
++; EG-NEXT:     MOV * T0.Z, 0.0,
++; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++entry:
++  %mul = mul i1 %a, %b
++  store i1 %mul, ptr addrspace(1) %out, align 4
++  ret void
++}
++
++define amdgpu_kernel void @v_mul_i1(ptr addrspace(1) %out, ptr addrspace(1) %in) {
++; SI-LABEL: v_mul_i1:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
++; SI-NEXT:    s_mov_b32 s7, 0xf000
++; SI-NEXT:    s_mov_b32 s6, -1
++; SI-NEXT:    s_mov_b32 s10, s6
++; SI-NEXT:    s_mov_b32 s11, s7
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mov_b32 s8, s2
++; SI-NEXT:    s_mov_b32 s9, s3
++; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
++; SI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:4
++; SI-NEXT:    s_mov_b32 s4, s0
++; SI-NEXT:    s_mov_b32 s5, s1
++; SI-NEXT:    s_waitcnt vmcnt(1)
++; SI-NEXT:    v_and_b32_e32 v0, 1, v0
++; SI-NEXT:    s_waitcnt vmcnt(0)
++; SI-NEXT:    v_and_b32_e32 v1, 1, v1
++; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
++; SI-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v1
++; SI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
++; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
++; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: v_mul_i1:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; VI-NEXT:    s_mov_b32 s7, 0xf000
++; VI-NEXT:    s_mov_b32 s6, -1
++; VI-NEXT:    s_mov_b32 s10, s6
++; VI-NEXT:    s_mov_b32 s11, s7
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mov_b32 s8, s2
++; VI-NEXT:    s_mov_b32 s9, s3
++; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
++; VI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:4
++; VI-NEXT:    s_mov_b32 s4, s0
++; VI-NEXT:    s_mov_b32 s5, s1
++; VI-NEXT:    s_waitcnt vmcnt(1)
++; VI-NEXT:    v_and_b32_e32 v0, 1, v0
++; VI-NEXT:    s_waitcnt vmcnt(0)
++; VI-NEXT:    v_and_b32_e32 v1, 1, v1
++; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
++; VI-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v1
++; VI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
++; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
++; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: v_mul_i1:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_mov_b32 s10, s6
++; GFX9-NEXT:    s_mov_b32 s11, s7
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mov_b32 s8, s2
++; GFX9-NEXT:    s_mov_b32 s9, s3
++; GFX9-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
++; GFX9-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:4
++; GFX9-NEXT:    s_mov_b32 s4, s0
++; GFX9-NEXT:    s_mov_b32 s5, s1
++; GFX9-NEXT:    s_waitcnt vmcnt(1)
++; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
++; GFX9-NEXT:    s_waitcnt vmcnt(0)
++; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
++; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
++; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v1
++; GFX9-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
++; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
++; GFX9-NEXT:    buffer_store_byte v0, off, s[4:7], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: v_mul_i1:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; GFX10-NEXT:    s_mov_b32 s2, -1
++; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s10, s2
++; GFX10-NEXT:    s_mov_b32 s11, s3
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mov_b32 s8, s6
++; GFX10-NEXT:    s_mov_b32 s9, s7
++; GFX10-NEXT:    s_clause 0x1
++; GFX10-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
++; GFX10-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:4
++; GFX10-NEXT:    s_mov_b32 s1, s5
++; GFX10-NEXT:    s_waitcnt vmcnt(1)
++; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
++; GFX10-NEXT:    s_waitcnt vmcnt(0)
++; GFX10-NEXT:    v_and_b32_e32 v1, 1, v1
++; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
++; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 1, v1
++; GFX10-NEXT:    s_and_b32 s0, vcc_lo, s0
++; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
++; GFX10-NEXT:    s_mov_b32 s0, s4
++; GFX10-NEXT:    buffer_store_byte v0, off, s[0:3], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: v_mul_i1:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
++; GFX11-NEXT:    s_mov_b32 s2, -1
++; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s10, s2
++; GFX11-NEXT:    s_mov_b32 s11, s3
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mov_b32 s8, s6
++; GFX11-NEXT:    s_mov_b32 s9, s7
++; GFX11-NEXT:    s_clause 0x1
++; GFX11-NEXT:    buffer_load_u8 v0, off, s[8:11], 0
++; GFX11-NEXT:    buffer_load_u8 v1, off, s[8:11], 0 offset:4
++; GFX11-NEXT:    s_mov_b32 s1, s5
++; GFX11-NEXT:    s_waitcnt vmcnt(1)
++; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
++; GFX11-NEXT:    s_waitcnt vmcnt(0)
++; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
++; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
++; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
++; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 1, v1
++; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
++; GFX11-NEXT:    s_and_b32 s0, vcc_lo, s0
++; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
++; GFX11-NEXT:    s_mov_b32 s0, s4
++; GFX11-NEXT:    buffer_store_b8 v0, off, s[0:3], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: v_mul_i1:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    TEX 1 @6
++; EG-NEXT:    ALU 12, @11, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 6:
++; EG-NEXT:     VTX_READ_8 T1.X, T0.X, 4, #1
++; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
++; EG-NEXT:    ALU clause starting at 10:
++; EG-NEXT:     MOV * T0.X, KC0[2].Z,
++; EG-NEXT:    ALU clause starting at 11:
++; EG-NEXT:     AND_INT T0.W, KC0[2].Y, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, T0.X, T1.X,
++; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
++; EG-NEXT:     AND_INT T1.W, PS, 1,
++; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
++; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
++; EG-NEXT:     LSHL T0.X, PV.W, PS,
++; EG-NEXT:     LSHL * T0.W, literal.x, PS,
++; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
++; EG-NEXT:     MOV T0.Y, 0.0,
++; EG-NEXT:     MOV * T0.Z, 0.0,
++; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++entry:
++  %b_ptr = getelementptr i32, ptr addrspace(1) %in, i32 1
++  %a = load i1, ptr addrspace(1) %in
++  %b = load i1, ptr addrspace(1) %b_ptr
++  %result = mul i1 %a, %b
++  store i1 %result, ptr addrspace(1) %out
++  ret void
++}
++
+ ; A standard 64-bit multiply.  The expansion should be around 6 instructions.
+ ; It would be difficult to match the expansion correctly without writing
+ ; a really complicated list of FileCheck expressions.  I don't want
+@@ -144,21 +1358,294 @@ define amdgpu_kernel void @v_mul_i32(ptr addrspace(1) %out, ptr addrspace(1) %in
+ ; so this test just uses FUNC-LABEL to make sure the compiler does not
+ ; crash with a 'failed to select' error.
+ 
+-; FUNC-LABEL: {{^}}s_mul_i64:
+-; GFX9PLUS-DAG: s_mul_i32
+-; GFX9PLUS-DAG: s_mul_hi_u32
+-; GFX9PLUS-DAG: s_mul_i32
+-; GFX9PLUS-DAG: s_mul_i32
+-; GFX9PLUS: s_endpgm
+ define amdgpu_kernel void @s_mul_i64(ptr addrspace(1) %out, i64 %a, i64 %b) nounwind {
++; SI-LABEL: s_mul_i64:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
++; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
++; SI-NEXT:    s_mov_b32 s3, 0xf000
++; SI-NEXT:    s_mov_b32 s2, -1
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mov_b32 s0, s4
++; SI-NEXT:    v_mov_b32_e32 v0, s8
++; SI-NEXT:    v_mul_hi_u32 v0, s6, v0
++; SI-NEXT:    s_mul_i32 s4, s6, s9
++; SI-NEXT:    s_mov_b32 s1, s5
++; SI-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
++; SI-NEXT:    s_mul_i32 s4, s7, s8
++; SI-NEXT:    v_add_i32_e32 v1, vcc, s4, v0
++; SI-NEXT:    s_mul_i32 s4, s6, s8
++; SI-NEXT:    v_mov_b32_e32 v0, s4
++; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: s_mul_i64:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; VI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
++; VI-NEXT:    s_mov_b32 s3, 0xf000
++; VI-NEXT:    s_mov_b32 s2, -1
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mov_b32 s0, s4
++; VI-NEXT:    v_mov_b32_e32 v0, s8
++; VI-NEXT:    v_mad_u64_u32 v[0:1], s[10:11], s6, v0, 0
++; VI-NEXT:    s_mul_i32 s4, s6, s9
++; VI-NEXT:    s_mov_b32 s1, s5
++; VI-NEXT:    v_add_u32_e32 v1, vcc, s4, v1
++; VI-NEXT:    s_mul_i32 s4, s7, s8
++; VI-NEXT:    v_add_u32_e32 v1, vcc, s4, v1
++; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: s_mul_i64:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; GFX9-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
++; GFX9-NEXT:    s_mov_b32 s3, 0xf000
++; GFX9-NEXT:    s_mov_b32 s2, -1
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mov_b32 s0, s4
++; GFX9-NEXT:    s_mov_b32 s1, s5
++; GFX9-NEXT:    s_mul_i32 s4, s6, s9
++; GFX9-NEXT:    s_mul_hi_u32 s5, s6, s8
++; GFX9-NEXT:    s_add_i32 s4, s5, s4
++; GFX9-NEXT:    s_mul_i32 s5, s7, s8
++; GFX9-NEXT:    s_add_i32 s4, s4, s5
++; GFX9-NEXT:    s_mul_i32 s5, s6, s8
++; GFX9-NEXT:    v_mov_b32_e32 v0, s5
++; GFX9-NEXT:    v_mov_b32_e32 v1, s4
++; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: s_mul_i64:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_clause 0x1
++; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mul_i32 s0, s6, s3
++; GFX10-NEXT:    s_mul_hi_u32 s1, s6, s2
++; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX10-NEXT:    s_add_i32 s0, s1, s0
++; GFX10-NEXT:    s_mul_i32 s1, s7, s2
++; GFX10-NEXT:    s_mul_i32 s2, s6, s2
++; GFX10-NEXT:    s_add_i32 s0, s0, s1
++; GFX10-NEXT:    v_mov_b32_e32 v0, s2
++; GFX10-NEXT:    v_mov_b32_e32 v1, s0
++; GFX10-NEXT:    s_mov_b32 s2, -1
++; GFX10-NEXT:    s_mov_b32 s0, s4
++; GFX10-NEXT:    s_mov_b32 s1, s5
++; GFX10-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: s_mul_i64:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_clause 0x1
++; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
++; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
++; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mul_i32 s1, s6, s1
++; GFX11-NEXT:    s_mul_hi_u32 s2, s6, s0
++; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
++; GFX11-NEXT:    s_add_i32 s1, s2, s1
++; GFX11-NEXT:    s_mul_i32 s2, s7, s0
++; GFX11-NEXT:    s_mul_i32 s0, s6, s0
++; GFX11-NEXT:    s_add_i32 s1, s1, s2
++; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
++; GFX11-NEXT:    s_mov_b32 s2, -1
++; GFX11-NEXT:    s_mov_b32 s0, s4
++; GFX11-NEXT:    s_mov_b32 s1, s5
++; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: s_mul_i64:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 7, @4, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    ALU clause starting at 4:
++; EG-NEXT:     MULHI * T0.X, KC0[2].W, KC0[3].Y,
++; EG-NEXT:     MULLO_INT * T0.Y, KC0[2].W, KC0[3].Z,
++; EG-NEXT:     ADD_INT T0.W, T0.X, PS,
++; EG-NEXT:     MULLO_INT * T0.X, KC0[3].X, KC0[3].Y,
++; EG-NEXT:     ADD_INT * T0.Y, PV.W, PS,
++; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++; EG-NEXT:     MULLO_INT * T0.X, KC0[2].W, KC0[3].Y,
++entry:
+   %mul = mul i64 %a, %b
+   store i64 %mul, ptr addrspace(1) %out, align 8
+   ret void
+ }
+ 
+-; FUNC-LABEL: {{^}}v_mul_i64:
+-; GCN: v_mul_lo_u32
+ define amdgpu_kernel void @v_mul_i64(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) {
++; SI-LABEL: v_mul_i64:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
++; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
++; SI-NEXT:    s_mov_b32 s3, 0xf000
++; SI-NEXT:    s_mov_b32 s2, -1
++; SI-NEXT:    s_mov_b32 s10, s2
++; SI-NEXT:    s_mov_b32 s11, s3
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mov_b32 s12, s6
++; SI-NEXT:    s_mov_b32 s13, s7
++; SI-NEXT:    s_mov_b32 s14, s2
++; SI-NEXT:    s_mov_b32 s15, s3
++; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
++; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[12:15], 0
++; SI-NEXT:    s_mov_b32 s0, s4
++; SI-NEXT:    s_mov_b32 s1, s5
++; SI-NEXT:    s_waitcnt vmcnt(0)
++; SI-NEXT:    v_mul_lo_u32 v1, v2, v1
++; SI-NEXT:    v_mul_hi_u32 v4, v2, v0
++; SI-NEXT:    v_mul_lo_u32 v3, v3, v0
++; SI-NEXT:    v_mul_lo_u32 v0, v2, v0
++; SI-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
++; SI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
++; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: v_mul_i64:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; VI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
++; VI-NEXT:    s_mov_b32 s3, 0xf000
++; VI-NEXT:    s_mov_b32 s2, -1
++; VI-NEXT:    s_mov_b32 s10, s2
++; VI-NEXT:    s_mov_b32 s11, s3
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mov_b32 s12, s6
++; VI-NEXT:    s_mov_b32 s13, s7
++; VI-NEXT:    s_mov_b32 s14, s2
++; VI-NEXT:    s_mov_b32 s15, s3
++; VI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
++; VI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[12:15], 0
++; VI-NEXT:    s_mov_b32 s0, s4
++; VI-NEXT:    s_mov_b32 s1, s5
++; VI-NEXT:    s_waitcnt vmcnt(0)
++; VI-NEXT:    v_mul_lo_u32 v4, v2, v1
++; VI-NEXT:    v_mad_u64_u32 v[1:2], s[6:7], v2, v0, 0
++; VI-NEXT:    v_mul_lo_u32 v0, v3, v0
++; VI-NEXT:    v_add_u32_e32 v2, vcc, v4, v2
++; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v0
++; VI-NEXT:    buffer_store_dwordx2 v[1:2], off, s[0:3], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: v_mul_i64:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; GFX9-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
++; GFX9-NEXT:    s_mov_b32 s3, 0xf000
++; GFX9-NEXT:    s_mov_b32 s2, -1
++; GFX9-NEXT:    s_mov_b32 s10, s2
++; GFX9-NEXT:    s_mov_b32 s11, s3
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mov_b32 s12, s6
++; GFX9-NEXT:    s_mov_b32 s13, s7
++; GFX9-NEXT:    s_mov_b32 s14, s2
++; GFX9-NEXT:    s_mov_b32 s15, s3
++; GFX9-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
++; GFX9-NEXT:    buffer_load_dwordx2 v[2:3], off, s[12:15], 0
++; GFX9-NEXT:    s_mov_b32 s0, s4
++; GFX9-NEXT:    s_mov_b32 s1, s5
++; GFX9-NEXT:    s_waitcnt vmcnt(0)
++; GFX9-NEXT:    v_mul_lo_u32 v1, v2, v1
++; GFX9-NEXT:    v_mul_hi_u32 v4, v2, v0
++; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v0
++; GFX9-NEXT:    v_mul_lo_u32 v0, v2, v0
++; GFX9-NEXT:    v_add_u32_e32 v1, v4, v1
++; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
++; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: v_mul_i64:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_clause 0x1
++; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
++; GFX10-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x34
++; GFX10-NEXT:    s_mov_b32 s2, -1
++; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s10, s2
++; GFX10-NEXT:    s_mov_b32 s11, s3
++; GFX10-NEXT:    s_mov_b32 s14, s2
++; GFX10-NEXT:    s_mov_b32 s15, s3
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mov_b32 s12, s6
++; GFX10-NEXT:    s_mov_b32 s13, s7
++; GFX10-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
++; GFX10-NEXT:    buffer_load_dwordx2 v[2:3], off, s[12:15], 0
++; GFX10-NEXT:    s_mov_b32 s0, s4
++; GFX10-NEXT:    s_mov_b32 s1, s5
++; GFX10-NEXT:    s_waitcnt vmcnt(0)
++; GFX10-NEXT:    v_mul_lo_u32 v1, v2, v1
++; GFX10-NEXT:    v_mul_hi_u32 v4, v2, v0
++; GFX10-NEXT:    v_mul_lo_u32 v3, v3, v0
++; GFX10-NEXT:    v_mul_lo_u32 v0, v2, v0
++; GFX10-NEXT:    v_add_nc_u32_e32 v1, v4, v1
++; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v3
++; GFX10-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: v_mul_i64:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_clause 0x1
++; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
++; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
++; GFX11-NEXT:    s_mov_b32 s10, -1
++; GFX11-NEXT:    s_mov_b32 s11, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s2, s10
++; GFX11-NEXT:    s_mov_b32 s3, s11
++; GFX11-NEXT:    s_mov_b32 s14, s10
++; GFX11-NEXT:    s_mov_b32 s15, s11
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mov_b32 s12, s6
++; GFX11-NEXT:    s_mov_b32 s13, s7
++; GFX11-NEXT:    buffer_load_b64 v[0:1], off, s[0:3], 0
++; GFX11-NEXT:    buffer_load_b64 v[2:3], off, s[12:15], 0
++; GFX11-NEXT:    s_mov_b32 s8, s4
++; GFX11-NEXT:    s_mov_b32 s9, s5
++; GFX11-NEXT:    s_waitcnt vmcnt(0)
++; GFX11-NEXT:    v_mul_lo_u32 v1, v2, v1
++; GFX11-NEXT:    v_mul_hi_u32 v4, v2, v0
++; GFX11-NEXT:    v_mul_lo_u32 v3, v3, v0
++; GFX11-NEXT:    v_mul_lo_u32 v0, v2, v0
++; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
++; GFX11-NEXT:    v_add_nc_u32_e32 v1, v4, v1
++; GFX11-NEXT:    v_add_nc_u32_e32 v1, v1, v3
++; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[8:11], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: v_mul_i64:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 1, @10, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    TEX 1 @6
++; EG-NEXT:    ALU 7, @12, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T2.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 6:
++; EG-NEXT:     VTX_READ_64 T1.XY, T1.X, 0, #1
++; EG-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
++; EG-NEXT:    ALU clause starting at 10:
++; EG-NEXT:     MOV T0.X, KC0[2].Z,
++; EG-NEXT:     MOV * T1.X, KC0[2].W,
++; EG-NEXT:    ALU clause starting at 12:
++; EG-NEXT:     MULHI * T0.Z, T0.X, T1.X,
++; EG-NEXT:     MULLO_INT * T0.W, T0.X, T1.Y,
++; EG-NEXT:     ADD_INT T0.W, T0.Z, PS,
++; EG-NEXT:     MULLO_INT * T0.Y, T0.Y, T1.X,
++; EG-NEXT:     ADD_INT * T0.Y, PV.W, PS,
++; EG-NEXT:     LSHR T2.X, KC0[2].Y, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, T0.X, T1.X,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++entry:
+   %a = load i64, ptr addrspace(1) %aptr, align 8
+   %b = load i64, ptr addrspace(1) %bptr, align 8
+   %mul = mul i64 %a, %b
+@@ -166,9 +1653,220 @@ define amdgpu_kernel void @v_mul_i64(ptr addrspace(1) %out, ptr addrspace(1) %ap
+   ret void
+ }
+ 
+-; FUNC-LABEL: {{^}}mul32_in_branch:
+-; GCN: s_mul_i32
+ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %a, i32 %b, i32 %c) {
++; SI-LABEL: mul32_in_branch:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_cmp_lg_u32 s2, 0
++; SI-NEXT:    s_cbranch_scc0 .LBB13_2
++; SI-NEXT:  ; %bb.1: ; %else
++; SI-NEXT:    s_mul_i32 s6, s2, s3
++; SI-NEXT:    s_mov_b64 s[4:5], 0
++; SI-NEXT:    s_branch .LBB13_3
++; SI-NEXT:  .LBB13_2:
++; SI-NEXT:    s_mov_b64 s[4:5], -1
++; SI-NEXT:    ; implicit-def: $sgpr6
++; SI-NEXT:  .LBB13_3: ; %Flow
++; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
++; SI-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mov_b64 vcc, vcc
++; SI-NEXT:    s_cbranch_vccnz .LBB13_5
++; SI-NEXT:  ; %bb.4: ; %if
++; SI-NEXT:    s_mov_b32 s7, 0xf000
++; SI-NEXT:    s_mov_b32 s6, -1
++; SI-NEXT:    s_mov_b32 s4, s2
++; SI-NEXT:    s_mov_b32 s5, s3
++; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
++; SI-NEXT:    s_branch .LBB13_6
++; SI-NEXT:  .LBB13_5:
++; SI-NEXT:    v_mov_b32_e32 v0, s6
++; SI-NEXT:  .LBB13_6: ; %endif
++; SI-NEXT:    s_mov_b32 s3, 0xf000
++; SI-NEXT:    s_mov_b32 s2, -1
++; SI-NEXT:    s_waitcnt vmcnt(0)
++; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: mul32_in_branch:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_cmp_lg_u32 s2, 0
++; VI-NEXT:    s_cbranch_scc0 .LBB13_2
++; VI-NEXT:  ; %bb.1: ; %else
++; VI-NEXT:    s_mul_i32 s6, s2, s3
++; VI-NEXT:    s_mov_b64 s[4:5], 0
++; VI-NEXT:    s_branch .LBB13_3
++; VI-NEXT:  .LBB13_2:
++; VI-NEXT:    s_mov_b64 s[4:5], -1
++; VI-NEXT:    ; implicit-def: $sgpr6
++; VI-NEXT:  .LBB13_3: ; %Flow
++; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; VI-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
++; VI-NEXT:    s_cbranch_vccnz .LBB13_5
++; VI-NEXT:  ; %bb.4: ; %if
++; VI-NEXT:    s_mov_b32 s7, 0xf000
++; VI-NEXT:    s_mov_b32 s6, -1
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mov_b32 s4, s2
++; VI-NEXT:    s_mov_b32 s5, s3
++; VI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
++; VI-NEXT:    s_branch .LBB13_6
++; VI-NEXT:  .LBB13_5:
++; VI-NEXT:    v_mov_b32_e32 v0, s6
++; VI-NEXT:  .LBB13_6: ; %endif
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_mov_b32 s3, 0xf000
++; VI-NEXT:    s_mov_b32 s2, -1
++; VI-NEXT:    s_waitcnt vmcnt(0)
++; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: mul32_in_branch:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_cmp_lg_u32 s2, 0
++; GFX9-NEXT:    s_cbranch_scc0 .LBB13_2
++; GFX9-NEXT:  ; %bb.1: ; %else
++; GFX9-NEXT:    s_mul_i32 s6, s2, s3
++; GFX9-NEXT:    s_mov_b64 s[4:5], 0
++; GFX9-NEXT:    s_branch .LBB13_3
++; GFX9-NEXT:  .LBB13_2:
++; GFX9-NEXT:    s_mov_b64 s[4:5], -1
++; GFX9-NEXT:    ; implicit-def: $sgpr6
++; GFX9-NEXT:  .LBB13_3: ; %Flow
++; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX9-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
++; GFX9-NEXT:    s_cbranch_vccnz .LBB13_5
++; GFX9-NEXT:  ; %bb.4: ; %if
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mov_b32 s4, s2
++; GFX9-NEXT:    s_mov_b32 s5, s3
++; GFX9-NEXT:    buffer_load_dword v0, off, s[4:7], 0
++; GFX9-NEXT:    s_branch .LBB13_6
++; GFX9-NEXT:  .LBB13_5:
++; GFX9-NEXT:    v_mov_b32_e32 v0, s6
++; GFX9-NEXT:  .LBB13_6: ; %endif
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mov_b32 s3, 0xf000
++; GFX9-NEXT:    s_mov_b32 s2, -1
++; GFX9-NEXT:    s_waitcnt vmcnt(0)
++; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: mul32_in_branch:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
++; GFX10-NEXT:    s_mov_b32 s4, 0
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_cmp_lg_u32 s2, 0
++; GFX10-NEXT:    s_cbranch_scc0 .LBB13_2
++; GFX10-NEXT:  ; %bb.1: ; %else
++; GFX10-NEXT:    s_mul_i32 s5, s2, s3
++; GFX10-NEXT:    s_branch .LBB13_3
++; GFX10-NEXT:  .LBB13_2:
++; GFX10-NEXT:    s_mov_b32 s4, -1
++; GFX10-NEXT:    ; implicit-def: $sgpr5
++; GFX10-NEXT:  .LBB13_3: ; %Flow
++; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
++; GFX10-NEXT:    s_andn2_b32 vcc_lo, exec_lo, s4
++; GFX10-NEXT:    s_cbranch_vccnz .LBB13_5
++; GFX10-NEXT:  ; %bb.4: ; %if
++; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mov_b32 s4, s2
++; GFX10-NEXT:    s_mov_b32 s5, s3
++; GFX10-NEXT:    buffer_load_dword v0, off, s[4:7], 0
++; GFX10-NEXT:    s_branch .LBB13_6
++; GFX10-NEXT:  .LBB13_5:
++; GFX10-NEXT:    v_mov_b32_e32 v0, s5
++; GFX10-NEXT:  .LBB13_6: ; %endif
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s2, -1
++; GFX10-NEXT:    s_waitcnt vmcnt(0)
++; GFX10-NEXT:    buffer_store_dword v0, off, s[0:3], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: mul32_in_branch:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x34
++; GFX11-NEXT:    s_mov_b32 s4, 0
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_cmp_lg_u32 s2, 0
++; GFX11-NEXT:    s_cbranch_scc0 .LBB13_2
++; GFX11-NEXT:  ; %bb.1: ; %else
++; GFX11-NEXT:    s_mul_i32 s5, s2, s3
++; GFX11-NEXT:    s_branch .LBB13_3
++; GFX11-NEXT:  .LBB13_2:
++; GFX11-NEXT:    s_mov_b32 s4, -1
++; GFX11-NEXT:    ; implicit-def: $sgpr5
++; GFX11-NEXT:  .LBB13_3: ; %Flow
++; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
++; GFX11-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
++; GFX11-NEXT:    s_cbranch_vccnz .LBB13_5
++; GFX11-NEXT:  ; %bb.4: ; %if
++; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s6, -1
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mov_b32 s4, s2
++; GFX11-NEXT:    s_mov_b32 s5, s3
++; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
++; GFX11-NEXT:    s_branch .LBB13_6
++; GFX11-NEXT:  .LBB13_5:
++; GFX11-NEXT:    v_mov_b32_e32 v0, s5
++; GFX11-NEXT:  .LBB13_6: ; %endif
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s2, -1
++; GFX11-NEXT:    s_waitcnt vmcnt(0)
++; GFX11-NEXT:    buffer_store_b32 v0, off, s[0:3], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: mul32_in_branch:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU_PUSH_BEFORE 3, @14, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    JUMP @3 POP:1
++; EG-NEXT:    ALU_POP_AFTER 4, @18, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    ALU_PUSH_BEFORE 2, @23, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    JUMP @8 POP:1
++; EG-NEXT:    ALU 0, @26, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    TEX 0 @12
++; EG-NEXT:    POP @8 POP:1
++; EG-NEXT:    ALU 1, @27, KC0[], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 12:
++; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
++; EG-NEXT:    ALU clause starting at 14:
++; EG-NEXT:     MOV T0.W, literal.x,
++; EG-NEXT:     SETNE_INT * T1.W, KC0[2].W, 0.0,
++; EG-NEXT:    1(1.401298e-45), 0(0.000000e+00)
++; EG-NEXT:     PRED_SETNE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0,
++; EG-NEXT:    ALU clause starting at 18:
++; EG-NEXT:     MOV T1.W, KC0[2].W,
++; EG-NEXT:     MOV * T2.W, KC0[3].X,
++; EG-NEXT:     MOV T0.W, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, PV.W, PS,
++; EG-NEXT:    0(0.000000e+00), 0(0.000000e+00)
++; EG-NEXT:    ALU clause starting at 23:
++; EG-NEXT:     MOV T1.W, KC0[2].Y,
++; EG-NEXT:     SETE_INT * T0.W, T0.W, 0.0,
++; EG-NEXT:     PRED_SETE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0,
++; EG-NEXT:    ALU clause starting at 26:
++; EG-NEXT:     MOV * T0.X, KC0[2].Z,
++; EG-NEXT:    ALU clause starting at 27:
++; EG-NEXT:     LSHR * T1.X, T1.W, literal.x,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+ entry:
+   %0 = icmp eq i32 %a, 0
+   br i1 %0, label %if, label %else
+@@ -187,12 +1885,227 @@ endif:
+   ret void
+ }
+ 
+-; FUNC-LABEL: {{^}}mul64_in_branch:
+-; SI-DAG: s_mul_i32
+-; SI-DAG: v_mul_hi_u32
+-; VI: v_mad_u64_u32
+-; GCN: s_endpgm
+ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(1) %in, i64 %a, i64 %b, i64 %c) {
++; SI-LABEL: mul64_in_branch:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
++; SI-NEXT:    s_mov_b64 s[8:9], 0
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    v_cmp_ne_u64_e64 s[10:11], s[4:5], 0
++; SI-NEXT:    s_and_b64 vcc, exec, s[10:11]
++; SI-NEXT:    s_cbranch_vccz .LBB14_4
++; SI-NEXT:  ; %bb.1: ; %else
++; SI-NEXT:    v_mov_b32_e32 v0, s6
++; SI-NEXT:    v_mul_hi_u32 v0, s4, v0
++; SI-NEXT:    s_mul_i32 s7, s4, s7
++; SI-NEXT:    s_mul_i32 s5, s5, s6
++; SI-NEXT:    s_mul_i32 s4, s4, s6
++; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
++; SI-NEXT:    v_add_i32_e32 v1, vcc, s5, v0
++; SI-NEXT:    v_mov_b32_e32 v0, s4
++; SI-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
++; SI-NEXT:    s_cbranch_vccnz .LBB14_3
++; SI-NEXT:  .LBB14_2: ; %if
++; SI-NEXT:    s_mov_b32 s7, 0xf000
++; SI-NEXT:    s_mov_b32 s6, -1
++; SI-NEXT:    s_mov_b32 s4, s2
++; SI-NEXT:    s_mov_b32 s5, s3
++; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
++; SI-NEXT:  .LBB14_3: ; %endif
++; SI-NEXT:    s_mov_b32 s3, 0xf000
++; SI-NEXT:    s_mov_b32 s2, -1
++; SI-NEXT:    s_waitcnt vmcnt(0)
++; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; SI-NEXT:    s_endpgm
++; SI-NEXT:  .LBB14_4:
++; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1
++; SI-NEXT:    s_branch .LBB14_2
++;
++; VI-LABEL: mul64_in_branch:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
++; VI-NEXT:    s_mov_b64 s[8:9], 0
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    s_cmp_lg_u64 s[4:5], 0
++; VI-NEXT:    s_cbranch_scc0 .LBB14_4
++; VI-NEXT:  ; %bb.1: ; %else
++; VI-NEXT:    v_mov_b32_e32 v0, s6
++; VI-NEXT:    v_mad_u64_u32 v[0:1], s[10:11], s4, v0, 0
++; VI-NEXT:    s_mul_i32 s4, s4, s7
++; VI-NEXT:    v_add_u32_e32 v1, vcc, s4, v1
++; VI-NEXT:    s_mul_i32 s4, s5, s6
++; VI-NEXT:    v_add_u32_e32 v1, vcc, s4, v1
++; VI-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
++; VI-NEXT:    s_cbranch_vccnz .LBB14_3
++; VI-NEXT:  .LBB14_2: ; %if
++; VI-NEXT:    s_mov_b32 s7, 0xf000
++; VI-NEXT:    s_mov_b32 s6, -1
++; VI-NEXT:    s_mov_b32 s4, s2
++; VI-NEXT:    s_mov_b32 s5, s3
++; VI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
++; VI-NEXT:  .LBB14_3: ; %endif
++; VI-NEXT:    s_mov_b32 s3, 0xf000
++; VI-NEXT:    s_mov_b32 s2, -1
++; VI-NEXT:    s_waitcnt vmcnt(0)
++; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; VI-NEXT:    s_endpgm
++; VI-NEXT:  .LBB14_4:
++; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1
++; VI-NEXT:    s_branch .LBB14_2
++;
++; GFX9-LABEL: mul64_in_branch:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
++; GFX9-NEXT:    s_mov_b64 s[8:9], 0
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_cmp_lg_u64 s[4:5], 0
++; GFX9-NEXT:    s_cbranch_scc0 .LBB14_3
++; GFX9-NEXT:  ; %bb.1: ; %else
++; GFX9-NEXT:    s_mul_i32 s7, s4, s7
++; GFX9-NEXT:    s_mul_hi_u32 s10, s4, s6
++; GFX9-NEXT:    s_add_i32 s7, s10, s7
++; GFX9-NEXT:    s_mul_i32 s5, s5, s6
++; GFX9-NEXT:    s_add_i32 s5, s7, s5
++; GFX9-NEXT:    s_mul_i32 s4, s4, s6
++; GFX9-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
++; GFX9-NEXT:    s_cbranch_vccnz .LBB14_4
++; GFX9-NEXT:  .LBB14_2: ; %if
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_mov_b32 s4, s2
++; GFX9-NEXT:    s_mov_b32 s5, s3
++; GFX9-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
++; GFX9-NEXT:    s_branch .LBB14_5
++; GFX9-NEXT:  .LBB14_3:
++; GFX9-NEXT:    ; implicit-def: $sgpr4_sgpr5
++; GFX9-NEXT:    s_branch .LBB14_2
++; GFX9-NEXT:  .LBB14_4:
++; GFX9-NEXT:    v_mov_b32_e32 v0, s4
++; GFX9-NEXT:    v_mov_b32_e32 v1, s5
++; GFX9-NEXT:  .LBB14_5: ; %endif
++; GFX9-NEXT:    s_mov_b32 s3, 0xf000
++; GFX9-NEXT:    s_mov_b32 s2, -1
++; GFX9-NEXT:    s_waitcnt vmcnt(0)
++; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: mul64_in_branch:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_cmp_lg_u64 s[4:5], 0
++; GFX10-NEXT:    s_cbranch_scc0 .LBB14_3
++; GFX10-NEXT:  ; %bb.1: ; %else
++; GFX10-NEXT:    s_mul_i32 s7, s4, s7
++; GFX10-NEXT:    s_mul_hi_u32 s8, s4, s6
++; GFX10-NEXT:    s_mul_i32 s5, s5, s6
++; GFX10-NEXT:    s_add_i32 s7, s8, s7
++; GFX10-NEXT:    s_mul_i32 s4, s4, s6
++; GFX10-NEXT:    s_add_i32 s5, s7, s5
++; GFX10-NEXT:    s_mov_b32 s6, 0
++; GFX10-NEXT:    s_cbranch_execnz .LBB14_4
++; GFX10-NEXT:  .LBB14_2: ; %if
++; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    s_mov_b32 s4, s2
++; GFX10-NEXT:    s_mov_b32 s5, s3
++; GFX10-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
++; GFX10-NEXT:    s_branch .LBB14_5
++; GFX10-NEXT:  .LBB14_3:
++; GFX10-NEXT:    s_mov_b32 s6, -1
++; GFX10-NEXT:    ; implicit-def: $sgpr4_sgpr5
++; GFX10-NEXT:    s_branch .LBB14_2
++; GFX10-NEXT:  .LBB14_4:
++; GFX10-NEXT:    v_mov_b32_e32 v0, s4
++; GFX10-NEXT:    v_mov_b32_e32 v1, s5
++; GFX10-NEXT:  .LBB14_5: ; %endif
++; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s2, -1
++; GFX10-NEXT:    s_waitcnt vmcnt(0)
++; GFX10-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: mul64_in_branch:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_cmp_lg_u64 s[4:5], 0
++; GFX11-NEXT:    s_cbranch_scc0 .LBB14_3
++; GFX11-NEXT:  ; %bb.1: ; %else
++; GFX11-NEXT:    s_mul_i32 s7, s4, s7
++; GFX11-NEXT:    s_mul_hi_u32 s8, s4, s6
++; GFX11-NEXT:    s_mul_i32 s5, s5, s6
++; GFX11-NEXT:    s_add_i32 s7, s8, s7
++; GFX11-NEXT:    s_mul_i32 s4, s4, s6
++; GFX11-NEXT:    s_add_i32 s5, s7, s5
++; GFX11-NEXT:    s_mov_b32 s6, 0
++; GFX11-NEXT:    s_cbranch_execnz .LBB14_4
++; GFX11-NEXT:  .LBB14_2: ; %if
++; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s6, -1
++; GFX11-NEXT:    s_mov_b32 s4, s2
++; GFX11-NEXT:    s_mov_b32 s5, s3
++; GFX11-NEXT:    buffer_load_b64 v[0:1], off, s[4:7], 0
++; GFX11-NEXT:    s_branch .LBB14_5
++; GFX11-NEXT:  .LBB14_3:
++; GFX11-NEXT:    s_mov_b32 s6, -1
++; GFX11-NEXT:    ; implicit-def: $sgpr4_sgpr5
++; GFX11-NEXT:    s_branch .LBB14_2
++; GFX11-NEXT:  .LBB14_4:
++; GFX11-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
++; GFX11-NEXT:  .LBB14_5: ; %endif
++; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s2, -1
++; GFX11-NEXT:    s_waitcnt vmcnt(0)
++; GFX11-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: mul64_in_branch:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU_PUSH_BEFORE 4, @14, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    JUMP @3 POP:1
++; EG-NEXT:    ALU_POP_AFTER 11, @19, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    ALU_PUSH_BEFORE 2, @31, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    JUMP @8 POP:1
++; EG-NEXT:    ALU 0, @34, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    TEX 0 @12
++; EG-NEXT:    POP @8 POP:1
++; EG-NEXT:    ALU 1, @35, KC0[], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 12:
++; EG-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
++; EG-NEXT:    ALU clause starting at 14:
++; EG-NEXT:     OR_INT T0.W, KC0[2].W, KC0[3].X,
++; EG-NEXT:     MOV * T1.W, literal.x,
++; EG-NEXT:    1(1.401298e-45), 0(0.000000e+00)
++; EG-NEXT:     SETNE_INT * T0.W, PV.W, 0.0,
++; EG-NEXT:     PRED_SETNE_INT * ExecMask,PredicateBit (MASKED), PV.W, 0.0,
++; EG-NEXT:    ALU clause starting at 19:
++; EG-NEXT:     MOV T0.W, KC0[2].W,
++; EG-NEXT:     MOV * T1.W, KC0[3].Z,
++; EG-NEXT:     MOV T2.W, KC0[3].Y,
++; EG-NEXT:     MULLO_INT * T0.X, PV.W, PS,
++; EG-NEXT:     MOV T1.W, KC0[3].X,
++; EG-NEXT:     MULHI * T0.Y, T0.W, PV.W,
++; EG-NEXT:     ADD_INT T3.W, PS, T0.X,
++; EG-NEXT:     MULLO_INT * T0.X, PV.W, T2.W,
++; EG-NEXT:     ADD_INT T0.Y, PV.W, PS,
++; EG-NEXT:     MOV T1.W, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, T0.W, T2.W,
++; EG-NEXT:    0(0.000000e+00), 0(0.000000e+00)
++; EG-NEXT:    ALU clause starting at 31:
++; EG-NEXT:     MOV T0.W, KC0[2].Y,
++; EG-NEXT:     SETE_INT * T1.W, T1.W, 0.0,
++; EG-NEXT:     PRED_SETE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0,
++; EG-NEXT:    ALU clause starting at 34:
++; EG-NEXT:     MOV * T0.X, KC0[2].Z,
++; EG-NEXT:    ALU clause starting at 35:
++; EG-NEXT:     LSHR * T1.X, T0.W, literal.x,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+ entry:
+   %0 = icmp eq i64 %a, 0
+   br i1 %0, label %if, label %else
+@@ -211,79 +2124,558 @@ endif:
+   ret void
+ }
+ 
+-; FIXME: Load dwordx4
+-; FUNC-LABEL: {{^}}s_mul_i128:
+-; GCN: s_load_dwordx4
+-; GCN: s_load_dwordx4
+-
+-; SI: v_mul_hi_u32
+-; SI: v_mul_hi_u32
+-; SI: s_mul_i32
+-; SI: v_mul_hi_u32
+-; SI: s_mul_i32
+-; SI: s_mul_i32
+-
+-; SI-DAG: s_mul_i32
+-; SI-DAG: v_mul_hi_u32
+-; SI-DAG: v_mul_hi_u32
+-; SI-DAG: s_mul_i32
+-; SI-DAG: s_mul_i32
+-; SI-DAG: v_mul_hi_u32
+-
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: s_mul_i32
+-; VI-DAG: s_mul_i32
+-; VI-DAG: s_mul_i32
+-; VI-DAG: s_mul_i32
+-
+-
+-; GCN: buffer_store_dwordx4
+ define amdgpu_kernel void @s_mul_i128(ptr addrspace(1) %out, [8 x i32], i128 %a, [8 x i32], i128 %b) nounwind #0 {
++; SI-LABEL: s_mul_i128:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x13
++; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x1f
++; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
++; SI-NEXT:    s_mov_b32 s3, 0xf000
++; SI-NEXT:    s_mov_b32 s2, -1
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    v_mov_b32_e32 v0, s6
++; SI-NEXT:    v_mul_hi_u32 v0, s8, v0
++; SI-NEXT:    v_mov_b32_e32 v1, s4
++; SI-NEXT:    v_mul_hi_u32 v1, s10, v1
++; SI-NEXT:    s_mul_i32 s7, s8, s7
++; SI-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
++; SI-NEXT:    s_mul_i32 s7, s10, s5
++; SI-NEXT:    s_mul_i32 s12, s9, s6
++; SI-NEXT:    s_mul_i32 s6, s8, s6
++; SI-NEXT:    v_add_i32_e32 v1, vcc, s7, v1
++; SI-NEXT:    s_mul_i32 s7, s11, s4
++; SI-NEXT:    v_add_i32_e32 v0, vcc, s12, v0
++; SI-NEXT:    v_add_i32_e32 v1, vcc, s7, v1
++; SI-NEXT:    s_mul_i32 s7, s10, s4
++; SI-NEXT:    v_mov_b32_e32 v2, s6
++; SI-NEXT:    v_add_i32_e32 v2, vcc, s7, v2
++; SI-NEXT:    v_addc_u32_e32 v0, vcc, v1, v0, vcc
++; SI-NEXT:    v_mov_b32_e32 v1, s8
++; SI-NEXT:    v_mul_hi_u32 v5, s4, v1
++; SI-NEXT:    v_mul_hi_u32 v1, s5, v1
++; SI-NEXT:    v_mov_b32_e32 v3, s9
++; SI-NEXT:    v_mul_hi_u32 v4, s4, v3
++; SI-NEXT:    s_mul_i32 s7, s5, s8
++; SI-NEXT:    v_add_i32_e32 v5, vcc, s7, v5
++; SI-NEXT:    s_mul_i32 s6, s4, s9
++; SI-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
++; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v5
++; SI-NEXT:    v_mul_hi_u32 v3, s5, v3
++; SI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
++; SI-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
++; SI-NEXT:    s_mul_i32 s5, s5, s9
++; SI-NEXT:    v_addc_u32_e64 v5, s[6:7], 0, 0, vcc
++; SI-NEXT:    v_add_i32_e32 v4, vcc, s5, v4
++; SI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
++; SI-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
++; SI-NEXT:    s_mul_i32 s4, s4, s8
++; SI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v0, vcc
++; SI-NEXT:    v_mov_b32_e32 v0, s4
++; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: s_mul_i128:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x4c
++; VI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x7c
++; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
++; VI-NEXT:    v_mov_b32_e32 v5, 0
++; VI-NEXT:    s_mov_b32 s3, 0xf000
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    v_mov_b32_e32 v0, s6
++; VI-NEXT:    v_mad_u64_u32 v[2:3], s[12:13], s8, v0, 0
++; VI-NEXT:    s_mul_i32 s7, s8, s7
++; VI-NEXT:    v_mov_b32_e32 v6, s8
++; VI-NEXT:    v_add_u32_e32 v3, vcc, s7, v3
++; VI-NEXT:    s_mul_i32 s12, s9, s6
++; VI-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], s4, v6, 0
++; VI-NEXT:    v_add_u32_e32 v3, vcc, s12, v3
++; VI-NEXT:    v_mov_b32_e32 v4, v1
++; VI-NEXT:    v_mad_u64_u32 v[6:7], s[6:7], s5, v6, v[4:5]
++; VI-NEXT:    v_mov_b32_e32 v8, s4
++; VI-NEXT:    v_mad_u64_u32 v[1:2], s[6:7], s10, v8, v[2:3]
++; VI-NEXT:    v_mov_b32_e32 v3, v7
++; VI-NEXT:    v_mov_b32_e32 v7, v5
++; VI-NEXT:    v_mov_b32_e32 v8, s9
++; VI-NEXT:    v_mad_u64_u32 v[4:5], s[6:7], s4, v8, v[6:7]
++; VI-NEXT:    s_mul_i32 s8, s11, s4
++; VI-NEXT:    v_add_u32_e32 v6, vcc, s8, v2
++; VI-NEXT:    v_mov_b32_e32 v2, v5
++; VI-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
++; VI-NEXT:    v_addc_u32_e64 v3, s[6:7], 0, 0, vcc
++; VI-NEXT:    s_mul_i32 s8, s10, s5
++; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], s5, v8, v[2:3]
++; VI-NEXT:    v_add_u32_e32 v5, vcc, s8, v6
++; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v1
++; VI-NEXT:    s_mov_b32 s2, -1
++; VI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
++; VI-NEXT:    v_mov_b32_e32 v1, v4
++; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: s_mul_i128:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x4c
++; GFX9-NEXT:    s_load_dwordx4 s[12:15], s[0:1], 0x7c
++; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
++; GFX9-NEXT:    s_mov_b32 s7, 0xf000
++; GFX9-NEXT:    s_mov_b32 s6, -1
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    s_mul_i32 s0, s12, s11
++; GFX9-NEXT:    s_mul_hi_u32 s1, s12, s10
++; GFX9-NEXT:    s_mul_i32 s2, s14, s9
++; GFX9-NEXT:    s_mul_hi_u32 s3, s14, s8
++; GFX9-NEXT:    s_add_i32 s0, s1, s0
++; GFX9-NEXT:    s_mul_i32 s1, s13, s10
++; GFX9-NEXT:    s_add_i32 s2, s3, s2
++; GFX9-NEXT:    s_mul_i32 s3, s15, s8
++; GFX9-NEXT:    s_add_i32 s0, s0, s1
++; GFX9-NEXT:    s_mul_i32 s1, s12, s10
++; GFX9-NEXT:    s_add_i32 s2, s2, s3
++; GFX9-NEXT:    s_mul_i32 s3, s14, s8
++; GFX9-NEXT:    s_add_u32 s3, s3, s1
++; GFX9-NEXT:    s_addc_u32 s2, s2, s0
++; GFX9-NEXT:    s_mul_i32 s14, s9, s12
++; GFX9-NEXT:    s_mul_hi_u32 s15, s8, s12
++; GFX9-NEXT:    s_mul_hi_u32 s11, s9, s12
++; GFX9-NEXT:    s_add_u32 s14, s14, s15
++; GFX9-NEXT:    s_mul_i32 s1, s8, s13
++; GFX9-NEXT:    s_addc_u32 s11, s11, 0
++; GFX9-NEXT:    s_mul_hi_u32 s10, s8, s13
++; GFX9-NEXT:    s_add_u32 s1, s1, s14
++; GFX9-NEXT:    s_addc_u32 s10, s10, 0
++; GFX9-NEXT:    s_add_u32 s10, s11, s10
++; GFX9-NEXT:    s_addc_u32 s11, 0, 0
++; GFX9-NEXT:    s_mul_hi_u32 s14, s9, s13
++; GFX9-NEXT:    s_mul_i32 s9, s9, s13
++; GFX9-NEXT:    s_add_u32 s9, s9, s10
++; GFX9-NEXT:    s_addc_u32 s10, s14, s11
++; GFX9-NEXT:    s_mov_b32 s0, 0
++; GFX9-NEXT:    s_add_u32 s9, s9, s3
++; GFX9-NEXT:    s_addc_u32 s10, s10, s2
++; GFX9-NEXT:    s_mul_i32 s2, s8, s12
++; GFX9-NEXT:    s_mov_b32 s3, s0
++; GFX9-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
++; GFX9-NEXT:    v_mov_b32_e32 v0, s0
++; GFX9-NEXT:    v_mov_b32_e32 v1, s1
++; GFX9-NEXT:    v_mov_b32_e32 v2, s9
++; GFX9-NEXT:    v_mov_b32_e32 v3, s10
++; GFX9-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: s_mul_i128:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_clause 0x1
++; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x4c
++; GFX10-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x7c
++; GFX10-NEXT:    s_mov_b32 s2, 0
++; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
++; GFX10-NEXT:    s_mov_b32 s13, s2
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_mul_i32 s3, s8, s7
++; GFX10-NEXT:    s_mul_hi_u32 s7, s8, s6
++; GFX10-NEXT:    s_mul_i32 s14, s10, s5
++; GFX10-NEXT:    s_mul_hi_u32 s15, s10, s4
++; GFX10-NEXT:    s_mul_i32 s12, s9, s6
++; GFX10-NEXT:    s_mul_i32 s11, s11, s4
++; GFX10-NEXT:    s_add_i32 s3, s7, s3
++; GFX10-NEXT:    s_add_i32 s7, s15, s14
++; GFX10-NEXT:    s_mul_i32 s6, s8, s6
++; GFX10-NEXT:    s_mul_i32 s10, s10, s4
++; GFX10-NEXT:    s_add_i32 s3, s3, s12
++; GFX10-NEXT:    s_add_i32 s7, s7, s11
++; GFX10-NEXT:    s_mul_i32 s19, s5, s8
++; GFX10-NEXT:    s_mul_hi_u32 s20, s4, s8
++; GFX10-NEXT:    s_add_u32 s6, s10, s6
++; GFX10-NEXT:    s_mul_hi_u32 s18, s5, s8
++; GFX10-NEXT:    s_addc_u32 s7, s7, s3
++; GFX10-NEXT:    s_mul_i32 s17, s4, s9
++; GFX10-NEXT:    s_add_u32 s3, s19, s20
++; GFX10-NEXT:    s_mul_hi_u32 s16, s4, s9
++; GFX10-NEXT:    s_mul_hi_u32 s21, s5, s9
++; GFX10-NEXT:    s_mul_i32 s5, s5, s9
++; GFX10-NEXT:    s_addc_u32 s9, s18, 0
++; GFX10-NEXT:    s_add_u32 s3, s17, s3
++; GFX10-NEXT:    s_addc_u32 s10, s16, 0
++; GFX10-NEXT:    s_mul_i32 s12, s4, s8
++; GFX10-NEXT:    s_add_u32 s4, s9, s10
++; GFX10-NEXT:    s_addc_u32 s8, 0, 0
++; GFX10-NEXT:    s_add_u32 s4, s5, s4
++; GFX10-NEXT:    s_addc_u32 s5, s21, s8
++; GFX10-NEXT:    s_add_u32 s4, s4, s6
++; GFX10-NEXT:    s_addc_u32 s5, s5, s7
++; GFX10-NEXT:    s_or_b64 s[2:3], s[12:13], s[2:3]
++; GFX10-NEXT:    v_mov_b32_e32 v2, s4
++; GFX10-NEXT:    v_mov_b32_e32 v0, s2
++; GFX10-NEXT:    v_mov_b32_e32 v1, s3
++; GFX10-NEXT:    v_mov_b32_e32 v3, s5
++; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX10-NEXT:    s_mov_b32 s2, -1
++; GFX10-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: s_mul_i128:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_clause 0x2
++; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x4c
++; GFX11-NEXT:    s_load_b128 s[8:11], s[0:1], 0x7c
++; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
++; GFX11-NEXT:    s_mov_b32 s2, 0
++; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
++; GFX11-NEXT:    s_mov_b32 s13, s2
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_mul_i32 s3, s8, s7
++; GFX11-NEXT:    s_mul_hi_u32 s7, s8, s6
++; GFX11-NEXT:    s_mul_i32 s14, s10, s5
++; GFX11-NEXT:    s_mul_hi_u32 s15, s10, s4
++; GFX11-NEXT:    s_mul_i32 s12, s9, s6
++; GFX11-NEXT:    s_mul_i32 s11, s11, s4
++; GFX11-NEXT:    s_add_i32 s3, s7, s3
++; GFX11-NEXT:    s_add_i32 s7, s15, s14
++; GFX11-NEXT:    s_mul_i32 s6, s8, s6
++; GFX11-NEXT:    s_mul_i32 s10, s10, s4
++; GFX11-NEXT:    s_add_i32 s3, s3, s12
++; GFX11-NEXT:    s_add_i32 s7, s7, s11
++; GFX11-NEXT:    s_mul_i32 s19, s5, s8
++; GFX11-NEXT:    s_mul_hi_u32 s20, s4, s8
++; GFX11-NEXT:    s_add_u32 s6, s10, s6
++; GFX11-NEXT:    s_mul_hi_u32 s18, s5, s8
++; GFX11-NEXT:    s_addc_u32 s7, s7, s3
++; GFX11-NEXT:    s_mul_i32 s17, s4, s9
++; GFX11-NEXT:    s_add_u32 s3, s19, s20
++; GFX11-NEXT:    s_mul_hi_u32 s16, s4, s9
++; GFX11-NEXT:    s_mul_hi_u32 s21, s5, s9
++; GFX11-NEXT:    s_mul_i32 s5, s5, s9
++; GFX11-NEXT:    s_addc_u32 s9, s18, 0
++; GFX11-NEXT:    s_add_u32 s3, s17, s3
++; GFX11-NEXT:    s_addc_u32 s10, s16, 0
++; GFX11-NEXT:    s_mul_i32 s12, s4, s8
++; GFX11-NEXT:    s_add_u32 s4, s9, s10
++; GFX11-NEXT:    s_addc_u32 s8, 0, 0
++; GFX11-NEXT:    s_add_u32 s4, s5, s4
++; GFX11-NEXT:    s_addc_u32 s5, s21, s8
++; GFX11-NEXT:    s_add_u32 s4, s4, s6
++; GFX11-NEXT:    s_addc_u32 s5, s5, s7
++; GFX11-NEXT:    s_or_b64 s[2:3], s[12:13], s[2:3]
++; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
++; GFX11-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v1, s3
++; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s5
++; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
++; GFX11-NEXT:    s_mov_b32 s2, -1
++; GFX11-NEXT:    buffer_store_b128 v[0:3], off, s[0:3], 0
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: s_mul_i128:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 41, @4, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    ALU clause starting at 4:
++; EG-NEXT:     MULLO_INT * T0.X, KC0[5].X, KC0[8].X,
++; EG-NEXT:     MULHI * T0.Y, KC0[5].X, KC0[8].X,
++; EG-NEXT:     MULLO_INT * T0.Z, KC0[8].Y, KC0[4].W,
++; EG-NEXT:     MULLO_INT * T0.W, KC0[8].X, KC0[5].Y,
++; EG-NEXT:     MULHI * T1.X, KC0[5].X, KC0[7].W,
++; EG-NEXT:     MULHI * T1.Y, KC0[4].W, KC0[8].X,
++; EG-NEXT:     MULHI * T1.Z, KC0[8].Y, KC0[4].W,
++; EG-NEXT:     MULLO_INT * T1.W, KC0[8].Y, KC0[5].X,
++; EG-NEXT:     MULHI * T2.X, KC0[7].W, KC0[5].Y,
++; EG-NEXT:     MULLO_INT * T2.Y, KC0[5].X, KC0[7].W,
++; EG-NEXT:     MULHI * T2.Z, KC0[4].W, KC0[7].W,
++; EG-NEXT:     ADD_INT T2.W, T2.Y, PS,
++; EG-NEXT:     MULLO_INT * T3.X, KC0[4].W, KC0[8].X,
++; EG-NEXT:     ADDC_UINT T2.Z, T2.Y, T2.Z,
++; EG-NEXT:     ADDC_UINT T3.W, PS, PV.W,
++; EG-NEXT:     MULLO_INT * T2.Y, KC0[7].W, KC0[5].Z,
++; EG-NEXT:     ADD_INT T2.X, T2.X, PS,
++; EG-NEXT:     ADD_INT T2.Y, T1.Z, T1.W,
++; EG-NEXT:     ADD_INT T1.Z, T1.Y, PV.W,
++; EG-NEXT:     ADD_INT T1.W, T1.X, PV.Z, BS:VEC_120/SCL_212
++; EG-NEXT:     MULLO_INT * T1.X, KC0[8].Z, KC0[4].W,
++; EG-NEXT:     ADD_INT T4.X, PV.W, PV.Z,
++; EG-NEXT:     ADDC_UINT T1.Y, PV.W, PV.Z,
++; EG-NEXT:     ADD_INT T1.Z, PV.Y, PS,
++; EG-NEXT:     ADD_INT T0.W, PV.X, T0.W,
++; EG-NEXT:     MULLO_INT * T1.X, KC0[7].W, KC0[5].Y,
++; EG-NEXT:     ADD_INT T2.Y, PV.Z, PV.W,
++; EG-NEXT:     ADDC_UINT T1.Z, T0.Z, PS,
++; EG-NEXT:     ADD_INT T0.W, T0.Y, PV.Y,
++; EG-NEXT:     ADDC_UINT * T1.W, T0.X, PV.X,
++; EG-NEXT:     ADD_INT T0.Y, T0.X, T4.X,
++; EG-NEXT:     ADD_INT T0.Z, T0.Z, T1.X, BS:VEC_021/SCL_122
++; EG-NEXT:     ADD_INT T0.W, PV.W, PS,
++; EG-NEXT:     ADD_INT * T1.W, PV.Y, PV.Z,
++; EG-NEXT:     ADD_INT T0.W, PV.W, PS,
++; EG-NEXT:     ADDC_UINT * T1.W, PV.Y, PV.Z,
++; EG-NEXT:     ADD_INT * T0.W, PV.W, PS,
++; EG-NEXT:     ADD_INT * T0.Z, T0.Y, T0.Z,
++; EG-NEXT:     ADD_INT * T0.Y, T3.X, T2.W,
++; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++; EG-NEXT:     MULLO_INT * T0.X, KC0[4].W, KC0[7].W,
++entry:
+   %mul = mul i128 %a, %b
+   store i128 %mul, ptr addrspace(1) %out
+   ret void
+ }
+ 
+-; FUNC-LABEL: {{^}}v_mul_i128:
+-; GCN: {{buffer|flat}}_load_dwordx4
+-; GCN: {{buffer|flat}}_load_dwordx4
+-
+-; SI-DAG: v_mul_lo_u32
+-; SI-DAG: v_mul_hi_u32
+-; SI-DAG: v_mul_hi_u32
+-; SI-DAG: v_mul_lo_u32
+-; SI-DAG: v_mul_hi_u32
+-; SI-DAG: v_mul_hi_u32
+-; SI-DAG: v_mul_lo_u32
+-; SI-DAG: v_mul_lo_u32
+-; SI-DAG: v_add_i32_e32
+-
+-; SI-DAG: v_mul_hi_u32
+-; SI-DAG: v_mul_lo_u32
+-; SI-DAG: v_mul_hi_u32
+-; SI-DAG: v_mul_lo_u32
+-; SI-DAG: v_mul_lo_u32
+-; SI-DAG: v_mul_lo_u32
+-; SI-DAG: v_mul_lo_u32
+-; SI-DAG: v_mul_lo_u32
+-
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: v_mad_u64_u32
+-; VI-DAG: v_mul_lo_u32
+-; VI-DAG: v_mul_lo_u32
+-; VI-DAG: v_mul_lo_u32
+-
+-; GCN: {{buffer|flat}}_store_dwordx4
+ define amdgpu_kernel void @v_mul_i128(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) #0 {
++; SI-LABEL: v_mul_i128:
++; SI:       ; %bb.0: ; %entry
++; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xb
++; SI-NEXT:    s_mov_b32 s7, 0xf000
++; SI-NEXT:    s_mov_b32 s6, 0
++; SI-NEXT:    v_lshlrev_b32_e32 v8, 4, v0
++; SI-NEXT:    v_mov_b32_e32 v9, 0
++; SI-NEXT:    s_waitcnt lgkmcnt(0)
++; SI-NEXT:    s_mov_b64 s[4:5], s[0:1]
++; SI-NEXT:    s_mov_b64 s[0:1], s[2:3]
++; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
++; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[8:9], s[4:7], 0 addr64
++; SI-NEXT:    buffer_load_dwordx4 v[4:7], v[8:9], s[0:3], 0 addr64
++; SI-NEXT:    s_waitcnt vmcnt(0)
++; SI-NEXT:    v_mul_lo_u32 v3, v4, v3
++; SI-NEXT:    v_mul_hi_u32 v10, v4, v2
++; SI-NEXT:    v_mul_lo_u32 v12, v6, v1
++; SI-NEXT:    v_mul_hi_u32 v13, v6, v0
++; SI-NEXT:    v_mul_lo_u32 v17, v1, v4
++; SI-NEXT:    v_mul_hi_u32 v18, v0, v4
++; SI-NEXT:    v_mul_lo_u32 v11, v5, v2
++; SI-NEXT:    v_mul_lo_u32 v7, v7, v0
++; SI-NEXT:    v_mul_hi_u32 v16, v1, v4
++; SI-NEXT:    v_mul_lo_u32 v15, v0, v5
++; SI-NEXT:    v_mul_hi_u32 v14, v0, v5
++; SI-NEXT:    v_mul_hi_u32 v19, v1, v5
++; SI-NEXT:    v_mul_lo_u32 v5, v1, v5
++; SI-NEXT:    v_add_i32_e32 v1, vcc, v10, v3
++; SI-NEXT:    v_add_i32_e32 v3, vcc, v13, v12
++; SI-NEXT:    v_mul_lo_u32 v2, v4, v2
++; SI-NEXT:    v_mul_lo_u32 v6, v6, v0
++; SI-NEXT:    v_mul_lo_u32 v0, v0, v4
++; SI-NEXT:    v_add_i32_e32 v4, vcc, v17, v18
++; SI-NEXT:    v_addc_u32_e32 v10, vcc, 0, v16, vcc
++; SI-NEXT:    v_add_i32_e32 v11, vcc, v1, v11
++; SI-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
++; SI-NEXT:    v_add_i32_e32 v1, vcc, v15, v4
++; SI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v14, vcc
++; SI-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
++; SI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v11, vcc
++; SI-NEXT:    v_add_i32_e32 v4, vcc, v10, v4
++; SI-NEXT:    v_addc_u32_e64 v6, s[4:5], 0, 0, vcc
++; SI-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
++; SI-NEXT:    v_addc_u32_e32 v5, vcc, v19, v6, vcc
++; SI-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
++; SI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
++; SI-NEXT:    buffer_store_dwordx4 v[0:3], v[8:9], s[0:3], 0 addr64
++; SI-NEXT:    s_endpgm
++;
++; VI-LABEL: v_mul_i128:
++; VI:       ; %bb.0: ; %entry
++; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x2c
++; VI-NEXT:    v_lshlrev_b32_e32 v2, 4, v0
++; VI-NEXT:    v_mov_b32_e32 v11, 0
++; VI-NEXT:    s_waitcnt lgkmcnt(0)
++; VI-NEXT:    v_mov_b32_e32 v1, s1
++; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
++; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
++; VI-NEXT:    v_mov_b32_e32 v3, s3
++; VI-NEXT:    v_add_u32_e32 v8, vcc, s2, v2
++; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v3, vcc
++; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
++; VI-NEXT:    flat_load_dwordx4 v[4:7], v[8:9]
++; VI-NEXT:    s_waitcnt vmcnt(0)
++; VI-NEXT:    v_mul_lo_u32 v10, v4, v3
++; VI-NEXT:    v_mad_u64_u32 v[12:13], s[0:1], v4, v2, 0
++; VI-NEXT:    v_mul_lo_u32 v14, v5, v2
++; VI-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v0, v4, 0
++; VI-NEXT:    v_mul_lo_u32 v15, v7, v0
++; VI-NEXT:    v_add_u32_e32 v7, vcc, v13, v10
++; VI-NEXT:    v_mov_b32_e32 v10, v3
++; VI-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], v1, v4, v[10:11]
++; VI-NEXT:    v_add_u32_e32 v13, vcc, v7, v14
++; VI-NEXT:    v_mov_b32_e32 v7, v4
++; VI-NEXT:    v_mov_b32_e32 v4, v11
++; VI-NEXT:    v_mad_u64_u32 v[12:13], s[0:1], v6, v0, v[12:13]
++; VI-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], v0, v5, v[3:4]
++; VI-NEXT:    v_add_u32_e32 v11, vcc, v15, v13
++; VI-NEXT:    v_mov_b32_e32 v0, v4
++; VI-NEXT:    v_mul_lo_u32 v10, v6, v1
++; VI-NEXT:    v_add_u32_e32 v6, vcc, v7, v0
++; VI-NEXT:    v_addc_u32_e64 v7, s[0:1], 0, 0, vcc
++; VI-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v1, v5, v[6:7]
++; VI-NEXT:    v_add_u32_e32 v5, vcc, v10, v11
++; VI-NEXT:    v_add_u32_e32 v4, vcc, v0, v12
++; VI-NEXT:    v_addc_u32_e32 v5, vcc, v1, v5, vcc
++; VI-NEXT:    flat_store_dwordx4 v[8:9], v[2:5]
++; VI-NEXT:    s_endpgm
++;
++; GFX9-LABEL: v_mul_i128:
++; GFX9:       ; %bb.0: ; %entry
++; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x2c
++; GFX9-NEXT:    v_lshlrev_b32_e32 v13, 4, v0
++; GFX9-NEXT:    v_mov_b32_e32 v10, 0
++; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX9-NEXT:    global_load_dwordx4 v[0:3], v13, s[0:1]
++; GFX9-NEXT:    global_load_dwordx4 v[4:7], v13, s[2:3]
++; GFX9-NEXT:    s_waitcnt vmcnt(0)
++; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[0:1], v0, v4, 0
++; GFX9-NEXT:    v_mul_lo_u32 v14, v5, v2
++; GFX9-NEXT:    v_mul_lo_u32 v15, v4, v3
++; GFX9-NEXT:    v_mad_u64_u32 v[11:12], s[0:1], v1, v4, v[9:10]
++; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v4, v2, 0
++; GFX9-NEXT:    v_mov_b32_e32 v4, v12
++; GFX9-NEXT:    v_mov_b32_e32 v12, v10
++; GFX9-NEXT:    v_mad_u64_u32 v[9:10], s[0:1], v0, v5, v[11:12]
++; GFX9-NEXT:    v_add3_u32 v3, v3, v15, v14
++; GFX9-NEXT:    v_mul_lo_u32 v17, v7, v0
++; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v6, v0, v[2:3]
++; GFX9-NEXT:    v_mov_b32_e32 v0, v10
++; GFX9-NEXT:    v_mul_lo_u32 v16, v6, v1
++; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v4, v0
++; GFX9-NEXT:    v_addc_co_u32_e64 v7, s[0:1], 0, 0, vcc
++; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v1, v5, v[6:7]
++; GFX9-NEXT:    v_add3_u32 v3, v17, v3, v16
++; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v0, v2
++; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, v1, v3, vcc
++; GFX9-NEXT:    global_store_dwordx4 v13, v[8:11], s[2:3]
++; GFX9-NEXT:    s_endpgm
++;
++; GFX10-LABEL: v_mul_i128:
++; GFX10:       ; %bb.0: ; %entry
++; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x2c
++; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 4, v0
++; GFX10-NEXT:    v_mov_b32_e32 v10, 0
++; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX10-NEXT:    s_clause 0x1
++; GFX10-NEXT:    global_load_dwordx4 v[0:3], v14, s[0:1]
++; GFX10-NEXT:    global_load_dwordx4 v[4:7], v14, s[2:3]
++; GFX10-NEXT:    s_waitcnt vmcnt(0)
++; GFX10-NEXT:    v_mad_u64_u32 v[8:9], s0, v0, v4, 0
++; GFX10-NEXT:    v_mul_lo_u32 v7, v7, v0
++; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s0, v1, v4, v[9:10]
++; GFX10-NEXT:    v_mov_b32_e32 v9, v12
++; GFX10-NEXT:    v_mov_b32_e32 v12, v10
++; GFX10-NEXT:    v_mul_lo_u32 v10, v5, v2
++; GFX10-NEXT:    v_mad_u64_u32 v[12:13], s0, v0, v5, v[11:12]
++; GFX10-NEXT:    v_mul_lo_u32 v11, v4, v3
++; GFX10-NEXT:    v_mad_u64_u32 v[2:3], s0, v4, v2, 0
++; GFX10-NEXT:    v_mov_b32_e32 v4, v13
++; GFX10-NEXT:    v_mul_lo_u32 v13, v6, v1
++; GFX10-NEXT:    v_add3_u32 v3, v3, v11, v10
++; GFX10-NEXT:    v_add_co_u32 v9, s0, v9, v4
++; GFX10-NEXT:    v_add_co_ci_u32_e64 v10, s0, 0, 0, s0
++; GFX10-NEXT:    v_mad_u64_u32 v[2:3], s0, v6, v0, v[2:3]
++; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v1, v5, v[9:10]
++; GFX10-NEXT:    v_mov_b32_e32 v9, v12
++; GFX10-NEXT:    v_add3_u32 v3, v7, v3, v13
++; GFX10-NEXT:    v_add_co_u32 v10, vcc_lo, v0, v2
++; GFX10-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, v1, v3, vcc_lo
++; GFX10-NEXT:    global_store_dwordx4 v14, v[8:11], s[2:3]
++; GFX10-NEXT:    s_endpgm
++;
++; GFX11-LABEL: v_mul_i128:
++; GFX11:       ; %bb.0: ; %entry
++; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x2c
++; GFX11-NEXT:    v_lshlrev_b32_e32 v16, 4, v0
++; GFX11-NEXT:    v_mov_b32_e32 v10, 0
++; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
++; GFX11-NEXT:    s_clause 0x1
++; GFX11-NEXT:    global_load_b128 v[0:3], v16, s[0:1]
++; GFX11-NEXT:    global_load_b128 v[4:7], v16, s[2:3]
++; GFX11-NEXT:    s_waitcnt vmcnt(0)
++; GFX11-NEXT:    v_mad_u64_u32 v[8:9], null, v0, v4, 0
++; GFX11-NEXT:    v_mul_lo_u32 v15, v5, v2
++; GFX11-NEXT:    v_mul_lo_u32 v3, v4, v3
++; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
++; GFX11-NEXT:    v_mad_u64_u32 v[11:12], null, v1, v4, v[9:10]
++; GFX11-NEXT:    v_dual_mov_b32 v9, v12 :: v_dual_mov_b32 v12, v10
++; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
++; GFX11-NEXT:    v_mad_u64_u32 v[13:14], null, v0, v5, v[11:12]
++; GFX11-NEXT:    v_mad_u64_u32 v[10:11], null, v4, v2, 0
++; GFX11-NEXT:    v_mul_lo_u32 v4, v6, v1
++; GFX11-NEXT:    v_mul_lo_u32 v12, v7, v0
++; GFX11-NEXT:    v_mov_b32_e32 v2, v14
++; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
++; GFX11-NEXT:    v_add3_u32 v11, v11, v3, v15
++; GFX11-NEXT:    v_add_co_u32 v2, s0, v9, v2
++; GFX11-NEXT:    v_mov_b32_e32 v9, v13
++; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s0
++; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
++; GFX11-NEXT:    v_mad_u64_u32 v[14:15], null, v6, v0, v[10:11]
++; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v1, v5, v[2:3]
++; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
++; GFX11-NEXT:    v_add3_u32 v0, v12, v15, v4
++; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v6, v14
++; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
++; GFX11-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, v7, v0, vcc_lo
++; GFX11-NEXT:    global_store_b128 v16, v[8:11], s[2:3]
++; GFX11-NEXT:    s_nop 0
++; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
++; GFX11-NEXT:    s_endpgm
++;
++; EG-LABEL: v_mul_i128:
++; EG:       ; %bb.0: ; %entry
++; EG-NEXT:    ALU 3, @10, KC0[CB0:0-32], KC1[]
++; EG-NEXT:    TEX 1 @6
++; EG-NEXT:    ALU 41, @14, KC0[], KC1[]
++; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
++; EG-NEXT:    CF_END
++; EG-NEXT:    PAD
++; EG-NEXT:    Fetch clause starting at 6:
++; EG-NEXT:     VTX_READ_128 T2.XYZW, T1.X, 0, #1
++; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
++; EG-NEXT:    ALU clause starting at 10:
++; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
++; EG-NEXT:    4(5.605194e-45), 0(0.000000e+00)
++; EG-NEXT:     ADD_INT T0.X, KC0[2].Z, PV.W,
++; EG-NEXT:     ADD_INT * T1.X, KC0[2].W, PV.W,
++; EG-NEXT:    ALU clause starting at 14:
++; EG-NEXT:     MULLO_INT * T1.Y, T0.Y, T2.Y,
++; EG-NEXT:     MULHI * T1.Z, T0.Y, T2.Y,
++; EG-NEXT:     MULLO_INT * T1.W, T2.Z, T0.X,
++; EG-NEXT:     MULLO_INT * T3.X, T2.Y, T0.Z,
++; EG-NEXT:     MULHI * T3.Y, T0.Y, T2.X,
++; EG-NEXT:     MULHI * T3.Z, T0.X, T2.Y,
++; EG-NEXT:     MULHI * T3.W, T2.Z, T0.X,
++; EG-NEXT:     MULLO_INT * T2.Z, T2.Z, T0.Y,
++; EG-NEXT:     MULHI * T4.X, T2.X, T0.Z,
++; EG-NEXT:     MULLO_INT * T0.Y, T0.Y, T2.X,
++; EG-NEXT:     MULHI * T4.Y, T0.X, T2.X,
++; EG-NEXT:     ADD_INT T4.W, T0.Y, PS,
++; EG-NEXT:     MULLO_INT * T2.Y, T0.X, T2.Y,
++; EG-NEXT:     ADDC_UINT T4.Z, T0.Y, T4.Y,
++; EG-NEXT:     ADDC_UINT T5.W, PS, PV.W,
++; EG-NEXT:     MULLO_INT * T0.Y, T2.X, T0.W,
++; EG-NEXT:     ADD_INT T4.X, T4.X, PS,
++; EG-NEXT:     ADD_INT T0.Y, T3.W, T2.Z,
++; EG-NEXT:     ADD_INT T2.Z, T3.Z, PV.W,
++; EG-NEXT:     ADD_INT T0.W, T3.Y, PV.Z,
++; EG-NEXT:     MULLO_INT * T2.W, T2.W, T0.X,
++; EG-NEXT:     ADD_INT T5.X, PV.W, PV.Z,
++; EG-NEXT:     ADDC_UINT T3.Y, PV.W, PV.Z,
++; EG-NEXT:     ADD_INT T2.Z, PV.Y, PS,
++; EG-NEXT:     ADD_INT T0.W, PV.X, T3.X,
++; EG-NEXT:     MULLO_INT * T0.Y, T2.X, T0.Z,
++; EG-NEXT:     ADD_INT T4.Y, PV.Z, PV.W,
++; EG-NEXT:     ADDC_UINT T0.Z, T1.W, PS,
++; EG-NEXT:     ADD_INT T0.W, T1.Z, PV.Y,
++; EG-NEXT:     ADDC_UINT * T2.W, T1.Y, PV.X,
++; EG-NEXT:     ADD_INT T1.Y, T1.Y, T5.X,
++; EG-NEXT:     ADD_INT T1.Z, T1.W, T0.Y,
++; EG-NEXT:     ADD_INT T0.W, PV.W, PS,
++; EG-NEXT:     ADD_INT * T1.W, PV.Y, PV.Z,
++; EG-NEXT:     ADD_INT T0.W, PV.W, PS,
++; EG-NEXT:     ADDC_UINT * T1.W, PV.Y, PV.Z,
++; EG-NEXT:     ADD_INT * T0.W, PV.W, PS,
++; EG-NEXT:     ADD_INT * T0.Z, T1.Y, T1.Z,
++; EG-NEXT:     ADD_INT * T0.Y, T2.Y, T4.W,
++; EG-NEXT:     LSHR T1.X, T1.X, literal.x,
++; EG-NEXT:     MULLO_INT * T0.X, T0.X, T2.X,
++; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
++entry:
+   %tid = call i32 @llvm.amdgcn.workitem.id.x()
+   %gep.a = getelementptr inbounds i128, ptr addrspace(1) %aptr, i32 %tid
+   %gep.b = getelementptr inbounds i128, ptr addrspace(1) %bptr, i32 %tid
+-- 
+2.31.1
+
diff --git a/var/spack/repos/builtin/packages/llvm-amdgpu/package.py b/var/spack/repos/builtin/packages/llvm-amdgpu/package.py
index 472b7f98b4c7f6..8f975b70720c19 100644
--- a/var/spack/repos/builtin/packages/llvm-amdgpu/package.py
+++ b/var/spack/repos/builtin/packages/llvm-amdgpu/package.py
@@ -16,7 +16,7 @@ class LlvmAmdgpu(CMakePackage):
 
     homepage = "https://github.com/RadeonOpenCompute/llvm-project"
     git = "https://github.com/RadeonOpenCompute/llvm-project.git"
-    url = "https://github.com/RadeonOpenCompute/llvm-project/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/llvm-project/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
     executables = [r"amdclang", r"amdclang\+\+", r"amdflang", r"clang.*", r"flang.*", "llvm-.*"]
     generator("ninja")
@@ -24,6 +24,10 @@ class LlvmAmdgpu(CMakePackage):
     maintainers("srekolam", "renjithravindrankannath", "haampie")
 
     version("master", branch="amd-stg-open")
+    version("5.6.1", sha256="045e43c0c4a3f4f2f1db9fb603a4f1ea3d56e128147e19ba17909eb57d7f08e5")
+    version("5.6.0", sha256="e922bd492b54d99e56ed88c81e2009ed6472059a180b10cc56ce1f9bd2d7b6ed")
+    version("5.5.1", sha256="7d7181f20f89cb0715191aa32914186c67a34258c13457055570d47e15296553")
+    version("5.5.0", sha256="5dc6c99f612b69ff73145bee17524e3712990100e16445b71634106acf7927cf")
     version("5.4.3", sha256="a844d3cc01613f6284a75d44db67c495ac1e9b600eacbb1eb13d2649f5d5404d")
     version("5.4.0", sha256="ff54f45a17723892cd775c1eaff9e5860527fcfd33d98759223c70e3362335bf")
     version("5.3.3", sha256="5296d5e474811c7d1e456cb6d5011db248b79b8d0512155e8a6c2aa5b5f12d38")
@@ -104,12 +108,6 @@ class LlvmAmdgpu(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
     variant(
         "rocm-device-libs",
         default=True,
@@ -135,13 +133,14 @@ class LlvmAmdgpu(CMakePackage):
     provides("libllvm@12", when="@3.9:4.2")
     provides("libllvm@13", when="@4.3:4.9")
     provides("libllvm@14", when="@5:5.2")
-    provides("libllvm@15", when="@5.3:")
+    provides("libllvm@15", when="@5.3:5.4")
+    provides("libllvm@16", when="@5.5:")
 
     depends_on("cmake@3.4.3:", type="build", when="@:3.8")
     depends_on("cmake@3.13.4:", type="build", when="@3.9.0:")
     depends_on("python", type="build")
     depends_on("z3", type="link")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("ncurses+termlib", type="link")
     depends_on("pkgconfig", type="build")
 
@@ -162,7 +161,18 @@ class LlvmAmdgpu(CMakePackage):
     # OpenMP clang toolchain looks for bitcode files in llvm/bin/../lib
     # as per 5.2.0 llvm code. It used to be llvm/bin/../lib/libdevice.
     # Below patch is to look in the old path.
-    patch("adjust-openmp-bitcode-directory-for-llvm-link.patch", when="@5.2.0:")
+    patch("adjust-openmp-bitcode-directory-for-llvm-link.patch", when="@5.2.0:5.6")
+
+    # Below patch is to set the flag -mcode-object-version=none until
+    # the below fix is available in device-libs release code.
+    # https://github.com/RadeonOpenCompute/ROCm-Device-Libs/commit/f0356159dbdc93ea9e545f9b61a7842f9c881fdf
+    patch("patch-llvm-5.5.0.patch", when="@5.5: +rocm-device-libs")
+
+    # i1 muls can sometimes happen after SCEV.
+    # They resulted in ISel failures because we were missing the patterns for them.
+    # This fix is targeting 6.1 rocm release.
+    # Need patch until https://github.com/llvm/llvm-project/pull/67291 is merged.
+    patch("001-Add-i1-mul-patterns.patch", when="@5.6")
 
     conflicts("^cmake@3.19.0")
 
@@ -171,6 +181,10 @@ class LlvmAmdgpu(CMakePackage):
 
     # Add device libs sources so they can be an external LLVM project
     for d_version, d_shasum in [
+        ("5.6.1", "f0dfab272ff936225bfa1e9dabeb3c5d12ce08b812bf53ffbddd2ddfac49761c"),
+        ("5.6.0", "efb5dcdca9b3a9fbe408d494fb4a23e0b78417eb5fa8eebd4a5d226088f28921"),
+        ("5.5.1", "3b5f6dd85f0e3371f6078da7b59bf77d5b210e30f1cc66ef1e2de6bbcb775833"),
+        ("5.5.0", "5ab95aeb9c8bed0514f96f7847e21e165ed901ed826cdc9382c14d199cbadbd3"),
         ("5.4.3", "f4f7281f2cea6d268fcc3662b37410957d4f0bc23e0df9f60b12eb0fcdf9e26e"),
         ("5.4.0", "d68813ded47179c39914c8d1b76af3dad8c714b10229d1e2246af67609473951"),
         ("5.3.3", "963c9a0561111788b55a8c3b492e2a5737047914752376226c97a28122a4d768"),
@@ -213,6 +227,44 @@ class LlvmAmdgpu(CMakePackage):
         when="@master +rocm-device-libs",
     )
 
+    for d_version, d_shasum in [
+        ("5.6.1", "4de9a57c2092edf9398d671c8a2c60626eb7daf358caf710da70d9c105490221"),
+        ("5.6.0", "30875d440df9d8481ffb24d87755eae20a0efc1114849a72619ea954f1e9206c"),
+    ]:
+        resource(
+            name="hsa-runtime",
+            placement="hsa-runtime",
+            url=f"https://github.com/RadeonOpenCompute/ROCR-Runtime/archive/rocm-{d_version}.tar.gz",
+            sha256=d_shasum,
+            when="@{0}".format(d_version),
+        )
+    resource(
+        name="hsa-runtime",
+        placement="hsa-runtime",
+        git="https://github.com/RadeonOpenCompute/ROCR-Runtime.git",
+        branch="master",
+        when="@master",
+    )
+
+    for d_version, d_shasum in [
+        ("5.6.1", "0a85d84619f98be26ca7a32c71f94ed3c4e9866133789eabb451be64ce739300"),
+        ("5.6.0", "9396a7238b547ee68146c669b10b9d5de8f1d76527c649133c75d8076a185a72"),
+    ]:
+        resource(
+            name="comgr",
+            placement="comgr",
+            url=f"https://github.com/RadeonOpenCompute/ROCm-CompilerSupport/archive/rocm-{d_version}.tar.gz",
+            sha256=d_shasum,
+            when="@{0}".format(d_version),
+        )
+    resource(
+        name="comgr",
+        placement="comgr",
+        git="https://github.com/RadeonOpenCompute/ROCm-CompilerSupport.git",
+        branch="amd-stg-open",
+        when="@master",
+    )
+
     def cmake_args(self):
         llvm_projects = ["clang", "lld", "clang-tools-extra", "compiler-rt"]
         llvm_runtimes = []
@@ -230,6 +282,7 @@ def cmake_args(self):
                 self.define("LIBCXXABI_ENABLE_STATIC", "ON"),
                 self.define("LIBCXXABI_INSTALL_STATIC_LIBRARY", "OFF"),
             ]
+        args.append(self.define("LLVM_ENABLE_RTTI", "ON"))
         if self.spec.satisfies("@4.3.0:4.5.2"):
             llvm_projects.append("libcxx")
             llvm_projects.append("libcxxabi")
@@ -272,7 +325,17 @@ def cmake_args(self):
         # Get the GCC prefix for LLVM.
         if self.compiler.name == "gcc":
             args.append(self.define("GCC_INSTALL_PREFIX", self.compiler.prefix))
-
+        if self.spec.satisfies("@5.4.3:"):
+            args.append("-DCMAKE_INSTALL_LIBDIR=lib")
+        if self.spec.satisfies("@5.5.0:"):
+            args.append("-DCLANG_DEFAULT_RTLIB=compiler-rt")
+            args.append("-DCLANG_DEFAULT_UNWINDLIB=libgcc")
+        if self.spec.satisfies("@5.6.0:"):
+            hsainc_path = os.path.join(self.stage.source_path, "hsa-runtime/src/inc")
+            comgrinc_path = os.path.join(self.stage.source_path, "comgr/lib/comgr/include")
+            args.append("-DSANITIZER_HSA_INCLUDE_PATH={0}".format(hsainc_path))
+            args.append("-DSANITIZER_COMGR_INCLUDE_PATH={0}".format(comgrinc_path))
+            args.append("-DSANITIZER_AMDGPU:Bool=ON")
         return args
 
     @run_after("install")
@@ -306,3 +369,13 @@ def determine_version(cls, path):
                     int(match.group(1)), int(match.group(2)), int(match.group(3))
                 )
         return detected_version
+
+    # Make sure that the compiler paths are in the LD_LIBRARY_PATH
+    def setup_run_environment(self, env):
+        llvm_amdgpu_home = self.spec["llvm-amdgpu"].prefix
+        env.prepend_path("LD_LIBRARY_PATH", llvm_amdgpu_home + "/llvm/lib")
+
+    # Make sure that the compiler paths are in the LD_LIBRARY_PATH
+    def setup_dependent_run_environment(self, env, dependent_spec):
+        llvm_amdgpu_home = self.spec["llvm-amdgpu"].prefix
+        env.prepend_path("LD_LIBRARY_PATH", llvm_amdgpu_home + "/llvm/lib")
diff --git a/var/spack/repos/builtin/packages/llvm-amdgpu/patch-llvm-5.5.0.patch b/var/spack/repos/builtin/packages/llvm-amdgpu/patch-llvm-5.5.0.patch
new file mode 100644
index 00000000000000..1495b6d3a39ac1
--- /dev/null
+++ b/var/spack/repos/builtin/packages/llvm-amdgpu/patch-llvm-5.5.0.patch
@@ -0,0 +1,38 @@
+From 7010d5da727825321d31863ceb9e2fe9eb22b5b9 Mon Sep 17 00:00:00 2001
+From: Renjith Ravindran 
+Date: Tue, 10 Oct 2023 05:16:47 +0000
+Subject: [PATCH] Condition check for enabling the flag
+ -mcode-object-version=none is failing in spack when device-libsis built with
+ llvm-amdgpu. The flag is required here as well as standalon build.
+
+---
+ rocm-device-libs/cmake/OCL.cmake | 13 ++-----------
+ 1 file changed, 2 insertions(+), 11 deletions(-)
+
+diff --git a/rocm-device-libs/cmake/OCL.cmake b/rocm-device-libs/cmake/OCL.cmake
+index 773c6f62e..30f60030b 100644
+--- a/rocm-device-libs/cmake/OCL.cmake
++++ b/rocm-device-libs/cmake/OCL.cmake
+@@ -30,17 +30,8 @@ if (WIN32)
+   set(CLANG_OCL_FLAGS ${CLANG_OCL_FLAGS} -fshort-wchar)
+ endif()
+ 
+-# Disable code object version module flag if available.
+-file(WRITE ${CMAKE_BINARY_DIR}/tmp.cl "")
+-execute_process (
+-  COMMAND ${LLVM_TOOLS_BINARY_DIR}/clang${EXE_SUFFIX} ${CLANG_OCL_FLAGS} -Xclang -mcode-object-version=none ${CMAKE_BINARY_DIR}/tmp.cl
+-  RESULT_VARIABLE TEST_CODE_OBJECT_VERSION_NONE_RESULT
+-  ERROR_QUIET
+-)
+-file(REMOVE ${CMAKE_BINARY_DIR}/tmp.cl)
+-if (NOT TEST_CODE_OBJECT_VERSION_NONE_RESULT)
+-  set(CLANG_OCL_FLAGS ${CLANG_OCL_FLAGS} -Xclang -mcode-object-version=none)
+-endif()
++# Disable code object version module flag.
++set(CLANG_OCL_FLAGS ${CLANG_OCL_FLAGS} -Xclang -mcode-object-version=none)
+ 
+ set (BC_EXT .bc)
+ set (LIB_SUFFIX ".lib${BC_EXT}")
+-- 
+2.31.1
+
diff --git a/var/spack/repos/builtin/packages/llvm-doe/package.py b/var/spack/repos/builtin/packages/llvm-doe/package.py
index 7937124d75ec1b..d3a91114ad55b6 100644
--- a/var/spack/repos/builtin/packages/llvm-doe/package.py
+++ b/var/spack/repos/builtin/packages/llvm-doe/package.py
@@ -252,6 +252,8 @@ def determine_version(cls, exe):
             output = compiler("--version", output=str, error=str)
             if "Apple" in output:
                 return None
+            if "AMD" in output:
+                return None
             match = version_regex.search(output)
             if match:
                 return match.group(match.lastindex)
diff --git a/var/spack/repos/builtin/packages/llvm-openmp-ompt/package.py b/var/spack/repos/builtin/packages/llvm-openmp-ompt/package.py
index cec56554eded12..930e9e7077972f 100644
--- a/var/spack/repos/builtin/packages/llvm-openmp-ompt/package.py
+++ b/var/spack/repos/builtin/packages/llvm-openmp-ompt/package.py
@@ -15,8 +15,8 @@ class LlvmOpenmpOmpt(CMakePackage):
     homepage = "https://github.com/OpenMPToolsInterface/LLVM-openmp"
     git = "https://github.com/OpenMPToolsInterface/LLVM-openmp.git"
 
-    # tr6_forwards branch
-    version("tr6_forwards", branch="tr6_forwards")
+    # tr6_forwards branch (last commit from 2017)
+    version("tr6_forwards", commit="4b29de49ce90cfb5c3cbc6bb7d91660b70bddb5d")
     version("3.9.2b2", commit="5cdca5dd3c0c336d42a335ca7cff622e270c9d47")
 
     # align-to-tr-rebased branch
diff --git a/var/spack/repos/builtin/packages/llvm/add-include-for-libelf-llvm-12-14.patch b/var/spack/repos/builtin/packages/llvm/add-include-for-libelf-llvm-12-14.patch
new file mode 100644
index 00000000000000..f585d08d010ae0
--- /dev/null
+++ b/var/spack/repos/builtin/packages/llvm/add-include-for-libelf-llvm-12-14.patch
@@ -0,0 +1,26 @@
+From e5f0939fde75f769c53c6c99c9ed6886e7d58c43 Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels 
+Date: Wed, 23 Aug 2023 11:35:23 +0200
+Subject: [PATCH] Add corresponding -I for libelf.h
+
+Funnily enough, it's added to `include_directories` in multiple places,
+except where it's necessary.
+---
+ openmp/libomptarget/plugins/amdgpu/CMakeLists.txt | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt b/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt
+index 92523c23f68b..52e1923f97b7 100644
+--- a/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt
++++ b/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt
+@@ -51,6 +51,7 @@ endif()
+ include_directories(
+   ${CMAKE_CURRENT_SOURCE_DIR}/impl
+   ${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
++  ${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR}
+ )
+ 
+ set(LIBOMPTARGET_DLOPEN_LIBHSA OFF)
+-- 
+2.39.2
+
diff --git a/var/spack/repos/builtin/packages/llvm/add-include-for-libelf-llvm-15.patch b/var/spack/repos/builtin/packages/llvm/add-include-for-libelf-llvm-15.patch
new file mode 100644
index 00000000000000..7a7d57d108f5a8
--- /dev/null
+++ b/var/spack/repos/builtin/packages/llvm/add-include-for-libelf-llvm-15.patch
@@ -0,0 +1,26 @@
+From 5e149c3ec8118ad8f3d20a30ce5d3fbac4072515 Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels 
+Date: Wed, 23 Aug 2023 11:35:23 +0200
+Subject: [PATCH] Add corresponding -I for libelf.h
+
+Funnily enough, it's added to `include_directories` in multiple places,
+except where it's necessary.
+---
+ openmp/libomptarget/plugins/amdgpu/CMakeLists.txt | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt b/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt
+index 66bf680d15c7..6be12ab6d7f8 100644
+--- a/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt
++++ b/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt
+@@ -99,6 +99,7 @@ target_include_directories(
+   PRIVATE
+   ${LIBOMPTARGET_INCLUDE_DIR}
+   ${CMAKE_CURRENT_SOURCE_DIR}/impl
++  ${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR}
+ )
+ 
+ 
+-- 
+2.39.2
+
diff --git a/var/spack/repos/builtin/packages/llvm/detection_test.yaml b/var/spack/repos/builtin/packages/llvm/detection_test.yaml
new file mode 100644
index 00000000000000..48e9d6751af20f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/llvm/detection_test.yaml
@@ -0,0 +1,56 @@
+paths:
+  - layout:
+      - executables:
+        - "bin/clang-3.9"
+        script: |
+          echo "clang version 3.9.1-19ubuntu1 (tags/RELEASE_391/rc2)"
+          echo "Target: x86_64-pc-linux-gnu"
+          echo "Thread model: posix"
+          echo "InstalledDir: /usr/bin"
+      - executables:
+        - "bin/clang++-3.9"
+        script: |
+          echo "clang version 3.9.1-19ubuntu1 (tags/RELEASE_391/rc2)"
+          echo "Target: x86_64-pc-linux-gnu"
+          echo "Thread model: posix"
+          echo "InstalledDir: /usr/bin"
+    results:
+      - spec: 'llvm@3.9.1 +clang~lld~lldb'
+  # Multiple LLVM packages in the same prefix
+  - layout:
+      - executables:
+        - "bin/clang-8"
+        - "bin/clang++-8"
+        script: |
+          echo "clang version 8.0.0-3~ubuntu18.04.2 (tags/RELEASE_800/final)"
+          echo "Target: x86_64-pc-linux-gnu"
+          echo "Thread model: posix"
+          echo "InstalledDir: /usr/bin"
+      - executables:
+        - "bin/ld.lld-8"
+        script: 'echo "LLD 8.0.0 (compatible with GNU linkers)"'
+      - executables:
+        - "bin/lldb"
+        script: 'echo "lldb version 8.0.0"'
+      - executables:
+        - "bin/clang-3.9"
+        - "bin/clang++-3.9"
+        script: |
+          echo "clang version 3.9.1-19ubuntu1 (tags/RELEASE_391/rc2)"
+          echo "Target: x86_64-pc-linux-gnu"
+          echo "Thread model: posix"
+          echo "InstalledDir: /usr/bin"
+    results:
+      - spec: 'llvm@8.0.0+clang+lld+lldb'
+      - spec: 'llvm@3.9.1+clang~lld~lldb'
+  # Apple Clang should not be detected
+  - layout:
+    - executables:
+      - "bin/clang"
+      - "bin/clang++"
+      script: |
+        echo "Apple clang version 11.0.0 (clang-1100.0.33.8)"
+        echo "Target: x86_64-apple-darwin19.5.0"
+        echo "Thread model: posix"
+        echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin"
+    results: []
diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py
index 9b537df57ae743..383871353fe9fa 100644
--- a/var/spack/repos/builtin/packages/llvm/package.py
+++ b/var/spack/repos/builtin/packages/llvm/package.py
@@ -35,6 +35,10 @@ class Llvm(CMakePackage, CudaPackage):
     family = "compiler"  # Used by lmod
 
     version("main", branch="main")
+    version("17.0.4", sha256="46200b79f52a02fe26d0a43fd856ab6ceff49ab2a0b7c240ac4b700a6ada700c")
+    version("17.0.3", sha256="1e3d9d04fb5fbd8d0080042ad72c7e2a5c68788b014b186647a604dbbdd625d2")
+    version("17.0.2", sha256="dcba3eb486973dce45b6edfe618f3f29b703ae7e6ef9df65182fb50fb6fe4235")
+    version("17.0.1", sha256="d51b10be66c10a6a81f4c594b554ffbf1063ffbadcb810af37d1f88d6e0b49dd")
     version("16.0.6", sha256="56b2f75fdaa95ad5e477a246d3f0d164964ab066b4619a01836ef08e475ec9d5")
     version("16.0.5", sha256="e0fbca476693fcafa125bc71c8535587b6d9950293122b66b262bb4333a03942")
     version("16.0.4", sha256="10c3fe1757d2e4f1cd7745dc548ecf687680a71824ec81701c38524c2a0753e2")
@@ -78,10 +82,6 @@ class Llvm(CMakePackage, CudaPackage):
     version("5.0.1", sha256="84ca454abf262579814a2a2b846569f6e0cb3e16dc33ca3642b4f1dff6fbafd3")
     version("5.0.0", sha256="1f1843315657a4371d8ca37f01265fa9aae17dbcf46d2d0a95c1fdb3c6a4bab6")
 
-    # NOTE: The debug version of LLVM is an order of magnitude larger than
-    # the release version, and may take up 20-30 GB of space. If you want
-    # to save space, build with `build_type=Release`.
-
     variant(
         "clang", default=True, description="Build the LLVM C/C++/Objective-C compiler frontend"
     )
@@ -225,6 +225,7 @@ class Llvm(CMakePackage, CudaPackage):
         description="Enable code-signing on macOS",
     )
     variant("python", default=False, description="Install python bindings")
+    variant("lua", default=True, description="Enable lua scripting inside lldb")
     variant("version_suffix", default="none", description="Add a symbol suffix")
     variant(
         "shlib_symbol_version",
@@ -235,6 +236,8 @@ class Llvm(CMakePackage, CudaPackage):
     variant("z3", default=False, description="Use Z3 for the clang static analyzer")
     conflicts("+z3", when="@:7")
     conflicts("+z3", when="~clang")
+    conflicts("+lua", when="@:10")
+    conflicts("+lua", when="~lldb")
 
     variant(
         "zstd",
@@ -243,6 +246,8 @@ class Llvm(CMakePackage, CudaPackage):
         description="Enable zstd support for static analyzer / lld",
     )
 
+    provides("libllvm@16", when="@16.0.0:16")
+    provides("libllvm@15", when="@15.0.0:15")
     provides("libllvm@14", when="@14.0.0:14")
     provides("libllvm@13", when="@13.0.0:13")
     provides("libllvm@12", when="@12.0.0:12")
@@ -281,23 +286,36 @@ class Llvm(CMakePackage, CudaPackage):
     depends_on("perl-data-dumper", type=("build"))
     depends_on("hwloc")
     depends_on("hwloc@2.0.1:", when="@13")
-    depends_on("elf", when="+cuda")  # libomptarget
-    depends_on("libffi", when="+libomptarget")  # libomptarget
+    with when("@:15"):
+        depends_on("elf", when="+cuda")
+        depends_on("elf", when="+libomptarget")
+    depends_on("libffi", when="+libomptarget")
 
     # llvm-config --system-libs libraries.
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # needs zstd cmake config file, which is not added when built with makefile.
     depends_on("zstd build_system=cmake", when="+zstd")
 
     # lldb dependencies
-    with when("+lldb +python"):
-        depends_on("swig")
+    with when("+lldb"):
+        depends_on("libedit")
+        depends_on("libxml2")
+        depends_on("lua@5.3", when="+lua")  # purposefully not a range
+        depends_on("ncurses")
+        depends_on("py-six", when="+python")
+        depends_on("swig", when="+lua")
+        depends_on("swig", when="+python")
+        depends_on("xz")
+
+    # Use ^swig cause it's triggered by both python & lua scripting in lldb
+    with when("^swig"):
         depends_on("swig@2:", when="@10:")
         depends_on("swig@3:", when="@12:")
-    depends_on("libedit", when="+lldb")
-    depends_on("ncurses", when="+lldb")
-    depends_on("py-six", when="+lldb+python")
+        depends_on("swig@4:", when="@17:")
+        # Commits f0a25fe0b746f56295d5c02116ba28d2f965c175 and
+        # 81fc5f7909a4ef5a8d4b5da2a10f77f7cb01ba63 fixed swig 4.1 support
+        depends_on("swig@:4.0", when="@:15")
 
     # gold support, required for some features
     depends_on("binutils+gold+ld+plugins+headers", when="+gold")
@@ -544,6 +562,19 @@ class Llvm(CMakePackage, CudaPackage):
         when="@13:14 compiler-rt=runtime",
     )
 
+    patch("add-include-for-libelf-llvm-12-14.patch", when="@12:14")
+    patch("add-include-for-libelf-llvm-15.patch", when="@15")
+
+    @when("@14:17")
+    def patch(self):
+        # https://github.com/llvm/llvm-project/pull/69458
+        filter_file(
+            r"${TERMINFO_LIB}",
+            r"${Terminfo_LIBRARIES}",
+            "lldb/source/Core/CMakeLists.txt",
+            string=True,
+        )
+
     # The functions and attributes below implement external package
     # detection for LLVM. See:
     #
@@ -582,6 +613,8 @@ def determine_version(cls, exe):
             output = compiler("--version", output=str, error=str)
             if "Apple" in output:
                 return None
+            if "AMD" in output:
+                return None
             match = version_regex.search(output)
             if match:
                 return match.group(match.lastindex)
@@ -722,14 +755,6 @@ def setup_build_environment(self, env):
                     os.symlink(bin, sym)
             env.prepend_path("PATH", self.stage.path)
 
-    def setup_run_environment(self, env):
-        if "+clang" in self.spec:
-            env.set("CC", join_path(self.spec.prefix.bin, "clang"))
-            env.set("CXX", join_path(self.spec.prefix.bin, "clang++"))
-        if "+flang" in self.spec:
-            env.set("FC", join_path(self.spec.prefix.bin, "flang"))
-            env.set("F77", join_path(self.spec.prefix.bin, "flang"))
-
     root_cmakelists_dir = "llvm"
 
     def cmake_args(self):
@@ -741,7 +766,6 @@ def cmake_args(self):
         cmake_args = [
             define("LLVM_REQUIRES_RTTI", True),
             define("LLVM_ENABLE_RTTI", True),
-            define("LLVM_ENABLE_EH", True),
             define("LLVM_ENABLE_LIBXML2", False),
             define("CLANG_DEFAULT_OPENMP_RUNTIME", "libomp"),
             define("PYTHON_EXECUTABLE", python.command.path),
@@ -750,6 +774,16 @@ def cmake_args(self):
             from_variant("LLVM_ENABLE_ZSTD", "zstd"),
         ]
 
+        # Flang does not support exceptions from core llvm.
+        # LLVM_ENABLE_EH=True when building flang will soon
+        # fail (with changes at the llvm-project level).
+        # Only enable exceptions in LLVM if we are *not*
+        # building flang.  FYI: LLVM <= 16.x will build flang
+        # successfully but the executable will suffer from
+        # link errors looking for C++ EH support.
+        if "+flang" not in spec:
+            cmake_args.append(define("LLVM_ENABLE_EH", True))
+
         version_suffix = spec.variants["version_suffix"].value
         if version_suffix != "none":
             cmake_args.append(define("LLVM_VERSION_SUFFIX", version_suffix))
@@ -781,13 +815,7 @@ def cmake_args(self):
                 ]
             )
             if "openmp=runtime" in spec:
-                cmake_args.extend(
-                    [
-                        define("LIBOMPTARGET_NVPTX_ENABLE_BCLIB", True),
-                        # work around bad libelf detection in libomptarget
-                        define("LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR", spec["elf"].prefix.include),
-                    ]
-                )
+                cmake_args.append(define("LIBOMPTARGET_NVPTX_ENABLE_BCLIB", True))
         else:
             # still build libomptarget but disable cuda
             cmake_args.extend(
@@ -803,13 +831,19 @@ def cmake_args(self):
 
         if "+lldb" in spec:
             projects.append("lldb")
-            cmake_args.append(define("LLDB_ENABLE_LIBEDIT", True))
-            cmake_args.append(define("LLDB_ENABLE_CURSES", True))
+            cmake_args.extend(
+                [
+                    define("LLDB_ENABLE_LIBEDIT", True),
+                    define("LLDB_ENABLE_CURSES", True),
+                    define("LLDB_ENABLE_LIBXML2", True),
+                    from_variant("LLDB_ENABLE_LUA", "lua"),
+                    define("LLDB_ENABLE_LZMA", True),
+                ]
+            )
             if spec["ncurses"].satisfies("+termlib"):
                 cmake_args.append(define("LLVM_ENABLE_TERMINFO", True))
             else:
                 cmake_args.append(define("LLVM_ENABLE_TERMINFO", False))
-            cmake_args.append(define("LLDB_ENABLE_LIBXML2", False))
             if spec.version >= Version("10"):
                 cmake_args.append(from_variant("LLDB_ENABLE_PYTHON", "python"))
             else:
@@ -885,6 +919,10 @@ def cmake_args(self):
         if self.spec.satisfies("~code_signing platform=darwin"):
             cmake_args.append(define("LLDB_USE_SYSTEM_DEBUGSERVER", True))
 
+        # LLDB test suite requires libc++
+        if "libcxx=none" in spec:
+            cmake_args.append(define("LLDB_INCLUDE_TESTS", False))
+
         # Enable building with CLT [and not require full Xcode]
         # https://github.com/llvm/llvm-project/issues/57037
         if self.spec.satisfies("@15.0.0: platform=darwin"):
@@ -895,7 +933,14 @@ def cmake_args(self):
 
         # Semicolon seperated list of runtimes to enable
         if runtimes:
-            cmake_args.append(define("LLVM_ENABLE_RUNTIMES", runtimes))
+            cmake_args.extend(
+                [
+                    define("LLVM_ENABLE_RUNTIMES", runtimes),
+                    define(
+                        "RUNTIMES_CMAKE_ARGS", [define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", True)]
+                    ),
+                ]
+            )
 
         return cmake_args
 
@@ -924,7 +969,6 @@ def post_install(self):
                 cmake_args.extend(
                     [
                         define("LIBOMPTARGET_NVPTX_ENABLE_BCLIB", True),
-                        define("LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR", spec["elf"].prefix.include),
                         self.stage.source_path + "/openmp",
                     ]
                 )
diff --git a/var/spack/repos/builtin/packages/lm-sensors/package.py b/var/spack/repos/builtin/packages/lm-sensors/package.py
index 31780f155fa0b1..9f4fe23a6a648d 100644
--- a/var/spack/repos/builtin/packages/lm-sensors/package.py
+++ b/var/spack/repos/builtin/packages/lm-sensors/package.py
@@ -31,6 +31,14 @@ class LmSensors(MakefilePackage):
     depends_on("flex", type="build")
     depends_on("perl", type="run")
 
+    @property
+    def build_targets(self):
+        targets = []
+
+        targets.append("CC={0}".format(spack_cc))
+
+        return targets
+
     def install(self, spec, prefix):
         make("install", "PREFIX={0}".format(prefix), "ETCDIR={0}/etc".format(prefix))
 
diff --git a/var/spack/repos/builtin/packages/lmdb/package.py b/var/spack/repos/builtin/packages/lmdb/package.py
index efd291ecbfd8b6..68a89213dd18d9 100644
--- a/var/spack/repos/builtin/packages/lmdb/package.py
+++ b/var/spack/repos/builtin/packages/lmdb/package.py
@@ -15,6 +15,7 @@ class Lmdb(MakefilePackage):
     homepage = "https://lmdb.tech/"
     url = "https://github.com/LMDB/lmdb/archive/LMDB_0.9.21.tar.gz"
 
+    version("0.9.31", sha256="dd70a8c67807b3b8532b3e987b0a4e998962ecc28643e1af5ec77696b081c9b0")
     version("0.9.29", sha256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb")
     version("0.9.24", sha256="44602436c52c29d4f301f55f6fd8115f945469b868348e3cddaf91ab2473ea26")
     version("0.9.22", sha256="f3927859882eb608868c8c31586bb7eb84562a40a6bf5cc3e13b6b564641ea28")
diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py
index f4ca15a3a3eaee..af3b8a7b6cbf3b 100644
--- a/var/spack/repos/builtin/packages/lmod/package.py
+++ b/var/spack/repos/builtin/packages/lmod/package.py
@@ -23,6 +23,7 @@ class Lmod(AutotoolsPackage):
 
     version("8.7.24", sha256="8451267652059b6507b652e1b563929ecf9b689ffb20830642085eb6a55bd539")
     version("8.7.20", sha256="c04deff7d2ca354610a362459a7aa9a1c642a095e45a4b0bb2471bb3254e85f4")
+    version("8.7.18", sha256="b9912caca1557dd0c17113bceb1a4952e0ae75331d38df6361601db3f80366af")
     version("8.7.2", sha256="5f44f3783496d2d597ced7531e1714c740dbb2883a7d16fde362135fb0b0fd96")
     version("8.6.18", sha256="3db1c665c35fb8beb78c02e40d56accd361d82b715df70b2a995bcb10fbc2c80")
     version("8.6.5", sha256="4a1823264187340be11104d82f8226905daa8149186fa8615dfc742b6d19c2ce")
diff --git a/var/spack/repos/builtin/packages/log4cxx/package.py b/var/spack/repos/builtin/packages/log4cxx/package.py
index 1ed4160489e3f0..ff2991fa5c615e 100644
--- a/var/spack/repos/builtin/packages/log4cxx/package.py
+++ b/var/spack/repos/builtin/packages/log4cxx/package.py
@@ -24,7 +24,7 @@ class Log4cxx(CMakePackage):
     depends_on("apr-util")
     depends_on("apr")
     depends_on("boost+thread+system", when="cxxstd=11")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("zip")
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/lordec/package.py b/var/spack/repos/builtin/packages/lordec/package.py
index 901942e5f2675e..f441c2859f4553 100644
--- a/var/spack/repos/builtin/packages/lordec/package.py
+++ b/var/spack/repos/builtin/packages/lordec/package.py
@@ -30,7 +30,7 @@ def url_for_version(self, version):
     # See https://github.com/spack/spack/pull/22303 for reference
     depends_on(Boost.with_default_variants)
     depends_on("gatb-core@1.4.1:", type=["build", "link", "run"])
-    depends_on("zlib", type=["build", "link"])
+    depends_on("zlib-api", type=["build", "link"])
 
     build_targets = ["clean", "all"]
 
diff --git a/var/spack/repos/builtin/packages/lorene/package.py b/var/spack/repos/builtin/packages/lorene/package.py
index b550c51cc975de..519e433e85f0b9 100644
--- a/var/spack/repos/builtin/packages/lorene/package.py
+++ b/var/spack/repos/builtin/packages/lorene/package.py
@@ -110,8 +110,9 @@ def build(self, spec, prefix):
                 )
 
     def install(self, spec, prefix):
-        mkdirp(prefix.lib)
         install_tree("Lib", prefix.lib)
+        install_tree("Export/C++/Include", prefix.include)
+        install_tree("C++/Include", prefix.include)
         mkdirp(prefix.bin)
         if "+bin_star" in spec:
             for exe in [
diff --git a/var/spack/repos/builtin/packages/lrzip/package.py b/var/spack/repos/builtin/packages/lrzip/package.py
index adcb6b99cbc96c..88b66f4fc33c0b 100644
--- a/var/spack/repos/builtin/packages/lrzip/package.py
+++ b/var/spack/repos/builtin/packages/lrzip/package.py
@@ -27,7 +27,7 @@ class Lrzip(Package):
 
     # depends_on('coreutils')
     depends_on("lzo")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2")
 
     def install(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/ltrace/package.py b/var/spack/repos/builtin/packages/ltrace/package.py
index a74c125d9bcc5f..1886e3d679c0d2 100644
--- a/var/spack/repos/builtin/packages/ltrace/package.py
+++ b/var/spack/repos/builtin/packages/ltrace/package.py
@@ -18,6 +18,8 @@ class Ltrace(AutotoolsPackage):
 
     conflicts("platform=darwin", msg="ltrace runs only on Linux.")
 
+    depends_on("elf", type="link")
+
     def configure_args(self):
         # Disable -Werror since some functions used by ltrace
         # have been deprecated in recent version of glibc
diff --git a/var/spack/repos/builtin/packages/lua-luajit-openresty/package.py b/var/spack/repos/builtin/packages/lua-luajit-openresty/package.py
index 31ae9aa55146bf..081e07fe6c2e10 100644
--- a/var/spack/repos/builtin/packages/lua-luajit-openresty/package.py
+++ b/var/spack/repos/builtin/packages/lua-luajit-openresty/package.py
@@ -13,11 +13,14 @@ class LuaLuajitOpenresty(LuaImplPackage):
     """Flast flexible JITed lua - OpenResty maintained fork"""
 
     homepage = "https://openresty.org/en/luajit.html"
-    url = "https://github.com/openresty/luajit2/archive/refs/tags/v2.1-20220111.tar.gz"
+    url = "https://github.com/openresty/luajit2/archive/refs/tags/v2.1-20230410.tar.gz"
 
     version(
         "2.1-20220111", sha256="1ad2e34b111c802f9d0cdf019e986909123237a28c746b21295b63c9e785d9c3"
     )
+    version(
+        "2.1-20230410", sha256="77bbcbb24c3c78f51560017288f3118d995fe71240aa379f5818ff6b166712ff"
+    )
 
     variant(
         "lualinks",
@@ -25,9 +28,7 @@ class LuaLuajitOpenresty(LuaImplPackage):
         description="add symlinks to make lua-luajit a drop-in lua replacement",
     )
 
-    provides("lua-lang@5.1", when="+lualinks")
-    conflicts("lua", when="+lualinks")
-    provides("luajit")
+    provides("luajit", "lua-lang@5.1", when="+lualinks")
     lua_version_override = "5.1"
 
     @run_after("install")
diff --git a/var/spack/repos/builtin/packages/lua-luajit/package.py b/var/spack/repos/builtin/packages/lua-luajit/package.py
index 8ac4fa0091b31c..dfe9f51cd0bd8b 100644
--- a/var/spack/repos/builtin/packages/lua-luajit/package.py
+++ b/var/spack/repos/builtin/packages/lua-luajit/package.py
@@ -33,9 +33,7 @@ class LuaLuajit(LuaImplPackage):
         description="add symlinks to make lua-luajit a drop-in lua replacement",
     )
 
-    provides("lua-lang@5.1", when="+lualinks")
-    conflicts("lua", when="+lualinks")
-    provides("luajit")
+    provides("luajit", "lua-lang@5.1", when="+lualinks")
     lua_version_override = "5.1"
     conflicts("platform=darwin", msg="luajit not supported on MacOS, see lua-luajit-openresty")
 
diff --git a/var/spack/repos/builtin/packages/lua/package.py b/var/spack/repos/builtin/packages/lua/package.py
index 60b13628d2fee8..ab65586205751d 100644
--- a/var/spack/repos/builtin/packages/lua/package.py
+++ b/var/spack/repos/builtin/packages/lua/package.py
@@ -53,7 +53,7 @@ def lua_share_dir(self):
         return os.path.join("share", self.lua_dir_name, self.__verdir())
 
     # luarocks needs unzip for some packages (e.g. lua-luaposix)
-    depends_on("unzip", type="run")
+    depends_on("unzip", type=("build", "run"))
 
     # luarocks needs a fetcher (curl/wget), unfortunately I have not found
     # how to force a choice for curl or wget, but curl seems the default.
diff --git a/var/spack/repos/builtin/packages/lulesh/package.py b/var/spack/repos/builtin/packages/lulesh/package.py
index 9282b3da66a883..6025944989092c 100644
--- a/var/spack/repos/builtin/packages/lulesh/package.py
+++ b/var/spack/repos/builtin/packages/lulesh/package.py
@@ -16,7 +16,7 @@ class Lulesh(MakefilePackage):
     homepage = "https://computing.llnl.gov/projects/co-design/lulesh"
     git = "https://github.com/LLNL/LULESH.git"
 
-    version("2.0.3", tag="2.0.3")
+    version("2.0.3", tag="2.0.3", commit="46c2a1d6db9171f9637d79f407212e0f176e8194")
 
     variant("mpi", default=True, description="Build with MPI support")
     variant("openmp", default=True, description="Build with OpenMP support")
diff --git a/var/spack/repos/builtin/packages/lvarray/package.py b/var/spack/repos/builtin/packages/lvarray/package.py
index b51b434a08bd9b..e5c9a3d507f41f 100644
--- a/var/spack/repos/builtin/packages/lvarray/package.py
+++ b/var/spack/repos/builtin/packages/lvarray/package.py
@@ -42,9 +42,15 @@ class Lvarray(CMakePackage, CudaPackage):
 
     version("develop", branch="develop", submodules=False)
     version("main", branch="main", submodules=False)
-    version("0.2.2", tag="v0.2.2", submodules=False)
-    version("0.2.1", tag="v0.2.1", submodules=False)
-    version("0.1.0", tag="v0.1.0", submodules=True)
+    version(
+        "0.2.2", tag="v0.2.2", commit="3f7bacf5a4e967d4628b860132f5c29673314d3a", submodules=False
+    )
+    version(
+        "0.2.1", tag="v0.2.1", commit="3fece6bbbeb172442275d1ad1a52d6b54cd53163", submodules=False
+    )
+    version(
+        "0.1.0", tag="v0.1.0", commit="0bf5f7d077de4a08f58db24baed207f9dba95f6e", submodules=True
+    )
 
     variant("shared", default=True, description="Build Shared Libs")
     variant("umpire", default=False, description="Build Umpire support")
diff --git a/var/spack/repos/builtin/packages/lz4/package.py b/var/spack/repos/builtin/packages/lz4/package.py
index edfd7958d59c7b..d12789f395bf36 100644
--- a/var/spack/repos/builtin/packages/lz4/package.py
+++ b/var/spack/repos/builtin/packages/lz4/package.py
@@ -20,6 +20,8 @@ class Lz4(CMakePackage, MakefilePackage):
     homepage = "https://lz4.github.io/lz4/"
     url = "https://github.com/lz4/lz4/archive/v1.9.2.tar.gz"
 
+    maintainers("AlexanderRichert-NOAA")
+
     version("1.9.4", sha256="0b0e3aa07c8c063ddf40b082bdf7e37a1562bda40a0ff5272957f3e987e0e54b")
     version("1.9.3", sha256="030644df4611007ff7dc962d981f390361e6c97a34e5cbc393ddfbe019ffe2c1")
     version("1.9.2", sha256="658ba6191fa44c92280d4aa2c271b0f4fbc0e34d249578dd05e50e76d0e5efcc")
@@ -40,7 +42,7 @@ class Lz4(CMakePackage, MakefilePackage):
         multi=True,
         description="Build shared libs, static libs or both",
     )
-    variant("pic", default=False, description="PIC")
+    variant("pic", default=True, description="Enable position-independent code (PIC)")
 
     def url_for_version(self, version):
         url = "https://github.com/lz4/lz4/archive"
@@ -79,10 +81,15 @@ def cmake_args(self):
         args.append(
             self.define("BUILD_STATIC_LIBS", True if "libs=static" in self.spec else False)
         )
+        args.append(self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"))
         return args
 
 
 class MakefileBuilder(MakefileBuilder):
+    def setup_build_environment(self, env):
+        if self.spec.satisfies("+pic"):
+            env.set("CFLAGS", self.pkg.compiler.cc_pic_flag)
+
     def build(self, pkg, spec, prefix):
         par = True
         if spec.compiler.name == "nvhpc":
diff --git a/var/spack/repos/builtin/packages/madx/package.py b/var/spack/repos/builtin/packages/madx/package.py
index 8534db82ccb947..4a11980ce2e79e 100644
--- a/var/spack/repos/builtin/packages/madx/package.py
+++ b/var/spack/repos/builtin/packages/madx/package.py
@@ -34,7 +34,7 @@ class Madx(CMakePackage):
     depends_on("cmake@2.8:", type="build")
 
     depends_on("libx11")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def cmake_args(self):
         args = [
diff --git a/var/spack/repos/builtin/packages/maeparser/package.py b/var/spack/repos/builtin/packages/maeparser/package.py
index 9ca021e334360f..97f141608f1fb8 100644
--- a/var/spack/repos/builtin/packages/maeparser/package.py
+++ b/var/spack/repos/builtin/packages/maeparser/package.py
@@ -24,7 +24,7 @@ class Maeparser(CMakePackage):
     )
 
     depends_on("boost +iostreams +filesystem +test")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def cmake_args(self):
         args = [self.define_from_variant("MAEPARSER_BUILD_SHARED_LIBS", "shared")]
diff --git a/var/spack/repos/builtin/packages/magics/package.py b/var/spack/repos/builtin/packages/magics/package.py
index 8d757bd9855436..49bf9ab2bc675f 100644
--- a/var/spack/repos/builtin/packages/magics/package.py
+++ b/var/spack/repos/builtin/packages/magics/package.py
@@ -75,7 +75,7 @@ class Magics(CMakePackage):
     # that files that make calls to png library get compiled and linked
     # unconditionally, which makes png a non-optional dependency (and
     # ENABLE_PNG always has to be set to ON).
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libpng")
 
     # GRIB support is non-optional, regardless of what the instruction says.
diff --git a/var/spack/repos/builtin/packages/magma/package.py b/var/spack/repos/builtin/packages/magma/package.py
index 257c82eff25d50..0aa38f91cdf493 100644
--- a/var/spack/repos/builtin/packages/magma/package.py
+++ b/var/spack/repos/builtin/packages/magma/package.py
@@ -23,6 +23,7 @@ class Magma(CMakePackage, CudaPackage, ROCmPackage):
     test_requires_compiler = True
 
     version("master", branch="master")
+    version("2.7.2", sha256="729bc1a70e518a7422fe7a3a54537a4741035a77be3349f66eac5c362576d560")
     version("2.7.1", sha256="d9c8711c047a38cae16efde74bee2eb3333217fd2711e1e9b8606cbbb4ae1a50")
     version("2.7.0", sha256="fda1cbc4607e77cacd8feb1c0f633c5826ba200a018f647f1c5436975b39fd18")
     version("2.6.2", sha256="75b554dab00903e2d10b972c913e50e7f88cbc62f3ae432b5a086c7e4eda0a71")
@@ -48,10 +49,12 @@ class Magma(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("hipsparse", when="+rocm")
     depends_on("python", when="@master", type="build")
 
-    conflicts("~cuda", when="~rocm", msg="Either CUDA or HIP support must be enabled")
-    conflicts("+rocm", when="+cuda", msg="CUDA must be disabled to support HIP (ROCm)")
-    conflicts("+rocm", when="@:2.5.4", msg="HIP support starts in version 2.6.0")
-    conflicts("cuda_arch=none", when="+cuda", msg="Please indicate a CUDA arch value or values")
+    conflicts("~cuda", when="~rocm", msg="magma: Either CUDA or HIP support must be enabled")
+    conflicts("+rocm", when="+cuda", msg="magma: CUDA must be disabled to support HIP (ROCm)")
+    conflicts("+rocm", when="@:2.5.4", msg="magma: HIP support starts in version 2.6.0")
+    conflicts(
+        "cuda_arch=none", when="+cuda", msg="magma: Please indicate a CUDA arch value or values"
+    )
 
     # currently not compatible with CUDA-11
     # https://bitbucket.org/icl/magma/issues/22/cuda-11-changes-issue
@@ -63,11 +66,11 @@ class Magma(CMakePackage, CudaPackage, ROCmPackage):
         conflicts("cuda_arch={}".format(target))
 
     # Some cuda_arch values had support added recently
-    conflicts("cuda_arch=37", when="@:2.5")
-    conflicts("cuda_arch=60", when="@:2.2")
-    conflicts("cuda_arch=70", when="@:2.2")
-    conflicts("cuda_arch=75", when="@:2.5.0")
-    conflicts("cuda_arch=80", when="@:2.5.3")
+    conflicts("cuda_arch=37", when="@:2.5", msg="magma: cuda_arch=37 needs a version > 2.5")
+    conflicts("cuda_arch=60", when="@:2.2", msg="magma: cuda_arch=60 needs a version > 2.2")
+    conflicts("cuda_arch=70", when="@:2.2", msg="magma: cuda_arch=70 needs a version > 2.2")
+    conflicts("cuda_arch=75", when="@:2.5.0", msg="magma: cuda_arch=75 needs a version > 2.5.0")
+    conflicts("cuda_arch=80", when="@:2.5.3", msg="magma: cuda_arch=80 needs a version > 2.5.3")
 
     patch("ibm-xl.patch", when="@2.2:2.5.0%xl")
     patch("ibm-xl.patch", when="@2.2:2.5.0%xl_r")
diff --git a/var/spack/repos/builtin/packages/malt/package.py b/var/spack/repos/builtin/packages/malt/package.py
index 5569e62d76a3f8..d5ecec6fad1963 100644
--- a/var/spack/repos/builtin/packages/malt/package.py
+++ b/var/spack/repos/builtin/packages/malt/package.py
@@ -15,10 +15,11 @@ class Malt(CMakePackage):
 
     # Project infos
     homepage = "https://memtt.github.io/malt"
-    url = "https://github.com/memtt/malt/archive/v1.2.1.tar.gz"
+    url = "https://github.com/memtt/malt/archive/v1.2.2.tar.gz"
     maintainers("svalat")
 
     # Versions
+    version("1.2.2", sha256="e19f49ad97bf2deedf0557eb00267f4dcf1c932c494dd07ada07fcdf5421935f")
     version("1.2.1", sha256="0e4c0743561f9fcc04dc83457386167a9851fc9289765f8b4f9390384ae3618a")
 
     # Variants
diff --git a/var/spack/repos/builtin/packages/mapnik/package.py b/var/spack/repos/builtin/packages/mapnik/package.py
index e1a3f1487ba34b..e49463d3341050 100644
--- a/var/spack/repos/builtin/packages/mapnik/package.py
+++ b/var/spack/repos/builtin/packages/mapnik/package.py
@@ -32,7 +32,7 @@ class Mapnik(AutotoolsPackage):
     # See https://github.com/spack/spack/pull/22303 for reference
     depends_on(Boost.with_default_variants)
     depends_on("icu4c")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("freetype")
     depends_on("libxml2")
     depends_on("harfbuzz")
diff --git a/var/spack/repos/builtin/packages/mapserver/package.py b/var/spack/repos/builtin/packages/mapserver/package.py
index 15bdbdcaba7643..f4f82b7cc250be 100644
--- a/var/spack/repos/builtin/packages/mapserver/package.py
+++ b/var/spack/repos/builtin/packages/mapserver/package.py
@@ -33,7 +33,7 @@ class Mapserver(CMakePackage):
     depends_on("libpng")
     depends_on("freetype")
     depends_on("jpeg")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("proj")
     depends_on("proj@:5", when="@:7.3")
     depends_on("proj@6:", when="@7.4:")
diff --git a/var/spack/repos/builtin/packages/mariadb-c-client/package.py b/var/spack/repos/builtin/packages/mariadb-c-client/package.py
index 740981c9a51dd6..148fc1e81199fa 100644
--- a/var/spack/repos/builtin/packages/mariadb-c-client/package.py
+++ b/var/spack/repos/builtin/packages/mariadb-c-client/package.py
@@ -22,6 +22,7 @@ class MariadbCClient(CMakePackage):
     list_url = "https://downloads.mariadb.com/Connectors/c/"
     list_depth = 1
 
+    version("3.3.7", sha256="975a9a862fed80f84e0206373f7ef05537aada5b65d99b71b36ab892b44240bf")
     version("3.3.5", sha256="ca72eb26f6db2befa77e48ff966f71bcd3cb44b33bd8bbb810b65e6d011c1e5c")
     version("3.3.4", sha256="486e5fdf976a8e7fadf583ae912128655e013ac575fa79b2d1af0fb8827a78ed")
     version("3.3.2", sha256="7e0722e07d30bb906fac9fe10fb582cde1e148e05a83d9ca7b6fcc884b68fbce")
@@ -63,7 +64,7 @@ class MariadbCClient(CMakePackage):
     depends_on("curl")
     depends_on("pcre")
     depends_on("openssl")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("krb5")
 
     # patch needed for cmake-3.20
diff --git a/var/spack/repos/builtin/packages/mariadb/package.py b/var/spack/repos/builtin/packages/mariadb/package.py
index 33780415991105..3acca3dbbf2020 100644
--- a/var/spack/repos/builtin/packages/mariadb/package.py
+++ b/var/spack/repos/builtin/packages/mariadb/package.py
@@ -52,7 +52,7 @@ class Mariadb(CMakePackage):
     depends_on("libedit")
     depends_on("libevent", when="+nonblocking")
     depends_on("ncurses")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("curl")
     depends_on("libxml2")
     depends_on("lz4")
diff --git a/var/spack/repos/builtin/packages/mash/package.py b/var/spack/repos/builtin/packages/mash/package.py
index 4f6ac179883ae5..7148bbe1e21638 100644
--- a/var/spack/repos/builtin/packages/mash/package.py
+++ b/var/spack/repos/builtin/packages/mash/package.py
@@ -27,6 +27,16 @@ class Mash(AutotoolsPackage):
     depends_on("capnproto")
     depends_on("gsl")
 
+    def patch(self):
+        if self.spec.satisfies("target=aarch64:"):
+            filter_file(
+                "CXXFLAGS += -include src/mash/memcpyLink.h -Wl,--wrap=memcpy",
+                "",
+                "Makefile.in",
+                string=True,
+            )
+            filter_file("CFLAGS += -include src/mash/memcpyLink.h", "", "Makefile.in", string=True)
+
     def configure_args(self):
         args = []
         args.append("--with-capnp=" + self.spec["capnproto"].prefix)
diff --git a/var/spack/repos/builtin/packages/masurca/package.py b/var/spack/repos/builtin/packages/masurca/package.py
index c3ee433c977090..24f42ebe48d93f 100644
--- a/var/spack/repos/builtin/packages/masurca/package.py
+++ b/var/spack/repos/builtin/packages/masurca/package.py
@@ -24,7 +24,7 @@ class Masurca(Package):
 
     depends_on("perl", type=("build", "run"))
     depends_on(Boost.with_default_variants)
-    depends_on("zlib")
+    depends_on("zlib-api")
     patch("arm.patch", when="target=aarch64:")
 
     def patch(self):
diff --git a/var/spack/repos/builtin/packages/mathematica/package.py b/var/spack/repos/builtin/packages/mathematica/package.py
index 719712e23ef691..069625a3d86fed 100644
--- a/var/spack/repos/builtin/packages/mathematica/package.py
+++ b/var/spack/repos/builtin/packages/mathematica/package.py
@@ -22,6 +22,11 @@ class Mathematica(Package):
     url = "file://{0}/Mathematica_12.0.0_LINUX.sh".format(os.getcwd())
     manual_download = True
 
+    version(
+        "13.0.1",
+        sha256="3672a920c1b4af1afd480733f6d67665baf8258757dfe59a6ed6d7440cf26dba",
+        expand=False,
+    )
     version(
         "12.2.0",
         sha256="3b6676a203c6adb7e9c418a5484b037974287b5be09c64e7dfea74ddc0e400d7",
diff --git a/var/spack/repos/builtin/packages/matio/package.py b/var/spack/repos/builtin/packages/matio/package.py
index 2451189ac9c225..9f3d237398c295 100644
--- a/var/spack/repos/builtin/packages/matio/package.py
+++ b/var/spack/repos/builtin/packages/matio/package.py
@@ -34,13 +34,13 @@ class Matio(AutotoolsPackage):
     variant("hdf5", default=True, description="support for version 7.3 mat files via hdf5")
     variant("shared", default=True, description="Enables the build of shared libraries.")
 
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("hdf5", when="+hdf5")
 
     def configure_args(self):
         args = []
         if "+zlib" in self.spec:
-            args.append("--with-zlib=%s" % self.spec["zlib"].prefix)
+            args.append("--with-zlib=%s" % self.spec["zlib-api"].prefix)
         if "+hdf5" in self.spec:
             args.append("--with-hdf5=%s" % self.spec["hdf5"].prefix)
         if "+shared" not in self.spec:
diff --git a/var/spack/repos/builtin/packages/mcutils/package.py b/var/spack/repos/builtin/packages/mcutils/package.py
index 0d513b4c677c7f..97d00223573def 100644
--- a/var/spack/repos/builtin/packages/mcutils/package.py
+++ b/var/spack/repos/builtin/packages/mcutils/package.py
@@ -15,21 +15,21 @@ class Mcutils(MakefilePackage):
 
     tags = ["hep"]
 
-    version("1.3.5", tag="mcutils-1.3.5")
-    version("1.3.4", tag="mcutils-1.3.4")
-    version("1.3.3", tag="mcutils-1.3.3")
-    version("1.3.2", tag="mcutils-1.3.2")
+    version("1.3.5", tag="mcutils-1.3.5", commit="d0e65bb7b6f80d6df50a71a25c54683b0db997a5")
+    version("1.3.4", tag="mcutils-1.3.4", commit="ddb527e5d03b8e4d13ae4e6d78fbcd7d35f7153b")
+    version("1.3.3", tag="mcutils-1.3.3", commit="638aabe930b05f8ecfe272bdd2f2a7ed65e5fc45")
+    version("1.3.2", tag="mcutils-1.3.2", commit="8228d443aa0877c067299d640704836e664dac51")
     version("1.3.1", tag="mcutils-1.3.1")
-    version("1.3.1", tag="mcutils-1.3.0")
-    version("1.2.1", tag="mcutils-1.2.1")
-    version("1.2.0", tag="mcutils-1.2.0")
-    version("1.1.2", tag="mcutils-1.1.2")
-    version("1.1.1", tag="mcutils-1.1.1")
-    version("1.1.0", tag="mcutils-1.1.0")
-    version("1.0.3", tag="mcutils-1.0.3")
-    version("1.0.2", tag="mcutils-1.0.2")
-    version("1.0.1", tag="mcutils-1.0.1")
-    version("1.0.0", tag="mcutils-1.0.0")
+    version("1.3.1", tag="mcutils-1.3.0", commit="e04693bf5aaa38b9cbe76aec94a3ffd2d466f1f6")
+    version("1.2.1", tag="mcutils-1.2.1", commit="3799336668c19ed86c93c82c815da6397597763f")
+    version("1.2.0", tag="mcutils-1.2.0", commit="c1ef0a2a0b09c9df16450c7b64da543119e3740f")
+    version("1.1.2", tag="mcutils-1.1.2", commit="cf212f61bb398cae1e987ff7e4c5537c1480be8d")
+    version("1.1.1", tag="mcutils-1.1.1", commit="c42d9123439fbcee512c23f853c60d6491b47fa0")
+    version("1.1.0", tag="mcutils-1.1.0", commit="15af9f40d8667030d9a68e007ac7e348157397d5")
+    version("1.0.3", tag="mcutils-1.0.3", commit="73a8e06256460e943af2336f80115d84630f6dd5")
+    version("1.0.2", tag="mcutils-1.0.2", commit="15e2daad2bfe7543e43b35421fffd177519c516f")
+    version("1.0.1", tag="mcutils-1.0.1", commit="85bb1c9e2761a7c70bdd18955d6cccc120d9c523")
+    version("1.0.0", tag="mcutils-1.0.0", commit="7ae9d007493ce65f5eac432d0ea6f730512a0a8a")
 
     depends_on("heputils", when="@1.1.0:")
 
diff --git a/var/spack/repos/builtin/packages/mdspan/package.py b/var/spack/repos/builtin/packages/mdspan/package.py
new file mode 100644
index 00000000000000..1e4978ca00d595
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mdspan/package.py
@@ -0,0 +1,44 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class Mdspan(CMakePackage):
+    """Extension of std::span to enable multidimensional arrays"""
+
+    homepage = "https://github.com/kokkos/mdspan/tree/stable"
+    url = "https://github.com/kokkos/mdspan/archive/refs/tags/mdspan-0.6.0.zip"
+    git = "https://github.com/kokkos/mdspan.git"
+
+    version("stable", branch="stable")
+    version("0.6.0", sha256="d6b7b9d4f472106df1d28729bd8383a8a7ea7938adf9f82d3be9c151344830d9")
+
+    variant("examples", default=True, description="Enable examples")
+    variant("tests", default=False, description="Enable tests")
+    variant("benchmarks", default=False, description="Enable benchmarks")
+
+    depends_on("benchmark", when="+benchmarks")
+    depends_on("googletest@main", when="+tests")
+
+    def cmake_args(self):
+        args = []
+
+        if self.spec.satisfies("+tests"):
+            args.append("-DMDSPAN_ENABLE_TESTS=ON")
+            args.append("-DMDSPAN_USE_SYSTEM_GTEST=ON")
+        if self.spec.satisfies("+bencmarks"):
+            args.append("-DMDSPAN_ENABLE_BENCHMARKS=ON")
+        if self.spec.satisfies("+examples"):
+            args.append("-DMDSPAN_ENABLE_EXAMPLES=ON")
+
+        args.append("-DCMAKE_CXX_FLAGS='-Wall -Wextra -pedantic'")
+        args.append("-DCMAKE_CXX_STANDARD=17")
+        args.append("-DMDSPAN_CXX_STANDARD=17")
+        args.append("-DCMAKE_CXX_COMPILER=g++")
+        args.append("-DCMAKE_CXX_EXTENSIONS=OFF")
+
+        return args
diff --git a/var/spack/repos/builtin/packages/mdsplus/package.py b/var/spack/repos/builtin/packages/mdsplus/package.py
index 0a3ef8637d5006..a4b39400cbd0d7 100644
--- a/var/spack/repos/builtin/packages/mdsplus/package.py
+++ b/var/spack/repos/builtin/packages/mdsplus/package.py
@@ -17,7 +17,12 @@ class Mdsplus(AutotoolsPackage):
 
     parallel = False
 
-    version("stable_release-7-96-17", tag="stable_release-7-96-17", submodules=True)
+    version(
+        "stable_release-7-96-17",
+        tag="stable_release-7-96-17",
+        commit="83928a157ee0a5875135aeee0996634ecb802523",
+        submodules=True,
+    )
 
     variant("java", default=True, description="Build java libraries and applications")
     variant("python", default=True, description="Install python module")
diff --git a/var/spack/repos/builtin/packages/mdtest/package.py b/var/spack/repos/builtin/packages/mdtest/package.py
index 771f440706b590..b822c70cd3c746 100644
--- a/var/spack/repos/builtin/packages/mdtest/package.py
+++ b/var/spack/repos/builtin/packages/mdtest/package.py
@@ -14,7 +14,7 @@ class Mdtest(Package):
     homepage = "https://github.com/LLNL/mdtest"
     git = "https://github.com/LLNL/mdtest.git"
 
-    version("1.9.3", commit="49f3f0")
+    version("1.9.3", commit="49f3f047c254c62848c23226d6f1afa5fc3c6583")
 
     depends_on("mpi")
 
diff --git a/var/spack/repos/builtin/packages/megahit/package.py b/var/spack/repos/builtin/packages/megahit/package.py
index cfc2ad269d7b45..cb96687d78e2b1 100644
--- a/var/spack/repos/builtin/packages/megahit/package.py
+++ b/var/spack/repos/builtin/packages/megahit/package.py
@@ -16,7 +16,7 @@ class Megahit(MakefilePackage):
     version("1.1.4", sha256="ecd64c8bfa516ef6b19f9b2961ede281ec814db836f1a91953c213c944e1575f")
     version("1.1.3", sha256="b6eefdee075aaf7a8f9090e2e8b08b770caff90aa43a255e0e220d82ce71c492")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     patch("amd.patch", when="target=aarch64:")
 
diff --git a/var/spack/repos/builtin/packages/meme/package.py b/var/spack/repos/builtin/packages/meme/package.py
index 1229847ee68116..3ebe9739b07864 100644
--- a/var/spack/repos/builtin/packages/meme/package.py
+++ b/var/spack/repos/builtin/packages/meme/package.py
@@ -15,6 +15,7 @@ class Meme(AutotoolsPackage):
     homepage = "https://meme-suite.org"
     url = "http://meme-suite.org/meme-software/5.1.1/meme-5.1.1.tar.gz"
 
+    version("5.5.4", sha256="cda6011c2b855bf2563c4e7a2c255e11e99b5b6e5e73736ff008942507580153")
     version("5.3.0", sha256="b2ddec9db972fcf77b29c7deb62df8b1dd8a6638c13c1aa06a5d563c4a7ff756")
     version("5.2.0", sha256="0cbf8c2172e9b6c07855b8aeec457f4825f0b132f8cbb11192880e2f6033f54f")
     version("5.1.1", sha256="38d73d256d431ad4eb7da2c817ce56ff2b4e26c39387ff0d6ada088938b38eb5")
@@ -22,35 +23,37 @@ class Meme(AutotoolsPackage):
     version("4.11.4", sha256="3e869ff57e327a9c8615dbef784e3f1095f7f7a0120cecd55efe10c3f2ee8eb3")
 
     variant("mpi", default=True, description="Enable MPI support")
-    variant("image-magick", default=False, description="Enable image-magick for png output")
+    variant("magick", default=False, description="Enable imagemagick for png output")
 
-    depends_on("zlib", type=("link"))
-    depends_on("libgcrypt", type=("link"))
-    depends_on("perl", type=("build", "run"))
-    depends_on("python@2.7:", type=("build", "run"))
-    depends_on("mpi", when="+mpi")
-    depends_on("imagemagick", type=("build", "run"), when="+image-magick")
+    # Perl runtime deps
+    depends_on("perl@5.16:", type=("build", "run"))
     depends_on("perl-xml-parser", type=("build", "run"))
-    depends_on("perl-xml-simple", when="@4.5.0:")
-    depends_on("libxml2", type=("build", "run"))
-    depends_on("libxslt", type=("build", "run"))
+    depends_on("perl-xml-simple", type=("build", "run"))
+    # Python runtime deps
+    depends_on("python", type=("build", "run"))
+    # Build deps
+    depends_on("zlib-ng")
+    depends_on("ghostscript")
+    depends_on("libxml2")
+    depends_on("libxslt")
+    # Variant deps
+    depends_on("imagemagick", type=("build", "run"), when="+magick")
+    depends_on("mpi", when="+mpi")
 
     patch("arm.patch", when="%arm")
 
+    def patch(self):
+        # Remove flags not recognized by the NVIDIA compiler
+        if self.spec.satisfies("%nvhpc"):
+            filter_file("-fno-common", "", "configure")
+            filter_file("-Wno-unused", "", "configure")
+
     def url_for_version(self, version):
         url = "http://meme-suite.org/meme-software/{0}/meme{1}{2}.tar.gz"
         sep = "-" if version >= Version("5.0.2") else "_"
         return url.format(version.up_to(3), sep, version)
 
     def configure_args(self):
-        spec = self.spec
-        args = []
-        if "~mpi" in spec:
-            args += ["--enable-serial"]
-        return args
-
-    def patch(self):
-        # Remove flags not recognized by the NVIDIA compiler
-        if self.spec.satisfies("%nvhpc"):
-            filter_file("-fno-common", "", "configure")
-            filter_file("-Wno-unused", "", "configure")
+        if "~mpi" in self.spec:
+            return ["--enable-serial"]
+        return []
diff --git a/var/spack/repos/builtin/packages/memsurfer/package.py b/var/spack/repos/builtin/packages/memsurfer/package.py
index 98e674bf01ebfb..c9408c1acdb03f 100644
--- a/var/spack/repos/builtin/packages/memsurfer/package.py
+++ b/var/spack/repos/builtin/packages/memsurfer/package.py
@@ -15,7 +15,7 @@ class Memsurfer(PythonPackage):
     git = "https://github.com/LLNL/MemSurfer.git"
     maintainers("bhatiaharsh")
 
-    version("1.0", tag="v1.0", submodules=True)
+    version("1.0", tag="v1.0", commit="93d114016cd3ef48950bc53cca0a6e9f70589361", submodules=True)
     version("master", branch="master", submodules=True)
     version("develop", branch="develop", submodules=True)
 
diff --git a/var/spack/repos/builtin/packages/mercury/package.py b/var/spack/repos/builtin/packages/mercury/package.py
index 1c531277d79ce9..a2cdad6f1b6874 100644
--- a/var/spack/repos/builtin/packages/mercury/package.py
+++ b/var/spack/repos/builtin/packages/mercury/package.py
@@ -17,6 +17,7 @@ class Mercury(CMakePackage):
     tags = ["e4s"]
 
     version("master", branch="master", submodules=True)
+    version("2.3.1", sha256="36182d49f2db7e2b075240cab4aaa1d4ec87a7756450c87643ededd1e6f16104")
     version("2.3.0", sha256="e9e62ce1bb2fd482f0e85ad75fa255d9750c6fed50ba441a03de93b3b8eae742")
     version("2.2.0", sha256="e66490cf63907c3959bbb2932b5aaf51d96a481b17f0935f409f3a862eff97f6")
     version("2.1.0", sha256="9a58437161e9273b1b1c484d2f1a477a89eea9afe84575415025d47656f3761b")
diff --git a/var/spack/repos/builtin/packages/mesa-demos/package.py b/var/spack/repos/builtin/packages/mesa-demos/package.py
index 4bdffb257b0524..6471be9c463be5 100644
--- a/var/spack/repos/builtin/packages/mesa-demos/package.py
+++ b/var/spack/repos/builtin/packages/mesa-demos/package.py
@@ -25,10 +25,10 @@ class MesaDemos(AutotoolsPackage):
         multi=False,
         description="The OpenGL provider to use",
     )
-    conflicts("osmesa", when="gl=glx")
-    conflicts("osmesa", when="gl=other")
-    conflicts("glx", when="gl=osmesa")
-    conflicts("glx", when="gl=other")
+    conflicts("^osmesa", when="gl=glx")
+    conflicts("^osmesa", when="gl=other")
+    conflicts("^glx", when="gl=osmesa")
+    conflicts("^glx", when="gl=other")
 
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
diff --git a/var/spack/repos/builtin/packages/mesa-glu/package.py b/var/spack/repos/builtin/packages/mesa-glu/package.py
index 8fdbb0bf7e8d80..b0c0ca2ca04548 100644
--- a/var/spack/repos/builtin/packages/mesa-glu/package.py
+++ b/var/spack/repos/builtin/packages/mesa-glu/package.py
@@ -25,10 +25,10 @@ class MesaGlu(AutotoolsPackage):
         multi=False,
         description="The OpenGL provider to use",
     )
-    conflicts("osmesa", when="gl=glx")
-    conflicts("osmesa", when="gl=other")
-    conflicts("glx", when="gl=osmesa")
-    conflicts("glx", when="gl=other")
+    conflicts("^osmesa", when="gl=glx")
+    conflicts("^osmesa", when="gl=other")
+    conflicts("^glx", when="gl=osmesa")
+    conflicts("^glx", when="gl=other")
 
     depends_on("gl@3:")
     depends_on("osmesa", when="gl=osmesa")
@@ -36,6 +36,10 @@ class MesaGlu(AutotoolsPackage):
 
     provides("glu@1.3")
 
+    # When using -std=c++17, using register long will throw an error. This
+    # patch switches all instances of register long to long to fix this.
+    patch("register-long.patch")
+
     def configure_args(self):
         args = ["--disable-libglvnd"]
 
diff --git a/var/spack/repos/builtin/packages/mesa-glu/register-long.patch b/var/spack/repos/builtin/packages/mesa-glu/register-long.patch
new file mode 100644
index 00000000000000..720aaba1aa59d0
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mesa-glu/register-long.patch
@@ -0,0 +1,28 @@
+From e5b42317078a1dbbf7ef9f1431ef6dcb711c7026 Mon Sep 17 00:00:00 2001
+From: Aiden Grossman 
+Date: Mon, 10 Jul 2023 17:59:18 -0700
+Subject: [PATCH] Change register long to long to fix compile error
+
+When using -std=c++17, using register long will throw an error. This
+patch switches all instances of register long to long to fix this.
+---
+ src/libnurbs/internals/varray.cc | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/libnurbs/internals/varray.cc b/src/libnurbs/internals/varray.cc
+index 1cb2354..41b3b18 100644
+--- a/src/libnurbs/internals/varray.cc
++++ b/src/libnurbs/internals/varray.cc
+@@ -73,8 +73,8 @@ Varray::~Varray( void )
+ inline void
+ Varray::update( Arc_ptr arc, long dir[2], REAL val )
+ {
+-    register long ds = sgn(arc->tail()[0] - arc->prev->tail()[0]);
+-    register long dt = sgn(arc->tail()[1] - arc->prev->tail()[1]);
++    long ds = sgn(arc->tail()[0] - arc->prev->tail()[0]);
++    long dt = sgn(arc->tail()[1] - arc->prev->tail()[1]);
+
+     if( dir[0] != ds || dir[1] != dt ) {
+ 	dir[0] = ds;
+--
+2.29.1
\ No newline at end of file
diff --git a/var/spack/repos/builtin/packages/mesa/package.py b/var/spack/repos/builtin/packages/mesa/package.py
index fbf3ce4652a857..e1e666a6bcd212 100644
--- a/var/spack/repos/builtin/packages/mesa/package.py
+++ b/var/spack/repos/builtin/packages/mesa/package.py
@@ -18,14 +18,17 @@ class Mesa(MesonPackage):
     git = "https://gitlab.freedesktop.org/mesa/mesa.git"
     url = "https://archive.mesa3d.org/mesa-20.2.1.tar.xz"
 
-    version("main", tag="main")
-    version("22.3.2", sha256="c15df758a8795f53e57f2a228eb4593c22b16dffd9b38f83901f76cd9533140b")
-    version("22.2.5", sha256="850f063146f8ebb262aec04f666c2c1e5623f2a1987dda24e4361b17b912c73b")
+    version("main", branch="main")
+
     version(
-        "22.1.6",
-        sha256="22ced061eb9adab8ea35368246c1995c09723f3f71653cd5050c5cec376e671a",
+        "23.0.3",
+        sha256="386362a5d80df3b096636b67f340e1ce67b705b44767d5bdd11d2ed1037192d5",
         preferred=True,
     )
+    version("23.0.2", sha256="1b7d3399fc6f16f030361f925d33ebc7600cbf98094582f54775b6a1180529e7")
+    version("22.3.2", sha256="c15df758a8795f53e57f2a228eb4593c22b16dffd9b38f83901f76cd9533140b")
+    version("22.2.5", sha256="850f063146f8ebb262aec04f666c2c1e5623f2a1987dda24e4361b17b912c73b")
+    version("22.1.6", sha256="22ced061eb9adab8ea35368246c1995c09723f3f71653cd5050c5cec376e671a")
     version("22.1.2", sha256="0971226b4a6a3d10cfc255736b33e4017e18c14c9db1e53863ac1f8ae0deb9ea")
     version("22.0.5", sha256="5ee2dc06eff19e19b2867f12eb0db0905c9691c07974f6253f2f1443df4c7a35")
     version("22.0.2", sha256="df4fa560dcce6680133067cd15b0505fc424ca703244ce9ab247c74d2fab6885")
@@ -49,11 +52,12 @@ class Mesa(MesonPackage):
     depends_on("cmake", type="build")
     depends_on("flex", type="build")
     depends_on("gettext", type="build")
-    depends_on("python@3:", type="build")
+    # Upperbound on 3.11 because distutils is used for checking py-mako
+    depends_on("python@3:3.11", type="build")
     depends_on("py-mako@0.8.0:", type="build")
     depends_on("unwind")
     depends_on("expat")
-    depends_on("zlib@1.2.3:")
+    depends_on("zlib-api")
 
     # Internal options
     variant("llvm", default=True, description="Enable LLVM.")
@@ -136,14 +140,14 @@ class Mesa(MesonPackage):
     # llvm::Module::setOverrideStackAlignment was added in LLVM 13.0.0, but forks based
     # on development versions of LLVM 13 may or may not have it. Use SFINAE to detect
     # the existence of the function and call it only if it is available.
-    patch("handle_missing_set_override_stack_alignment.patch", when="@21.2.3:")
+    patch("handle_missing_set_override_stack_alignment.patch", when="@21.2.3:22.3")
 
     # ROCm 5.3.0 is providing llvm15. Gallivm coroutine is disabled in mesa upstream version
     # for llvm-15. Until mesa release is available with this changes below patch is required
     # in order to move on with ROCm 5.3.0 and ROCm 5.4.0.
     # The revised patch was part of https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17518/diffs.
 
-    patch("0001-disable-gallivm-coroutine-for-libllvm15.patch", when="@22.1.2: ^libllvm@15:")
+    patch("0001-disable-gallivm-coroutine-for-libllvm15.patch", when="@22.1.2:22.3 ^libllvm@15")
 
     # Explicitly use the llvm-config tool
     def patch(self):
diff --git a/var/spack/repos/builtin/packages/mesa18/package.py b/var/spack/repos/builtin/packages/mesa18/package.py
index 663b1138a518a8..f7ba4e156baddb 100644
--- a/var/spack/repos/builtin/packages/mesa18/package.py
+++ b/var/spack/repos/builtin/packages/mesa18/package.py
@@ -21,7 +21,7 @@ class Mesa18(AutotoolsPackage):
     # whatever version of LLVM you're using.
     git = "https://gitlab.freedesktop.org/mesa/mesa.git"
 
-    version("18.3.6", tag="mesa-18.3.6")
+    version("18.3.6", tag="mesa-18.3.6", commit="11049bcff86da8013a4f63bd68daf637e3af22f3")
 
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
@@ -36,7 +36,7 @@ class Mesa18(AutotoolsPackage):
     depends_on("python@:3.8", type="build")  # https://github.com/spack/spack/issues/28219
     depends_on("py-mako@0.8.0:", type="build")
     depends_on("libxml2")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("expat")
     depends_on("ncurses+termlib")
 
diff --git a/var/spack/repos/builtin/packages/meshtool/package.py b/var/spack/repos/builtin/packages/meshtool/package.py
index 4a4dbdfe6f9f52..df74760ce9e978 100644
--- a/var/spack/repos/builtin/packages/meshtool/package.py
+++ b/var/spack/repos/builtin/packages/meshtool/package.py
@@ -16,9 +16,9 @@ class Meshtool(MakefilePackage):
 
     version("master", branch="master", preferred=True)
     # Version to use with openCARP releases
-    version("oc13.0", commit="867431d")
-    version("oc12.0", commit="867431d")
-    version("oc11.0", commit="867431d")
+    version("oc13.0", commit="867431d6bde35ad41104f611aa57130ef58cfb79")
+    version("oc12.0", commit="867431d6bde35ad41104f611aa57130ef58cfb79")
+    version("oc11.0", commit="867431d6bde35ad41104f611aa57130ef58cfb79")
     version("oc10.0", commit="6c5cfbd067120901f15a04bf63beec409bda6dc9")
     version("oc9.0", commit="6c5cfbd067120901f15a04bf63beec409bda6dc9")
     version("oc8.2", commit="6c5cfbd067120901f15a04bf63beec409bda6dc9")
diff --git a/var/spack/repos/builtin/packages/meson/package.py b/var/spack/repos/builtin/packages/meson/package.py
index 5e58eb6b80844e..cabafbe0011b7b 100644
--- a/var/spack/repos/builtin/packages/meson/package.py
+++ b/var/spack/repos/builtin/packages/meson/package.py
@@ -18,7 +18,12 @@ class Meson(PythonPackage):
 
     maintainers("eli-schwartz", "michaelkuhn")
 
+    version("1.2.2", sha256="1caa0ef6082e311bdca9836e7907f548b8c3f041a42ed41f0ff916b83ac7dddd")
+    version("1.2.1", sha256="e1f3b32b636cc86496261bd89e63f00f206754697c7069788b62beed5e042713")
+    version("1.2.0", sha256="603489f0aaa6305f806c6cc4a4455a965f22290fc74f65871f589b002110c790")
+    version("1.1.1", sha256="1c3b9e1a3a36b51adb5de498d582fd5cbf6763fadbcf151de9f2a762e02bd2e6")
     version("1.1.0", sha256="f29a3e14062043d75e82d16f1e41856e6b1ed7a7c016e10c7b13afa7ee6364cc")
+    version("1.0.2", sha256="1f1239c3091668643f7d2086663d6afd8cc87fbab84fe7462bc18b9ba6d65de8")
     version("1.0.1", sha256="4ab3a5c0060dc22bdefb04507efc6c38acb910e91bcd467a38e1fa211e5a6cfe")
     version("1.0.0", sha256="a2ada84d43c7e57400daee80a880a1f5003d062b2cb6c9be1747b0db38f2eb8d")
     version("0.64.1", sha256="1d12a4bc1cf3ab18946d12cf0b6452e5254ada1ad50aacc97f87e2cccd7da315")
@@ -81,6 +86,9 @@ class Meson(PythonPackage):
     # https://github.com/mesonbuild/meson/pull/9850
     patch("oneapi.patch", when="@0.62:0.63 %oneapi")
 
+    # Python 3.12 detection support
+    patch("python-3.12-support.patch", when="@1.1:1.2.2")
+
     executables = ["^meson$"]
 
     @classmethod
diff --git a/var/spack/repos/builtin/packages/meson/python-3.12-support.patch b/var/spack/repos/builtin/packages/meson/python-3.12-support.patch
new file mode 100644
index 00000000000000..727976ff26234b
--- /dev/null
+++ b/var/spack/repos/builtin/packages/meson/python-3.12-support.patch
@@ -0,0 +1,283 @@
+From 5f96e35b873d6230970fd63ba2e706bbd3f4e26f Mon Sep 17 00:00:00 2001
+From: Eli Schwartz 
+Date: Fri, 8 Sep 2023 16:54:48 -0400
+Subject: [PATCH 1/6] python dependency: ensure that setuptools doesn't inject
+ itself into distutils
+
+We do not use setuptools for anything, and only lightly use distutils.
+Unpredictable issues can occur due to setuptools monkey-patching, which
+interferes with our intended use. Tell setuptools to simply never get
+involved.
+
+Note: while it's otherwise possible to check if the probe is run using
+sys.executable and avoid forking, setuptools unconditionally injects
+itself at startup in a way that requires subprocess isolation to
+disable.
+
+(cherry picked from commit 9f610ad5b72ea91de2d7aeb6f3266d0a7477062e)
+---
+ mesonbuild/dependencies/python.py | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/mesonbuild/dependencies/python.py b/mesonbuild/dependencies/python.py
+index 160772888..f04494674 100644
+--- a/mesonbuild/dependencies/python.py
++++ b/mesonbuild/dependencies/python.py
+@@ -113,7 +113,9 @@ class BasicPythonExternalProgram(ExternalProgram):
+ 
+         with importlib.resources.path('mesonbuild.scripts', 'python_info.py') as f:
+             cmd = self.get_command() + [str(f)]
+-            p, stdout, stderr = mesonlib.Popen_safe(cmd)
++            env = os.environ.copy()
++            env['SETUPTOOLS_USE_DISTUTILS'] = 'stdlib'
++            p, stdout, stderr = mesonlib.Popen_safe(cmd, env=env)
+ 
+         try:
+             info = json.loads(stdout)
+-- 
+2.39.2
+
+
+From cb4e62a8c55118988babac8b8254e0af1dc9698b Mon Sep 17 00:00:00 2001
+From: Eli Schwartz 
+Date: Mon, 21 Nov 2022 20:47:14 -0500
+Subject: [PATCH 2/6] python module: stop using distutils schemes on
+ sufficiently new Debian
+
+Since 3.10.3, Debian finally started patching sysconfig with custom
+paths, instead of just distutils. This means we can now go use that
+instead. It reduces our reliance on the deprecated distutils module.
+
+Partial fix for #7702
+
+(cherry picked from commit 40f897fa92f7d3cc43788d3000733310ce77cf0c)
+---
+ mesonbuild/scripts/python_info.py | 32 +++++++++++++++++++++++--------
+ 1 file changed, 24 insertions(+), 8 deletions(-)
+
+diff --git a/mesonbuild/scripts/python_info.py b/mesonbuild/scripts/python_info.py
+index 9c3a0791a..65597b121 100755
+--- a/mesonbuild/scripts/python_info.py
++++ b/mesonbuild/scripts/python_info.py
+@@ -13,7 +13,6 @@ if sys.path[0].endswith('scripts'):
+     del sys.path[0]
+ 
+ import json, os, sysconfig
+-import distutils.command.install
+ 
+ def get_distutils_paths(scheme=None, prefix=None):
+     import distutils.dist
+@@ -37,15 +36,32 @@ def get_distutils_paths(scheme=None, prefix=None):
+ # default scheme to a custom one pointing to /usr/local and replacing
+ # site-packages with dist-packages.
+ # See https://github.com/mesonbuild/meson/issues/8739.
+-# XXX: We should be using sysconfig, but Debian only patches distutils.
++#
++# We should be using sysconfig, but before 3.10.3, Debian only patches distutils.
++# So we may end up falling back.
+ 
+-if 'deb_system' in distutils.command.install.INSTALL_SCHEMES:
+-    paths = get_distutils_paths(scheme='deb_system')
+-    install_paths = get_distutils_paths(scheme='deb_system', prefix='')
+-else:
+-    paths = sysconfig.get_paths()
++def get_install_paths():
++    if sys.version_info >= (3, 10):
++        scheme = sysconfig.get_default_scheme()
++    else:
++        scheme = sysconfig._get_default_scheme()
++
++    if sys.version_info >= (3, 10, 3):
++        if 'deb_system' in sysconfig.get_scheme_names():
++            scheme = 'deb_system'
++    else:
++        import distutils.command.install
++        if 'deb_system' in distutils.command.install.INSTALL_SCHEMES:
++            paths = get_distutils_paths(scheme='deb_system')
++            install_paths = get_distutils_paths(scheme='deb_system', prefix='')
++            return paths, install_paths
++
++    paths = sysconfig.get_paths(scheme=scheme)
+     empty_vars = {'base': '', 'platbase': '', 'installed_base': ''}
+-    install_paths = sysconfig.get_paths(vars=empty_vars)
++    install_paths = sysconfig.get_paths(scheme=scheme, vars=empty_vars)
++    return paths, install_paths
++
++paths, install_paths = get_install_paths()
+ 
+ def links_against_libpython():
+     from distutils.core import Distribution, Extension
+-- 
+2.39.2
+
+
+From c179c18765514d5c37737dec996b4c91cb31477f Mon Sep 17 00:00:00 2001
+From: Eli Schwartz 
+Date: Mon, 2 Oct 2023 16:40:15 -0400
+Subject: [PATCH 3/6] python module: refactor pypy detection into a consistent
+ variable
+
+(cherry picked from commit 3d3a10ef022284c8377bd9f8e1b1adec73c50d95)
+---
+ mesonbuild/scripts/python_info.py | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/mesonbuild/scripts/python_info.py b/mesonbuild/scripts/python_info.py
+index 65597b121..d17b3a376 100755
+--- a/mesonbuild/scripts/python_info.py
++++ b/mesonbuild/scripts/python_info.py
+@@ -72,6 +72,8 @@ def links_against_libpython():
+ variables = sysconfig.get_config_vars()
+ variables.update({'base_prefix': getattr(sys, 'base_prefix', sys.prefix)})
+ 
++is_pypy = '__pypy__' in sys.builtin_module_names
++
+ if sys.version_info < (3, 0):
+     suffix = variables.get('SO')
+ elif sys.version_info < (3, 8, 7):
+@@ -88,7 +90,7 @@ print(json.dumps({
+   'install_paths': install_paths,
+   'version': sysconfig.get_python_version(),
+   'platform': sysconfig.get_platform(),
+-  'is_pypy': '__pypy__' in sys.builtin_module_names,
++  'is_pypy': is_pypy,
+   'is_venv': sys.prefix != variables['base_prefix'],
+   'link_libpython': links_against_libpython(),
+   'suffix': suffix,
+-- 
+2.39.2
+
+
+From 3c493dae4bd8410bfb09e8f654605f65e15d8e66 Mon Sep 17 00:00:00 2001
+From: Eli Schwartz 
+Date: Tue, 22 Nov 2022 22:56:10 -0500
+Subject: [PATCH 4/6] python module: stop using distutils "link to libpython"
+ probe on recent python
+
+On python >=3.8, this information is expected to be encoded in the
+sysconfig vars.
+
+In distutils, it is always necessary to link to libpython on Windows;
+for posix platforms, it depends on the value of LIBPYTHON (which is the
+library to link to, possibly the empty string) as generated by
+configure.ac and embedded into python.pc and python-config.sh, and then
+coded a second time in the distutils python sources.
+
+There are a couple of caveats which have ramifications for Cygwin and
+Android:
+
+- python.pc and python-config.sh disagree with distutils when python is
+  not built shared. In that case, the former act the same as a shared
+  build, while the latter *never* links to libpython
+
+- python.pc disagrees with python-config.sh and distutils when python is
+  built shared. The former never links to libpython, while the latter do
+
+The disagreement is resolved in favor of distutils' behavior in all
+cases, and python.pc is correct for our purposes on python 3.12; see:
+https://github.com/python/cpython/pull/100356
+https://github.com/python/cpython/pull/100967
+
+Although it was not backported to older releases, Cygwin at least has
+always patched in a fix for python.pc, which behavior is now declared
+canonical. We can reliably assume it is always correct.
+
+This is the other half of the fix for #7702
+
+(cherry picked from commit 2d6c10908b3771216e7ce086af1ee4dc77e698c2)
+---
+ mesonbuild/scripts/python_info.py | 17 +++++++++++++----
+ 1 file changed, 13 insertions(+), 4 deletions(-)
+
+diff --git a/mesonbuild/scripts/python_info.py b/mesonbuild/scripts/python_info.py
+index d17b3a376..a3f3d3535 100755
+--- a/mesonbuild/scripts/python_info.py
++++ b/mesonbuild/scripts/python_info.py
+@@ -64,10 +64,19 @@ def get_install_paths():
+ paths, install_paths = get_install_paths()
+ 
+ def links_against_libpython():
+-    from distutils.core import Distribution, Extension
+-    cmd = Distribution().get_command_obj('build_ext')
+-    cmd.ensure_finalized()
+-    return bool(cmd.get_libraries(Extension('dummy', [])))
++    # on versions supporting python-embed.pc, this is the non-embed lib
++    #
++    # PyPy is not yet up to 3.12 and work is still pending to export the
++    # relevant information (it doesn't automatically provide arbitrary
++    # Makefile vars)
++    if sys.version_info >= (3, 8) and not is_pypy:
++        variables = sysconfig.get_config_vars()
++        return bool(variables.get('LIBPYTHON', 'yes'))
++    else:
++        from distutils.core import Distribution, Extension
++        cmd = Distribution().get_command_obj('build_ext')
++        cmd.ensure_finalized()
++        return bool(cmd.get_libraries(Extension('dummy', [])))
+ 
+ variables = sysconfig.get_config_vars()
+ variables.update({'base_prefix': getattr(sys, 'base_prefix', sys.prefix)})
+-- 
+2.39.2
+
+
+From ae44d9a379faca6274db819be44ffca3e0159f56 Mon Sep 17 00:00:00 2001
+From: Eli Schwartz 
+Date: Mon, 2 Oct 2023 23:51:57 -0400
+Subject: [PATCH 5/6] tests: fix test case to not import distutils on python
+ 3.12
+
+Testing the correctness of the `modules: ` kwarg can be done with other
+guaranteed stdlib modules that are even more guaranteed since they
+didn't get deprecated for removal.
+
+(cherry picked from commit ecf261330c498783760cbde00b613b7469f8d3c0)
+---
+ test cases/python/5 modules kwarg/meson.build | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/test cases/python/5 modules kwarg/meson.build b/test cases/python/5 modules kwarg/meson.build
+index 9751adaab..41a9a4fae 100644
+--- a/test cases/python/5 modules kwarg/meson.build	
++++ b/test cases/python/5 modules kwarg/meson.build	
+@@ -1,7 +1,7 @@
+ project('python kwarg')
+ 
+ py = import('python')
+-prog_python = py.find_installation('python3', modules : ['distutils'])
++prog_python = py.find_installation('python3', modules : ['os', 'sys', 're'])
+ assert(prog_python.found() == true, 'python not found when should be')
+ prog_python = py.find_installation('python3', modules : ['thisbetternotexistmod'], required : false)
+ assert(prog_python.found() == false, 'python not found but reported as found')
+-- 
+2.39.2
+
+
+From d9abf4a97dc182b3c57204a792000d620f9f941e Mon Sep 17 00:00:00 2001
+From: Eli Schwartz 
+Date: Tue, 3 Oct 2023 00:22:25 -0400
+Subject: [PATCH 6/6] mark the PyPI metadata as supporting python 3.12
+
+meson itself runs okay on 3.12, and the last issue for *probing* against
+3.12 is solved. Tests pass here locally.
+
+(cherry picked from commit 880f21281ee359e01de659fe7d45549d19e6b84d)
+---
+ setup.cfg | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/setup.cfg b/setup.cfg
+index dfaba76dd..2f2962eed 100644
+--- a/setup.cfg
++++ b/setup.cfg
+@@ -30,6 +30,7 @@ classifiers =
+   Programming Language :: Python :: 3.9
+   Programming Language :: Python :: 3.10
+   Programming Language :: Python :: 3.11
++  Programming Language :: Python :: 3.12
+   Topic :: Software Development :: Build Tools
+ long_description = Meson is a cross-platform build system designed to be both as fast and as user friendly as possible. It supports many languages and compilers, including GCC, Clang, PGI, Intel, and Visual Studio. Its build definitions are written in a simple non-Turing complete DSL.
+ 
+-- 
+2.39.2
+
diff --git a/var/spack/repos/builtin/packages/met/package.py b/var/spack/repos/builtin/packages/met/package.py
index c1646e3dcbab51..750422dc763509 100644
--- a/var/spack/repos/builtin/packages/met/package.py
+++ b/var/spack/repos/builtin/packages/met/package.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -14,10 +14,12 @@ class Met(AutotoolsPackage):
     configurable methods to compute statistics and diagnostics"""
 
     homepage = "https://dtcenter.org/community-code/model-evaluation-tools-met"
+    git = "https://github.com/dtcenter/MET"
     url = "https://github.com/dtcenter/MET/archive/refs/tags/v11.0.1.tar.gz"
 
     maintainers("AlexanderRichert-NOAA", "climbfuji")
 
+    version("develop", branch="develop")
     version("11.1.0", sha256="e2e371ae1f49185ff8bf08201b1a3e90864a467aa3369b04132d231213c3c9e5")
     version("11.0.2", sha256="f720d15e1d6c235c9a41fd97dbeb0eb1082fb8ae99e1bcdcb5e51be9b50bdfbf")
     version("11.0.1", sha256="48d471ad4634f1b969d9358c51925ce36bf0a1cec5312a6755203a4794b81646")
@@ -37,7 +39,7 @@ class Met(AutotoolsPackage):
 
     depends_on("gsl")
     depends_on("bufr")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("netcdf-c")
     depends_on("netcdf-cxx4")
     depends_on("g2c", when="+grib2")
@@ -50,7 +52,7 @@ class Met(AutotoolsPackage):
     depends_on("cairo", when="+graphics")
     depends_on("freetype", when="+graphics")
 
-    depends_on("python@3.6.3:", when="+python", type=("build", "run"))
+    depends_on("python@3.6.3:", when="+python")
     depends_on("py-netcdf4", when="+python", type=("build", "run"))
     depends_on("py-numpy", when="+python", type=("build", "run"))
     depends_on("py-xarray", when="+python", type=("build", "run"))
@@ -59,8 +61,10 @@ class Met(AutotoolsPackage):
     patch("openmp_shape_patch.patch", when="@10.1.0")
 
     # https://github.com/JCSDA/spack-stack/issues/615
-    patch("apple-clang-string-cast-operator.patch", when="@10.1.1:11.0.99 %apple-clang@14:")
-    patch("apple-clang-no-register.patch", when="@10.1.1:11.0.99 %apple-clang@14:")
+    # TODO(srherbener) Apple clang 14.x is getting pickier! When these updates are
+    # merged into the MET code base, the following two patches can be removed.
+    patch("apple-clang-string-cast-operator.patch", when="@10.1.1:11.0 %apple-clang@14:")
+    patch("apple-clang-no-register.patch", when="@10.1.1:11.0 %apple-clang@14:")
 
     def url_for_version(self, version):
         if version < Version("11"):
@@ -102,7 +106,7 @@ def setup_build_environment(self, env):
             ldflags.append(nc_config("--libs", "--static", output=str).strip())
             libs.append(nc_config("--libs", "--static", output=str).strip())
 
-        zlib = spec["zlib"]
+        zlib = spec["zlib-api"]
         cppflags.append("-D__64BIT__")
         ldflags.append("-L" + zlib.prefix.lib)
         libs.append("-lz")
@@ -160,26 +164,18 @@ def setup_build_environment(self, env):
 
     def configure_args(self):
         args = []
-        spec = self.spec
-
-        if "+grib2" in spec:
-            args.append("--enable-grib2")
-
-        if "+python" in spec:
-            args.append("--enable-python")
-
-        if "~openmp" in spec:
-            args.append("--disable-openmp")
-
-        if "+lidar2nc" in spec:
-            args.append("--enable-lidar2nc")
-
-        if "+modis" in spec:
-            args.append("--enable-modis")
-
-        if "+graphics" in spec:
-            args.append("--enable-mode_graphics")
+        args.extend(self.enable_or_disable("grib2"))
+        args.extend(self.enable_or_disable("python"))
+        args.extend(self.enable_or_disable("openmp"))
+        args.extend(self.enable_or_disable("lidar2nc"))
+        args.extend(self.enable_or_disable("modis"))
+        args.extend(self.enable_or_disable("mode_graphics", variant="graphics"))
 
         if self.spec.satisfies("%apple-clang@14:"):
             args.append("CXXFLAGS=-std=gnu++17")
+
         return args
+
+
+#    def setup_run_environment(self, env):
+#        env.set('MET_BASE', self.prefix)
diff --git a/var/spack/repos/builtin/packages/metabat/package.py b/var/spack/repos/builtin/packages/metabat/package.py
index 08846c107770e4..c623bd87bf97cb 100644
--- a/var/spack/repos/builtin/packages/metabat/package.py
+++ b/var/spack/repos/builtin/packages/metabat/package.py
@@ -37,7 +37,7 @@ class Metabat(CMakePackage):
     # See https://github.com/spack/spack/pull/22303 for reference
     depends_on(Boost.with_default_variants, type=("build", "run"))
     depends_on("perl", type="run")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("ncurses", type="link")
 
     def setup_build_environment(self, env):
diff --git a/var/spack/repos/builtin/packages/metal/package.py b/var/spack/repos/builtin/packages/metal/package.py
new file mode 100644
index 00000000000000..24325c744b753e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/metal/package.py
@@ -0,0 +1,25 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Metal(CMakePackage):
+    """METAL is a tool for the meta-analysis of genome-wide association studies"""
+
+    homepage = "https://genome.sph.umich.edu/wiki/METAL"
+    url = "https://github.com/statgen/METAL/archive/refs/tags/2020-05-05.tar.gz"
+
+    version(
+        "2020-05-05", sha256="0ffa2419ca2ab43766e7e6e8c97822c8ce1f5b6233fb5f992d1b1be1955fede7"
+    )
+
+    depends_on("cmake@3.1:", type="build")
+    depends_on("zlib-ng")
+
+    @run_after("install")
+    def mv_binary(self):
+        with working_dir(self.build_directory):
+            install_tree("bin", self.prefix.bin)
diff --git a/var/spack/repos/builtin/packages/metplus/package.py b/var/spack/repos/builtin/packages/metplus/package.py
index efa2fdcdc68180..facae3f5928fa8 100644
--- a/var/spack/repos/builtin/packages/metplus/package.py
+++ b/var/spack/repos/builtin/packages/metplus/package.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -13,10 +13,12 @@ class Metplus(Package):
     """
 
     homepage = "https://dtcenter.org/community-code/metplus"
+    git = "https://github.com/dtcenter/METplus"
     url = "https://github.com/dtcenter/METplus/archive/refs/tags/v4.1.0.tar.gz"
 
     maintainers("AlexanderRichert-NOAA", "climbfuji")
 
+    version("develop", branch="develop")
     version("5.1.0", sha256="e80df2d1059176a453b7991a9f123cb5a187cc7ba7f48a75313b92c7a0e68474")
     version("5.0.1", sha256="0e22b4f6791496551d99f68247d382b2af02c90b34c172a64c6f060e774bdced")
     version("5.0.0", sha256="59d519bd062559b4cece9f8672e2e282b200057bc77e2e0937414003d8f2dd50")
diff --git a/var/spack/repos/builtin/packages/mfem/mfem-4.6.patch b/var/spack/repos/builtin/packages/mfem/mfem-4.6.patch
new file mode 100644
index 00000000000000..94f1f863f8d267
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mfem/mfem-4.6.patch
@@ -0,0 +1,12 @@
+diff --git a/general/kdtree.hpp b/general/kdtree.hpp
+index eebbdaa27..b35a33ea4 100644
+--- a/general/kdtree.hpp
++++ b/general/kdtree.hpp
+@@ -17,6 +17,7 @@
+ #include 
+ #include 
+ #include 
++#include 
+ #include 
+ 
+ namespace mfem
diff --git a/var/spack/repos/builtin/packages/mfem/package.py b/var/spack/repos/builtin/packages/mfem/package.py
index b3b8daad4b22f3..f4821e63c2ba0f 100644
--- a/var/spack/repos/builtin/packages/mfem/package.py
+++ b/var/spack/repos/builtin/packages/mfem/package.py
@@ -6,6 +6,7 @@
 import os
 import shutil
 import sys
+from platform import machine
 
 from spack.package import *
 
@@ -48,6 +49,13 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     # other version.
     version("develop", branch="master")
 
+    version(
+        "4.6.0",
+        sha256="5fa9465b5bec56bfb777a4d2826fba48d85fbace4aed8b64a2fd4059bf075b15",
+        url="https://bit.ly/mfem-4-6",
+        extension="tar.gz",
+    )
+
     version(
         "4.5.2",
         sha256="7003c908c8265810ff97cb37531521b3aed24959975833a01ea05adfdb36e0f7",
@@ -101,7 +109,9 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     )
 
     # Tagged development version used by the laghos package:
-    version("3.4.1-laghos-v2.0", tag="laghos-v2.0")
+    version(
+        "3.4.1-laghos-v2.0", tag="laghos-v2.0", commit="4cb8d2cbc19aac5528df650b4a41c54158d1d2c2"
+    )
 
     version(
         "3.4.0",
@@ -118,7 +128,9 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     )
 
     # Tagged development version used by the laghos package:
-    version("3.3.1-laghos-v1.0", tag="laghos-v1.0")
+    version(
+        "3.3.1-laghos-v1.0", tag="laghos-v1.0", commit="e1f466e6bf80d5f8449b9e1585c414bad4704195"
+    )
 
     version(
         "3.3",
@@ -282,6 +294,11 @@ class Mfem(Package, CudaPackage, ROCmPackage):
             "sundials@5.4.0:+cuda cuda_arch={0}".format(sm_),
             when="@4.2.0:+sundials+cuda cuda_arch={0}".format(sm_),
         )
+    for gfx in ROCmPackage.amdgpu_targets:
+        depends_on(
+            "sundials@5.7.0:+rocm amdgpu_target={0}".format(gfx),
+            when="@4.6.0:+sundials+rocm amdgpu_target={0}".format(gfx),
+        )
     depends_on("pumi", when="+pumi~shared")
     depends_on("pumi+shared", when="+pumi+shared")
     depends_on("pumi@2.2.3:2.2.5", when="@4.2.0:4.3.0+pumi")
@@ -292,6 +309,20 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on("gslib@1.0.7:", when="@4.3.0:+gslib")
     depends_on("suite-sparse", when="+suite-sparse")
     depends_on("superlu-dist", when="+superlu-dist")
+    # Propagate 'cuda_arch' to 'superlu-dist' without propagating the '+cuda'
+    # variant so we can build 'mfem+cuda+superlu-dist ^superlu-dist~cuda':
+    for sm_ in CudaPackage.cuda_arch_values:
+        depends_on(
+            "superlu-dist+cuda cuda_arch={0}".format(sm_),
+            when="+superlu-dist+cuda cuda_arch={0} ^superlu-dist+cuda".format(sm_),
+        )
+    # Propagate 'amdgpu_target' to 'superlu-dist' without propagating the '+rocm'
+    # variant so we can build 'mfem+rocm+superlu-dist ^superlu-dist~rocm':
+    for gfx in ROCmPackage.amdgpu_targets:
+        depends_on(
+            "superlu-dist+rocm amdgpu_target={0}".format(gfx),
+            when="+superlu-dist+rocm amdgpu_target={0} ^superlu-dist+rocm".format(gfx),
+        )
     depends_on("strumpack@3.0.0:", when="+strumpack~shared")
     depends_on("strumpack@3.0.0:+shared", when="+strumpack+shared")
     for sm_ in CudaPackage.cuda_arch_values:
@@ -334,7 +365,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on("mpfr", when="+mpfr")
     depends_on("netcdf-c@4.1.3:", when="+netcdf")
     depends_on("unwind", when="+libunwind")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("gnutls", when="+gnutls")
     depends_on("conduit@0.3.1:,master:", when="+conduit")
     depends_on("conduit+mpi", when="+conduit+mpi")
@@ -441,6 +472,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     # upstream.
     patch("mfem-4.0.0-makefile-syntax-fix.patch", when="@4.0.0")
     patch("mfem-4.5.patch", when="@4.5.0")
+    patch("mfem-4.6.patch", when="@4.6.0")
 
     phases = ["configure", "build", "install"]
 
@@ -852,11 +884,11 @@ def find_optional_library(name, prefix):
 
         if "+zlib" in spec:
             if "@:3.3.2" in spec:
-                options += ["ZLIB_DIR=%s" % spec["zlib"].prefix]
+                options += ["ZLIB_DIR=%s" % spec["zlib-api"].prefix]
             else:
                 options += [
-                    "ZLIB_OPT=-I%s" % spec["zlib"].prefix.include,
-                    "ZLIB_LIB=%s" % ld_flags_from_library_list(spec["zlib"].libs),
+                    "ZLIB_OPT=-I%s" % spec["zlib-api"].prefix.include,
+                    "ZLIB_LIB=%s" % ld_flags_from_library_list(spec["zlib-api"].libs),
                 ]
 
         if "+mpfr" in spec:
@@ -891,10 +923,27 @@ def find_optional_library(name, prefix):
                 "CUDA_CXX=%s" % join_path(spec["cuda"].prefix, "bin", "nvcc"),
                 "CUDA_ARCH=sm_%s" % cuda_arch,
             ]
+            # Check if we are using a CUDA installation where the math libs are
+            # in a separate directory:
+            culibs = ["libcusparse"]
+            cuda_libs = find_optional_library(culibs, spec["cuda"].prefix)
+            if not cuda_libs:
+                p0 = os.path.realpath(join_path(spec["cuda"].prefix, "bin", "nvcc"))
+                p0 = os.path.dirname(p0)
+                p1 = os.path.dirname(p0)
+                while p1 != p0:
+                    cuda_libs = find_optional_library(culibs, join_path(p1, "math_libs"))
+                    if cuda_libs:
+                        break
+                    p0, p1 = p1, os.path.dirname(p1)
+                if not cuda_libs:
+                    raise InstallError("Required CUDA libraries not found: %s" % culibs)
+                options += ["CUDA_LIB=%s" % ld_flags_from_library_list(cuda_libs)]
 
         if "+rocm" in spec:
             amdgpu_target = ",".join(spec.variants["amdgpu_target"].value)
             options += ["HIP_CXX=%s" % spec["hip"].hipcc, "HIP_ARCH=%s" % amdgpu_target]
+            hip_headers = HeaderList([])
             hip_libs = LibraryList([])
             # To use a C++ compiler that supports -xhip flag one can use
             # something like this:
@@ -905,7 +954,7 @@ def find_optional_library(name, prefix):
             #   hip_libs += find_libraries("libamdhip64", spec["hip"].prefix.lib)
             if "^hipsparse" in spec:  # hipsparse is needed @4.4.0:+rocm
                 hipsparse = spec["hipsparse"]
-                options += ["HIP_OPT=%s" % hipsparse.headers.cpp_flags]
+                hip_headers += hipsparse.headers
                 hip_libs += hipsparse.libs
                 # Note: MFEM's defaults.mk wants to find librocsparse.* in
                 # $(HIP_DIR)/lib, so we set HIP_DIR to be $ROCM_PATH when using
@@ -915,11 +964,16 @@ def find_optional_library(name, prefix):
                     options += ["HIP_DIR=%s" % env["ROCM_PATH"]]
                 else:
                     options += ["HIP_DIR=%s" % hipsparse["rocsparse"].prefix]
+            if "^rocthrust" in spec and not spec["hip"].external:
+                # petsc+rocm needs the rocthrust header path
+                hip_headers += spec["rocthrust"].headers
             if "%cce" in spec:
                 # We assume the proper Cray CCE module (cce) is loaded:
-                craylibs_path = env["CRAYLIBS_" + env["MACHTYPE"].capitalize()]
+                craylibs_path = env["CRAYLIBS_" + machine().upper()]
                 craylibs = ["libmodules", "libfi", "libcraymath", "libf", "libu", "libcsup"]
                 hip_libs += find_libraries(craylibs, craylibs_path)
+            if hip_headers:
+                options += ["HIP_OPT=%s" % hip_headers.cpp_flags]
             if hip_libs:
                 options += ["HIP_LIB=%s" % ld_flags_from_library_list(hip_libs)]
 
@@ -1170,6 +1224,8 @@ def sundials_components(self):
                 sun_comps += ",nvecparhyp,nvecparallel"
         if "+cuda" in spec and "+cuda" in spec["sundials"]:
             sun_comps += ",nveccuda"
+        if "+rocm" in spec and "+rocm" in spec["sundials"]:
+            sun_comps += ",nvechip"
         return sun_comps
 
     @property
diff --git a/var/spack/repos/builtin/packages/mfem/test_builds.sh b/var/spack/repos/builtin/packages/mfem/test_builds.sh
index 13e72e7d558a5f..cb658dd59cc468 100755
--- a/var/spack/repos/builtin/packages/mfem/test_builds.sh
+++ b/var/spack/repos/builtin/packages/mfem/test_builds.sh
@@ -14,9 +14,9 @@ rocm_arch="gfx908"
 spack_jobs=''
 # spack_jobs='-j 128'
 
-mfem='mfem@4.5.2'${compiler}
+mfem='mfem@4.6.0'${compiler}
 # mfem_dev='mfem@develop'${compiler}
-mfem_dev='mfem@4.5.2'${compiler}
+mfem_dev='mfem@4.6.0'${compiler}
 
 backends='+occa+raja+libceed'
 backends_specs='^occa~cuda ^raja~openmp'
@@ -24,15 +24,16 @@ backends_specs='^occa~cuda ^raja~openmp'
 # ~fortran is needed for Cray Fortran linking with tcmalloc*
 conduit_spec='^conduit~fortran'
 # petsc spec
-petsc_spec='^petsc+suite-sparse+mumps'
-petsc_spec_cuda='^petsc+cuda+suite-sparse+mumps'
-# superlu-dist specs
-superlu_spec_cuda='^superlu-dist+cuda cuda_arch='"${cuda_arch}"
-superlu_spec_rocm='^superlu-dist+rocm amdgpu_target='"${rocm_arch}"
+petsc_spec='^petsc+mumps'
+petsc_spec_cuda='^petsc+cuda+mumps'
+petsc_spec_rocm='^petsc+rocm+mumps'
 # strumpack spec without cuda (use version > 6.3.1)
 strumpack_spec='^strumpack~slate~openmp~cuda'
 strumpack_cuda_spec='^strumpack+cuda~slate~openmp'
 strumpack_rocm_spec='^strumpack+rocm~slate~openmp~cuda'
+# superlu specs with cuda and rocm
+superlu_cuda_spec='^superlu-dist+cuda'
+superlu_rocm_spec='^superlu-dist+rocm'
 
 builds=(
     # preferred version:
@@ -138,7 +139,7 @@ builds_cuda=(
         +superlu-dist+strumpack+suite-sparse+gslib+petsc+slepc \
         +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         ^raja+cuda+openmp ^hiop+shared'" $strumpack_cuda_spec"' \
-        '"$superlu_spec_cuda $petsc_spec_cuda $conduit_spec"
+        '"$superlu_cuda_spec $petsc_spec_cuda $conduit_spec"
 
     # hypre with cuda:
     # TODO: restore '+libceed' when the libCEED CUDA unit tests take less time.
@@ -150,7 +151,7 @@ builds_cuda=(
         +superlu-dist+strumpack+suite-sparse+gslib \
         +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         ^raja+cuda+openmp ^hiop+shared ^hypre+cuda \
-        '" $superlu_spec_cuda $strumpack_cuda_spec $conduit_spec"
+        '" $strumpack_cuda_spec $superlu_cuda_spec $conduit_spec"
 
     #
     # same builds as above with ${mfem_dev}
@@ -175,7 +176,7 @@ builds_cuda=(
         +superlu-dist+strumpack+suite-sparse+gslib+petsc+slepc \
         +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         ^raja+cuda+openmp ^hiop+shared'" $strumpack_cuda_spec"' \
-        '"$superlu_spec_cuda $petsc_spec_cuda $conduit_spec"
+        '"$superlu_cuda_spec $petsc_spec_cuda $conduit_spec"
 
     # hypre with cuda:
     # TODO: restore '+libceed' when the libCEED CUDA unit tests take less time.
@@ -187,7 +188,7 @@ builds_cuda=(
         +superlu-dist+strumpack+suite-sparse+gslib \
         +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
         ^raja+cuda+openmp ^hiop+shared ^hypre+cuda \
-        '"$superlu_spec_cuda $strumpack_cuda_spec $conduit_spec"
+        '"$strumpack_cuda_spec $superlu_cuda_spec $conduit_spec"
 )
 
 
@@ -203,15 +204,14 @@ builds_rocm=(
         ^raja+rocm~openmp ^occa~cuda~openmp ^hypre+rocm'
 
     # hypre without rocm:
-    # TODO: add "+petsc+slepc $petsc_spec_rocm" when it is supported.
     # TODO: add back '+hiop' when it is no longer linked with tcmalloc* through
     #       its magma dependency.
     # TODO: add back '+ginkgo' when the Ginkgo example works.
     ${mfem}'+rocm+openmp+raja+occa+libceed amdgpu_target='"${rocm_arch}"' \
-        +superlu-dist+strumpack+suite-sparse+gslib \
+        +superlu-dist+strumpack+suite-sparse+gslib+petsc+slepc \
         +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
         ^raja+rocm~openmp ^occa~cuda'" $strumpack_rocm_spec"' \
-        '"$superlu_spec_rocm $conduit_spec"
+        '"$superlu_rocm_spec $petsc_spec_rocm $conduit_spec"
 
     # hypre with rocm:
     # TODO: add back "+petsc+slepc $petsc_spec_rocm" when it works.
@@ -223,7 +223,7 @@ builds_rocm=(
         +superlu-dist+strumpack+suite-sparse+gslib \
         +pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
         ^raja+rocm~openmp ^occa~cuda ^hypre+rocm \
-        '"$strumpack_rocm_spec $superlu_spec_rocm $conduit_spec"
+        '"$strumpack_rocm_spec $superlu_rocm_spec $conduit_spec"
 
     #
     # same builds as above with ${mfem_dev}
diff --git a/var/spack/repos/builtin/packages/mgard/package.py b/var/spack/repos/builtin/packages/mgard/package.py
index a73129d97d74aa..411dd0c9b9c6e2 100644
--- a/var/spack/repos/builtin/packages/mgard/package.py
+++ b/var/spack/repos/builtin/packages/mgard/package.py
@@ -26,21 +26,31 @@ class Mgard(CMakePackage, CudaPackage):
     version("2021-11-12", commit="3c05c80a45a51bb6cc5fb5fffe7b1b16787d3366")
     version("2020-10-01", commit="b67a0ac963587f190e106cc3c0b30773a9455f7a")
 
-    variant("serial", when="@2022-11-18:", default=True)
-    variant("openmp", when="@2022-11-18:", default=True)
-    variant("timing", when="@2022-11-18:", default=False)
-    variant("unstructured", when="@2022-11-18:", default=False)
+    variant(
+        "serial",
+        when="@2022-11-18:",
+        default=True,
+        description="Enable the classic non-parallel implmementation",
+    )
+    variant("openmp", when="@2022-11-18:", default=True, description="Enable OpenMP support")
+    variant("timing", when="@2022-11-18:", default=False, description="Enable profile timings")
+    variant(
+        "unstructured",
+        when="@2022-11-18:",
+        default=False,
+        description="Enable experimental unstructured mesh support",
+    )
 
     depends_on("python", type=("build",), when="@2022-11-18:")
     depends_on("sed", type=("build",), when="@2022-11-18:")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("pkgconfig", type=("build",), when="@2022-11-18:")
     depends_on("zstd")
     depends_on("protobuf@:3.21.12", when="@2022-11-18:")
     depends_on("libarchive", when="@2021-11-12:")
     depends_on("tclap", when="@2021-11-12")
     depends_on("yaml-cpp", when="@2021-11-12:")
-    depends_on("cmake@3.19:")
+    depends_on("cmake@3.19:", type="build")
     depends_on("nvcomp@2.2.0:", when="@2022-11-18:+cuda")
     depends_on("nvcomp@2.0.2", when="@:2021-11-12+cuda")
     conflicts("cuda_arch=none", when="+cuda")
@@ -49,6 +59,12 @@ class Mgard(CMakePackage, CudaPackage):
     )
     conflicts("%gcc@:7", when="@2022-11-18:", msg="requires std::optional and other c++17 things")
 
+    def flag_handler(self, name, flags):
+        if name == "cxxflags":
+            if self.spec.satisfies("@2020-10-01 %oneapi@2023:"):
+                flags.append("-Wno-error=c++11-narrowing")
+        return (flags, None, None)
+
     def cmake_args(self):
         spec = self.spec
         args = ["-DBUILD_TESTING=OFF"]
diff --git a/var/spack/repos/builtin/packages/mgis/package.py b/var/spack/repos/builtin/packages/mgis/package.py
index fc0c93836e74df..7910537426ce23 100644
--- a/var/spack/repos/builtin/packages/mgis/package.py
+++ b/var/spack/repos/builtin/packages/mgis/package.py
@@ -46,6 +46,7 @@ class Mgis(CMakePackage):
     variant("c", default=True, description="Enables c bindings")
     variant("fortran", default=True, description="Enables fortran bindings")
     variant("python", default=True, description="Enables python bindings")
+    variant("static", default=False, description="Enables static libraries")
     variant(
         "build_type",
         default="Release",
@@ -104,4 +105,7 @@ def cmake_args(self):
             # adding path to boost
             args.append("-DBOOST_ROOT={0}".format(self.spec["boost"].prefix))
 
+        if "+static" in self.spec:
+            args.append("-Denable-static=ON")
+
         return args
diff --git a/var/spack/repos/builtin/packages/micromamba/package.py b/var/spack/repos/builtin/packages/micromamba/package.py
index 58523ccc95cfa2..7b8f2a0645cfc8 100644
--- a/var/spack/repos/builtin/packages/micromamba/package.py
+++ b/var/spack/repos/builtin/packages/micromamba/package.py
@@ -45,8 +45,8 @@ class Micromamba(CMakePackage):
         depends_on("yaml-cpp", type="link")
         depends_on("libreproc+cxx+shared", type="link")
         depends_on("tl-expected@2022-11-24", type="link")
-        depends_on("fmt", type="link")
-        depends_on("spdlog", type="link")
+        depends_on("fmt@9.1.0", type="link")
+        depends_on("spdlog@1.11.0", type="link")
 
         # https://github.com/mamba-org/mamba/blob/micromamba-1.0.0/libmamba/include/mamba/core/validate.hpp#L13
         depends_on("nlohmann-json", type="link")
@@ -78,8 +78,8 @@ class Micromamba(CMakePackage):
         depends_on("yaml-cpp", type="link")
         depends_on("libreproc+cxx", type="link")
         depends_on("tl-expected@2022-11-24", type="link")
-        depends_on("fmt", type="link")
-        depends_on("spdlog", type="link")
+        depends_on("fmt@9.1.0", type="link")
+        depends_on("spdlog@1.11.0", type="link")
         depends_on("nlohmann-json", type="link")
         depends_on("cpp-termcolor", type="link")
         depends_on("cli11@2.2:", type="link")
@@ -110,7 +110,7 @@ class Micromamba(CMakePackage):
             depends_on("bzip2", type="link")
             depends_on("lz4", type="link")
             depends_on("zstd", type="link")
-            depends_on("zlib", type="link")
+            depends_on("zlib-api", type="link")
             depends_on("xz libs=static", type="link")
             depends_on("lzo", type="link")
             depends_on("libsolv+conda~shared", type="link")
diff --git a/var/spack/repos/builtin/packages/migraphx/0004-restrict-python2.7-usage-for-5.5.0.patch b/var/spack/repos/builtin/packages/migraphx/0004-restrict-python2.7-usage-for-5.5.0.patch
new file mode 100644
index 00000000000000..8dfb4fc13ad3cd
--- /dev/null
+++ b/var/spack/repos/builtin/packages/migraphx/0004-restrict-python2.7-usage-for-5.5.0.patch
@@ -0,0 +1,25 @@
+From 3edc823a73a020df7e674a9d5493a4b690798fa4 Mon Sep 17 00:00:00 2001
+From: sreenivasa murthy kolam 
+Date: Tue, 9 May 2023 10:16:20 +0000
+Subject: [PATCH] restrict python2.7 usage for 5.5.0
+
+---
+ cmake/PythonModules.cmake | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/cmake/PythonModules.cmake b/cmake/PythonModules.cmake
+index b5818ce..b4bfbb3 100755
+--- a/cmake/PythonModules.cmake
++++ b/cmake/PythonModules.cmake
+@@ -76,7 +76,7 @@ function(py_add_module NAME)
+     )
+ 
+ endfunction()
+-set(PYTHON_SEARCH_VERSIONS 2.7 3.5 3.6 3.7 3.8 3.9 3.10)
++set(PYTHON_SEARCH_VERSIONS 3.5 3.6 3.7 3.8 3.9 3.10)
+ set(PYTHON_DISABLE_VERSIONS "" CACHE STRING "")
+ foreach(PYTHON_DISABLE_VERSION ${PYTHON_DISABLE_VERSIONS})
+     list(REMOVE_ITEM PYTHON_SEARCH_VERSIONS ${PYTHON_DISABLE_VERSION})
+-- 
+2.17.1
+
diff --git a/var/spack/repos/builtin/packages/migraphx/0005-Adding-half-include-directory-path-migraphx.patch b/var/spack/repos/builtin/packages/migraphx/0005-Adding-half-include-directory-path-migraphx.patch
new file mode 100644
index 00000000000000..b11445bdca57d9
--- /dev/null
+++ b/var/spack/repos/builtin/packages/migraphx/0005-Adding-half-include-directory-path-migraphx.patch
@@ -0,0 +1,48 @@
+From 612664789657444daa88f8f28a183928e01595d0 Mon Sep 17 00:00:00 2001
+From: Renjith Ravindran 
+Date: Mon, 25 Sep 2023 19:30:19 +0000
+Subject: [PATCH] Adding-half-include-directory-path
+
+---
+ CMakeLists.txt            | 4 +++-
+ cmake/PythonModules.cmake | 2 +-
+ 2 files changed, 4 insertions(+), 2 deletions(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 4601cdd..9cd48ad 100755
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -51,7 +51,7 @@ set(CMAKE_BUILD_RPATH "${CMAKE_BINARY_DIR}/lib")
+ project(migraphx)
+ find_package(ROCM REQUIRED)
+ 
+-find_path(HALF_INCLUDE_DIR half.hpp PATH_SUFFIXES half)
++find_path(HALF_INCLUDE_DIR half.hpp)
+ if (NOT HALF_INCLUDE_DIR)
+     message(FATAL_ERROR "Could not find half.hpp - Please check that the install path of half.hpp has been added to CMAKE_PREFIX_PATH")
+ else()
+@@ -272,6 +272,8 @@ add_subdirectory(docs)
+ add_subdirectory(test)
+ add_subdirectory(tools)
+ 
++target_include_directories(migraphx PUBLIC "${NLOHMANN_JSON_INCLUDE} ${HALF_INCLUDE_DIR}")
++
+ set(DEST_DIR ${CMAKE_BINARY_DIR})
+ file(GLOB backend_files ${CMAKE_SOURCE_DIR}/src/py/backend/*.py)
+ file(MAKE_DIRECTORY ${DEST_DIR}/lib/onnx_migraphx)
+diff --git a/cmake/PythonModules.cmake b/cmake/PythonModules.cmake
+index b5818ce..b4bfbb3 100755
+--- a/cmake/PythonModules.cmake
++++ b/cmake/PythonModules.cmake
+@@ -76,7 +76,7 @@ function(py_add_module NAME)
+     )
+ 
+ endfunction()
+-set(PYTHON_SEARCH_VERSIONS 2.7 3.5 3.6 3.7 3.8 3.9 3.10)
++set(PYTHON_SEARCH_VERSIONS 3.5 3.6 3.7 3.8 3.9 3.10)
+ set(PYTHON_DISABLE_VERSIONS "" CACHE STRING "")
+ foreach(PYTHON_DISABLE_VERSION ${PYTHON_DISABLE_VERSIONS})
+     list(REMOVE_ITEM PYTHON_SEARCH_VERSIONS ${PYTHON_DISABLE_VERSION})
+-- 
+2.31.1
+
diff --git a/var/spack/repos/builtin/packages/migraphx/package.py b/var/spack/repos/builtin/packages/migraphx/package.py
index 5a1f0372426635..81bf1bff2b3818 100644
--- a/var/spack/repos/builtin/packages/migraphx/package.py
+++ b/var/spack/repos/builtin/packages/migraphx/package.py
@@ -13,12 +13,16 @@ class Migraphx(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/AMDMIGraphX"
     git = "https://github.com/ROCmSoftwarePlatform/AMDMIGraphX.git"
-    url = "https://github.com/ROCmSoftwarePlatform/AMDMIGraphX/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/AMDMIGraphX/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
     libraries = ["libmigraphx"]
 
+    version("5.6.1", sha256="b108c33f07572ffd880b20f6de06f1934ab2a1b41ae69095612322ac412fa91c")
+    version("5.6.0", sha256="eaec90535d62002fd5bb264677ad4a7e30c55f18d2a287680d0495c7e60432b2")
+    version("5.5.1", sha256="e71c4744f8ef6a1a99c179bbad94b8fe9bd7686eaa9397f376b70988c3341f0c")
+    version("5.5.0", sha256="6084eb596b170f5e38f22b5fa37e66aa43a8cbc626712c9f03cde48c8fecfc8f")
     version("5.4.3", sha256="f83e7bbe5d6d0951fb2cf0abf7e8b3530e9a5e45f7cec6d760da055d6905d568")
     version("5.4.0", sha256="b6e7f4a1bf445ea0dae644ed5722369cde66fbee82a5917722f5d3f8c48b0a8c")
     version("5.3.3", sha256="91d91902bbedd5e1951a231e8e5c9a328360b128c731912ed17c8059df38e02a")
@@ -108,25 +112,21 @@ def url_for_version(self, version):
 
         return url
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
-    patch("0001-Adding-nlohmann-json-include-directory.patch", when="@3.9.0:")
+    patch("0001-Adding-nlohmann-json-include-directory.patch", when="@3.9.0:5.5")
     # Restrict Python 2.7 usage to fix the issue below
     # https://github.com/spack/spack/issues/24429
     patch("0002-restrict-python-2.7-usage.patch", when="@3.9.0:5.1.3")
-    patch("0003-restrict-python-2.7-usage.patch", when="@5.2.0:")
+    patch("0003-restrict-python-2.7-usage.patch", when="@5.2.0:5.4")
+    patch("0004-restrict-python2.7-usage-for-5.5.0.patch", when="@5.5.0")
+    patch("0005-Adding-half-include-directory-path-migraphx.patch", when="@5.6.0:")
 
     depends_on("cmake@3.5:", type="build")
     depends_on("protobuf", type="link")
     depends_on("blaze", type="build")
     depends_on("nlohmann-json", type="link")
     depends_on("msgpack-c", type="link")
-    depends_on("half@1.12.0", type="link")
+    depends_on("half@1.12.0", type="link", when="@:5.5")
+    depends_on("half@2:", when="@5.6:")
     depends_on("python@3.5:", type="build")
     depends_on("py-pybind11", type="build", when="@:4.0.0")
     depends_on("py-pybind11@2.6:", type="build", when="@4.1.0:")
@@ -156,6 +156,10 @@ def url_for_version(self, version):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocm-cmake@%s:" % ver, type="build", when="@" + ver)
         depends_on("hip@" + ver, when="@" + ver)
@@ -183,12 +187,23 @@ def determine_version(cls, lib):
         return ver
 
     def cmake_args(self):
+        spec = self.spec
+        abspath = spec["abseil-cpp"].prefix.include
         args = ["-DCMAKE_CXX_COMPILER={0}/bin/clang++".format(self.spec["llvm-amdgpu"].prefix)]
         if "@3.9.0:" in self.spec:
             args.append(
                 "-DNLOHMANN_JSON_INCLUDE={0}".format(self.spec["nlohmann-json"].prefix.include)
             )
-
         if self.spec["cmake"].satisfies("@3.16.0:"):
             args += self.cmake_python_hints
+        if "@5.5.0:" in self.spec:
+            args.append(self.define("CMAKE_CXX_FLAGS", "-I{0}".format(abspath)))
         return args
+
+    def test(self):
+        if self.spec.satisfies("@:5.5.0"):
+            print("Skipping: stand-alone tests")
+            return
+        test_dir = join_path(self.spec["migraphx"].prefix, "bin")
+        with working_dir(test_dir, create=True):
+            self.run_test("UnitTests")
diff --git a/var/spack/repos/builtin/packages/migrate/package.py b/var/spack/repos/builtin/packages/migrate/package.py
index 5db76564f2ed10..d4aa59698e34f2 100644
--- a/var/spack/repos/builtin/packages/migrate/package.py
+++ b/var/spack/repos/builtin/packages/migrate/package.py
@@ -22,7 +22,7 @@ class Migrate(AutotoolsPackage):
     depends_on("automake", type="build")
     depends_on("libtool", type="build")
     depends_on("m4", type="build")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     depends_on("openmpi", type=("build", "link", "run"), when="+mpi")
 
diff --git a/var/spack/repos/builtin/packages/millepede/package.py b/var/spack/repos/builtin/packages/millepede/package.py
index bec7e307aa2ca7..6cc9b5a9ada53d 100644
--- a/var/spack/repos/builtin/packages/millepede/package.py
+++ b/var/spack/repos/builtin/packages/millepede/package.py
@@ -21,7 +21,7 @@ class Millepede(MakefilePackage):
     version("04-13-03", sha256="669a6e46a6f02ba3c78b2760e2ffb2c90d25b582ccd1a5c0770eef81c7bcbbe9")
     version("04-11-01", sha256="9869eb84d8d07cecfab15c396f3faa36aef10906e39f8641c48b58e0325b3205")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         make("install", "PREFIX=" + prefix)
diff --git a/var/spack/repos/builtin/packages/mimalloc/package.py b/var/spack/repos/builtin/packages/mimalloc/package.py
index e764d2e92d4ec8..fed6e5bf783543 100644
--- a/var/spack/repos/builtin/packages/mimalloc/package.py
+++ b/var/spack/repos/builtin/packages/mimalloc/package.py
@@ -17,7 +17,17 @@ class Mimalloc(CMakePackage):
     version("dev-slice", branch="dev-slice")
     version("dev", branch="dev")
     version("master", branch="master")
+    version("2.1.2", sha256="2b1bff6f717f9725c70bf8d79e4786da13de8a270059e4ba0bdd262ae7be46eb")
+    version("2.1.1", sha256="38b9660d0d1b8a732160191609b64057d8ccc3811ab18b7607bc93ca63a6010f")
+    version("2.1.0", sha256="86e5e53e38bace59a9eb20d27e7bd7c5f448cb246a887d4f99478fa4809731fc")
+    version("2.0.9", sha256="4a29edae32a914a706715e2ac8e7e4109e25353212edeed0888f4e3e15db5850")
+    version("2.0.7", sha256="f23aac6c73594e417af50cb38f1efed88ef1dc14a490f0eff07c7f7b079810a4")
     version("2.0.6", sha256="9f05c94cc2b017ed13698834ac2a3567b6339a8bde27640df5a1581d49d05ce5")
+    version("1.8.2", sha256="4058d53d6ceb75862f32c30a6ee686c3cbb5e965b2c324b828ca454f7fe064f9")
+    version("1.8.1", sha256="7aaa54c3ed1feac90b6187eb93108e808660c6e103b0fa6a7e2fabb58c3147d5")
+    version("1.8.0", sha256="4bb69b49821499256e7d9b2a47427c4661f5ad3f1547a21f7bdea7e3bcbc60f8")
+    version("1.7.9", sha256="45e05be518363d32b2cdcce1a1fac3580895ea2e4524e1a3c7e71145cb58659f")
+    version("1.7.7", sha256="0f6663be1e1764851bf9563fcf7a6b3330e23b933eb4737dd07e3289b87895fe")
     version("1.7.6", sha256="d74f86ada2329016068bc5a243268f1f555edd620b6a7d6ce89295e7d6cf18da")
 
     depends_on("cmake@3.0:", type="build")
@@ -105,4 +115,9 @@ def cmake_args(self):
             for lib in self.libs_values
         ]
         args += [self.define_from_variant("MI_%s" % k.upper(), k) for k in self.mimalloc_options]
+
+        # Use LTO also for non-Intel compilers please. This can be removed when they
+        # bump cmake_minimum_required to VERSION 3.9.
+        if "+ipo" in self.spec:
+            args.append("-DCMAKE_POLICY_DEFAULT_CMP0069=NEW")
         return args
diff --git a/var/spack/repos/builtin/packages/minc-toolkit/package.py b/var/spack/repos/builtin/packages/minc-toolkit/package.py
index 43c88269b29f39..40e793d24da885 100644
--- a/var/spack/repos/builtin/packages/minc-toolkit/package.py
+++ b/var/spack/repos/builtin/packages/minc-toolkit/package.py
@@ -29,7 +29,7 @@ class MincToolkit(CMakePackage):
     depends_on("perl-mni-perllib", type=("build", "run"))
     depends_on("flex", type=("build", "run"))  # e.g minccalc depends on libfl.so
     depends_on("bison", type="build")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("freeglut", when="+visualisation")
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/mindthegap/package.py b/var/spack/repos/builtin/packages/mindthegap/package.py
index d14ce5cd2aac8f..73d6d307e7cb21 100644
--- a/var/spack/repos/builtin/packages/mindthegap/package.py
+++ b/var/spack/repos/builtin/packages/mindthegap/package.py
@@ -16,10 +16,14 @@ class Mindthegap(CMakePackage):
 
     maintainers("snehring")
 
-    version("2.3.0", tag="v2.3.0", submodules=True)
-    version("2.0.2", tag="v2.0.2", submodules=True)
+    version(
+        "2.3.0", tag="v2.3.0", commit="fe85c308434a4ad1ae7977dad67e966abc2bf93e", submodules=True
+    )
+    version(
+        "2.0.2", tag="v2.0.2", commit="8401af2a2bce9997396fbf0a04757ca7c887a1da", submodules=True
+    )
 
     depends_on("cmake@3.1:", type="build", when="@2.3.0")
     depends_on("cmake@2.6:", type="build", when="@2.0.2")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/miniasm/package.py b/var/spack/repos/builtin/packages/miniasm/package.py
index 0518dceb0f5bfb..4cddf247e15c02 100644
--- a/var/spack/repos/builtin/packages/miniasm/package.py
+++ b/var/spack/repos/builtin/packages/miniasm/package.py
@@ -15,7 +15,7 @@ class Miniasm(MakefilePackage):
 
     version("2018-3-30", commit="55cf0189e2f7d5bda5868396cebe066eec0a9547")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         install_tree(".", prefix.bin)
diff --git a/var/spack/repos/builtin/packages/minimap2/package.py b/var/spack/repos/builtin/packages/minimap2/package.py
index b7f7dc119706a6..e84bb540449e06 100644
--- a/var/spack/repos/builtin/packages/minimap2/package.py
+++ b/var/spack/repos/builtin/packages/minimap2/package.py
@@ -20,7 +20,7 @@ class Minimap2(PythonPackage):
     version("2.2", sha256="7e8683aa74c4454a8cfe3821f405c4439082e24c152b4b834fdb56a117ecaed9")
 
     conflicts("target=aarch64:", when="@:2.10")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("py-setuptools", type="build")
     depends_on("py-cython", type="build")
 
diff --git a/var/spack/repos/builtin/packages/miniocli/package.py b/var/spack/repos/builtin/packages/miniocli/package.py
new file mode 100644
index 00000000000000..5857f4d4872e63
--- /dev/null
+++ b/var/spack/repos/builtin/packages/miniocli/package.py
@@ -0,0 +1,49 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class Miniocli(MakefilePackage):
+    """MinIO Client (mc) provides a modern alternative to UNIX commands
+    like ls, cat, cp, mirror, diff etc. It supports filesystems and
+    Amazon S3 compatible cloud storage service (AWS Signature v2 and v4)."""
+
+    homepage = "https://docs.min.io/docs/minio-client-complete-guide.html"
+    url = "https://github.com/minio/mc/archive/refs/tags/RELEASE.2022-02-02T02-03-24Z.tar.gz"
+
+    version(
+        "2023-06-28",
+        sha256="033a80439474595665bdbc3ec72b059dc9e69e99db85fe6820877ad8973a080b",
+        url="https://github.com/minio/mc/archive/refs/tags/RELEASE.2023-06-28T21-54-17Z.tar.gz",
+    )
+    version(
+        "2022-05-04",
+        sha256="2b935c3744228978c93c14186b65a6078e4ffc267f32cdb8c994eff2dda95a6e",
+        url="https://github.com/minio/mc/archive/RELEASE.2022-05-04T06-07-55Z.tar.gz",
+    )
+    version(
+        "2022-03-31",
+        sha256="3b983ea1cc50768b0826989dba931044ac3f8e841cc09aefed217301c92fa8a3",
+        url="https://github.com/minio/mc/archive/RELEASE.2022-03-31T04-55-30Z.tar.gz",
+    )
+    # versions of 2022-03-* or later require  github.com/shirou/gopsutil/v3 and
+    # this is generating an error
+    version(
+        "2022-02-02",
+        sha256="2d4a64c17935d40d0e325761cc214b2efceb19ce006101c192da9b31f8920a97",
+        url="https://github.com/minio/mc/archive/RELEASE.2022-02-02T02-03-24Z.tar.gz",
+    )
+    version(
+        "2022-01-05",
+        sha256="d5dbd32b7a7f79baace09dd6518121798d2fcbb84b81046b61ff90f980c8f963",
+        url="https://github.com/minio/mc/archive/RELEASE.2022-01-05T23-52-51Z.tar.gz",
+    )
+
+    depends_on("go", type="build")
+
+    def install(self, spec, prefix):
+        go("build")
+        mkdirp(prefix.bin)
+        install("mc", prefix.bin)
diff --git a/var/spack/repos/builtin/packages/minivite/package.py b/var/spack/repos/builtin/packages/minivite/package.py
index 0de582bcdb961a..702f701dd4c6e1 100644
--- a/var/spack/repos/builtin/packages/minivite/package.py
+++ b/var/spack/repos/builtin/packages/minivite/package.py
@@ -17,8 +17,8 @@ class Minivite(MakefilePackage):
     git = "https://github.com/Exa-Graph/miniVite.git"
 
     version("develop", branch="master")
-    version("1.0", tag="v1.0")
-    version("1.1", tag="v1.1")
+    version("1.0", tag="v1.0", commit="65ccaa8a4ec0b4bea516e2abdafbeb2f8a5f0c94")
+    version("1.1", tag="v1.1", commit="23476d9d41eb8a17bf4108ac56852dacda89b253")
 
     variant("openmp", default=True, description="Build with OpenMP support")
     variant("opt", default=True, description="Optimization flags")
diff --git a/var/spack/repos/builtin/packages/minizip/package.py b/var/spack/repos/builtin/packages/minizip/package.py
index 580cadfe650de1..25242a60d9b586 100644
--- a/var/spack/repos/builtin/packages/minizip/package.py
+++ b/var/spack/repos/builtin/packages/minizip/package.py
@@ -20,7 +20,7 @@ class Minizip(AutotoolsPackage):
     depends_on("autoconf", type="build")
     depends_on("libtool", type="build")
     depends_on("m4", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # error: implicit declaration of function 'mkdir' is invalid in C99
     patch("implicit.patch", when="%apple-clang@12:")
diff --git a/var/spack/repos/builtin/packages/miopen-hip/package.py b/var/spack/repos/builtin/packages/miopen-hip/package.py
index 98c610f52b2f4c..6ab2967ac24b23 100644
--- a/var/spack/repos/builtin/packages/miopen-hip/package.py
+++ b/var/spack/repos/builtin/packages/miopen-hip/package.py
@@ -14,12 +14,15 @@ class MiopenHip(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/MIOpen"
     git = "https://github.com/ROCmSoftwarePlatform/MIOpen.git"
-    url = "https://github.com/ROCmSoftwarePlatform/MIOpen/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/MIOpen/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
     libraries = ["libMIOpen"]
-
+    version("5.6.1", sha256="ff627d68ed9e52433a3c808b5d3ff179a398b77ce81b00cfea7b2c4da5162c6c")
+    version("5.6.0", sha256="d620ddab5b488bdf81242654fefa337c6b71dc410c2ff26d30a4ee86a8d22d11")
+    version("5.5.1", sha256="2cd75071b8ee876c69a94f028b6c8a9346d6d2fde7d4b64e6d635f3b6c994262")
+    version("5.5.0", sha256="791087242551669e546225e36123c21663f0dad14dbcfd6d0ce0e7bad0ab0de1")
     version("5.4.3", sha256="37ffe2ed3d7942da8ea2f6bdb85c7a2f58e3ccd31767db158a322769d3604efd")
     version("5.4.0", sha256="b4153791f9eeee4cbc5534bc6ad8b32c0947bcd38e08b77ebe144065a4fa5456")
     version("5.3.3", sha256="7efc98215d23a2caaf212378c37e9a6484f54a4ed3e9660719286e4f287d3715")
@@ -100,13 +103,6 @@ class MiopenHip(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3.5:", type="build")
     depends_on("pkgconfig", type="build")
 
@@ -118,7 +114,7 @@ class MiopenHip(CMakePackage):
     depends_on("bzip2")
     depends_on("sqlite")
     depends_on("half")
-    depends_on("zlib", when="@3.9.0:")
+    depends_on("zlib-api", when="@3.9.0:")
 
     patch("0001-Add-rocm-path-and-rocm-device-lib-path-flags.patch", when="@3.9.0:5.0.2")
     patch("miopen-hip-include-nlohmann-include-directory.patch", when="@5.4.0:")
@@ -147,6 +143,10 @@ class MiopenHip(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocm-cmake@%s:" % ver, type="build", when="@" + ver)
         depends_on("hip@" + ver, when="@" + ver)
@@ -156,13 +156,16 @@ class MiopenHip(CMakePackage):
     for ver in ["5.1.0", "5.1.3", "5.2.0", "5.2.1", "5.2.3", "5.3.0", "5.3.3"]:
         depends_on("mlirmiopen@" + ver, when="@" + ver)
 
-    for ver in ["5.4.0", "5.4.3"]:
-        depends_on("rocmlir@" + ver, when="@" + ver)
+    for ver in ["5.5.1", "5.6.0", "5.6.1"]:
         depends_on("nlohmann-json", type="link")
+        depends_on("composable-kernel@" + ver, when="@" + ver)
+    for ver in ["5.4.0", "5.4.3", "5.5.0"]:
+        depends_on("nlohmann-json", type="link")
+        depends_on("rocmlir@" + ver, when="@" + ver)
 
     def setup_build_environment(self, env):
         if "@3.9.0:" in self.spec:
-            lib_dir = self.spec["zlib"].libs.directories[0]
+            lib_dir = self.spec["zlib-api"].libs.directories[0]
             env.prepend_path("LIBRARY_PATH", lib_dir)
 
     def get_bitcode_dir(self):
@@ -203,11 +206,19 @@ def cmake_args(self):
         if self.spec.satisfies("@5.1.0:5.3"):
             mlir_inc = spec["mlirmiopen"].prefix.include
             args.append(self.define("CMAKE_CXX_FLAGS", "-I{0}".format(mlir_inc)))
-        # TODO: need to turn on composable-kernel to on at a later date
-        # requires a new recipe for composable-kernel
         if self.spec.satisfies("@5.4.0:"):
-            args.append(self.define("MIOPEN_USE_COMPOSABLEKERNEL", "OFF"))
             args.append(
                 "-DNLOHMANN_JSON_INCLUDE={0}".format(self.spec["nlohmann-json"].prefix.include)
             )
+        if self.spec.satisfies("@5.4.0:5.5.0"):
+            args.append(self.define("MIOPEN_USE_COMPOSABLEKERNEL", "OFF"))
+            args.append(self.define("MIOPEN_USE_MLIR", "ON"))
+            args.append(self.define("MIOPEN_ENABLE_AI_KERNEL_TUNING", "OFF"))
+        if self.spec.satisfies("@5.5.1:"):
+            args.append(self.define("MIOPEN_USE_COMPOSABLEKERNEL", "ON"))
+            args.append(self.define("MIOPEN_ENABLE_AI_KERNEL_TUNING", "OFF"))
+            args.append(self.define("MIOPEN_USE_MLIR", "OFF"))
+        args.append(
+            "-DNLOHMANN_JSON_INCLUDE={0}".format(self.spec["nlohmann-json"].prefix.include)
+        )
         return args
diff --git a/var/spack/repos/builtin/packages/miopen-opencl/package.py b/var/spack/repos/builtin/packages/miopen-opencl/package.py
index d0fe124bc01eb8..73808b43e47de2 100644
--- a/var/spack/repos/builtin/packages/miopen-opencl/package.py
+++ b/var/spack/repos/builtin/packages/miopen-opencl/package.py
@@ -14,12 +14,14 @@ class MiopenOpencl(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/MIOpen"
     git = "https://github.com/ROCmSoftwarePlatform/MIOpen.git"
-    url = "https://github.com/ROCmSoftwarePlatform/MIOpen/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/MIOpen/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
     libraries = ["libMIOpen"]
 
+    version("5.5.1", sha256="2cd75071b8ee876c69a94f028b6c8a9346d6d2fde7d4b64e6d635f3b6c994262")
+    version("5.5.0", sha256="791087242551669e546225e36123c21663f0dad14dbcfd6d0ce0e7bad0ab0de1")
     version("5.4.3", sha256="37ffe2ed3d7942da8ea2f6bdb85c7a2f58e3ccd31767db158a322769d3604efd")
     version("5.4.0", sha256="b4153791f9eeee4cbc5534bc6ad8b32c0947bcd38e08b77ebe144065a4fa5456")
     version("5.3.3", sha256="7efc98215d23a2caaf212378c37e9a6484f54a4ed3e9660719286e4f287d3715")
@@ -100,13 +102,6 @@ class MiopenOpencl(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3.5:", type="build")
     depends_on("boost@1.67.0:1.73.0", type="link")
 
@@ -146,6 +141,8 @@ class MiopenOpencl(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
     ]:
         depends_on("rocm-cmake@%s:" % ver, type="build", when="@" + ver)
         depends_on("rocm-opencl@" + ver, when="@" + ver)
@@ -154,7 +151,7 @@ class MiopenOpencl(CMakePackage):
     for ver in ["5.1.0", "5.1.3", "5.2.0", "5.2.1", "5.2.3", "5.3.0", "5.3.3"]:
         depends_on("mlirmiopen@" + ver, when="@" + ver)
 
-    for ver in ["5.4.0", "5.4.3"]:
+    for ver in ["5.4.0", "5.4.3", "5.5.0", "5.5.1"]:
         depends_on("nlohmann-json", type="link")
         depends_on("rocblas", type="link")
         depends_on("rocmlir@" + ver, when="@" + ver)
diff --git a/var/spack/repos/builtin/packages/miopen-tensile/package.py b/var/spack/repos/builtin/packages/miopen-tensile/package.py
index 9ffbfcad0d75a9..eb559972107038 100644
--- a/var/spack/repos/builtin/packages/miopen-tensile/package.py
+++ b/var/spack/repos/builtin/packages/miopen-tensile/package.py
@@ -44,7 +44,13 @@ class MiopenTensile(CMakePackage):
 
     tensile_architecture = ("all", "gfx906", "gfx908", "gfx803", "gfx900")
 
-    variant("tensile_architecture", default="all", values=tensile_architecture, multi=True)
+    variant(
+        "tensile_architecture",
+        default="all",
+        description="AMD GPU architecture",
+        values=tensile_architecture,
+        multi=True,
+    )
     variant(
         "build_type",
         default="Release",
diff --git a/var/spack/repos/builtin/packages/miopengemm/package.py b/var/spack/repos/builtin/packages/miopengemm/package.py
index 321ca281568fc6..808c63f14c3138 100644
--- a/var/spack/repos/builtin/packages/miopengemm/package.py
+++ b/var/spack/repos/builtin/packages/miopengemm/package.py
@@ -14,7 +14,7 @@ class Miopengemm(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/MIOpenGEMM"
     git = "https://github.com/ROCmSoftwarePlatform/MIOpenGEMM.git"
-    url = "https://github.com/ROCmSoftwarePlatform/MIOpenGEMM/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/MIOpenGEMM/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
@@ -26,6 +26,8 @@ def url_for_version(self, version):
         url = "https://github.com/ROCmSoftwarePlatform/MIOpenGEMM/archive/rocm-{0}.tar.gz"
         return url.format(version)
 
+    version("5.5.1", sha256="a997b560521641e7173613cf547ecde5d15ac6fac1786d392b0f133c91f99a40")
+    version("5.5.0", sha256="ffd9775129564662b338952588057a088f7e9723b4a9a766b2dd96fdc0992c26")
     version("5.4.3", sha256="5051051cab60ca0f6347a981da6c9dbeddf8b0de698d4e5409a0db0c622acafc")
     version("5.4.0", sha256="a39faa8f4ab73e0cd6505a667bf10c07f93b9612af0711405c65043c4755129d")
     version("5.3.3", sha256="4a9c92bebe59bf6e08bd48861b68b1801d9e8dc406250dc8637d36614a5884c8")
@@ -106,13 +108,6 @@ def url_for_version(self, version):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3:", type="build")
     depends_on("rocm-cmake@3.5.0", type="build", when="@1.1.6")
     depends_on("rocm-opencl@3.5.0", when="@1.1.6")
@@ -140,6 +135,8 @@ def url_for_version(self, version):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
     ]:
         depends_on("rocm-cmake@" + ver, type="build", when="@" + ver)
         depends_on("rocm-opencl@" + ver, when="@" + ver)
diff --git a/var/spack/repos/builtin/packages/mivisionx/package.py b/var/spack/repos/builtin/packages/mivisionx/package.py
index 7d15d26aefda4a..b298160520f53f 100644
--- a/var/spack/repos/builtin/packages/mivisionx/package.py
+++ b/var/spack/repos/builtin/packages/mivisionx/package.py
@@ -13,7 +13,7 @@ class Mivisionx(CMakePackage):
 
     homepage = "https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX"
     git = "https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX.git"
-    url = "https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/archive/rocm-5.4.0.tar.gz"
+    url = "https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/archive/rocm-5.5.0.tar.gz"
 
     maintainers("srekolam", "renjithravindrankannath")
     tags = ["rocm"]
@@ -25,6 +25,10 @@ def url_for_version(self, version):
         url = "https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/archive/rocm-{0}.tar.gz"
         return url.format(version)
 
+    version("5.6.1", sha256="b2ff95c1488e244f379482631dae4f9ab92d94a513d180e03607aa1e184b5b0a")
+    version("5.6.0", sha256="34c184e202b1a6da2398b66e33c384d5bafd8f8291089c18539715c5cb73eb1f")
+    version("5.5.1", sha256="e8209f87a57c4222003a936240e7152bbfa496862113358f29d4c3e80d4cdf56")
+    version("5.5.0", sha256="af266550ecccad80f08954f23e47e8264eb338b0928a5314bd6efca349fc5a14")
     version("5.4.3", sha256="4da82974962a70c326ce2427c664517b1efdff436efe222e6bc28817c222a082")
     version("5.4.0", sha256="caa28a30972704ddbf1a87cefdc0b0a35381d369961c43973d473a1573bd35cc")
     version("5.3.3", sha256="378fafcb327e17e0e11fe1d1029d1740d84aaef0fd59614ed7376499b3d716f6")
@@ -105,12 +109,6 @@ def url_for_version(self, version):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
     # Adding 2 variants OPENCL ,HIP which HIP as default. earlier to 5.0.0,OPENCL
     # was the default but has change dto HIP from 5.0.0 onwards.
     # when tested with HIP as true for versions before 5.1.0, build errors were encountered
@@ -119,6 +117,8 @@ def url_for_version(self, version):
     variant("opencl", default=False, description="Use OPENCL as the backend")
     variant("hip", default=True, description="Use HIP as backend")
 
+    conflicts("+opencl", when="@5.6.0:")
+
     def patch(self):
         if self.spec.satisfies("@4.2.0"):
             filter_file(
@@ -182,7 +182,7 @@ def patch(self):
 
     depends_on("cmake@3.5:", type="build")
     depends_on("ffmpeg@:4", type="build", when="@:5.3")
-    depends_on("ffmpeg@4.4:", type="build", when="@5.4:")
+    depends_on("ffmpeg@4.4", type="build", when="@5.4:")
     depends_on("protobuf@:3", type="build")
     depends_on(
         "opencv@:3.4"
@@ -236,6 +236,8 @@ def patch(self):
             "5.3.3",
             "5.4.0",
             "5.4.3",
+            "5.5.0",
+            "5.5.1",
         ]:
             depends_on("rocm-opencl@" + ver, when="@" + ver)
             depends_on("miopengemm@" + ver, when="@" + ver)
@@ -255,11 +257,19 @@ def patch(self):
             "5.3.3",
             "5.4.0",
             "5.4.3",
+            "5.5.0",
+            "5.5.1",
+            "5.6.0",
+            "5.6.1",
         ]:
             depends_on("miopen-hip@" + ver, when="@" + ver)
-        for ver in ["5.3.3", "5.4.0", "5.4.3"]:
+        for ver in ["5.3.3", "5.4.0", "5.4.3", "5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
             depends_on("migraphx@" + ver, when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+        depends_on("python@3.5:", type="build")
+
     def flag_handler(self, name, flags):
         spec = self.spec
         protobuf = spec["protobuf"].prefix.include
diff --git a/var/spack/repos/builtin/packages/mlirmiopen/package.py b/var/spack/repos/builtin/packages/mlirmiopen/package.py
index 312341e4a639ae..4515385b4af5b4 100644
--- a/var/spack/repos/builtin/packages/mlirmiopen/package.py
+++ b/var/spack/repos/builtin/packages/mlirmiopen/package.py
@@ -44,7 +44,7 @@ def patch(self):
 
     depends_on("python", type="build")
     depends_on("z3", type="link")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("ncurses+termlib", type="link")
     depends_on("bzip2")
     depends_on("sqlite")
diff --git a/var/spack/repos/builtin/packages/mlpack/package.py b/var/spack/repos/builtin/packages/mlpack/package.py
index 30a50132d9f681..5088e187062107 100644
--- a/var/spack/repos/builtin/packages/mlpack/package.py
+++ b/var/spack/repos/builtin/packages/mlpack/package.py
@@ -70,7 +70,7 @@ class Mlpack(CMakePackage):
     def cmake_args(self):
         args = [
             self.define("BUILD_CLI_EXECUTABLES", True),
-            self.define_from_variant("BUILD_GO_BINDINGS", "go"),
+            # self.define_from_variant("BUILD_GO_BINDINGS", "go"),
             self.define_from_variant("BUILD_JULIA_BINDINGS", "julia"),
             self.define_from_variant("BUILD_PYTHON_BINDINGS", "python"),
             self.define_from_variant("BUILD_R_BINDINGS", "r"),
diff --git a/var/spack/repos/builtin/packages/moab/package.py b/var/spack/repos/builtin/packages/moab/package.py
index 721bac23a20625..2c7262c16660f9 100644
--- a/var/spack/repos/builtin/packages/moab/package.py
+++ b/var/spack/repos/builtin/packages/moab/package.py
@@ -18,12 +18,16 @@ class Moab(AutotoolsPackage):
 
     homepage = "https://bitbucket.org/fathomteam/moab"
     git = "https://bitbucket.org/fathomteam/moab.git"
-    url = "https://ftp.mcs.anl.gov/pub/fathom/moab-5.0.0.tar.gz"
+    url = "https://ftp.mcs.anl.gov/pub/fathom/moab-5.5.0.tar.gz"
 
     maintainers("vijaysm", "iulian787")
 
     version("develop", branch="develop")
     version("master", branch="master")
+    version("5.5.0", sha256="58969f8a1b209ec9036c08c53a6b7078b368eb3bf99d0368a4de5a2f2a8db678")
+    version("5.4.1", sha256="3625e25321bf37f88d98438f5d56c280b2774172602d8b6eb6c34eedf37686fc")
+    version("5.4.0", sha256="a30d2a1911fbf214ae0175b0856e0475c0077dc51ea5914c850d631155a72952")
+    version("5.3.1", sha256="2404fab2d84f87be72b57cfef5ea237bfa444aaca059e66a158f22134956fe54")
     version("5.3.0", sha256="51c31ccbcaa76d9658a44452b9a39f076b795b27a1c9f408fc3d0eea97e032ef")
     version("5.2.1", sha256="60d31762be3f0e5c89416c764e844ec88dac294169b59a5ead3c316b50f85c29")
     version("5.2.0", sha256="805ed3546deff39e076be4d1f68aba1cf0dda8c34ce43e1fc179d1aff57c5d5d")
@@ -36,22 +40,23 @@ class Moab(AutotoolsPackage):
     version("4.9.0", sha256="267a7c05da847e4ea856db2c649a5484fb7bdc132ab56721ca50ee69a7389f4d")
     version("4.8.2", sha256="b105cff42930058dc14eabb9a25e979df7289b175732fe319d2494e83e09e968")
 
-    variant("mpi", default=True, description="enable mpi support")
-    variant("hdf5", default=True, description="Required to enable the hdf5 (default I/O) format")
-    variant("netcdf", default=False, description="Required to enable the ExodusII reader/writer.")
-    variant("pnetcdf", default=False, description="Enable pnetcdf (AKA parallel-netcdf) support")
-    variant("zoltan", default=False, description="Enable zoltan support")
-    variant("cgm", default=False, description="Enable common geometric module")
-    variant("metis", default=True, description="Enable metis link")
-    variant("parmetis", default=True, description="Enable parmetis link")
-    variant("irel", default=False, description="Enable irel interface")
-    variant("fbigeom", default=False, description="Enable fbigeom interface")
+    variant("mpi", default=True, description="Enable MPI parallelism support")
+    variant("hdf5", default=True, description="Enable the HDF5 (default I/O) format")
+    variant("netcdf", default=False, description="Enable the ExodusII reader/writer support")
+    variant("pnetcdf", default=False, description="Enable parallel-Netcdf library support")
+    variant("zoltan", default=True, description="Enable Zoltan partitioner support")
+    variant("eigen", default=True, description="Enable Eigen template library for linear algebra")
+    variant("cgm", default=False, description="Enable the Common Geometry Module")
+    variant("metis", default=False, description="Enable Metis partitioner")
+    variant("parmetis", default=False, description="Enable Parmetis partitioner")
+    variant("irel", default=False, description="Enable iRel interface")
+    variant("fbigeom", default=False, description="Enable FBiGeom interface")
     variant("coupler", default=True, description="Enable mbcoupler tool")
-    variant("dagmc", default=False, description="Enable dagmc tool")
+    variant("dagmc", default=False, description="Enable DagMC tool")
 
-    variant("debug", default=False, description="enable debug symbols")
+    variant("debug", default=False, description="Enable debug symbols in libraries")
     variant("shared", default=False, description="Enables the build of shared libraries")
-    variant("fortran", default=True, description="Enable Fortran support")
+    variant("fortran", default=False, description="Enable Fortran support")
 
     conflicts("+irel", when="~cgm")
     conflicts("+pnetcdf", when="~mpi")
@@ -80,6 +85,7 @@ class Moab(AutotoolsPackage):
     depends_on("cgm", when="+cgm")
     depends_on("metis", when="+metis")
     depends_on("parmetis", when="+parmetis")
+    depends_on("eigen", when="+eigen")
     # FIXME it seems that zoltan needs to be built without fortran
     depends_on("zoltan~fortran", when="+zoltan")
 
@@ -126,6 +132,8 @@ def configure_args(self):
 
         options.append("--with-blas=%s" % spec["blas"].libs.ld_flags)
         options.append("--with-lapack=%s" % spec["lapack"].libs.ld_flags)
+        if "+eigen" in spec:
+            options.append("--with-eigen3=%s" % spec["eigen"].prefix.include.eigen3)
 
         if "+hdf5" in spec:
             options.append("--with-hdf5=%s" % spec["hdf5"].prefix)
diff --git a/var/spack/repos/builtin/packages/mochi-margo/package.py b/var/spack/repos/builtin/packages/mochi-margo/package.py
index d0ffb47c6a4123..5caff944789edf 100644
--- a/var/spack/repos/builtin/packages/mochi-margo/package.py
+++ b/var/spack/repos/builtin/packages/mochi-margo/package.py
@@ -17,6 +17,7 @@ class MochiMargo(AutotoolsPackage):
     maintainers("carns", "mdorier", "fbudin69500")
 
     version("main", branch="main")
+    version("0.14.1", sha256="69229a9126b76aff7fd47e25c4a8f72804f101c5c603c4e4ef93f4fb7a1b6662")
     version("0.14.0", sha256="ff0e3fa786630b63280606243c35f1ea3a25fa2ba6f08bf9065cab9fcc7fa1c7")
     version("0.13.1", sha256="cff1decb94089cd0f9c0930b02092838679827b09ce4a2f3a359d59caee28782")
     version("0.13", sha256="9a5a4aa81ceb10e010fbad6c7bb8d39d082fe6e61ed33b2b2d2b056917f401d8")
diff --git a/var/spack/repos/builtin/packages/modeltest-ng/package.py b/var/spack/repos/builtin/packages/modeltest-ng/package.py
index 55d1bd1ff603d0..7a2b4dcc2c2d0e 100644
--- a/var/spack/repos/builtin/packages/modeltest-ng/package.py
+++ b/var/spack/repos/builtin/packages/modeltest-ng/package.py
@@ -13,8 +13,9 @@ class ModeltestNg(CMakePackage):
     url = "https://github.com/ddarriba/modeltest/archive/refs/tags/v0.1.7.tar.gz"
     git = "https://github.com/ddarriba/modeltest.git"
 
-    maintainers("dorton21")
+    maintainers("snehring")
 
+    version("20220721", commit="1066356b984100897b8bd38ac771c5c950984c01", submodules=True)
     version("0.1.7", commit="cc028888f1d4222aaa53b99c6b02cd934a279001", submodules=True)
 
     variant("mpi", default=False, description="Enable MPI")
@@ -24,5 +25,12 @@ class ModeltestNg(CMakePackage):
     depends_on("flex", type="build")
     depends_on("openmpi", when="+mpi")
 
+    # 40217: ICE by gcc-toolset-12-gcc-12.2.1-7.4.el8.aarch64 of Rocky Linux 8.8:
+    conflicts("%gcc@12.2.0:12.2", when="target=aarch64:", msg="ICE with gcc@12.2 on aarch64")
+
+    requires(
+        "@20220721:", when="target=aarch64:", msg="Support for aarch64 was added after 20220721."
+    )
+
     def cmake_args(self):
         return [self.define_from_variant("ENABLE_MPI", "mpi")]
diff --git a/var/spack/repos/builtin/packages/mold/package.py b/var/spack/repos/builtin/packages/mold/package.py
index f964162427e078..61b4de3f6dd0b7 100644
--- a/var/spack/repos/builtin/packages/mold/package.py
+++ b/var/spack/repos/builtin/packages/mold/package.py
@@ -14,13 +14,15 @@ class Mold(CMakePackage):
 
     maintainers("jabcross")
 
+    version("2.1.0", sha256="a32bec1282671b18ea4691855aed925ea2f348dfef89cb7689cd81273ea0c5df")
+    version("2.0.0", sha256="2ae8a22db09cbff626df74c945079fa29c1e5f60bbe02502dcf69191cf43527b")
     version("1.11.0", sha256="99318eced81b09a77e4c657011076cc8ec3d4b6867bd324b8677974545bc4d6f")
     version("1.7.1", sha256="fa2558664db79a1e20f09162578632fa856b3cde966fbcb23084c352b827dfa9")
 
     depends_on("mimalloc")
+    depends_on("zlib-api")
     depends_on("openssl")
     depends_on("tbb")
-    depends_on("zlib")
 
     def cmake_args(self):
         args = []
diff --git a/var/spack/repos/builtin/packages/molgw/package.py b/var/spack/repos/builtin/packages/molgw/package.py
index a0f07aca2ae10a..91026c7abe0b00 100644
--- a/var/spack/repos/builtin/packages/molgw/package.py
+++ b/var/spack/repos/builtin/packages/molgw/package.py
@@ -39,6 +39,8 @@ class Molgw(MakefilePackage):
 
     # enforce scalapack-capable mkl when asking +scalapack (and using intel-oneapi-mkl)
     depends_on("intel-oneapi-mkl+cluster", when="+scalapack ^intel-oneapi-mkl")
+    # enforce threaded mkl when asking +openmp (and using intel-oneapi-mkl)
+    depends_on("intel-oneapi-mkl threads=openmp", when="+openmp ^intel-oneapi-mkl")
     # enforce threaded openblas when asking +openmp (and using openblas)
     depends_on("openblas threads=openmp", when="+openmp ^openblas")
 
@@ -76,7 +78,7 @@ def edit(self, spec, prefix):
         flags["PREFIX"] = prefix
 
         # Set LAPACK and SCALAPACK
-        if "^mkl" in spec:
+        if spec["lapack"].name not in INTEL_MATH_LIBRARIES:
             flags["LAPACK"] = self._get_mkl_ld_flags(spec)
         else:
             flags["LAPACK"] = spec["lapack"].libs.ld_flags + " " + spec["blas"].libs.ld_flags
@@ -103,7 +105,7 @@ def edit(self, spec, prefix):
         if "+scalapack" in spec:
             flags["CPPFLAGS"] = flags.get("CPPFLAGS", "") + " -DHAVE_SCALAPACK -DHAVE_MPI "
 
-        if "^mkl" in spec:
+        if spec["lapack"].name in INTEL_MATH_LIBRARIES:
             flags["CPPFLAGS"] = flags.get("CPPFLAGS", "") + " -DHAVE_MKL "
 
         # Write configuration file
diff --git a/var/spack/repos/builtin/packages/mongo-c-driver/package.py b/var/spack/repos/builtin/packages/mongo-c-driver/package.py
index 5794ec0ccfe0dc..94c92d9c023a99 100644
--- a/var/spack/repos/builtin/packages/mongo-c-driver/package.py
+++ b/var/spack/repos/builtin/packages/mongo-c-driver/package.py
@@ -14,6 +14,7 @@ class MongoCDriver(Package):
 
     maintainers("michaelkuhn")
 
+    version("1.24.4", sha256="2f4a3e8943bfe3b8672c2053f88cf74acc8494dc98a45445f727901eee141544")
     version("1.23.3", sha256="c8f951d4f965d455f37ae2e10b72914736fc0f25c4ffc14afc3cbadd1a574ef6")
     version("1.21.0", sha256="840ff79480070f98870743fbb332e2c10dd021b6b9c952d08010efdda4d70ee4")
     version("1.17.6", sha256="8644deec7ae585e8d12566978f2017181e883f303a028b5b3ccb83c91248b150")
@@ -52,24 +53,30 @@ class MongoCDriver(Package):
     depends_on("pkgconfig", type="build")
 
     # When updating mongo-c-driver, libbson has to be kept in sync.
-    depends_on("libbson@1.23.0:1.23", when="@1.23")
-    depends_on("libbson@1.21.0:1.21", when="@1.21")
-    depends_on("libbson@1.17.0:1.17", when="@1.17")
-    depends_on("libbson@1.16.0:1.16", when="@1.16")
-    depends_on("libbson@1.9.0:1.9", when="@1.9")
-    depends_on("libbson@1.8.0:1.8", when="@1.8")
-    depends_on("libbson@1.7.0:1.7", when="@1.7")
-    depends_on("libbson@1.6.0:1.6", when="@1.6")
+    depends_on("libbson@1.24", when="@1.24")
+    depends_on("libbson@1.23", when="@1.23")
+    depends_on("libbson@1.21", when="@1.21")
+    depends_on("libbson@1.17", when="@1.17")
+    depends_on("libbson@1.16", when="@1.16")
+    depends_on("libbson@1.9", when="@1.9")
+    depends_on("libbson@1.8", when="@1.8")
+    depends_on("libbson@1.7", when="@1.7")
+    depends_on("libbson@1.6", when="@1.6")
 
     depends_on("openssl", when="+ssl")
     depends_on("snappy", when="+snappy")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("zstd", when="+zstd")
 
     def cmake_args(self):
         spec = self.spec
 
-        args = ["-DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF", "-DENABLE_BSON=SYSTEM"]
+        args = ["-DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF"]
+
+        if spec.satisfies("@1.24:"):
+            args.append("-DUSE_SYSTEM_LIBBSON=ON")
+        else:
+            args.append("-DENABLE_BSON=SYSTEM")
 
         if "+ssl" in spec:
             args.append("-DENABLE_SSL=OPENSSL")
diff --git a/var/spack/repos/builtin/packages/mongodb/package.py b/var/spack/repos/builtin/packages/mongodb/package.py
new file mode 100644
index 00000000000000..d1f25e29e2a980
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mongodb/package.py
@@ -0,0 +1,118 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+# ----------------------------------------------------------------------------
+import os
+
+from spack.package import *
+
+
+class Mongodb(SConsPackage):
+    """MongoDB is a source-available cross-platform document-oriented database
+    program. Classified as a NoSQL database program, MongoDB uses JSON-like
+    documents with optional schemas."""
+
+    homepage = "https://www.mongodb.com/"
+
+    maintainers("DaxLynch")
+
+    version("6.2", git="https://github.com/mongodb/mongo.git", branch="v6.2")
+
+    requires(
+        "%gcc", "%clang", policy="one_of", msg=" builds only with GCC or Clang"
+    )
+    depends_on("xz")
+    depends_on("curl")
+    depends_on("py-cryptography@36.0.1")
+    depends_on("py-requirements-parser")
+    depends_on("py-psutil@:5.8")
+    depends_on("py-pymongo@3.9:4.0")
+    depends_on("py-pyyaml@3:6")
+    depends_on("py-requests@2.0.0:2.26.0")
+    depends_on("py-typing-extensions@3.7.4:")
+
+    depends_on("py-cheetah3@:3.2.6")
+    depends_on("py-packaging@:21.3")
+    depends_on("py-regex@:2021.11.10")
+    depends_on("py-setuptools")
+    depends_on("ninja@1.10.0")
+
+    depends_on("py-distro@1.5.0")
+    depends_on("py-gitpython@3.1.7")
+    depends_on("py-pydantic@1.8.2")
+    depends_on("py-dnspython")
+
+    def build(self, spec, prefix):
+        pass  # This specific scons only uses the install phase
+
+    def install(self, spec, prefix):
+        library_dirs = []
+        include_dirs = []
+        # Sometimes scons does not detect curl or ninja, so these arrays
+        # get the include and lib directories and then are explicitly
+        # passed it to scons
+
+        # Scons fails to find the the python packages, even when linking
+        # with them with -I package-prefix/lib/python3.10/site_packages/
+        # (this is one of the options for scons). To work around this, I
+        # symlink package-prefix/lib/python3.X/site_packages/package/ to
+        # python-prefix/lib/python3.X/site_package/package/ I then
+        # remove these after the install.
+        python_prefix_lib = self.spec["python"].prefix.lib  # python-prefix/lib
+        lib_contents = os.scandir(python_prefix_lib)
+        python_version = ""
+        python_site_packages = ""
+        for entry in lib_contents:
+            if entry.is_dir() and entry.name.startswith("python3"):
+                # this gets the path python-prefix/lib/python3.X
+                python_version = entry.name
+                # sets the version as python3.X
+                python_site_packages = os.path.join(entry.path, "site-packages")
+                # python-prefix/lib/python3.X/site-packages
+
+        for dep in spec.dependencies(deptype="link"):  # iterate through the dependencies
+            query = self.spec[dep.name]
+            lib = query.prefix.lib
+            if dep.name in ["curl", "ninja", "xz"]:
+                # For the non python packages, we just extract the
+                # package-prefix/lib and package-prefix/include
+                try:
+                    library_dirs.extend(query.libs.directories)
+                    include_dirs.extend(query.headers.directories)
+                except Exception:
+                    pass
+            else:
+                dependency_site_packages = os.path.join(lib, python_version, "site-packages")
+                # package-prefix/lib/python3.X/site-packages
+                for entry in os.scandir(dependency_site_packages):
+                    # iterates through files in site-packages
+                    try:
+                        os.symlink(entry.path, os.path.join(python_site_packages, entry.name))
+                    except Exception:
+                        pass
+
+        # PYTHONDIRS="-I" + " -I".join(os.environ["PYTHONPATH"].split(":"))
+        # ^This is an attempt to pass the python directories directly to scons.
+        # Just add PYTHONDIRS as an argument to scons. It sadly does not work :(
+        # but feel free to try it.To get more information on the options for scons,
+        # do spack load scons, and then scons --help in the mongodb repo
+        LINKFLAGS = "-L" + " -L".join(library_dirs)
+        CXXFLAGS = "-I" + " -I".join(include_dirs)
+        scons(
+            "DESTDIR=%s" % prefix,
+            "install-mongod",
+            "--disable-warnings-as-errors",
+            "MONGO_VERSION=6.2.0",
+            "CC=%s" % self.compiler.cc,
+            "CXX=%s" % self.compiler.cxx,
+            "CCFLAGS=%s" % CXXFLAGS + " " + LINKFLAGS,
+            "LINKFLAGS=%s" % LINKFLAGS,
+        )
+
+        prefix_lib_python_site_package = os.scandir(
+            os.path.join(python_prefix_lib, python_version, "site-packages")
+        )  # python-prefix/lib/python3.X/site-packages
+        for entry in prefix_lib_python_site_package:  # remove symlinks after install
+            if entry.is_symlink():
+                os.unlink(entry.path)
diff --git a/var/spack/repos/builtin/packages/mosesdecoder/package.py b/var/spack/repos/builtin/packages/mosesdecoder/package.py
index b26fb816224d14..60112a6327f8b9 100644
--- a/var/spack/repos/builtin/packages/mosesdecoder/package.py
+++ b/var/spack/repos/builtin/packages/mosesdecoder/package.py
@@ -22,5 +22,5 @@ class Mosesdecoder(Package):
     depends_on("tcl")
     depends_on("gcc")
     depends_on("boost")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("python")
diff --git a/var/spack/repos/builtin/packages/mosh/package.py b/var/spack/repos/builtin/packages/mosh/package.py
index 705d390959f17c..bea6f2a08f333b 100644
--- a/var/spack/repos/builtin/packages/mosh/package.py
+++ b/var/spack/repos/builtin/packages/mosh/package.py
@@ -23,7 +23,7 @@ class Mosh(AutotoolsPackage):
 
     depends_on("protobuf")
     depends_on("ncurses")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("openssl")
 
     depends_on("pkgconfig", type="build")
diff --git a/var/spack/repos/builtin/packages/mothur/package.py b/var/spack/repos/builtin/packages/mothur/package.py
index 9263be74e8de7e..fe36e1cbd13080 100644
--- a/var/spack/repos/builtin/packages/mothur/package.py
+++ b/var/spack/repos/builtin/packages/mothur/package.py
@@ -38,7 +38,7 @@ class Mothur(MakefilePackage):
     depends_on("readline")
     depends_on("vsearch@2.13.5:", type="run")
     depends_on("usearch", type="run")
-    depends_on("zlib", when="+boost")
+    depends_on("zlib-api", when="+boost")
 
     def edit(self, spec, prefix):
         filter_file(r"^.*DMOTHUR_TOOLS.*$", "", "Makefile")
diff --git a/var/spack/repos/builtin/packages/mpi-test-suite/package.py b/var/spack/repos/builtin/packages/mpi-test-suite/package.py
new file mode 100644
index 00000000000000..239650655af900
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mpi-test-suite/package.py
@@ -0,0 +1,42 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class MpiTestSuite(AutotoolsPackage):
+    """The MPI-testsuite was initially developed for the use with PACX-MPI and has been extended
+    within the Open MPI project.
+
+    The main focus is on:
+    - High degree of code coverage through combinations of tests.
+    - Easy maintainability,
+    - Easy integration of new tests,
+    - Rich underlying functionality for flexible tests (convenience functions for datatypes, comms
+      and checking),
+    - Only a single binary (for single, since expensive MPI_Init/MPI_Finalize) to make it as quick
+      and easy as possible to run automatically
+    """
+
+    homepage = "https://github.com/open-mpi/mpi-test-suite"
+    url = "https://github.com/open-mpi/mpi-test-suite/archive/refs/tags/v1.1.1.tar.gz"
+
+    maintainers("jcortial-safran")
+
+    version("1.1.1", sha256="4cb7bdbdafa0855dab96d996f863b5d364c935e678c057ada3c8869c3666e926")
+
+    depends_on("autoconf", type="build")
+    depends_on("automake@1.14:", type="build")
+    depends_on("libtool", type="build")
+    depends_on("m4", type="build")
+    depends_on("gengetopt", type="build")
+
+    depends_on("mpi")
+
+    def autoreconf(self, spec, prefix):
+        autoreconf("--install", "--verbose", "--force")
+
+    def configure_args(self):
+        args = ["CC=%s" % self.spec["mpi"].mpicc]
+        return args
diff --git a/var/spack/repos/builtin/packages/mpibind/package.py b/var/spack/repos/builtin/packages/mpibind/package.py
index 67a9f7e4b5b84d..a9a34476aa55c0 100644
--- a/var/spack/repos/builtin/packages/mpibind/package.py
+++ b/var/spack/repos/builtin/packages/mpibind/package.py
@@ -24,9 +24,9 @@ class Mpibind(AutotoolsPackage):
     # AC_INIT would be missing the version argument,
     # which is derived with git.
     version("master", branch="master", get_full_repo=True)
-    version("0.8.0", commit="ff38b9d", no_cache=True)
-    version("0.7.0", commit="3c437a9", no_cache=True)
-    version("0.5.0", commit="8698f07", no_cache=True)
+    version("0.8.0", commit="ff38b9dcd150ca1e8a8796835d8e1e1847b3ba68", no_cache=True)
+    version("0.7.0", commit="3c437a97cd841b9c13abfbe1062a0285e1a29d3e", no_cache=True)
+    version("0.5.0", commit="8698f07412232e4dd4de4802b508374dc0de48c9", no_cache=True)
 
     variant("cuda", default=False, description="Build w/support for NVIDIA GPUs.")
     variant("rocm", default=False, description="Build w/support for AMD GPUs.")
diff --git a/var/spack/repos/builtin/packages/mpich/mpich32_411_CFI_configure.patch b/var/spack/repos/builtin/packages/mpich/mpich32_411_CFI_configure.patch
new file mode 100644
index 00000000000000..c0fce33abf0c4e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mpich/mpich32_411_CFI_configure.patch
@@ -0,0 +1,10 @@
+--- a/configure
++++ b/configure
+@@ -39449,6 +39449,7 @@ int foo_c(CFI_cdesc_t * a_desc, CFI_cdesc_t * b_desc)
+ 
+ void test_assumed_rank_async_impl_c(CFI_cdesc_t * a_desc)
+ {
++	CFI_is_contiguous(a_desc);
+ 	return;
+ }
+ 
diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py
index c4f0627eea7c6c..80bb05b28e8a77 100644
--- a/var/spack/repos/builtin/packages/mpich/package.py
+++ b/var/spack/repos/builtin/packages/mpich/package.py
@@ -32,7 +32,10 @@ def url_for_version(self, version):
             url = "https://www.mpich.org/static/downloads/{0}/mpich-{0}.tar.gz"
         return url.format(version)
 
+    keep_werror = "specific"
+
     version("develop", submodules=True)
+    version("4.1.2", sha256="3492e98adab62b597ef0d292fb2459b6123bc80070a8aa0a30be6962075a12f0")
     version("4.1.1", sha256="ee30471b35ef87f4c88f871a5e2ad3811cd9c4df32fd4f138443072ff4284ca2")
     version("4.1", sha256="8b1ec63bc44c7caa2afbb457bc5b3cd4a70dbe46baba700123d67c48dc5ab6a0")
     version("4.0.3", sha256="17406ea90a6ed4ecd5be39c9ddcbfac9343e6ab4f77ac4e8c5ebe4a3e3b6c501")
@@ -101,15 +104,6 @@ def url_for_version(self, version):
     variant("argobots", default=False, description="Enable Argobots support")
     variant("fortran", default=True, description="Enable Fortran support")
 
-    variant(
-        "two_level_namespace",
-        default=False,
-        description="""Build shared libraries and programs
-built with the mpicc/mpifort/etc. compiler wrappers
-with '-Wl,-commons,use_dylibs' and without
-'-Wl,-flat_namespace'.""",
-    )
-
     variant(
         "vci",
         default=False,
@@ -177,7 +171,7 @@ def url_for_version(self, version):
     patch(
         "https://github.com/pmodels/mpich/commit/8a851b317ee57366cd15f4f28842063d8eff4483.patch?full_index=1",
         sha256="d2dafc020941d2d8cab82bc1047e4a6a6d97736b62b06e2831d536de1ac01fd0",
-        when="@3.3:3.3.99 +hwloc",
+        when="@3.3 +hwloc",
     )
 
     # fix MPI_Barrier segmentation fault
@@ -215,6 +209,18 @@ def url_for_version(self, version):
         patch("mpich34_yaksa_hindexed.patch", when="datatype-engine=yaksa")
         patch("mpich34_yaksa_hindexed.patch", when="datatype-engine=auto device=ch4")
 
+    # Fix false positive result of the configure time check for CFI support
+    # https://github.com/pmodels/mpich/pull/6537
+    # https://github.com/pmodels/mpich/issues/6505
+    with when("@3.2.2:4.1.1"):
+        # Apply the patch from the upstream repo in case we have to run the autoreconf stage:
+        patch(
+            "https://github.com/pmodels/mpich/commit/d901a0b731035297dd6598888c49322e2a05a4e0.patch?full_index=1",
+            sha256="de0de41ec42ac5f259ea02f195eea56fba84d72b0b649a44c947eab6632995ab",
+        )
+        # Apply the changes to the configure script to skip the autoreconf stage if possible:
+        patch("mpich32_411_CFI_configure.patch")
+
     depends_on("findutils", type="build")
     depends_on("pkgconfig", type="build")
 
@@ -250,14 +256,14 @@ def url_for_version(self, version):
     # building from git requires regenerating autotools files
     depends_on("automake@1.15:", when="@develop", type="build")
     depends_on("libtool@2.4.4:", when="@develop", type="build")
-    depends_on("m4", when="@develop", type="build"),
+    depends_on("m4", when="@develop", type="build")
     depends_on("autoconf@2.67:", when="@develop", type="build")
 
-    # building with "+hwloc" also requires regenerating autotools files
-    depends_on("automake@1.15:", when="@3.3:3.3.99 +hwloc", type="build")
-    depends_on("libtool@2.4.4:", when="@3.3:3.3.99 +hwloc", type="build")
-    depends_on("m4", when="@3.3:3.3.99 +hwloc", type="build"),
-    depends_on("autoconf@2.67:", when="@3.3:3.3.99 +hwloc", type="build")
+    # building with "+hwloc' also requires regenerating autotools files
+    depends_on("automake@1.15:", when="@3.3 +hwloc", type="build")
+    depends_on("libtool@2.4.4:", when="@3.3 +hwloc", type="build")
+    depends_on("m4", when="@3.3 +hwloc", type="build")
+    depends_on("autoconf@2.67:", when="@3.3 +hwloc", type="build")
 
     # MPICH's Yaksa submodule requires python to configure
     depends_on("python@3.0:", when="@develop", type="build")
@@ -467,7 +473,7 @@ def setup_dependent_package(self, module, dependent_spec):
     def autoreconf(self, spec, prefix):
         """Not needed usually, configure should be already there"""
         # If configure exists nothing needs to be done
-        if os.path.exists(self.configure_abs_path) and not spec.satisfies("@3.3:3.3.99 +hwloc"):
+        if os.path.exists(self.configure_abs_path) and not spec.satisfies("@3.3 +hwloc"):
             return
         # Else bootstrap with autotools
         bash = which("bash")
@@ -495,6 +501,7 @@ def die_without_fortran(self):
     def configure_args(self):
         spec = self.spec
         config_args = [
+            "--disable-maintainer-mode",
             "--disable-silent-rules",
             "--enable-shared",
             "--with-pm={0}".format("hydra" if "+hydra" in spec else "no"),
@@ -504,6 +511,10 @@ def configure_args(self):
             "--with-yaksa={0}".format(spec["yaksa"].prefix if "^yaksa" in spec else "embedded"),
         ]
 
+        # see https://github.com/pmodels/mpich/issues/5530
+        if spec.platform == "darwin":
+            config_args.append("--enable-two-level-namespace")
+
         # hwloc configure option changed in 4.0
         if spec.satisfies("@4.0:"):
             config_args.append(
@@ -586,9 +597,6 @@ def configure_args(self):
             config_args.append("--with-thread-package=argobots")
             config_args.append("--with-argobots=" + spec["argobots"].prefix)
 
-        if "+two_level_namespace" in spec:
-            config_args.append("--enable-two-level-namespace")
-
         if "+vci" in spec:
             config_args.append("--enable-thread-cs=per-vci")
             config_args.append("--with-ch4-max-vcis=default")
diff --git a/var/spack/repos/builtin/packages/mpifileutils/package.py b/var/spack/repos/builtin/packages/mpifileutils/package.py
index 7ea08345734f19..094c7b73e606dc 100644
--- a/var/spack/repos/builtin/packages/mpifileutils/package.py
+++ b/var/spack/repos/builtin/packages/mpifileutils/package.py
@@ -22,7 +22,7 @@ class Mpifileutils(Package):
 
     tags = ["e4s"]
 
-    version("develop", branch="master")
+    version("develop", branch="main")
     version("0.11.1", sha256="e2cba53309b5b3ee581b6ff82e4e66f54628370cce694c34224ed947fece32d4")
     version("0.11", sha256="f5dc1b39077b3c04f79b2c335c4fd80306f8c57ecfbcacbb82cf532caf02b5fd")
     version("0.10.1", sha256="4c8409ef4140f6f557d0e93f0c1267baf5d893c203b29fb7a33d9bc3c5a5d25c")
diff --git a/var/spack/repos/builtin/packages/mpl/package.py b/var/spack/repos/builtin/packages/mpl/package.py
index 612049bad2c8e5..74f03225566ffe 100644
--- a/var/spack/repos/builtin/packages/mpl/package.py
+++ b/var/spack/repos/builtin/packages/mpl/package.py
@@ -15,9 +15,9 @@ class Mpl(CMakePackage):
     maintainers("rabauke")
 
     version("develop", branch="master")
-    version("0.3.0", tag="v0.3.0")
-    version("0.2.1", tag="v0.2.1")
-    version("0.2.0", tag="v0.2.0")
-    version("0.1", tag="v0.1")
+    version("0.3.0", tag="v0.3.0", commit="e6bd4926914127f3609a14474aa4a9c4fabbff0b")
+    version("0.2.1", tag="v0.2.1", commit="5bee297b453d7b66a803453bfc6884611a36c4d0")
+    version("0.2.0", tag="v0.2.0", commit="f322352c93627c1b91d8efb1c4ee2e4873aed016")
+    version("0.1", tag="v0.1", commit="970d0f3436ddbfcf2eba12c5bc7f4f7660e433ca")
 
     depends_on("mpi")
diff --git a/var/spack/repos/builtin/packages/mrtrix3/package.py b/var/spack/repos/builtin/packages/mrtrix3/package.py
index 1cb3ae3d03da53..2a59d7ec22a8e9 100644
--- a/var/spack/repos/builtin/packages/mrtrix3/package.py
+++ b/var/spack/repos/builtin/packages/mrtrix3/package.py
@@ -28,7 +28,7 @@ class Mrtrix3(Package):
     depends_on("glu")
     depends_on("qt+opengl@4.7:")
     depends_on("eigen")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libtiff")
     depends_on("fftw")
 
diff --git a/var/spack/repos/builtin/packages/mruby/config.rb b/var/spack/repos/builtin/packages/mruby/config.rb
new file mode 100644
index 00000000000000..318c1972304686
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mruby/config.rb
@@ -0,0 +1,83 @@
+MRuby::Build.new do |conf|
+  # load specific toolchain settings
+  conf.toolchain
+  if ENV.key?('MRUBY_ENABLE_CXX_EXCEPTION')
+    conf.enable_cxx_exception
+  end
+
+  # Use mrbgems
+  # conf.gem 'examples/mrbgems/ruby_extension_example'
+  # conf.gem 'examples/mrbgems/c_extension_example' do |g|
+  #   g.cc.flags << '-g' # append cflags in this gem
+  # end
+  # conf.gem 'examples/mrbgems/c_and_ruby_extension_example'
+  # conf.gem :core => 'mruby-eval'
+  # conf.gem :mgem => 'mruby-onig-regexp'
+  # conf.gem :github => 'mattn/mruby-onig-regexp'
+  # conf.gem :git => 'git@github.com:mattn/mruby-onig-regexp.git', :branch => 'master', :options => '-v'
+
+  # include the GEM box
+  conf.gembox 'full-core'
+
+  # C compiler settings
+  # conf.cc do |cc|
+  #   cc.command = ENV['CC'] || 'gcc'
+  #   cc.flags = [ENV['CFLAGS'] || %w()]
+  #   cc.include_paths = ["#{root}/include"]
+  #   cc.defines = %w()
+  #   cc.option_include_path = %q[-I"%s"]
+  #   cc.option_define = '-D%s'
+  #   cc.compile_options = %Q[%{flags} -MMD -o "%{outfile}" -c "%{infile}"]
+  # end
+
+  # mrbc settings
+  # conf.mrbc do |mrbc|
+  #   mrbc.compile_options = "-g -B%{funcname} -o-" # The -g option is required for line numbers
+  # end
+
+  # Linker settings
+  # conf.linker do |linker|
+  #   linker.command = ENV['LD'] || 'gcc'
+  #   linker.flags = [ENV['LDFLAGS'] || []]
+  #   linker.flags_before_libraries = []
+  #   linker.libraries = %w()
+  #   linker.flags_after_libraries = []
+  #   linker.library_paths = []
+  #   linker.option_library = '-l%s'
+  #   linker.option_library_path = '-L%s'
+  #   linker.link_options = "%{flags} -o "%{outfile}" %{objs} %{libs}"
+  # end
+
+  # Archiver settings
+  # conf.archiver do |archiver|
+  #   archiver.command = ENV['AR'] || 'ar'
+  #   archiver.archive_options = 'rs "%{outfile}" %{objs}'
+  # end
+
+  # Parser generator settings
+  # conf.yacc do |yacc|
+  #   yacc.command = ENV['YACC'] || 'bison'
+  #   yacc.compile_options = %q[-o "%{outfile}" "%{infile}"]
+  # end
+
+  # gperf settings
+  # conf.gperf do |gperf|
+  #   gperf.command = 'gperf'
+  #   gperf.compile_options = %q[-L ANSI-C -C -p -j1 -i 1 -g -o -t -N mrb_reserved_word -k"1,3,$" "%{infile}" > "%{outfile}"]
+  # end
+
+  # file extensions
+  # conf.exts do |exts|
+  #   exts.object = '.o'
+  #   exts.executable = '' # '.exe' if Windows
+  #   exts.library = '.a'
+  # end
+
+  # file separator
+  # conf.file_separator = '/'
+
+  # Turn on `enable_debug` for better debugging
+  # conf.enable_debug
+  conf.enable_bintest
+  conf.enable_test
+end
diff --git a/var/spack/repos/builtin/packages/mruby/package.py b/var/spack/repos/builtin/packages/mruby/package.py
new file mode 100644
index 00000000000000..f6285d1b793ac9
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mruby/package.py
@@ -0,0 +1,56 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Mruby(Package):
+    """mruby is the lightweight implementation of the Ruby language complying
+    to (part of) the ISO standard. Its syntax is Ruby 2.x compatible."""
+
+    homepage = "https://mruby.org/"
+    url = "https://github.com/mruby/mruby/archive/refs/tags/3.0.0.tar.gz"
+    git = "https://github.com/mruby/mruby.git"
+
+    maintainers = ["mdorier"]
+
+    version("master", branch="master")
+    version("3.2.0", sha256="3c198e4a31d31fe8524013066fac84a67fe6cd6067d92c25a1c79089744cb608")
+    version("3.1.0", sha256="64ce0a967028a1a913d3dfc8d3f33b295332ab73be6f68e96d0f675f18c79ca8")
+    version("3.0.0", sha256="95b798cdd931ef29d388e2b0b267cba4dc469e8722c37d4ef8ee5248bc9075b0")
+    version("2.1.2", sha256="4dc0017e36d15e81dc85953afb2a643ba2571574748db0d8ede002cefbba053b")
+    version("2.1.1", sha256="bb27397ee9cb7e0ddf4ff51caf5b0a193d636b7a3c52399684c8c383b41c362a")
+    version("2.1.0", sha256="d6733742a07e553c52ab71df08b0604b3b571768bbc0c2729fbf0389d1bb5d13")
+
+    variant("cxx_exception", description="Enable C++ exceptions", default=False, when="@3.1.0:")
+
+    depends_on("ruby@3.0.0:", type=("build"))
+    depends_on("bison", type=("build"))
+
+    def patch(self):
+        """Create a config.rb file for rake to use"""
+        import os
+
+        here = os.path.dirname(os.path.abspath(__file__))
+        copy(os.path.join(here, "config.rb"), os.path.join("build_config", "spack.rb"))
+
+    def install(self, spec, prefix):
+        import os
+
+        rb = spec["ruby"]
+        env["MRUBY_CONFIG"] = os.path.join("build_config", "spack.rb")
+        env["GEM_PATH"] = os.path.join(
+            rb.prefix, "lib", "ruby", "gems", str(rb.version.up_to(2)) + ".0"
+        )
+        if "+cxx_exception" in spec:
+            env["MRUBY_ENABLE_CXX_EXCEPTION"] = "ON"
+        rake()
+        build_path = os.path.join("build", "host")
+        for d in ["include", "lib", "bin", "mrblib", "mrbgems"]:
+            prefix_d = os.path.join(prefix, d)
+            build_path_d = os.path.join(build_path, d)
+            mkdirp(prefix_d)
+            install_tree(build_path_d, prefix_d)
+        install_tree("include", prefix.include)
diff --git a/var/spack/repos/builtin/packages/mtn/package.py b/var/spack/repos/builtin/packages/mtn/package.py
new file mode 100644
index 00000000000000..6e2111251fa499
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mtn/package.py
@@ -0,0 +1,31 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class Mtn(MakefilePackage):
+    """Movie Thumbnailer is CLI thumbnail generator using FFmpeg."""
+
+    homepage = "https://gitlab.com/movie_thumbnailer/mtn"
+    url = "https://gitlab.com/movie_thumbnailer/mtn/-/archive/v3.4.2/mtn-v3.4.2.tar.gz"
+
+    maintainers("ledif")
+
+    version("3.4.2", sha256="19b2076c00f5b0ad70c2467189b17f335c6e7ece5d1a01ed8910779f6a5ca52a")
+
+    depends_on("ffmpeg")
+    depends_on("libgd")
+
+    def build(self, spec, prefix):
+        src_dir = join_path(self.build_directory, "src")
+        with working_dir(src_dir):
+            make()
+
+    def install(self, spec, prefix):
+        src_dir = join_path(self.build_directory, "src")
+        with working_dir(src_dir):
+            make(f"PREFIX={prefix}", "install")
diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py
index 32bcaf3a812ac1..f9a210407dc8dc 100644
--- a/var/spack/repos/builtin/packages/mumps/package.py
+++ b/var/spack/repos/builtin/packages/mumps/package.py
@@ -223,7 +223,7 @@ def write_makefile_inc(self):
         # As of version 5.2.0, MUMPS is able to take advantage
         # of the GEMMT BLAS extension. MKL and amdblis are the only
         # known BLAS implementation supported.
-        if "@5.2.0: ^mkl" in self.spec:
+        if self.spec["blas"].name in INTEL_MATH_LIBRARIES and self.spec.satifies("@5.2.0:"):
             optf.append("-DGEMMT_AVAILABLE")
 
         if "@5.2.0: ^amdblis@3.0:" in self.spec:
diff --git a/var/spack/repos/builtin/packages/muse/package.py b/var/spack/repos/builtin/packages/muse/package.py
index 12d7cbe3501f33..685e020a65747f 100644
--- a/var/spack/repos/builtin/packages/muse/package.py
+++ b/var/spack/repos/builtin/packages/muse/package.py
@@ -14,7 +14,7 @@ class Muse(MakefilePackage):
 
     version("1.0-rc", sha256="b48b8be0044a2249bdc0b625fe0192c65089c598bbd1b1142902dfa81e804023")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     def install(self, spec, prefix):
         mkdir(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/musl/package.py b/var/spack/repos/builtin/packages/musl/package.py
index 8ff70c2f53037e..ddaba97eb8e381 100644
--- a/var/spack/repos/builtin/packages/musl/package.py
+++ b/var/spack/repos/builtin/packages/musl/package.py
@@ -26,6 +26,7 @@ class Musl(MakefilePackage):
     homepage = "https://www.musl-libc.org"
     url = "https://www.musl-libc.org/releases/musl-1.1.23.tar.gz"
 
+    version("1.2.4", sha256="7a35eae33d5372a7c0da1188de798726f68825513b7ae3ebe97aaaa52114f039")
     version("1.2.3", sha256="7d5b0b6062521e4627e099e4c9dc8248d32a30285e959b7eecaa780cf8cfd4a4")
     version("1.2.2", sha256="9b969322012d796dc23dda27a35866034fa67d8fb67e0e2c45c913c3d43219dd")
     version("1.2.1", sha256="68af6e18539f646f9c41a3a2bb25be4a5cfa5a8f65f0bb647fd2bbfdf877e84b")
diff --git a/var/spack/repos/builtin/packages/must/package.py b/var/spack/repos/builtin/packages/must/package.py
index 6cca6c9cf278b0..5d6c36bb044e84 100644
--- a/var/spack/repos/builtin/packages/must/package.py
+++ b/var/spack/repos/builtin/packages/must/package.py
@@ -19,11 +19,10 @@ class Must(CMakePackage):
 
     maintainers("jgalarowicz", "dmont")
 
-    version("1.8.0-rc1", sha256="49fd2487fbd1aa41f4252c7e37efebd3f6ff48218c88e82f34b88d59348fe406")
-    version(
-        "1.8-preview", sha256="67b4b061db7a893e22a6610e2085072716d11738bc6cc3cb3ffd60d6833e8bad"
-    )
+    version("1.9.0", sha256="24998f4ca6bce718d69347de90798600f2385c21266c2d1dd39a87dd8bd1fba4")
+    version("1.8.0", sha256="9754fefd2e4c8cba812f8b56a5dd929bc84aa599b2509305e1eb8518be0a8a39")
     version("1.7.2", sha256="616c54b7487923959df126ac4b47ae8c611717d679fe7ec29f57a89bf0e2e0d0")
+
     variant("test", default=False, description="Enable must internal tests")
     variant("tsan", default=True, description="Enable thread sanitizer")
     variant("graphviz", default=False, description="Use to generate graphs")
diff --git a/var/spack/repos/builtin/packages/mvapich/package.py b/var/spack/repos/builtin/packages/mvapich/package.py
index 1022639a1a80d5..68df26215aeb61 100644
--- a/var/spack/repos/builtin/packages/mvapich/package.py
+++ b/var/spack/repos/builtin/packages/mvapich/package.py
@@ -100,7 +100,7 @@ class Mvapich(AutotoolsPackage):
     depends_on("findutils", type="build")
     depends_on("bison", type="build")
     depends_on("pkgconfig", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libpciaccess", when=(sys.platform != "darwin"))
     depends_on("libxml2")
     depends_on("cuda", when="+cuda")
diff --git a/var/spack/repos/builtin/packages/mvapich2/fix-torque.patch b/var/spack/repos/builtin/packages/mvapich2/fix-torque.patch
new file mode 100644
index 00000000000000..0f12f1931eccc7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mvapich2/fix-torque.patch
@@ -0,0 +1,27 @@
+diff -ruN spack-src/src/pm/hydra/configure.ac spack-src-patched/src/pm/hydra/configure.ac
+--- spack-src/src/pm/hydra/configure.ac	2022-05-16 16:58:22.000000000 +0000
++++ spack-src-patched/src/pm/hydra/configure.ac	2023-07-20 15:23:21.802299913 +0000
+@@ -306,15 +306,16 @@
+ 		    available_launchers="$available_launchers pbs"
+ 		    PAC_APPEND_FLAG([-ltorque],[WRAPPER_LIBS])
+                 fi
+-		PAC_SET_HEADER_LIB_PATH(pbs)
+ 		PAC_PUSH_FLAG(LIBS)
+-		PAC_CHECK_HEADER_LIB(tm.h, pbs, tm_init, have_pbs_launcher=yes,
++                if test "have_pbs_launcher" = "no" ; then
++		  PAC_CHECK_HEADER_LIB(tm.h, pbs, tm_init, have_pbs_launcher=yes,
+ 					   have_pbs_launcher=no)
+-		PAC_POP_FLAG(LIBS)
+-                if test "$have_pbs_launcher" = "yes" ; then
+-		    available_launchers="$available_launchers pbs"
+-		    PAC_APPEND_FLAG([-lpbs],[WRAPPER_LIBS])
+-                    AC_DEFINE(HAVE_PBS_PRO, 1, [Define if PBS Pro support is enabled])
++		  PAC_POP_FLAG(LIBS)
++                  if test "$have_pbs_launcher" = "yes" ; then
++		      available_launchers="$available_launchers pbs"
++		      PAC_APPEND_FLAG([-lpbs],[WRAPPER_LIBS])
++                      AC_DEFINE(HAVE_PBS_PRO, 1, [Define if PBS Pro support is enabled])
++                  fi
+                 fi
+ 		available_rmks="$available_rmks pbs"
+ 		;;
diff --git a/var/spack/repos/builtin/packages/mvapich2/mpir_attr_delete_list_segfault.patch b/var/spack/repos/builtin/packages/mvapich2/mpir_attr_delete_list_segfault.patch
new file mode 100644
index 00000000000000..ee48dd8e0f9784
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mvapich2/mpir_attr_delete_list_segfault.patch
@@ -0,0 +1,18 @@
+--- a/src/mpi/attr/attrutil.c
++++ b/src/mpi/attr/attrutil.c
+@@ -266,6 +266,7 @@
+ 	   corresponding keyval */
+ 	/* Still to do: capture any error returns but continue to 
+ 	   process attributes */
++    if (p->keyval) {
+ 	mpi_errno = MPIR_Call_attr_delete( handle, p );
+ 
+ 	/* We must also remove the keyval reference.  If the keyval
+@@ -282,6 +283,7 @@
+ 		MPIU_Handle_obj_free( &MPID_Keyval_mem, p->keyval );
+ 	    }
+ 	}
++	}
+ 	
+ 	MPIU_Handle_obj_free( &MPID_Attr_mem, p );
+ 	
diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py
index 90520ceda8832f..c1f67620f38fd7 100644
--- a/var/spack/repos/builtin/packages/mvapich2/package.py
+++ b/var/spack/repos/builtin/packages/mvapich2/package.py
@@ -117,10 +117,11 @@ class Mvapich2(AutotoolsPackage):
         values=auto_or_any_combination_of("lustre", "gpfs", "nfs", "ufs"),
     )
 
+    depends_on("automake@1.15", type="build")  # needed for torque patch
     depends_on("findutils", type="build")
     depends_on("bison", type="build")
     depends_on("pkgconfig", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libpciaccess", when=(sys.platform != "darwin"))
     depends_on("libxml2")
     depends_on("cuda", when="+cuda")
@@ -133,8 +134,14 @@ class Mvapich2(AutotoolsPackage):
     depends_on("libfabric", when="fabrics=nemesisofi")
     depends_on("slurm", when="process_managers=slurm")
 
+    # Fix segmentation fault in `MPIR_Attr_delete_list`:
+    # .
+    patch("mpir_attr_delete_list_segfault.patch", when="@2.3.7")
+
     conflicts("fabrics=psm2", when="@:2.1")  # psm2 support was added at version 2.2
 
+    patch("fix-torque.patch", when="@2.3.7-1")
+
     filter_compiler_wrappers("mpicc", "mpicxx", "mpif77", "mpif90", "mpifort", relative_root="bin")
 
     @classmethod
diff --git a/var/spack/repos/builtin/packages/mysql-connector-c/fix-cmake.patch b/var/spack/repos/builtin/packages/mysql-connector-c/fix-cmake.patch
new file mode 100644
index 00000000000000..e9010c3a7b662b
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mysql-connector-c/fix-cmake.patch
@@ -0,0 +1,12 @@
+--- a/cmake/install_macros.cmake
++++ /dev/null
+@@ -362,8 +362,8 @@
+         CONFIGURATIONS Release RelWithDebInfo
+         COMPONENT ${ARG_COMPONENT}
+         OPTIONAL)
+-    ENDIF()
+   ENDFOREACH()
++  ENDIF()
+ 
+ ENDFUNCTION()
+ 
diff --git a/var/spack/repos/builtin/packages/mysql-connector-c/package.py b/var/spack/repos/builtin/packages/mysql-connector-c/package.py
new file mode 100644
index 00000000000000..8b24b649d03e7f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mysql-connector-c/package.py
@@ -0,0 +1,23 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class MysqlConnectorC(CMakePackage):
+    """MySQL Connector/C, the C interface for communicating with MySQL servers.
+
+    Connector/C is a client library that implements the C API for client/server
+    communication. It is a standalone replacement for the MySQL client library
+    shipped with MySQL Server distributions."""
+
+    homepage = "https://dev.mysql.com/downloads/connector/c/"
+    url = "https://dev.mysql.com/get/Downloads/Connector-C/mysql-connector-c-6.1.11-src.tar.gz"
+
+    depends_on("cmake")
+
+    patch("fix-cmake.patch", when="@6.1.11")
+
+    version("6.1.11", sha256="c8664851487200162b38b6f3c8db69850bd4f0e4c5ff5a6d161dbfb5cb76b6c4")
diff --git a/var/spack/repos/builtin/packages/mysql/package.py b/var/spack/repos/builtin/packages/mysql/package.py
index a5d6bc0be5cd83..a06b616eca739e 100644
--- a/var/spack/repos/builtin/packages/mysql/package.py
+++ b/var/spack/repos/builtin/packages/mysql/package.py
@@ -16,6 +16,7 @@ class Mysql(CMakePackage):
     homepage = "https://www.mysql.com/"
     url = "https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.15.tar.gz"
 
+    version("8.2.0", sha256="8ad3e3f1c5ae2154be638acf556e8981f702d052acc5957e776df1722c211979")
     version("8.0.29", sha256="512170fa6f78a694d6f18d197e999d2716ee68dc541d7644dd922a3663407266")
     version("8.0.19", sha256="a62786d67b5e267eef928003967b4ccfe362d604b80f4523578e0688f5b9f834")
     version("8.0.18", sha256="4cb39a315298eb243c25c53c184b3682b49c2a907a1d8432ba0620534806ade8")
@@ -57,6 +58,7 @@ class Mysql(CMakePackage):
         multi=False,
         description="Use the specified C++ standard when building.",
     )
+    variant("download_boost", default=False, description="Download own boost libs", when="@5.7:")
 
     # 5.7.X cannot be compiled client-only.
     conflicts("+client_only", when="@5.7.0:5.7")
@@ -64,6 +66,8 @@ class Mysql(CMakePackage):
     # std::byte.
     conflicts("cxxstd=17", when="@8.0.0:~client_only")
 
+    requires("cxxstd=17", when="@8.2:")
+
     provides("mysql-client")
 
     # https://dev.mysql.com/doc/refman/8.0/en/source-installation.html
@@ -81,52 +85,54 @@ class Mysql(CMakePackage):
     depends_on("pkgconfig", type="build", when="@5.7.0:")
     depends_on("doxygen", type="build", when="@8.0.0:")
 
-    # Each version of MySQL requires a specific version of boost
-    # See BOOST_PACKAGE_NAME in cmake/boost.cmake
-    # 8.0.29
-    depends_on("boost@1.77.0 cxxstd=98", type="build", when="@8.0.29 cxxstd=98")
-    depends_on("boost@1.77.0 cxxstd=11", type="build", when="@8.0.29 cxxstd=11")
-    depends_on("boost@1.77.0 cxxstd=14", type="build", when="@8.0.29 cxxstd=14")
-    depends_on("boost@1.77.0 cxxstd=17", type="build", when="@8.0.29 cxxstd=17")
-    # 8.0.19
-    depends_on("boost@1.70.0 cxxstd=98", type="build", when="@8.0.19 cxxstd=98")
-    depends_on("boost@1.70.0 cxxstd=11", type="build", when="@8.0.19 cxxstd=11")
-    depends_on("boost@1.70.0 cxxstd=14", type="build", when="@8.0.19 cxxstd=14")
-    depends_on("boost@1.70.0 cxxstd=17", type="build", when="@8.0.19 cxxstd=17")
-    # 8.0.16--8.0.18
-    depends_on("boost@1.69.0 cxxstd=98", type="build", when="@8.0.16:8.0.18 cxxstd=98")
-    depends_on("boost@1.69.0 cxxstd=11", type="build", when="@8.0.16:8.0.18 cxxstd=11")
-    depends_on("boost@1.69.0 cxxstd=14", type="build", when="@8.0.16:8.0.18 cxxstd=14")
-    depends_on("boost@1.69.0 cxxstd=17", type="build", when="@8.0.16:8.0.18 cxxstd=17")
-    # 8.0.14--8.0.15
-    depends_on("boost@1.68.0 cxxstd=98", type="build", when="@8.0.14:8.0.15 cxxstd=98")
-    depends_on("boost@1.68.0 cxxstd=11", type="build", when="@8.0.14:8.0.15 cxxstd=11")
-    depends_on("boost@1.68.0 cxxstd=14", type="build", when="@8.0.14:8.0.15 cxxstd=14")
-    depends_on("boost@1.68.0 cxxstd=17", type="build", when="@8.0.14:8.0.15 cxxstd=17")
-    # 8.0.12--8.0.13
-    depends_on("boost@1.67.0 cxxstd=98", type="build", when="@8.0.12:8.0.13 cxxstd=98")
-    depends_on("boost@1.67.0 cxxstd=11", type="build", when="@8.0.12:8.0.13 cxxstd=11")
-    depends_on("boost@1.67.0 cxxstd=14", type="build", when="@8.0.12:8.0.13 cxxstd=14")
-    depends_on("boost@1.67.0 cxxstd=17", type="build", when="@8.0.12:8.0.13 cxxstd=17")
-    # 8.0.11
-    depends_on("boost@1.66.0 cxxstd=98", type="build", when="@8.0.11 cxxstd=98")
-    depends_on("boost@1.66.0 cxxstd=11", type="build", when="@8.0.11 cxxstd=11")
-    depends_on("boost@1.66.0 cxxstd=14", type="build", when="@8.0.11 cxxstd=14")
-    depends_on("boost@1.66.0 cxxstd=17", type="build", when="@8.0.11 cxxstd=17")
-    # 5.7.X
-    depends_on("boost@1.59.0 cxxstd=98", when="@5.7.0:5.7 cxxstd=98")
-    depends_on("boost@1.59.0 cxxstd=11", when="@5.7.0:5.7 cxxstd=11")
-    depends_on("boost@1.59.0 cxxstd=14", when="@5.7.0:5.7 cxxstd=14")
-    depends_on("boost@1.59.0 cxxstd=17", when="@5.7.0:5.7 cxxstd=17")
-
-    # TODO: replace this with an explicit list of components of Boost,
-    # for instance depends_on('boost +filesystem')
-    # See https://github.com/spack/spack/pull/22303 for reference
-    depends_on(Boost.with_default_variants, when="@5.7:")
+    with when("~download_boost"):
+        # Each version of MySQL requires a specific version of boost
+        # See BOOST_PACKAGE_NAME in cmake/boost.cmake
+        # 8.0.29
+        depends_on("boost@1.77.0 cxxstd=98", type="build", when="@8.0.29: cxxstd=98")
+        depends_on("boost@1.77.0 cxxstd=11", type="build", when="@8.0.29: cxxstd=11")
+        depends_on("boost@1.77.0 cxxstd=14", type="build", when="@8.0.29: cxxstd=14")
+        depends_on("boost@1.77.0 cxxstd=17", type="build", when="@8.0.29: cxxstd=17")
+        # 8.0.19
+        depends_on("boost@1.70.0 cxxstd=98", type="build", when="@8.0.19 cxxstd=98")
+        depends_on("boost@1.70.0 cxxstd=11", type="build", when="@8.0.19 cxxstd=11")
+        depends_on("boost@1.70.0 cxxstd=14", type="build", when="@8.0.19 cxxstd=14")
+        depends_on("boost@1.70.0 cxxstd=17", type="build", when="@8.0.19 cxxstd=17")
+        # 8.0.16--8.0.18
+        depends_on("boost@1.69.0 cxxstd=98", type="build", when="@8.0.16:8.0.18 cxxstd=98")
+        depends_on("boost@1.69.0 cxxstd=11", type="build", when="@8.0.16:8.0.18 cxxstd=11")
+        depends_on("boost@1.69.0 cxxstd=14", type="build", when="@8.0.16:8.0.18 cxxstd=14")
+        depends_on("boost@1.69.0 cxxstd=17", type="build", when="@8.0.16:8.0.18 cxxstd=17")
+        # 8.0.14--8.0.15
+        depends_on("boost@1.68.0 cxxstd=98", type="build", when="@8.0.14:8.0.15 cxxstd=98")
+        depends_on("boost@1.68.0 cxxstd=11", type="build", when="@8.0.14:8.0.15 cxxstd=11")
+        depends_on("boost@1.68.0 cxxstd=14", type="build", when="@8.0.14:8.0.15 cxxstd=14")
+        depends_on("boost@1.68.0 cxxstd=17", type="build", when="@8.0.14:8.0.15 cxxstd=17")
+        # 8.0.12--8.0.13
+        depends_on("boost@1.67.0 cxxstd=98", type="build", when="@8.0.12:8.0.13 cxxstd=98")
+        depends_on("boost@1.67.0 cxxstd=11", type="build", when="@8.0.12:8.0.13 cxxstd=11")
+        depends_on("boost@1.67.0 cxxstd=14", type="build", when="@8.0.12:8.0.13 cxxstd=14")
+        depends_on("boost@1.67.0 cxxstd=17", type="build", when="@8.0.12:8.0.13 cxxstd=17")
+        # 8.0.11
+        depends_on("boost@1.66.0 cxxstd=98", type="build", when="@8.0.11 cxxstd=98")
+        depends_on("boost@1.66.0 cxxstd=11", type="build", when="@8.0.11 cxxstd=11")
+        depends_on("boost@1.66.0 cxxstd=14", type="build", when="@8.0.11 cxxstd=14")
+        depends_on("boost@1.66.0 cxxstd=17", type="build", when="@8.0.11 cxxstd=17")
+        # 5.7.X
+        depends_on("boost@1.59.0 cxxstd=98", when="@5.7.0:5.7 cxxstd=98")
+        depends_on("boost@1.59.0 cxxstd=11", when="@5.7.0:5.7 cxxstd=11")
+        depends_on("boost@1.59.0 cxxstd=14", when="@5.7.0:5.7 cxxstd=14")
+        depends_on("boost@1.59.0 cxxstd=17", when="@5.7.0:5.7 cxxstd=17")
+
+        # TODO: replace this with an explicit list of components of Boost,
+        # for instance depends_on('boost +filesystem')
+        # See https://github.com/spack/spack/pull/22303 for reference
+        depends_on(Boost.with_default_variants, when="@5.7:")
 
     depends_on("rpcsvc-proto")
     depends_on("ncurses")
     depends_on("openssl")
+    depends_on("openssl@1", when="@:8.0.29")
     depends_on("libtirpc", when="@5.7.0: platform=linux")
     depends_on("libedit", type=["build", "run"])
     depends_on("perl", type=["build", "test"], when="@:7")
@@ -160,11 +166,18 @@ def cmake_args(self):
         options = []
         if "boost" in spec:
             options.append("-DBOOST_ROOT={0}".format(spec["boost"].prefix))
+            options.append("-DWITH_BOOST={0}".format(spec["boost"].prefix))
+        if spec.satisfies("+download_boost"):
+            options.append(
+                "-DWITH_BOOST={0}".format(os.path.join(self.build_directory, "boostdir"))
+            )
+            options.append("-DDOWNLOAD_BOOST=1")
         if "+client_only" in self.spec:
             options.append("-DWITHOUT_SERVER:BOOL=ON")
         options.append("-DWITH_EDITLINE=system")
         options.append("-Dlibedit_INCLUDE_DIR={0}".format(spec["libedit"].prefix.include))
         options.append("-Dlibedit_LIBRARY={0}".format(spec["libedit"].libs.directories[0]))
+        options.append("-DFORCE_UNSUPPORTED_COMPILER=ON")
         return options
 
     def _fix_dtrace_shebang(self, env):
diff --git a/var/spack/repos/builtin/packages/nag/package.py b/var/spack/repos/builtin/packages/nag/package.py
index 5d6775c9ed02a3..1bc2c618e65cf7 100644
--- a/var/spack/repos/builtin/packages/nag/package.py
+++ b/var/spack/repos/builtin/packages/nag/package.py
@@ -17,10 +17,21 @@ class Nag(Package):
     homepage = "https://www.nag.com/nagware/np.asp"
     maintainers("skosukhin")
 
-    version("7.1", sha256="18640737b232cebeb532ba36187675cdaf36d5b1fc235a780fc9e588c19a3ed2")
-    version("7.0", sha256="6d509208533d79139e5a9f879b7b93e7b58372b78d404d51f35e491ecbaa54c7")
-    version("6.2", sha256="9b60f6ffa4f4be631079676963e74eea25e8824512e5c864eb06758b2a3cdd2d")
-    version("6.1", sha256="32580e0004e6798abf1fa52f0070281b28abeb0da2387530a4cc41218e813c7c")
+    version("7.1.7125", sha256="738ed9ed943ebeb05d337cfdc603b9c88b8642b3d0cafea8d2872f36201adb37")
+    version(
+        "7.1.7101",
+        sha256="18640737b232cebeb532ba36187675cdaf36d5b1fc235a780fc9e588c19a3ed2",
+        url="file://{0}/npl6a71na_amd64.tgz".format(os.getcwd()),
+        deprecated=True,
+    )
+    version("7.0.7048", sha256="6d509208533d79139e5a9f879b7b93e7b58372b78d404d51f35e491ecbaa54c7")
+    version("6.2.6252", sha256="9b60f6ffa4f4be631079676963e74eea25e8824512e5c864eb06758b2a3cdd2d")
+    version(
+        "6.1.6136",
+        sha256="32580e0004e6798abf1fa52f0070281b28abeb0da2387530a4cc41218e813c7c",
+        url="file://{0}/npl6a61na_amd64.tgz".format(os.getcwd()),
+        deprecated=True,
+    )
 
     # Licensing
     license_required = True
@@ -37,7 +48,7 @@ def url_for_version(self, version):
         # TODO: url and checksum are architecture dependent
         # TODO: We currently only support x86_64
         url = "https://www.nag.com/downloads/impl/npl6a{0}na_amd64.tgz"
-        return url.format(version.joined)
+        return url.format(version.up_to(2).joined)
 
     def install(self, spec, prefix):
         # Set installation directories
diff --git a/var/spack/repos/builtin/packages/namd/package.py b/var/spack/repos/builtin/packages/namd/package.py
index 8e61984708c595..c273da3dbe5d95 100644
--- a/var/spack/repos/builtin/packages/namd/package.py
+++ b/var/spack/repos/builtin/packages/namd/package.py
@@ -24,6 +24,7 @@ class Namd(MakefilePackage, CudaPackage):
     maintainers("jcphill")
 
     version("master", branch="master")
+    version("3.0b3", sha256="20c32b6161f9c376536e3cb97c3bfe5367e1baaaace3c716ff79831fc2eb8199")
     version("2.15a2", sha256="8748cbaa93fc480f92fc263d9323e55bce6623fc693dbfd4a40f59b92669713e")
     version("2.15a1", branch="master", tag="release-2-15-alpha-1")
     # Same as above, but lets you use a local file instead of git
@@ -53,6 +54,7 @@ class Namd(MakefilePackage, CudaPackage):
     )
 
     variant("avxtiles", when="target=x86_64_v4:", default=False, description="Enable avxtiles")
+    variant("single_node_gpu", default=False, description="Single node GPU")
 
     # init_tcl_pointers() declaration and implementation are inconsistent
     # "src/colvarproxy_namd.C", line 482: error: inherited member is not
@@ -62,7 +64,8 @@ class Namd(MakefilePackage, CudaPackage):
     # Handle change in python-config for python@3.8:
     patch("namd-python38.patch", when="interface=python ^python@3.8:")
 
-    depends_on("charmpp@6.10.1:6", when="@2.14:")
+    depends_on("charmpp@7.0.0:", when="@3.0b3")
+    depends_on("charmpp@6.10.1:6", when="@2.14:2")
     depends_on("charmpp@6.8.2", when="@2.13")
     depends_on("charmpp@6.7.1", when="@2.12")
 
@@ -88,7 +91,11 @@ class Namd(MakefilePackage, CudaPackage):
 
     def _copy_arch_file(self, lib):
         config_filename = "arch/{0}.{1}".format(self.arch, lib)
-        copy("arch/Linux-x86_64.{0}".format(lib), config_filename)
+        if self.arch == "linux-aarch64":
+            copy("arch/Linux-ARM64.{0}".format(lib), config_filename)
+        else:
+            copy("arch/Linux-x86_64.{0}".format(lib), config_filename)
+
         if lib == "tcl":
             filter_file(
                 r"-ltcl8\.5", "-ltcl{0}".format(self.spec["tcl"].version.up_to(2)), config_filename
@@ -259,6 +266,11 @@ def edit(self, spec, prefix):
                 "CUDADIR={0}".format(spec["cuda"].prefix),
                 join_path("arch", self.arch + ".cuda"),
             )
+            for cuda_arch in spec.variants["cuda_arch"].value:
+                opts.extend(["--cuda-gencode", f"arch=compute_{cuda_arch},code=sm_{cuda_arch}"])
+
+            if "+single_node_gpu" in spec:
+                opts.extend(["--with-single-node-cuda"])
 
         config = Executable("./config")
 
@@ -279,7 +291,10 @@ def edit(self, spec, prefix):
     def install(self, spec, prefix):
         with working_dir(self.build_directory):
             mkdirp(prefix.bin)
-            install("namd2", prefix.bin)
+            if spec.version < Version("3"):
+                install("namd2", prefix.bin)
+            else:
+                install("namd3", prefix.bin)
             install("psfgen", prefix.bin)
 
             # I'm not sure this is a good idea or if an autoload of the charm
diff --git a/var/spack/repos/builtin/packages/nauty/package.py b/var/spack/repos/builtin/packages/nauty/package.py
index eec5b90bab0776..6c6e54efe3a9b0 100644
--- a/var/spack/repos/builtin/packages/nauty/package.py
+++ b/var/spack/repos/builtin/packages/nauty/package.py
@@ -75,7 +75,7 @@ class Nauty(AutotoolsPackage):
     depends_on("libtool", type="build", when="@2.6r7")
     depends_on("pkgconfig", type="build")
     depends_on("help2man", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("gmp")
 
     @property
diff --git a/var/spack/repos/builtin/packages/ncbi-toolkit/package.py b/var/spack/repos/builtin/packages/ncbi-toolkit/package.py
index bb2db9813ce05e..cadaf37711edc8 100644
--- a/var/spack/repos/builtin/packages/ncbi-toolkit/package.py
+++ b/var/spack/repos/builtin/packages/ncbi-toolkit/package.py
@@ -52,7 +52,7 @@ class NcbiToolkit(AutotoolsPackage):
     depends_on("pcre")
     depends_on("giflib")
     depends_on("sqlite@3.6.6:")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("samtools")
     depends_on("bamtools")
     depends_on("berkeley-db")
diff --git a/var/spack/repos/builtin/packages/ncbi-vdb/package.py b/var/spack/repos/builtin/packages/ncbi-vdb/package.py
index 5e655abf64fc91..1e5ac39ff0d1ae 100644
--- a/var/spack/repos/builtin/packages/ncbi-vdb/package.py
+++ b/var/spack/repos/builtin/packages/ncbi-vdb/package.py
@@ -14,8 +14,8 @@ class NcbiVdb(CMakePackage):
     homepage = "https://github.com/ncbi/ncbi-vdb"
     git = "https://github.com/ncbi/ncbi-vdb.git"
 
-    version("3.0.2", tag="3.0.2")
-    version("3.0.0", tag="3.0.0")
+    version("3.0.2", tag="3.0.2", commit="c4aa19632714c2f04af07505721fb16c71bba3d5")
+    version("3.0.0", tag="3.0.0", commit="2222d7727122d0cbad93344dd6a9044abff34280")
 
     depends_on("openjdk")
     depends_on("flex@2.6:")
diff --git a/var/spack/repos/builtin/packages/nccl/package.py b/var/spack/repos/builtin/packages/nccl/package.py
index 99e569dbdddc09..51f10ca7eec3a7 100644
--- a/var/spack/repos/builtin/packages/nccl/package.py
+++ b/var/spack/repos/builtin/packages/nccl/package.py
@@ -17,7 +17,9 @@ class Nccl(MakefilePackage, CudaPackage):
     maintainers("adamjstewart")
     libraries = ["libnccl.so"]
 
-    version("2.18.3-1", sha256="b4f5d7d9eea2c12e32e7a06fe138b2cfc75969c6d5c473aa6f819a792db2fc96")
+    version("2.19.3-1", sha256="1c5474553afedb88e878c772f13d6f90b9226b3f2971dfa6f873adb9443100c2")
+    version("2.18.5-1", sha256="16ac98f3e926c024ce48e10ab220e19ce734adc48c423cfd55ad6f509bd1179f")
+    version("2.18.3-1", sha256="6477d83c9edbb34a0ebce6d751a1b32962bc6415d75d04972b676c6894ceaef9")
     version("2.18.1-1", sha256="0e4ede5cf8df009bff5aeb3a9f194852c03299ae5664b5a425b43358e7a9eef2")
     version("2.17.1-1", sha256="1311a6fd7cd44ad6d4523ba03065ce694605843fd30a5c0f77aa3d911abe706d")
     version("2.16.2-1", sha256="7f7c738511a8876403fc574d13d48e7c250d934d755598d82e14bab12236fc64")
diff --git a/var/spack/repos/builtin/packages/ncio/package.py b/var/spack/repos/builtin/packages/ncio/package.py
index 05acb1666243f6..4615c5b21cf70c 100644
--- a/var/spack/repos/builtin/packages/ncio/package.py
+++ b/var/spack/repos/builtin/packages/ncio/package.py
@@ -14,9 +14,11 @@ class Ncio(CMakePackage):
 
     homepage = "https://github.com/NOAA-EMC/NCEPLIBS-ncio"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-ncio/archive/refs/tags/v1.0.0.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-ncio"
 
     maintainers("edwardhartnett", "AlexanderRichert-NOAA", "Hang-Lei-NOAA")
 
+    version("develop", branch="develop")
     version("1.1.2", sha256="2e4506fe3176344f28e837f2f69bcbd3dda51a64cacf33af0e435b13abe094fc")
     version("1.1.1", sha256="c1cfeb3d731a2ae858bdfc02b2914fd6c8eac5cc76c7e67eb2580ae5e7500a2b")
     version("1.1.0", sha256="9de05cf3b8b1291010197737666cede3d621605806379b528d2146c4f02d08f6")
@@ -30,3 +32,7 @@ def setup_run_environment(self, env):
         env.set("NCIO_LIB", lib[0])
         env.set("NCIO_INC", join_path(self.prefix, "include"))
         env.set("NCIO_LIBDIR", lib[0])
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/ncl/package.py b/var/spack/repos/builtin/packages/ncl/package.py
index 294363f6c98166..f858ca68a6a714 100644
--- a/var/spack/repos/builtin/packages/ncl/package.py
+++ b/var/spack/repos/builtin/packages/ncl/package.py
@@ -49,6 +49,8 @@ class Ncl(Package):
         when="%gcc@10:",
         sha256="64f3502c9deab48615a4cbc26073173081c0774faf75778b044d251e45d238f7",
     )
+    # g2clib does not have a ymakefile. This patch avoids a benign ymake error.
+    patch("ymake-grib.patch", when="+grib")
 
     # This installation script is implemented according to this manual:
     # http://www.ncl.ucar.edu/Download/build_from_src.shtml
@@ -58,6 +60,7 @@ class Ncl(Package):
     variant("triangle", default=True, description="Enable Triangle support.")
     variant("udunits2", default=True, description="Enable UDUNITS-2 support.")
     variant("openmp", default=True, description="Enable OpenMP support.")
+    variant("grib", default=True, description="Enable GRIB support.")
 
     # Non-optional dependencies according to the manual:
     depends_on("jpeg")
@@ -104,6 +107,7 @@ class Ncl(Package):
     depends_on("hdf", when="+hdf4")
     depends_on("gdal@:2.4", when="+gdal")
     depends_on("udunits", when="+udunits2")
+    depends_on("jasper@2.0.32", when="+grib")
 
     # We need src files of triangle to appear in ncl's src tree if we want
     # triangle's features.
@@ -143,6 +147,11 @@ def install(self, spec, prefix):
         self.prepare_src_tree()
         make("Everything", parallel=False)
 
+        # Build system may fail without errors, so check for main program.
+        exes = os.listdir(self.spec.prefix.bin)
+        if "ncl" not in exes:
+            raise RuntimeError("Installation failed (ncl executable was not created)")
+
     def setup_run_environment(self, env):
         env.set("NCARG_ROOT", self.spec.prefix)
         env.set("ESMFBINDIR", self.spec["esmf"].prefix.bin)
@@ -173,6 +182,14 @@ def prepare_site_config(self):
             fc_flags.append("-fallow-argument-mismatch")
             cc_flags.append("-fcommon")
 
+        if self.spec.satisfies("+grib"):
+            gribline = (
+                "#define GRIB2lib %s/external/g2clib-1.6.0/libgrib2c.a -ljasper -lpng -lz -ljpeg\n"
+                % self.stage.source_path
+            )
+        else:
+            gribline = ""
+
         with open("./config/Spack", "w") as f:
             f.writelines(
                 [
@@ -200,7 +217,8 @@ def prepare_site_config(self):
                         if len(fc_flags) > 0
                         else ""
                     ),
-                    "#define BuildShared NO",
+                    "#define BuildShared NO\n",
+                    gribline,
                 ]
             )
 
@@ -244,6 +262,11 @@ def prepare_install_config(self):
                 ]
             )
 
+        gribinc = (
+            " " + self.stage.source_path + "/external/g2clib-1.6.0/"
+            if self.spec.satisfies("+grib")
+            else ""
+        )
         config_answers.extend(
             [
                 # Build Triangle support (optional) into NCL
@@ -257,7 +280,7 @@ def prepare_install_config(self):
                 # Build EEMD support (optional) into NCL?
                 "n\n",
                 # Build Udunits-2 support (optional) into NCL?
-                "y\n" if "+uduints2" in self.spec else "n\n",
+                "y\n" if "+udunits2" in self.spec else "n\n",
                 # Build Vis5d+ support (optional) into NCL?
                 "n\n",
                 # Build HDF-EOS2 support (optional) into NCL?
@@ -267,17 +290,22 @@ def prepare_install_config(self):
                 # Build HDF-EOS5 support (optional) into NCL?
                 "n\n",
                 # Build GRIB2 support (optional) into NCL?
-                "n\n",
+                "y\n" if self.spec.satisfies("+grib") else "n\n",
                 # Enter local library search path(s) :
                 self.spec["fontconfig"].prefix.lib
                 + " "
                 + self.spec["pixman"].prefix.lib
                 + " "
                 + self.spec["bzip2"].prefix.lib
+                + (
+                    (" " + self.spec["jasper"].prefix.lib64)
+                    if self.spec.satisfies("+grib")
+                    else ""
+                )
                 + "\n",
                 # Enter local include search path(s) :
                 # All other paths will be passed by the Spack wrapper.
-                self.spec["freetype"].headers.directories[0] + "\n",
+                self.spec["freetype"].headers.directories[0] + gribinc + "\n",
                 # Go back and make more changes or review?
                 "n\n",
                 # Save current configuration?
@@ -318,3 +346,13 @@ def delete_files(*filenames):
                     os.remove(filename)
                 except OSError as e:
                     raise InstallError("Failed to delete file %s: %s" % (e.filename, e.strerror))
+
+    @when("+grib")
+    def patch(self):
+        filter_file("image.inmem_=1;", "", "external/g2clib-1.6.0/enc_jpeg2000.c")
+        filter_file("SUBDIRS = ", "SUBDIRS = g2clib-1.6.0 ", "external/yMakefile")
+        filter_file(
+            "INC=.*",
+            "INC=%s" % self.spec["jasper"].prefix.include,
+            "external/g2clib-1.6.0/makefile",
+        )
diff --git a/var/spack/repos/builtin/packages/ncl/ymake-grib.patch b/var/spack/repos/builtin/packages/ncl/ymake-grib.patch
new file mode 100644
index 00000000000000..40b9163599b397
--- /dev/null
+++ b/var/spack/repos/builtin/packages/ncl/ymake-grib.patch
@@ -0,0 +1,11 @@
+--- a/config/ymake
++++ b/config/ymake
+@@ -226,7 +226,7 @@
+ else if (-e "yMakefile") then
+         set ymakefile="yMakefile"
+         set outfile = "Makefile"
+-else
++else if (! -e "makefile") then
+         echo "$0 : no y(mM)akefile found" > /dev/tty
+         exit 1
+ endif
diff --git a/var/spack/repos/builtin/packages/nco/package.py b/var/spack/repos/builtin/packages/nco/package.py
index 4b53a1b296ac0a..d20a4255f6b137 100644
--- a/var/spack/repos/builtin/packages/nco/package.py
+++ b/var/spack/repos/builtin/packages/nco/package.py
@@ -66,6 +66,10 @@ def configure_args(self):
             config_args.append("CFLAGS=-O1")
             config_args.append("CXXFLAGS=-O1")
 
+        ncconfig = which("nc-config")
+        nc_libs = ncconfig("--static", "--libs", output=str).strip()
+        config_args.append(f"LIBS={nc_libs}")
+
         return config_args
 
     def setup_build_environment(self, env):
diff --git a/var/spack/repos/builtin/packages/ncview/package.py b/var/spack/repos/builtin/packages/ncview/package.py
index 8fa0e5fdaa224b..4af51198c6fe37 100644
--- a/var/spack/repos/builtin/packages/ncview/package.py
+++ b/var/spack/repos/builtin/packages/ncview/package.py
@@ -12,8 +12,8 @@ class Ncview(AutotoolsPackage):
     """Simple viewer for NetCDF files."""
 
     homepage = "https://cirrus.ucsd.edu/ncview/"
-    url = "ftp://cirrus.ucsd.edu/pub/ncview/ncview-2.1.7.tar.gz"
 
+    version("2.1.9", sha256="e2317ac094af62f0adcf68421d70658209436aae344640959ec8975a645891af")
     version("2.1.8", sha256="e8badc507b9b774801288d1c2d59eb79ab31b004df4858d0674ed0d87dfc91be")
     version("2.1.7", sha256="a14c2dddac0fc78dad9e4e7e35e2119562589738f4ded55ff6e0eca04d682c82")
 
@@ -34,7 +34,7 @@ def configure_args(self):
         config_args.append("--with-udunits2_libdir={}".format(spec["udunits"].prefix.lib))
 
         # Use the same C compiler that was used for netcdf-c
-        cc = subprocess.check_output(['nc-config', '--cc']).decode().rstrip('\n')
+        cc = subprocess.check_output(["nc-config", "--cc"]).decode().rstrip("\n")
         config_args.append("CC={}".format(cc))
 
         return config_args
@@ -49,3 +49,9 @@ def patch(self):
                 "if false; then",
                 patched_file,
             )
+
+    def url_for_version(self, version):
+        if version >= Version("2.1.9"):
+            return "https://cirrus.ucsd.edu/~pierce/ncview/ncview-2.1.9.tar.gz"
+        else:
+            return "ftp://cirrus.ucsd.edu/pub/ncview/ncview-2.1.7.tar.gz"
diff --git a/var/spack/repos/builtin/packages/neko/package.py b/var/spack/repos/builtin/packages/neko/package.py
index 64ecf28f24a481..5023660db3bf9f 100644
--- a/var/spack/repos/builtin/packages/neko/package.py
+++ b/var/spack/repos/builtin/packages/neko/package.py
@@ -16,6 +16,7 @@ class Neko(AutotoolsPackage, CudaPackage, ROCmPackage):
     url = "https://github.com/ExtremeFLOW/neko/releases/download/v0.3.2/neko-0.3.2.tar.gz"
     maintainers("njansson")
 
+    version("0.6.1", sha256="6282baaf9c8a201669e274cba23c37922f7ad701ba20ef086442e48f00dabf29")
     version("0.6.0", sha256="ce37c7cea1a7bf1bf554c5717aa7fed35bbd079ff68c2fc9d3529facc717e31a")
     version("0.5.2", sha256="8873f5ada106f92f21c9bb13ea8164550bccde9301589b9e7f1c1a82a2efe2b8")
     version("0.5.1", sha256="8b176bcc9f2d4a6804b68dd93a2f5e02e2dfa986d5c88063bbc72d39e9659cc4")
@@ -27,6 +28,7 @@ class Neko(AutotoolsPackage, CudaPackage, ROCmPackage):
     version("develop", branch="develop")
     variant("parmetis", default=False, description="Build with support for parmetis")
     variant("xsmm", default=False, description="Build with support for libxsmm")
+    variant("gslib", default=False, when="@develop", description="Build with support for gslib")
 
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
@@ -38,6 +40,8 @@ class Neko(AutotoolsPackage, CudaPackage, ROCmPackage):
     depends_on("mpi")
     depends_on("blas")
     depends_on("lapack")
+    depends_on("json-fortran", when="@develop")
+    depends_on("gslib", when="+gslib")
 
     def configure_args(self):
         args = []
@@ -46,6 +50,7 @@ def configure_args(self):
         args += self.with_or_without("parmetis", variant="parmetis", activation_value="prefix")
         args += self.with_or_without("metis", variant="parmetis", activation_value="prefix")
         args += self.with_or_without("libxsmm", variant="xsmm")
+        args += self.with_or_without("gslib", variant="gslib", activation_value="prefix")
         args += self.with_or_without("cuda", activation_value="prefix")
         rocm_fn = lambda x: self.spec["hip"].prefix
         args += self.with_or_without("hip", variant="rocm", activation_value=rocm_fn)
diff --git a/var/spack/repos/builtin/packages/nekrs/package.py b/var/spack/repos/builtin/packages/nekrs/package.py
index 721e3476db8f7d..5e109b3880323a 100644
--- a/var/spack/repos/builtin/packages/nekrs/package.py
+++ b/var/spack/repos/builtin/packages/nekrs/package.py
@@ -30,7 +30,7 @@ class Nekrs(Package, CudaPackage, ROCmPackage):
 
     maintainers("thilinarmtb", "stgeke")
 
-    version("21.0", tag="v21.0")
+    version("21.0", tag="v21.0", commit="bcd890bf3f9fb4d91224c83aeda75c33570f1eaa")
 
     variant("opencl", default=False, description="Activates support for OpenCL")
 
diff --git a/var/spack/repos/builtin/packages/nemsio/package.py b/var/spack/repos/builtin/packages/nemsio/package.py
index 0c2538ce469687..713ddba6e5f5d4 100644
--- a/var/spack/repos/builtin/packages/nemsio/package.py
+++ b/var/spack/repos/builtin/packages/nemsio/package.py
@@ -15,13 +15,15 @@ class Nemsio(CMakePackage):
 
     homepage = "https://noaa-emc.github.io/NCEPLIBS-nemsio"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-nemsio/archive/refs/tags/v2.5.2.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-nemsio"
 
-    maintainers("t-brown", "edwardhartnett", "AlexanderRichert-NOAA", "Hang-Lei-NOAA")
+    maintainers("edwardhartnett", "AlexanderRichert-NOAA", "Hang-Lei-NOAA")
 
     variant("mpi", default=True, description="Build Nemsio with MPI")
     # Nemsio 2.5.3 and below require MPI
     conflicts("~mpi", when="@:2.5.3")
 
+    version("develop", branch="develop")
     version("2.5.4", sha256="186a5f37d509d280c0237d4917db86ad676c5dd12d8a892df0333a10e751e481")
     version("2.5.3", sha256="3fe8a781fc96197803d369cafe0138f3a5cbbca9816a7f8fd57567a1719a4d49")
     version("2.5.2", sha256="c59e9379969690de8d030cbf4bbbbe3726faf13c304f3b88b0f6aec1496d2c08")
@@ -42,3 +44,7 @@ def cmake_args(self):
             args.append(self.define("CMAKE_Fortran_COMPILER", self.spec["mpi"].mpifc))
 
         return args
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/nemsiogfs/package.py b/var/spack/repos/builtin/packages/nemsiogfs/package.py
index 5656831c957cd8..e01e7e808f8d26 100644
--- a/var/spack/repos/builtin/packages/nemsiogfs/package.py
+++ b/var/spack/repos/builtin/packages/nemsiogfs/package.py
@@ -14,9 +14,15 @@ class Nemsiogfs(CMakePackage):
 
     homepage = "https://github.com/NOAA-EMC/NCEPLIBS-nemsiogfs"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-nemsiogfs/archive/refs/tags/v2.5.3.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-nemsiogfs"
 
     maintainers("AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
 
+    version("develop", branch="develop")
     version("2.5.3", sha256="bf84206b08c8779787bef33e4aba18404df05f8b2fdd20fc40b3af608ae4b9af")
 
     depends_on("nemsio")
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/neovim/package.py b/var/spack/repos/builtin/packages/neovim/package.py
index ac5407ca04402e..ff59c4f539f0f2 100644
--- a/var/spack/repos/builtin/packages/neovim/package.py
+++ b/var/spack/repos/builtin/packages/neovim/package.py
@@ -16,7 +16,9 @@ class Neovim(CMakePackage):
     maintainers("albestro", "trws")
 
     version("master", branch="master")
-    version("stable", tag="stable")
+    version("stable", tag="stable", commit="d772f697a281ce9c58bf933997b87c7f27428a60")
+    version("0.9.4", sha256="148356027ee8d586adebb6513a94d76accc79da9597109ace5c445b09d383093")
+    version("0.9.2", sha256="06b8518bad4237a28a67a4fbc16ec32581f35f216b27f4c98347acee7f5fb369")
     version("0.9.1", sha256="8db17c2a1f4776dcda00e59489ea0d98ba82f7d1a8ea03281d640e58d8a3a00e")
     version("0.9.0", sha256="39d79107c54d2f3babcad2cd157c399241c04f6e75e98c18e8afaf2bb5e82937")
     version("0.8.3", sha256="adf45ff160e1d89f519b6114732eba03485ae469beb27919b0f7a4f6b44233c1")
@@ -86,7 +88,7 @@ class Neovim(CMakePackage):
 
     # dependencies to allow regular lua to work
     depends_on("lua-ffi", when="^lua", type=("link", "run"))
-    depends_on("lua-bitlib", type="link", when="^lua")
+    depends_on("lua-bitlib", type=("link", "run"), when="^lua")
 
     # base dependencies
     depends_on("cmake@3.0:", type="build")
@@ -119,7 +121,7 @@ class Neovim(CMakePackage):
     with when("@0.6:"):
         depends_on("cmake@3.10:", type="build")
         depends_on("gperf@3.1:", type="link")
-        conflicts("libiconv@:1.14")
+        conflicts("^libiconv@:1.14")
         depends_on("libtermkey@0.22:", type="link")
         depends_on("libvterm@0.1.4:", type="link")
         depends_on("msgpack-c@3.0.0:", type="link")
diff --git a/var/spack/repos/builtin/packages/netcdf-c/package.py b/var/spack/repos/builtin/packages/netcdf-c/package.py
index 0d0b8e4c8b08c1..f111264e811725 100644
--- a/var/spack/repos/builtin/packages/netcdf-c/package.py
+++ b/var/spack/repos/builtin/packages/netcdf-c/package.py
@@ -241,7 +241,8 @@ class NetcdfC(CMakePackage, AutotoolsPackage):
     # https://docs.unidata.ucar.edu/nug/current/getting_and_building_netcdf.html), zlib 1.2.5 or
     # later is required for netCDF-4 compression. However, zlib became a direct dependency only
     # starting NetCDF 4.9.0 (for the deflate plugin):
-    depends_on("zlib@1.2.5:", when="@4.9.0:+shared")
+    depends_on("zlib-api", when="@4.9.0:+shared")
+    depends_on("zlib@1.2.5:", when="^zlib")
 
     # Use the vendored bzip2 on Windows:
     for __p in ["darwin", "cray", "linux"]:
@@ -433,7 +434,7 @@ def configure_args(self):
                     extra_libs.append(hdf["szip"].libs)
                 if "+external-xdr" in hdf:
                     extra_libs.append(hdf["rpc"].libs)
-                extra_libs.append(hdf["zlib"].libs)
+                extra_libs.append(hdf["zlib-api"].libs)
 
         hdf5 = self.spec["hdf5:hl"]
         lib_search_dirs.extend(hdf5.libs.directories)
@@ -443,7 +444,7 @@ def configure_args(self):
             extra_libs.append(hdf5["zlib"].libs)
 
         if self.spec.satisfies("@4.9.0:+shared"):
-            lib_search_dirs.extend(self.spec["zlib"].libs.directories)
+            lib_search_dirs.extend(self.spec["zlib-api"].libs.directories)
         else:
             # Prevent overlinking to zlib:
             config_args.append("ac_cv_search_deflate=")
diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py
index e021ee00454ce9..4ea183ee8071bb 100644
--- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py
+++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py
@@ -6,7 +6,6 @@
 import glob
 import os
 from shutil import Error, copyfile
-import subprocess
 
 from spack.package import *
 
@@ -22,6 +21,7 @@ class NetcdfFortran(AutotoolsPackage):
 
     maintainers("skosukhin", "WardF")
 
+    version("4.6.1", sha256="b50b0c72b8b16b140201a020936aa8aeda5c79cf265c55160986cd637807a37a")
     version("4.6.0", sha256="198bff6534cc85a121adc9e12f1c4bc53406c403bda331775a1291509e7b2f23")
     version("4.5.4", sha256="0a19b26a2b6e29fab5d29d7d7e08c24e87712d09a5cafeea90e16e0a2ab86b81")
     version("4.5.3", sha256="123a5c6184336891e62cf2936b9f2d1c54e8dee299cfd9d2c1a1eb05dd668a74")
@@ -144,10 +144,8 @@ def configure_args(self):
 
         return config_args
 
-    @when("@:4.4.5")
     def check(self):
-        with working_dir(self.build_directory):
-            make("check", parallel=False)
+        make("check", parallel=self.spec.satisfies("@4.5:"))
 
     @run_after("install")
     def cray_module_filenames(self):
diff --git a/var/spack/repos/builtin/packages/netcdf95/package.py b/var/spack/repos/builtin/packages/netcdf95/package.py
index 07e336d9d366f7..2a5b43c71ed5fa 100644
--- a/var/spack/repos/builtin/packages/netcdf95/package.py
+++ b/var/spack/repos/builtin/packages/netcdf95/package.py
@@ -15,6 +15,6 @@ class Netcdf95(CMakePackage):
 
     maintainers("RemiLacroix-IDRIS")
 
-    version("0.3", tag="v0.3", submodules=True)
+    version("0.3", tag="v0.3", commit="5b8db6bb66a22b6a080589ee1c11521ee3cae550", submodules=True)
 
     depends_on("netcdf-fortran")
diff --git a/var/spack/repos/builtin/packages/netdata/package.py b/var/spack/repos/builtin/packages/netdata/package.py
index a2ce76110c1fc5..8395b518dd30c2 100644
--- a/var/spack/repos/builtin/packages/netdata/package.py
+++ b/var/spack/repos/builtin/packages/netdata/package.py
@@ -27,7 +27,7 @@ class Netdata(AutotoolsPackage):
     depends_on("openssl")
     depends_on("python@3:", type=("build", "run"))
     depends_on("uuid")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def setup_run_environment(self, env):
         env.prepend_path("PATH", self.prefix.sbin)
diff --git a/var/spack/repos/builtin/packages/netgen/package.py b/var/spack/repos/builtin/packages/netgen/package.py
index 421acb3e3565e9..1c890ef079067f 100644
--- a/var/spack/repos/builtin/packages/netgen/package.py
+++ b/var/spack/repos/builtin/packages/netgen/package.py
@@ -24,7 +24,7 @@ class Netgen(AutotoolsPackage):
     variant("gui", default=False, description="enable gui")
     variant("metis", default=False, description="use metis for partitioning")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("mpi", when="+mpi")
     depends_on("oce+X11", when="+oce")
     depends_on("metis", when="+metis")
diff --git a/var/spack/repos/builtin/packages/netpbm/package.py b/var/spack/repos/builtin/packages/netpbm/package.py
index 0e0cf67d9aad45..18b77e64d963e6 100644
--- a/var/spack/repos/builtin/packages/netpbm/package.py
+++ b/var/spack/repos/builtin/packages/netpbm/package.py
@@ -46,7 +46,7 @@ class Netpbm(MakefilePackage):
 
     # These are general pre-requisites indicated at
     # http://netpbm.sourceforge.net/prereq.html
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("jpeg")
     depends_on("libtiff")
     depends_on("libpng")
@@ -147,7 +147,7 @@ def edit(self, spec, prefix):
             if not pkg_config("x11", "--exists"):
                 config.append("X11LIB={0}".format(spec["libx11"].libs.ld_flags))
                 config.append("X11HDR_DIR={0}".format(spec["libx11"].headers.directories[0]))
-                config.append("ZLIB={0}".format(spec["zlib"].libs.ld_flags))
+                config.append("ZLIB={0}".format(spec["zlib-api"].libs.ld_flags))
         config.append("NETPBM_DOCURL = http://netpbm.sourceforge.net/doc/")
         if spec.target.family == "x86_64":
             config.append("WANT_SSE = Y")
diff --git a/var/spack/repos/builtin/packages/neuron/package.py b/var/spack/repos/builtin/packages/neuron/package.py
index 8df73f48b817e5..2f388329b7a4fd 100644
--- a/var/spack/repos/builtin/packages/neuron/package.py
+++ b/var/spack/repos/builtin/packages/neuron/package.py
@@ -21,9 +21,15 @@ class Neuron(CMakePackage):
     maintainers("pramodk", "nrnhines", "iomaganaris", "alexsavulescu")
 
     version("develop", branch="master", submodules="True")
-    version("8.0.0", tag="8.0.0", submodules="True")
-    version("7.8.2", tag="7.8.2", submodules="True")
-    version("7.8.1", tag="7.8.1", submodules="True")
+    version(
+        "8.0.0", tag="8.0.0", commit="429d11ef34b1d860b3ddbfffc9f7960acb399b0c", submodules="True"
+    )
+    version(
+        "7.8.2", tag="7.8.2", commit="09b151ecb2b3984335c265932dc6ba3e4fcb318e", submodules="True"
+    )
+    version(
+        "7.8.1", tag="7.8.1", commit="47cd8c85aa3fde5dbb7319facd6f475438235d7d", submodules="True"
+    )
 
     variant("coreneuron", default=False, description="Enable CoreNEURON as submodule")
     variant("cross-compile", default=False, description="Build for cross-compile environment")
diff --git a/var/spack/repos/builtin/packages/nextdenovo/package.py b/var/spack/repos/builtin/packages/nextdenovo/package.py
new file mode 100644
index 00000000000000..51748f0e3a489d
--- /dev/null
+++ b/var/spack/repos/builtin/packages/nextdenovo/package.py
@@ -0,0 +1,29 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Nextdenovo(MakefilePackage):
+    """NextDenovo is a string graph-based de novo assembler for long reads."""
+
+    homepage = "https://nextdenovo.readthedocs.io/en/latest/index.html"
+    url = "https://github.com/Nextomics/NextDenovo/archive/refs/tags/2.5.2.tar.gz"
+
+    version("2.5.2", sha256="f1d07c9c362d850fd737c41e5b5be9d137b1ef3f1aec369dc73c637790611190")
+
+    depends_on("python", type="run")
+    depends_on("py-paralleltask", type="run")
+
+    def edit(self, spec, prefix):
+        makefile = FileFilter("Makefile")
+        makefile.filter(r"^TOP_DIR.*", "TOP_DIR={0}".format(self.build_directory))
+        runfile = FileFilter("nextDenovo")
+        runfile.filter(r"^SCRIPT_PATH.*", "SCRIPT_PATH = '{0}'".format(prefix))
+
+    def install(self, spec, prefix):
+        install_tree("bin", prefix.bin)
+        install("nextDenovo", prefix.bin)
+        install_tree("lib", prefix.lib)
diff --git a/var/spack/repos/builtin/packages/nextflow/package.py b/var/spack/repos/builtin/packages/nextflow/package.py
index bac1d267843630..379a3c9ba3307a 100644
--- a/var/spack/repos/builtin/packages/nextflow/package.py
+++ b/var/spack/repos/builtin/packages/nextflow/package.py
@@ -14,6 +14,16 @@ class Nextflow(Package):
 
     maintainers("dialvarezs", "marcodelapierre")
 
+    version(
+        "23.10.0",
+        sha256="4b7fba61ecc6d53a6850390bb435455a54ae4d0c3108199f88b16b49e555afdd",
+        expand=False,
+    )
+    version(
+        "23.04.3",
+        sha256="258714c0772db3cab567267e8441c5b72102381f6bd58fc6957c2972235be7e0",
+        expand=False,
+    )
     version(
         "23.04.1",
         sha256="5de3e09117ca648b2b50778d3209feb249b35de0f97cdbcf52c7d92c7a96415c",
diff --git a/var/spack/repos/builtin/packages/nghttp2/package.py b/var/spack/repos/builtin/packages/nghttp2/package.py
index fe9d4f94e38a8e..2de551d8b5fa49 100644
--- a/var/spack/repos/builtin/packages/nghttp2/package.py
+++ b/var/spack/repos/builtin/packages/nghttp2/package.py
@@ -13,6 +13,7 @@ class Nghttp2(AutotoolsPackage):
     homepage = "https://nghttp2.org/"
     url = "https://github.com/nghttp2/nghttp2/releases/download/v1.26.0/nghttp2-1.26.0.tar.gz"
 
+    version("1.57.0", sha256="1e3258453784d3b7e6cc48d0be087b168f8360b5d588c66bfeda05d07ad39ffd")
     version("1.52.0", sha256="9877caa62bd72dde1331da38ce039dadb049817a01c3bdee809da15b754771b8")
     version("1.51.0", sha256="2a0bef286f65b35c24250432e7ec042441a8157a5b93519412d9055169d9ce54")
     version("1.50.0", sha256="d162468980dba58e54e31aa2cbaf96fd2f0890e6dd141af100f6bd1b30aa73c6")
diff --git a/var/spack/repos/builtin/packages/nginx/package.py b/var/spack/repos/builtin/packages/nginx/package.py
index 169d02a04f5e55..efc1cf01084a12 100644
--- a/var/spack/repos/builtin/packages/nginx/package.py
+++ b/var/spack/repos/builtin/packages/nginx/package.py
@@ -25,7 +25,7 @@ class Nginx(AutotoolsPackage):
     depends_on("openssl")
     depends_on("openssl@:1", when="@:1.21.2")
     depends_on("pcre")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     conflicts("%gcc@8:", when="@:1.14")
 
diff --git a/var/spack/repos/builtin/packages/ngmerge/package.py b/var/spack/repos/builtin/packages/ngmerge/package.py
index c8721d554e8d60..7a3cc8f403f61c 100644
--- a/var/spack/repos/builtin/packages/ngmerge/package.py
+++ b/var/spack/repos/builtin/packages/ngmerge/package.py
@@ -14,7 +14,7 @@ class Ngmerge(MakefilePackage):
 
     version("0.3", sha256="5928f727feebd0d1bcdbee0e631ba06fbe9ce88328bd58b6c8bf4e54cc742ac3")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/ngmlr/package.py b/var/spack/repos/builtin/packages/ngmlr/package.py
index cc52f69bf9fe34..278b7a0443e491 100644
--- a/var/spack/repos/builtin/packages/ngmlr/package.py
+++ b/var/spack/repos/builtin/packages/ngmlr/package.py
@@ -17,7 +17,7 @@ class Ngmlr(CMakePackage):
     version("0.2.7", sha256="5126a6b3e726cac0da0713883daac688f38587f118428247a9a3ace5a55b29aa")
     version("0.2.5", sha256="719944a35cc7ff9c321eedbf3385a7375ce2301f609b3fd7be0a850cabbb028b")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("sse2neon", when="target=aarch64:")
 
     patch("for_aarch64.patch", when="target=aarch64:")
diff --git a/var/spack/repos/builtin/packages/ngspice/package.py b/var/spack/repos/builtin/packages/ngspice/package.py
index c826b24052d635..08bbbd712f49e5 100644
--- a/var/spack/repos/builtin/packages/ngspice/package.py
+++ b/var/spack/repos/builtin/packages/ngspice/package.py
@@ -18,6 +18,7 @@ class Ngspice(AutotoolsPackage):
 
     # Master version by default adds the experimental adms feature
     version("master", branch="master")
+    version("41", sha256="1ce219395d2f50c33eb223a1403f8318b168f1e6d1015a7db9dbf439408de8c4")
     version("40", sha256="e303ca7bc0f594e2d6aa84f68785423e6bf0c8dad009bb20be4d5742588e890d")
     version("39", sha256="bf94e811eaad8aaf05821d036a9eb5f8a65d21d30e1cab12701885e09618d771")
     version("38", sha256="2c3e22f6c47b165db241cf355371a0a7558540ab2af3f8b5eedeeb289a317c56")
@@ -52,6 +53,7 @@ class Ngspice(AutotoolsPackage):
     variant("openmp", default=False, description="Compile with multi-threading support")
     variant("readline", default=True, description="Build readline support (for bin)")
     variant("fft", default=True, description="Use external fftw lib")
+    variant("osdi", default=False, description="Use osdi/OpenVAF")
 
     depends_on("fftw-api@3:~mpi~openmp", when="+fft~openmp")
     depends_on("fftw-api@3:~mpi+openmp", when="+fft+openmp")
@@ -120,6 +122,8 @@ def configure_args(self):
             args.append("--enable-openmp")
         if "~fft" in spec:
             args.append("--with-fftw3=no")
+        if "+osdi" in spec:
+            args.append("--enable-osdi")
         if "darwin" in spec.architecture:
             args.append("--enable-pss")
         if "@master" in spec:
diff --git a/var/spack/repos/builtin/packages/nimrod-aai/package.py b/var/spack/repos/builtin/packages/nimrod-aai/package.py
index 862a07b496f656..1ad7101e1a918c 100644
--- a/var/spack/repos/builtin/packages/nimrod-aai/package.py
+++ b/var/spack/repos/builtin/packages/nimrod-aai/package.py
@@ -14,29 +14,37 @@ class NimrodAai(CMakePackage):
 
     homepage = "https://gitlab.com/NIMRODteam/nimrod-abstract"
     url = (
-        "https://gitlab.com/NIMRODteam/nimrod-abstract/-/archive/23.6/nimrod-abstract-23.6.tar.gz"
+        "https://gitlab.com/NIMRODteam/nimrod-abstract/-/archive/23.9/nimrod-abstract-23.9.tar.gz"
     )
     git = "https://gitlab.com/NIMRODteam/nimrod-abstract.git"
 
     maintainers("jacobrking")
 
     version("main", branch="main")
+    version("23.9", sha256="212d591c5a5e7a394b56a5cf2f92cc69feafc49dd5f042fa95eeb6441649390b")
     version("23.6", sha256="1794b89a5a64ff2b3c548818b90d17eef85d819ba4f63a76c41a682d5b76c14f")
 
-    variant("debug", default=False)
-    variant("openacc", default=False)
-    variant("openacc_autocompare", default=False)
-    variant("enable_shared", default=True)
-    variant("mpi", default=False)
-    variant("time_level1", default=False)
-    variant("time_level2", default=False)
-    variant("nvtx_profile", default=False)
-    variant("openacc_cc", default="native")
-    variant("trap_fp_exceptions", default=False)
+    variant("debug", default=False, description="Whether to enable debug code")
+    variant("openacc", default=False, description="Whether to enable OpenACC")
+    variant(
+        "openacc_autocompare", default=False, description="Whether to enable OpenACC autocompare"
+    )
+    variant("enable_shared", default=True, description="Whether to build and use shared libraries")
+    variant("mpi", default=False, description="Whether to enable MPI")
+    variant("time_level1", default=False, description="Whether to add timings at level 1")
+    variant("time_level2", default=False, description="Whether to add timings at level 2")
+    variant("nvtx_profile", default=False, description="Whether to enable NVTX profiling")
+    variant("openacc_cc", default="native", description="OpenACC compute capability")
+    variant(
+        "trap_fp_exceptions",
+        default=False,
+        description="Whether to enable trapping of floating point exceptions",
+    )
 
     depends_on("cmake", type="build")
-    depends_on("hdf5+fortran", type="build")
     depends_on("mpi", when="+mpi")
+    depends_on("hdf5+fortran~mpi", type="build", when="~mpi")
+    depends_on("hdf5+fortran+mpi", type="build", when="+mpi")
 
     def cmake_args(self):
         args = [
@@ -56,3 +64,9 @@ def cmake_args(self):
             ]
             args.append(addl_args)
         return args
+
+    @run_after("build")
+    @on_package_attributes(run_tests=True)
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            ctest("--output-on-failure")
diff --git a/var/spack/repos/builtin/packages/ninja-fortran/package.py b/var/spack/repos/builtin/packages/ninja-fortran/package.py
index 77bd0f0f3a738b..ae58406c3f85e5 100644
--- a/var/spack/repos/builtin/packages/ninja-fortran/package.py
+++ b/var/spack/repos/builtin/packages/ninja-fortran/package.py
@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from spack.build_environment import MakeExecutable, determine_number_of_jobs
 from spack.package import *
 from spack.util.executable import which_string
 
diff --git a/var/spack/repos/builtin/packages/ninja/package.py b/var/spack/repos/builtin/packages/ninja/package.py
index 8614b2c1b56fe1..bb30e4fbbd8752 100644
--- a/var/spack/repos/builtin/packages/ninja/package.py
+++ b/var/spack/repos/builtin/packages/ninja/package.py
@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import sys
 
-from spack.build_environment import MakeExecutable, determine_number_of_jobs
 from spack.package import *
 from spack.util.executable import which_string
 
diff --git a/var/spack/repos/builtin/packages/nlcglib/package.py b/var/spack/repos/builtin/packages/nlcglib/package.py
index b8869ed0ab03af..5d899b17ed10a6 100644
--- a/var/spack/repos/builtin/packages/nlcglib/package.py
+++ b/var/spack/repos/builtin/packages/nlcglib/package.py
@@ -21,8 +21,8 @@ class Nlcglib(CMakePackage, CudaPackage, ROCmPackage):
     version("1.0b", sha256="086c46f06a117f267cbdf1df4ad42a8512689a9610885763f463469fb15e82dc")
     version("0.9", sha256="8d5bc6b85ee714fb3d6480f767e7f43e5e7d569116cf60e48f533a7f50a37a08")
 
-    variant("openmp", default=True)
-    variant("tests", default=False)
+    variant("openmp", default=True, description="Use OpenMP")
+    variant("tests", default=False, description="Build tests")
     variant(
         "build_type",
         default="Release",
diff --git a/var/spack/repos/builtin/packages/nmap/package.py b/var/spack/repos/builtin/packages/nmap/package.py
index bd8f6b7f924b91..3eacd0343cfefd 100644
--- a/var/spack/repos/builtin/packages/nmap/package.py
+++ b/var/spack/repos/builtin/packages/nmap/package.py
@@ -61,7 +61,7 @@ class Nmap(AutotoolsPackage):
 
     depends_on("libssh2@1.10")
     depends_on("pcre@8")
-    depends_on("zlib@1.2")
+    depends_on("zlib-api")
 
     def configure_args(self):
         args = []
diff --git a/var/spack/repos/builtin/packages/node-js/package.py b/var/spack/repos/builtin/packages/node-js/package.py
index dc9fb2684a38d3..160913b33a36d8 100644
--- a/var/spack/repos/builtin/packages/node-js/package.py
+++ b/var/spack/repos/builtin/packages/node-js/package.py
@@ -72,7 +72,7 @@ class NodeJs(Package):
     # depends_on('bash-completion', when="+bash-completion")
     depends_on("icu4c", when="+icu4c")
     depends_on("openssl@1.1:", when="+openssl")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
 
     phases = ["configure", "build", "install"]
 
@@ -137,8 +137,8 @@ def configure_args(self):
             args.extend(
                 [
                     "--shared-zlib",
-                    "--shared-zlib-includes={0}".format(self.spec["zlib"].prefix.include),
-                    "--shared-zlib-libpath={0}".format(self.spec["zlib"].prefix.lib),
+                    "--shared-zlib-includes={0}".format(self.spec["zlib-api"].prefix.include),
+                    "--shared-zlib-libpath={0}".format(self.spec["zlib-api"].prefix.lib),
                 ]
             )
 
diff --git a/var/spack/repos/builtin/packages/notmuch/package.py b/var/spack/repos/builtin/packages/notmuch/package.py
index 0ba10baba45cc9..506aaf8243d131 100644
--- a/var/spack/repos/builtin/packages/notmuch/package.py
+++ b/var/spack/repos/builtin/packages/notmuch/package.py
@@ -17,7 +17,7 @@ class Notmuch(AutotoolsPackage):
 
     version("0.23.7", sha256="f11bb10d71945f6c3f16d23117afc70810aa485878e66bb4bf43cc3f08038913")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("talloc")
     depends_on("gmime@2.6:")
     depends_on("xapian-core")
diff --git a/var/spack/repos/builtin/packages/nss/package.py b/var/spack/repos/builtin/packages/nss/package.py
index ffa435777381d7..43e156deb8507a 100644
--- a/var/spack/repos/builtin/packages/nss/package.py
+++ b/var/spack/repos/builtin/packages/nss/package.py
@@ -28,7 +28,7 @@ class Nss(MakefilePackage):
 
     depends_on("nspr@4.24:")
     depends_on("sqlite")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     parallel = False
 
diff --git a/var/spack/repos/builtin/packages/numactl/package.py b/var/spack/repos/builtin/packages/numactl/package.py
index 9e272f6d1d8502..5afa3512a92ad0 100644
--- a/var/spack/repos/builtin/packages/numactl/package.py
+++ b/var/spack/repos/builtin/packages/numactl/package.py
@@ -31,7 +31,7 @@ class Numactl(AutotoolsPackage):
 
     # Numactl has hardcoded minimum versions for libtool,
     # libtool@develop returns UNKOWN as a version tag and fails
-    conflicts("libtool@develop")
+    conflicts("^libtool@develop")
 
     # Numerous errors when trying to build on darwin
     conflicts("platform=darwin")
diff --git a/var/spack/repos/builtin/packages/numaprof/package.py b/var/spack/repos/builtin/packages/numaprof/package.py
index 22a8fa512d94ba..9b734a4115cc1c 100644
--- a/var/spack/repos/builtin/packages/numaprof/package.py
+++ b/var/spack/repos/builtin/packages/numaprof/package.py
@@ -20,6 +20,7 @@ class Numaprof(CMakePackage):
     maintainers("svalat")
 
     # Versions
+    version("1.1.5", sha256="7c479cc6d39f2fe685532b9aaeb9efce8153350177fdcc24133e447dd0776323")
     version("1.1.4", sha256="96cc5e153895f43d8be58e052433c9e7c9842071cc6bf915b3b1b346908cbbff")
 
     # Variants
diff --git a/var/spack/repos/builtin/packages/nvhpc/package.py b/var/spack/repos/builtin/packages/nvhpc/package.py
index ba5cfd2ac1cd70..578813f80668cb 100644
--- a/var/spack/repos/builtin/packages/nvhpc/package.py
+++ b/var/spack/repos/builtin/packages/nvhpc/package.py
@@ -21,6 +21,34 @@
 #  - package key must be in the form '{os}-{arch}' where 'os' is in the
 #    format returned by platform.system() and 'arch' by platform.machine()
 _versions = {
+    "23.9": {
+        "Linux-aarch64": (
+            "dd32ae4233438adb71b2b4f8891f04802fdf90f67036ecf18bfde1b6043a03c3",
+            "https://developer.download.nvidia.com/hpc-sdk/23.9/nvhpc_2023_239_Linux_aarch64_cuda_multi.tar.gz",
+        ),
+        "Linux-ppc64le": (
+            "984d61695499db098fd32be8345c1f7d7c637ea3bdb29cef17aad656f16b000f",
+            "https://developer.download.nvidia.com/hpc-sdk/23.9/nvhpc_2023_239_Linux_ppc64le_cuda_multi.tar.gz",
+        ),
+        "Linux-x86_64": (
+            "ecf343ecad2398e21c8d7f24a580b2932348017dfd8ea38c1ef31b37114b2d4b",
+            "https://developer.download.nvidia.com/hpc-sdk/23.9/nvhpc_2023_239_Linux_x86_64_cuda_multi.tar.gz",
+        ),
+    },
+    "23.7": {
+        "Linux-aarch64": (
+            "d3b9b674045e6e17156b298941be4e1e1e7dea6a3c1938f14ad653b180860ff2",
+            "https://developer.download.nvidia.com/hpc-sdk/23.7/nvhpc_2023_237_Linux_aarch64_cuda_multi.tar.gz",
+        ),
+        "Linux-ppc64le": (
+            "67b137cf67e2c8556ef3952d1ee35f4966c9d1968626825924fb8e4b198a532b",
+            "https://developer.download.nvidia.com/hpc-sdk/23.7/nvhpc_2023_237_Linux_ppc64le_cuda_multi.tar.gz",
+        ),
+        "Linux-x86_64": (
+            "fea91d95ff18bca1ce7afde50371caa02001ade8bed6ddfc5ff70862ccbebece",
+            "https://developer.download.nvidia.com/hpc-sdk/23.7/nvhpc_2023_237_Linux_x86_64_cuda_multi.tar.gz",
+        ),
+    },
     "23.5": {
         "Linux-aarch64": (
             "3af202ad36bbf205b2af56aabe63b971c01b5ec0e82a02effb3c4928f63bc657",
@@ -322,6 +350,8 @@ class Nvhpc(Package):
     maintainers("samcmill")
     tags = ["e4s"]
 
+    skip_version_audit = ["platform=darwin"]
+
     for ver, packages in _versions.items():
         key = "{0}-{1}".format(platform.system(), platform.machine())
         pkg = packages.get(key)
@@ -339,16 +369,15 @@ class Nvhpc(Package):
     )
     variant("lapack", default=True, description="Enable LAPACK")
     variant("mpi", default=False, description="Enable MPI")
+    variant(
+        "default_cuda", default="default", description="Default CUDA version, for example 11.8"
+    )
 
     provides("blas", when="+blas")
     provides("lapack", when="+lapack")
     provides("mpi", when="+mpi")
 
-    # TODO: effectively gcc is a direct dependency of nvhpc, but we cannot express that
-    #  properly. For now, add conflicts for non-gcc compilers instead.
-    for __compiler in spack.compilers.supported_compilers():
-        if __compiler != "gcc":
-            conflicts("%{0}".format(__compiler), msg="nvhpc must be installed with %gcc")
+    requires("%gcc", msg="nvhpc must be installed with %gcc")
 
     def _version_prefix(self):
         return join_path(self.prefix, "Linux_%s" % self.spec.target.family, self.version)
@@ -357,6 +386,8 @@ def setup_build_environment(self, env):
         env.set("NVHPC_SILENT", "true")
         env.set("NVHPC_ACCEPT_EULA", "accept")
         env.set("NVHPC_INSTALL_DIR", self.prefix)
+        if self.spec.variants["default_cuda"].value != "default":
+            env.set("NVHPC_DEFAULT_CUDA", self.spec.variants["default_cuda"].value)
 
         if self.spec.variants["install_type"].value == "network":
             local_dir = join_path(self._version_prefix(), "share_objects")
diff --git a/var/spack/repos/builtin/packages/nwchem/package.py b/var/spack/repos/builtin/packages/nwchem/package.py
index aeb55ab31901ee..ba93ca219cdbef 100644
--- a/var/spack/repos/builtin/packages/nwchem/package.py
+++ b/var/spack/repos/builtin/packages/nwchem/package.py
@@ -53,13 +53,14 @@ class Nwchem(Package):
     depends_on("mpi")
     depends_on("scalapack")
     depends_on("fftw-api")
-    depends_on("python@3", type=("build", "link", "run"))
+    depends_on("python@3:3.9", type=("build", "link", "run"), when="@:7.0.2")
+    depends_on("python@3", type=("build", "link", "run"), when="@7.2.0:")
 
     def install(self, spec, prefix):
         scalapack = spec["scalapack"].libs
         lapack = spec["lapack"].libs
         blas = spec["blas"].libs
-        fftw = spec["fftw-api"].libs
+        fftw = spec["fftw-api:double,float"].libs
         # see https://nwchemgit.github.io/Compiling-NWChem.html
         args = []
         args.extend(
diff --git a/var/spack/repos/builtin/packages/oce/package.py b/var/spack/repos/builtin/packages/oce/package.py
index cd40190b96f79a..d0f35684b25ed7 100644
--- a/var/spack/repos/builtin/packages/oce/package.py
+++ b/var/spack/repos/builtin/packages/oce/package.py
@@ -36,7 +36,7 @@ class Oce(Package):
     with when("+tbb"):
         depends_on("tbb")
         depends_on("intel-tbb@:2020 build_system=makefile", when="^intel-tbb")
-        conflicts("intel-oneapi-tbb@2021.1:")
+        conflicts("^intel-oneapi-tbb@2021.1:")
 
     # There is a bug in OCE which appears with Clang (version?) or GCC 6.0
     # and has to do with compiler optimization, see
diff --git a/var/spack/repos/builtin/packages/octa/package.py b/var/spack/repos/builtin/packages/octa/package.py
index 4df81387293d58..0f46bb1027c9c4 100644
--- a/var/spack/repos/builtin/packages/octa/package.py
+++ b/var/spack/repos/builtin/packages/octa/package.py
@@ -25,7 +25,7 @@ class Octa(AutotoolsPackage):
     depends_on("libtool", type="build")
     depends_on("libjpeg", type="link")
     depends_on("libpng", type="link")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("jogl")
     depends_on("python")
     depends_on("gnuplot", type="run")
diff --git a/var/spack/repos/builtin/packages/octave-quaternion/package.py b/var/spack/repos/builtin/packages/octave-quaternion/package.py
index 663f463f620244..6e17326ebcb206 100644
--- a/var/spack/repos/builtin/packages/octave-quaternion/package.py
+++ b/var/spack/repos/builtin/packages/octave-quaternion/package.py
@@ -16,5 +16,5 @@ class OctaveQuaternion(OctavePackage, SourceforgePackage):
     version("2.4.0", sha256="4c2d4dd8f1d213f080519c6f9dfbbdca068087ee0411122b16e377e0f4641610")
     version("2.2.2", sha256="261d51657bc729c8f9fe915532d91e75e48dce2af2b298781e78cc93a5067cbd")
 
-    conflicts("octave@6:")
+    conflicts("^octave@6:")
     extends("octave@3.8.0:5.2.0")
diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py
index 8aa0f87ece48f7..90dbdb44786ee2 100644
--- a/var/spack/repos/builtin/packages/octave/package.py
+++ b/var/spack/repos/builtin/packages/octave/package.py
@@ -52,28 +52,28 @@ class Octave(AutotoolsPackage, GNUMirrorPackage):
     patch("patch_4.2.1_inline.diff", when="@4.2.1")
 
     # Variants
-    variant("readline", default=True)
-    variant("bz2", default=True)
-    variant("arpack", default=False)
-    variant("curl", default=False)
-    variant("fftw", default=False)
-    variant("fltk", default=False)
-    variant("fontconfig", default=False)
-    variant("freetype", default=False)
-    variant("glpk", default=False)
-    variant("gl2ps", default=False)
-    variant("gnuplot", default=False)
-    variant("magick", default=False)
-    variant("hdf5", default=False)
-    variant("jdk", default=False)
-    variant("llvm", default=False)
-    variant("opengl", default=False)
-    variant("qhull", default=False)
-    variant("qrupdate", default=False)
-    variant("qscintilla", default=False)
-    variant("qt", default=False)
-    variant("suitesparse", default=False)
-    variant("zlib", default=False)
+    variant("readline", default=True, description="Use readline")
+    variant("bz2", default=True, description="Use bzip2")
+    variant("arpack", default=False, description="Use arpack")
+    variant("curl", default=False, description="Use curl")
+    variant("fftw", default=False, description="Use FFTW3")
+    variant("fltk", default=False, description="Use FLTK")
+    variant("fontconfig", default=False, description="Use fontconfig")
+    variant("freetype", default=False, description="Use freetype")
+    variant("glpk", default=False, description="Use GLPK")
+    variant("gl2ps", default=False, description="Use GL2PS")
+    variant("gnuplot", default=False, description="Use gnuplot")
+    variant("magick", default=False, description="Use magick")
+    variant("hdf5", default=False, description="Use HDF5")
+    variant("jdk", default=False, description="Use JDK")
+    variant("llvm", default=False, description="Use LLVM")
+    variant("opengl", default=False, description="Use OpenGL")
+    variant("qhull", default=False, description="Use qhull")
+    variant("qrupdate", default=False, description="Use qrupdate")
+    variant("qscintilla", default=False, description="Use QScintill")
+    variant("qt", default=False, description="Use Qt")
+    variant("suitesparse", default=False, description="Use SuiteSparse")
+    variant("zlib", default=False, description="Use zlib")
 
     # Required dependencies
     depends_on("blas")
@@ -109,7 +109,7 @@ class Octave(AutotoolsPackage, GNUMirrorPackage):
     depends_on("qscintilla", when="+qscintilla")
     depends_on("qt+opengl", when="+qt")
     depends_on("suite-sparse", when="+suitesparse")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
 
     def patch(self):
         # Filter mkoctfile.in.cc to use underlying compilers and not
@@ -167,7 +167,7 @@ def configure_args(self):
         config_args = []
 
         # Required dependencies
-        if "^mkl" in spec and "gfortran" in self.compiler.fc:
+        if spec["lapack"].name in INTEL_MATH_LIBRARIES and "gfortran" in self.compiler.fc:
             mkl_re = re.compile(r"(mkl_)intel(_i?lp64\b)")
             config_args.extend(
                 [
@@ -339,8 +339,8 @@ def configure_args(self):
         if "+zlib" in spec:
             config_args.extend(
                 [
-                    "--with-z-includedir=%s" % spec["zlib"].prefix.include,
-                    "--with-z-libdir=%s" % spec["zlib"].prefix.lib,
+                    "--with-z-includedir=%s" % spec["zlib-api"].prefix.include,
+                    "--with-z-libdir=%s" % spec["zlib-api"].prefix.lib,
                 ]
             )
         else:
diff --git a/var/spack/repos/builtin/packages/octopus/package.py b/var/spack/repos/builtin/packages/octopus/package.py
index 4a6bb6501fc2e9..3ccd8719a1758b 100644
--- a/var/spack/repos/builtin/packages/octopus/package.py
+++ b/var/spack/repos/builtin/packages/octopus/package.py
@@ -93,13 +93,14 @@ class Octopus(AutotoolsPackage, CudaPackage):
     depends_on("libxc@2:4", when="@8:9")
     depends_on("libxc@5.1.0:", when="@10:")
     depends_on("libxc@5.1.0:", when="@develop")
+    depends_on("netcdf-fortran", when="+netcdf")  # NetCDF fortran lib without mpi variant
     with when("+mpi"):  # list all the parallel dependencies
         depends_on("fftw@3:+mpi+openmp", when="@8:9")  # FFT library
         depends_on("fftw-api@3:+mpi+openmp", when="@10:")
         depends_on("libvdwxc+mpi", when="+libvdwxc")
         depends_on("arpack-ng+mpi", when="+arpack")
         depends_on("elpa+mpi", when="+elpa")
-        depends_on("netcdf-fortran ^netcdf-c+mpi", when="+netcdf")
+        depends_on("netcdf-c+mpi", when="+netcdf")  # Link dependency of NetCDF fortran lib
         depends_on("berkeleygw@2.1+mpi", when="+berkeleygw")
 
     with when("~mpi"):  # list all the serial dependencies
@@ -108,7 +109,7 @@ class Octopus(AutotoolsPackage, CudaPackage):
         depends_on("libvdwxc~mpi", when="+libvdwxc")
         depends_on("arpack-ng~mpi", when="+arpack")
         depends_on("elpa~mpi", when="+elpa")
-        depends_on("netcdf-fortran ^netcdf-c~~mpi", when="+netcdf")
+        depends_on("netcdf-c~~mpi", when="+netcdf")  # Link dependency of NetCDF fortran lib
         depends_on("berkeleygw@2.1~mpi", when="+berkeleygw")
 
     depends_on("etsf-io", when="+etsf-io")
@@ -158,7 +159,7 @@ def configure_args(self):
 
         if "^fftw" in spec:
             args.append("--with-fftw-prefix=%s" % spec["fftw"].prefix)
-        elif "^mkl" in spec:
+        elif spec["fftw-api"].name in INTEL_MATH_LIBRARIES:
             # As of version 10.0, Octopus depends on fftw-api instead
             # of FFTW. If FFTW is not in the dependency tree, then
             # it ought to be MKL as it is currently the only providers
@@ -192,8 +193,8 @@ def configure_args(self):
         if "+scalapack" in spec:
             args.extend(
                 [
-                    "--with-blacs=%s" % spec["scalapack"].libs,
-                    "--with-scalapack=%s" % spec["scalapack"].libs,
+                    f"--with-blacs={spec['scalapack'].libs.ld_flags}",
+                    f"--with-scalapack={spec['scalapack'].libs.ld_flags}",
                 ]
             )
 
diff --git a/var/spack/repos/builtin/packages/odgi/package.py b/var/spack/repos/builtin/packages/odgi/package.py
new file mode 100644
index 00000000000000..459ab59bac3b24
--- /dev/null
+++ b/var/spack/repos/builtin/packages/odgi/package.py
@@ -0,0 +1,48 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Odgi(CMakePackage):
+    """Optimized dynamic genome/graph implementation.
+    Odgi provides an efficient and succinct dynamic DNA sequence graph model, as
+    well as a host of algorithms that allow the use of such graphs in
+    bioinformatic analyses.
+    """
+
+    homepage = "https://github.com/pangenome/odgi"
+    git = "https://github.com/pangenome/odgi.git"
+
+    # notify when the package is updated.
+    maintainers("tbhaxor", "EbiArnie")
+
+    # <<< Versions list starts here
+    version("0.8.3", commit="34f006f31c3f6b35a1eb8d58a4edb1c458583de3", submodules=True)
+    # >>> Versions list ends here
+
+    # compilation problem with ninja
+    generator("make", default="make")
+
+    # the range is required to successfully build the program
+    requires("%gcc", msg="Package odgi depends on the gcc C++ compiler")
+    conflicts(
+        "%gcc@:9.2,13:", msg="Unsupported compiler version. Recommended range is 9.3 -> 12.x"
+    )
+
+    # <<< Dependencies list starts here
+    depends_on("python")
+    depends_on("py-pybind11")
+    depends_on("sdsl-lite")
+    depends_on("libdivsufsort")
+    depends_on("jemalloc")
+    # >>> Dependencies list ends here
+
+    def cmake_args(self):
+        args = [
+            "-DCMAKE_CXX_STANDARD_REQUIRED:BOOL=ON",
+            "-DPYTHON_EXECUTABLE:FILEPATH={0}".format(self.spec["python"].command),
+        ]
+        return args
diff --git a/var/spack/repos/builtin/packages/of-catalyst/package.py b/var/spack/repos/builtin/packages/of-catalyst/package.py
index 2e3c6874ff1ad7..3bfca4e7d38a3b 100644
--- a/var/spack/repos/builtin/packages/of-catalyst/package.py
+++ b/var/spack/repos/builtin/packages/of-catalyst/package.py
@@ -26,7 +26,7 @@ class OfCatalyst(CMakePackage):
     git = "https://develop.openfoam.com/Community/catalyst.git"
 
     version("develop", branch="develop")
-    version("1806", tag="v1806")
+    version("1806", tag="v1806", commit="d97babec3581bad413fd602e17fcd4bc1e312d26")
 
     variant("full", default=False, description="Build against paraview (full) or catalyst (light)")
 
diff --git a/var/spack/repos/builtin/packages/omega-h/package.py b/var/spack/repos/builtin/packages/omega-h/package.py
index 86c8993c061527..3d8c952afd4007 100644
--- a/var/spack/repos/builtin/packages/omega-h/package.py
+++ b/var/spack/repos/builtin/packages/omega-h/package.py
@@ -19,6 +19,11 @@ class OmegaH(CMakePackage, CudaPackage):
     maintainers("cwsmith")
     tags = ["e4s"]
     version("main", branch="main")
+    version(
+        "scorec.10.7.0",
+        commit="0e5de8618c3370f702e08c1b1af476dbbc118892",
+        git="https://github.com/SCOREC/omega_h.git",
+    )
     version(
         "scorec.10.6.0",
         commit="f376fad4741b55a4b2482218eb3437d719b7c72e",
@@ -59,18 +64,18 @@ class OmegaH(CMakePackage, CudaPackage):
     depends_on("mpi", when="+mpi")
     depends_on("trilinos +kokkos", when="+trilinos")
     depends_on("kokkos", when="+kokkos")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     # Note: '+cuda' and 'cuda_arch' variants are added by the CudaPackage
     depends_on("cuda", when="+cuda")
     conflicts(
-        "cuda@11.2",
+        "^cuda@11.2",
         when="@scorec.10.1.0:",
         msg="Thrust is broken in CUDA = 11.2.* see https://github.com/sandialabs/omega_h/issues/366",
     )
     # the sandia repo has a fix for cuda > 11.2 support
     #  see github.com/sandialabs/omega_h/pull/373
     conflicts(
-        "cuda@11.2",
+        "^cuda@11.2",
         when="@:9.34.4",
         msg="Thrust is broken in CUDA = 11.2.* see https://github.com/sandialabs/omega_h/issues/366",
     )
@@ -126,7 +131,7 @@ def cmake_args(self):
             args.append("-DOmega_h_USE_Kokkos:BOOL=ON")
         if "+zlib" in self.spec:
             args.append("-DOmega_h_USE_ZLIB:BOOL=ON")
-            args.append("-DZLIB_ROOT:PATH={0}".format(self.spec["zlib"].prefix))
+            args.append("-DZLIB_ROOT:PATH={0}".format(self.spec["zlib-api"].prefix))
         else:
             args.append("-DOmega_h_USE_ZLIB:BOOL=OFF")
         if "+examples" in self.spec:
diff --git a/var/spack/repos/builtin/packages/oommf/package.py b/var/spack/repos/builtin/packages/oommf/package.py
index f7127f073e5573..a3a4a6aeac00ed 100644
--- a/var/spack/repos/builtin/packages/oommf/package.py
+++ b/var/spack/repos/builtin/packages/oommf/package.py
@@ -168,15 +168,15 @@ def test_env(self):
     def configure(self, spec, prefix):
         # change into directory with source code
         with working_dir(self.get_oommf_source_root()):
-            configure = Executable("./oommf.tcl pimake distclean")
-            configure()
-            configure2 = Executable("./oommf.tcl pimake upgrade")
-            configure2()
+            configure = Executable("./oommf.tcl")
+            configure("pimake", "distclean")
+            configure2 = Executable("./oommf.tcl")
+            configure2("pimake", "upgrade")
 
     def build(self, spec, prefix):
         with working_dir(self.get_oommf_source_root()):
-            make = Executable("./oommf.tcl pimake ")
-            make()
+            make = Executable("./oommf.tcl")
+            make("pimake")
 
     def install(self, spec, prefix):
         # keep a copy of all the tcl files and everything oommf created.
diff --git a/var/spack/repos/builtin/packages/op2-dsl/package.py b/var/spack/repos/builtin/packages/op2-dsl/package.py
index 71f02476641528..64f2d57bed31a9 100644
--- a/var/spack/repos/builtin/packages/op2-dsl/package.py
+++ b/var/spack/repos/builtin/packages/op2-dsl/package.py
@@ -17,7 +17,7 @@ class Op2Dsl(MakefilePackage, CudaPackage):
     maintainers("gihanmudalige", "reguly", "bozbez")
 
     version("master", branch="master")
-    version("1.1.0", tag="v1.1.0")
+    version("1.1.0", tag="v1.1.0", commit="22c13b425976e32a6c904f3a5a95ffb761680eb3")
 
     build_directory = "op2"
 
diff --git a/var/spack/repos/builtin/packages/opa-psm2/package.py b/var/spack/repos/builtin/packages/opa-psm2/package.py
index 9aab41575001d6..38d1ab5e972c64 100644
--- a/var/spack/repos/builtin/packages/opa-psm2/package.py
+++ b/var/spack/repos/builtin/packages/opa-psm2/package.py
@@ -6,7 +6,7 @@
 from spack.package import *
 
 
-class OpaPsm2(MakefilePackage):
+class OpaPsm2(MakefilePackage, CudaPackage):
     """Omni-Path Performance Scaled Messaging 2 (PSM2) library"""
 
     homepage = "https://github.com/cornelisnetworks/opa-psm2"
@@ -31,6 +31,7 @@ class OpaPsm2(MakefilePackage):
     variant("avx2", default=True, description="Enable AVX2 instructions")
 
     depends_on("numactl")
+    depends_on("cuda@8:", when="+cuda")
 
     # patch to get the Makefile to use the spack compiler wrappers
     patch(
@@ -45,6 +46,8 @@ def setup_build_environment(self, env):
             # this variable must be set when we use the Intel compilers to
             # ensure that the proper flags are set
             env.set("CCARCH", "icc")
+        if "+cuda" in self.spec:
+            env.set("PSM_CUDA", "1")
 
     def edit(self, spec, prefix):
         # Change the makefile so libraries and includes are not
diff --git a/var/spack/repos/builtin/packages/open3d/package.py b/var/spack/repos/builtin/packages/open3d/package.py
index 60d0ed4eaa220e..887ae7a27ed926 100644
--- a/var/spack/repos/builtin/packages/open3d/package.py
+++ b/var/spack/repos/builtin/packages/open3d/package.py
@@ -15,7 +15,9 @@ class Open3d(CMakePackage, CudaPackage):
     url = "https://github.com/isl-org/Open3D/archive/refs/tags/v0.13.0.tar.gz"
     git = "https://github.com/isl-org/Open3D.git"
 
-    version("0.13.0", tag="v0.13.0", submodules=True)
+    version(
+        "0.13.0", tag="v0.13.0", commit="c3f9de224e13838a72da0e5565a7ba51038b0f11", submodules=True
+    )
 
     variant("python", default=False, description="Build the Python module")
 
diff --git a/var/spack/repos/builtin/packages/openbabel/package.py b/var/spack/repos/builtin/packages/openbabel/package.py
index 37be524cc8f9de..4991e347cfce9e 100644
--- a/var/spack/repos/builtin/packages/openbabel/package.py
+++ b/var/spack/repos/builtin/packages/openbabel/package.py
@@ -19,11 +19,11 @@ class Openbabel(CMakePackage):
     maintainers("RMeli")
 
     version("master", branch="master")
-    version("3.1.1", tag="openbabel-3-1-1")
-    version("3.1.0", tag="openbabel-3-1-0")
-    version("3.0.0", tag="openbabel-3-0-0")
-    version("2.4.1", tag="openbabel-2-4-1")
-    version("2.4.0", tag="openbabel-2-4-0")
+    version("3.1.1", tag="openbabel-3-1-1", commit="cbd4db43f8908b874864280fdc03bf92569eebc1")
+    version("3.1.0", tag="openbabel-3-1-0", commit="1e593abc1edf47352d5e8a0887654edf69a2f5f3")
+    version("3.0.0", tag="openbabel-3-0-0", commit="49f9cfb32bd0bc6ea440639d338123eb27accbe2")
+    version("2.4.1", tag="openbabel-2-4-1", commit="701f6049c483b1349118c2ff736a7f609a84dedd")
+    version("2.4.0", tag="openbabel-2-4-0", commit="087f33320e6796f39e6a1da04f4de7ec46bec4af")
 
     variant("python", default=True, description="Build Python bindings")
     variant("gui", default=True, description="Build with GUI")
@@ -44,7 +44,7 @@ class Openbabel(CMakePackage):
     depends_on("pango", when="+cairo")  # custom cairo requires custom pango
     depends_on("eigen@3.0:")  # required if using the language bindings
     depends_on("libxml2")  # required to read/write CML files, XML formats
-    depends_on("zlib")  # required to support reading gzipped files
+    depends_on("zlib-api")  # required to support reading gzipped files
     depends_on("rapidjson")  # required to support JSON
     depends_on("libsm")
     depends_on("uuid")
diff --git a/var/spack/repos/builtin/packages/openblas/ifort-msvc.patch b/var/spack/repos/builtin/packages/openblas/ifort-msvc.patch
new file mode 100644
index 00000000000000..dd4c97fdd8691b
--- /dev/null
+++ b/var/spack/repos/builtin/packages/openblas/ifort-msvc.patch
@@ -0,0 +1,13 @@
+diff -ruN spack-src/cmake/fc.cmake spack-src-new/cmake/fc.cmake
+--- spack-src/cmake/fc.cmake	2023-04-01 14:18:01.000000000 -0600
++++ spack-src-new/cmake/fc.cmake	2023-06-06 09:34:12.921982500 -0600
+@@ -89,6 +89,9 @@
+ 
+ if (${F_COMPILER} STREQUAL "INTEL")
+   set(CCOMMON_OPT "${CCOMMON_OPT} -DF_INTERFACE_INTEL")
++  if (MSVC)
++    set(FCOMMON_OPT "${FCOMMON_OPT} -names:uppercase -assume:underscore")
++  endif ()  
+   if (INTERFACE64)
+     set(FCOMMON_OPT "${FCOMMON_OPT} -i8")
+   endif ()
diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py
index b003de9f6afd8f..cd3bb0cdd4b348 100644
--- a/var/spack/repos/builtin/packages/openblas/package.py
+++ b/var/spack/repos/builtin/packages/openblas/package.py
@@ -6,11 +6,13 @@
 import os
 import re
 
+import spack.build_systems.cmake
+import spack.build_systems.makefile
 from spack.package import *
 from spack.package_test import compare_output_file, compile_c_and_execute
 
 
-class Openblas(MakefilePackage):
+class Openblas(CMakePackage, MakefilePackage):
     """OpenBLAS: An optimized BLAS library"""
 
     homepage = "https://www.openblas.net"
@@ -19,7 +21,7 @@ class Openblas(MakefilePackage):
     )
     git = "https://github.com/OpenMathLib/OpenBLAS.git"
 
-    libraries = ["libopenblas"]
+    libraries = ["libopenblas", "openblas"]
 
     version("develop", branch="develop")
     version("0.3.24", sha256="ceadc5065da97bd92404cac7254da66cc6eb192679cf1002098688978d4d5132")
@@ -92,11 +94,13 @@ class Openblas(MakefilePackage):
     )
 
     # virtual dependency
-    provides("blas")
-    provides("lapack")
+    provides("blas", "lapack")
     provides("lapack@3.9.1:", when="@0.3.15:")
     provides("lapack@3.7.0", when="@0.2.20")
 
+    # https://github.com/xianyi/OpenBLAS/pull/2519/files
+    patch("ifort-msvc.patch", when="%msvc")
+
     # https://github.com/OpenMathLib/OpenBLAS/pull/3712
     patch("cce.patch", when="@0.3.20 %cce")
 
@@ -219,6 +223,8 @@ class Openblas(MakefilePackage):
 
     depends_on("perl", type="build")
 
+    build_system("makefile", "cmake", default="makefile")
+
     def flag_handler(self, name, flags):
         spec = self.spec
         iflags = []
@@ -255,6 +261,30 @@ def check_compilers(self):
                 + " has no Fortran compiler added in spack. Add it or use openblas~fortran!"
             )
 
+    @property
+    def headers(self):
+        # The only public headers for cblas and lapacke in
+        # openblas are cblas.h and lapacke.h. The remaining headers are private
+        # headers either included in one of these two headers, or included in
+        # one of the source files implementing functions declared in these
+        # headers.
+        return find_headers(["cblas", "lapacke"], self.prefix.include)
+
+    @property
+    def libs(self):
+        spec = self.spec
+
+        # Look for openblas{symbol_suffix}
+        name = ["libopenblas", "openblas"]
+        search_shared = bool(spec.variants["shared"].value)
+        suffix = spec.variants["symbol_suffix"].value
+        if suffix != "none":
+            name = [x + suffix for x in name]
+
+        return find_libraries(name, spec.prefix, shared=search_shared, recursive=True)
+
+
+class MakefileBuilder(spack.build_systems.makefile.MakefileBuilder):
     @staticmethod
     def _read_targets(target_file):
         """Parse a list of available targets from the OpenBLAS/TargetList.txt
@@ -310,7 +340,7 @@ def _microarch_target_args(self):
                 if microarch.name in available_targets:
                     break
 
-        if self.version >= Version("0.3"):
+        if self.spec.version >= Version("0.3"):
             # 'ARCH' argument causes build errors in older OpenBLAS
             # see https://github.com/spack/spack/issues/15385
             arch_name = microarch.family.name
@@ -385,9 +415,9 @@ def make_defs(self):
 
         if "~shared" in self.spec:
             if "+pic" in self.spec:
-                make_defs.append("CFLAGS={0}".format(self.compiler.cc_pic_flag))
+                make_defs.append("CFLAGS={0}".format(self.pkg.compiler.cc_pic_flag))
                 if "~fortran" not in self.spec:
-                    make_defs.append("FFLAGS={0}".format(self.compiler.f77_pic_flag))
+                    make_defs.append("FFLAGS={0}".format(self.pkg.compiler.f77_pic_flag))
             make_defs += ["NO_SHARED=1"]
         # fix missing _dggsvd_ and _sggsvd_
         if self.spec.satisfies("@0.2.16"):
@@ -451,28 +481,6 @@ def make_defs(self):
 
         return make_defs
 
-    @property
-    def headers(self):
-        # As in netlib-lapack, the only public headers for cblas and lapacke in
-        # openblas are cblas.h and lapacke.h. The remaining headers are private
-        # headers either included in one of these two headers, or included in
-        # one of the source files implementing functions declared in these
-        # headers.
-        return find_headers(["cblas", "lapacke"], self.prefix.include)
-
-    @property
-    def libs(self):
-        spec = self.spec
-
-        # Look for openblas{symbol_suffix}
-        name = "libopenblas"
-        search_shared = bool(spec.variants["shared"].value)
-        suffix = spec.variants["symbol_suffix"].value
-        if suffix != "none":
-            name += suffix
-
-        return find_libraries(name, spec.prefix, shared=search_shared, recursive=True)
-
     @property
     def build_targets(self):
         return ["-s"] + self.make_defs + ["all"]
@@ -508,3 +516,28 @@ def check_install(self):
 
         output = compile_c_and_execute(source_file, [include_flags], link_flags.split())
         compare_output_file(output, blessed_file)
+
+
+class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder):
+    def cmake_args(self):
+        cmake_defs = [self.define("TARGET", "GENERIC")]
+        if self.spec.satisfies("platform=windows"):
+            cmake_defs += [
+                self.define("DYNAMIC_ARCH", "OFF"),
+                self.define("BUILD_WITHOUT_LAPACK", "ON"),
+            ]
+
+        if "~fortran" in self.spec:
+            cmake_defs += [self.define("NOFORTRAN", "ON")]
+
+        if "+shared" in self.spec:
+            cmake_defs += [self.define("BUILD_SHARED_LIBS", "ON")]
+
+        if self.spec.satisfies("threads=openmp"):
+            cmake_defs += [self.define("USE_OPENMP", "ON"), self.define("USE_THREAD", "ON")]
+        elif self.spec.satisfies("threads=pthreads"):
+            cmake_defs += [self.define("USE_OPENMP", "OFF"), self.define("USE_THREAD", "ON")]
+        else:
+            cmake_defs += [self.define("USE_OPENMP", "OFF"), self.define("USE_THREAD", "OFF")]
+
+        return cmake_defs
diff --git a/var/spack/repos/builtin/packages/opencarp/package.py b/var/spack/repos/builtin/packages/opencarp/package.py
index 7d975f1dd1ac6e..1c9683bd5d1b08 100644
--- a/var/spack/repos/builtin/packages/opencarp/package.py
+++ b/var/spack/repos/builtin/packages/opencarp/package.py
@@ -18,14 +18,34 @@ class Opencarp(CMakePackage):
 
     maintainers("MarieHouillon")
 
-    version("13.0", commit="e1e0deca", submodules=False, no_cache=True, preferred=True)
-    version("12.0", commit="a34c11af", submodules=False, no_cache=True)
-    version("11.0", commit="fd8419d5", submodules=False, no_cache=True)
-    version("10.0", commit="7aec7900", submodules=False, no_cache=True)
-    version("9.0", commit="c0167599", submodules=False, no_cache=True)
-    version("8.2", commit="dbfd16fd", submodules=False, no_cache=True)
-    version("8.1", commit="28eb2e97", submodules=False, no_cache=True)
-    version("7.0", commit="78da9195", submodules=False, no_cache=True)
+    version(
+        "13.0",
+        commit="e1e0deca7eddcfd210835f54430361c85a97a5a4",
+        submodules=False,
+        no_cache=True,
+        preferred=True,
+    )
+    version(
+        "12.0", commit="a34c11af3e8c2afd6e123e586a446c6993e0b039", submodules=False, no_cache=True
+    )
+    version(
+        "11.0", commit="fd8419d5c7649060c9447adf2dbee1723a8af9db", submodules=False, no_cache=True
+    )
+    version(
+        "10.0", commit="7aec7900b3efa6cfe8b27a13fafcb99fd6ff5c8e", submodules=False, no_cache=True
+    )
+    version(
+        "9.0", commit="c01675994df46b8b39c80e001590f9cfaf43cd87", submodules=False, no_cache=True
+    )
+    version(
+        "8.2", commit="dbfd16fdd472375694190b4c7802c0bfba114146", submodules=False, no_cache=True
+    )
+    version(
+        "8.1", commit="28eb2e978f276e7e998719a3f6d436fcb87e482a", submodules=False, no_cache=True
+    )
+    version(
+        "7.0", commit="78da91952738b45760bcbc34610814a83c8c6299", submodules=False, no_cache=True
+    )
     version("master", branch="master", submodules=False, no_cache=True)
 
     variant("carputils", default=False, description="Installs the carputils framework")
@@ -40,7 +60,7 @@ class Opencarp(CMakePackage):
     depends_on("gengetopt")
     depends_on("pkgconfig")
     depends_on("python")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("perl")
 
     depends_on("py-carputils", when="+carputils", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/opencascade/package.py b/var/spack/repos/builtin/packages/opencascade/package.py
index c275a9f43065fe..6179a6af917819 100644
--- a/var/spack/repos/builtin/packages/opencascade/package.py
+++ b/var/spack/repos/builtin/packages/opencascade/package.py
@@ -136,7 +136,7 @@ class Opencascade(CMakePackage):
     depends_on("tk", when="+tk")
     depends_on("gl")
 
-    conflicts("vtk@9.2", when="@:7.7.0 +vtk")
+    conflicts("^vtk@9.2", when="@:7.7.0 +vtk")
 
     def url_for_version(self, version):
         url = (
diff --git a/var/spack/repos/builtin/packages/opencv/package.py b/var/spack/repos/builtin/packages/opencv/package.py
index b07eed756bb71c..d8db8f53626117 100644
--- a/var/spack/repos/builtin/packages/opencv/package.py
+++ b/var/spack/repos/builtin/packages/opencv/package.py
@@ -19,6 +19,8 @@ class Opencv(CMakePackage, CudaPackage):
     maintainers("bvanessen", "adamjstewart")
 
     version("master", branch="master")
+    version("4.8.0", sha256="cbf47ecc336d2bff36b0dcd7d6c179a9bb59e805136af6b9670ca944aef889bd")
+    version("4.7.0", sha256="8df0079cdbe179748a18d44731af62a245a45ebf5085223dc03133954c662973")
     version("4.6.0", sha256="1ec1cba65f9f20fe5a41fda1586e01c70ea0c9a6d7b67c9e13edf0cfe2239277")
     version("4.5.5", sha256="a1cfdcf6619387ca9e232687504da996aaa9f7b5689986b8331ec02cb61d28ad")
     version("4.5.4", sha256="c20bb83dd790fc69df9f105477e24267706715a9d3c705ca1e7f613c7b3bad3d")
@@ -63,6 +65,8 @@ class Opencv(CMakePackage, CudaPackage):
         "4.5.4",
         "4.5.5",
         "4.6.0",
+        "4.7.0",
+        "4.8.0",
     ]
     for cv in contrib_vers:
         resource(
@@ -691,6 +695,7 @@ class Opencv(CMakePackage, CudaPackage):
         "mfx",
         "ngraph",
         "nvcuvid",  # disabled, details: https://github.com/opencv/opencv/issues/14850
+        "nvcuvenc",  # disabled, depends on nvcuvid being enabled
         "opencl_svm",
         "openclamdblas",
         "openclamdfft",
@@ -727,7 +732,7 @@ class Opencv(CMakePackage, CudaPackage):
     depends_on("cmake@3.5.1:", type="build")
     depends_on("python@2.7:2.8,3.2:", type="build")
     depends_on("java", type="build")
-    depends_on("zlib@1.2.3:")
+    depends_on("zlib-api")
 
     # Optional 3rd party components (dependencies)
     depends_on("clp", when="+clp")
@@ -740,8 +745,9 @@ class Opencv(CMakePackage, CudaPackage):
     depends_on("cudnn@:7.3", when="@3.3.1:3.4+cudnn")
     depends_on("cudnn@:6", when="@:3.3.0+cudnn")
     depends_on("eigen", when="+eigen")
-    depends_on("ffmpeg+avresample", when="+ffmpeg")
-    depends_on("ffmpeg@:4+avresample", when="@:4.5+ffmpeg")
+    depends_on("ffmpeg", when="+ffmpeg")
+    depends_on("ffmpeg@:5", when="@:4.7+ffmpeg")
+    depends_on("ffmpeg@:4+avresample", when="@:4.6+ffmpeg")
     depends_on("gdal", when="+gdal")
     depends_on("gtkplus", when="+gtk")
     depends_on("hpx", when="+hpx")
@@ -937,7 +943,7 @@ def cmake_args(self):
             args.append(self.define("ENABLE_VSX", True))
 
         # Media I/O
-        zlib = spec["zlib"]
+        zlib = spec["zlib-api"]
         args.extend(
             [
                 self.define("BUILD_ZLIB", False),
diff --git a/var/spack/repos/builtin/packages/opendatadetector/package.py b/var/spack/repos/builtin/packages/opendatadetector/package.py
index 6aaba8d19ac9cb..d54d3013a905f2 100644
--- a/var/spack/repos/builtin/packages/opendatadetector/package.py
+++ b/var/spack/repos/builtin/packages/opendatadetector/package.py
@@ -18,8 +18,9 @@ class Opendatadetector(CMakePackage):
     tags = ["hep"]
 
     version("main", branch="main")
-    version("v2", tag="v2")
-    version("v1", tag="v1")
+    version("v3.0.0", tag="v3.0.0", commit="e3b1eceae96fd5dddf10223753964c570ee868c9")
+    version("v2", tag="v2", commit="7041ae086dff4ee4a8d5b65f5d9559acc6dbec47")
+    version("v1", tag="v1", commit="81c43c6511723c13c15327479082d3dcfa1947c7")
 
     depends_on("dd4hep")
     depends_on("root")
diff --git a/var/spack/repos/builtin/packages/openexr/package.py b/var/spack/repos/builtin/packages/openexr/package.py
index 36c6ede9446df0..af9f4b02b6383b 100644
--- a/var/spack/repos/builtin/packages/openexr/package.py
+++ b/var/spack/repos/builtin/packages/openexr/package.py
@@ -3,18 +3,23 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+from spack.build_systems.cmake import CMakeBuilder
 from spack.package import *
 
 
-class Openexr(CMakePackage):
+class Openexr(CMakePackage, AutotoolsPackage):
     """OpenEXR Graphics Tools (high dynamic-range image file format)"""
 
     homepage = "https://www.openexr.com/"
     url = "https://github.com/AcademySoftwareFoundation/openexr/archive/refs/tags/v3.1.5.tar.gz"
 
     # New versions should come from github now
+    version("3.2.0", sha256="b1b200606640547fceff0d3ebe01ac05c4a7ae2a131be7e9b3e5b9f491ef35b3")
+    version("3.1.11", sha256="06b4a20d0791b5ec0f804c855d320a0615ce8445124f293616a086e093f1f1e1")
     version("3.1.7", sha256="78dbca39115a1c526e6728588753955ee75fa7f5bb1a6e238bed5b6d66f91fd7")
     version("3.1.5", sha256="93925805c1fc4f8162b35f0ae109c4a75344e6decae5a240afdfce25f8a433ec")
+    version("2.5.9", sha256="05bb9c2da3ff3508eee51c30f59c7f2c59bf068f3b636d12d5991e8bbaf13e01")
+    version("2.4.2", sha256="8e5bfd89f4ae1221f84216a163003edddf0d37b8aac4ee42b46edb55544599b9")
     version(
         "2.3.0",
         sha256="fd6cb3a87f8c1a233be17b94c74799e6241d50fc5efd4df75c7a4b9cf4e25ea6",
@@ -62,31 +67,24 @@ class Openexr(CMakePackage):
         url="http://download.savannah.nongnu.org/releases/openexr/openexr-1.3.2.tar.gz",
     )
 
-    variant("debug", default=False, description="Builds a debug version of the libraries")
-
-    depends_on("cmake@3.12:", when="@3:", type="build")
-    depends_on("pkgconfig", when="@:2", type="build")
+    depends_on("pkgconfig", type="build")
     depends_on("imath", when="@3:")
     depends_on("ilmbase", when="@:2")
-    depends_on("zlib")
+    depends_on("zlib-api")
+    depends_on("libdeflate", when="@3.2:")
+
+    conflicts("@:2.5.8 %gcc@13:")
 
-    @property
-    def build_directory(self):
-        if self.spec.satisfies("@3:"):
-            return super().build_directory
-        else:
-            return "."
+    # Build system
+    build_system(
+        conditional("cmake", when="@2.4:"), conditional("autotools", when="@:2.3"), default="cmake"
+    )
 
-    def configure_args(self):
-        args = ["--prefix=" + self.prefix]
+    with when("build_system=cmake"):
+        depends_on("cmake@3.12:", type="build")
 
-        if "+debug" in self.spec:
-            args.append("--enable-debug")
-        else:
-            args.append("--disable-debug")
 
+class CMakeBuilder(CMakeBuilder):
+    def cmake_args(self):
+        args = [self.define("BUILD_TESTING", self.pkg.run_tests)]
         return args
-
-    @when("@:2")
-    def cmake(self, spec, prefix):
-        configure(*self.configure_args())
diff --git a/var/spack/repos/builtin/packages/openfast/package.py b/var/spack/repos/builtin/packages/openfast/package.py
index 522a3a3a696e24..77c46d891c33c1 100644
--- a/var/spack/repos/builtin/packages/openfast/package.py
+++ b/var/spack/repos/builtin/packages/openfast/package.py
@@ -16,21 +16,21 @@ class Openfast(CMakePackage):
 
     version("develop", branch="dev")
     version("master", branch="main")
-    version("3.4.1", tag="v3.4.1")
-    version("3.4.0", tag="v3.4.0")
-    version("3.3.0", tag="v3.3.0")
-    version("3.2.1", tag="v3.2.1")
-    version("3.2.0", tag="v3.2.0")
-    version("3.1.0", tag="v3.1.0")
-    version("3.0.0", tag="v3.0.0")
-    version("2.6.0", tag="v2.6.0")
-    version("2.5.0", tag="v2.5.0")
-    version("2.4.0", tag="v2.4.0")
-    version("2.3.0", tag="v2.3.0")
-    version("2.2.0", tag="v2.2.0")
-    version("2.1.0", tag="v2.1.0")
-    version("2.0.0", tag="v2.0.0")
-    version("1.0.0", tag="v1.0.0")
+    version("3.4.1", tag="v3.4.1", commit="18704086dad861ab13daf804825da7c4b8d59428")
+    version("3.4.0", tag="v3.4.0", commit="e8ec53f9c7f9d3f6a13bfb61dba12a0ca04d8a2f")
+    version("3.3.0", tag="v3.3.0", commit="5f3fb6ef74f48e75ca94000090737a41866fb264")
+    version("3.2.1", tag="v3.2.1", commit="08fffef240461a8334596179f1de462be43ad3e9")
+    version("3.2.0", tag="v3.2.0", commit="90a1ffb626baf398d89681b9422bdbfef11cd3ad")
+    version("3.1.0", tag="v3.1.0", commit="3456a645581456883e44d441eb285ed688e98797")
+    version("3.0.0", tag="v3.0.0", commit="42a5a8196529ae0349eda6d797a79461c2c03ff0")
+    version("2.6.0", tag="v2.6.0", commit="bbbb1ca7b28a4ba411613b5c85f5de02f8316754")
+    version("2.5.0", tag="v2.5.0", commit="718d46f707d78e85edf1b49d3b1a63e8e23e1aae")
+    version("2.4.0", tag="v2.4.0", commit="ff33ca1cf65f2e13c1de0ab78cc2396ec4a47ce0")
+    version("2.3.0", tag="v2.3.0", commit="f2419c5d1c23caad9146b95a103d89e9dcaefe30")
+    version("2.2.0", tag="v2.2.0", commit="e4faf27b774982df274b87c0570e4b58c4a13fe3")
+    version("2.1.0", tag="v2.1.0", commit="f147b80521eff90c19f065eabeceac13de39ac59")
+    version("2.0.0", tag="v2.0.0", commit="0769598a17e19b3ccd00a85cde389995f55024a8")
+    version("1.0.0", tag="v1.0.0", commit="e788b9b18bd5ed96ea59d4bc0812d461bc430cfe")
 
     variant("shared", default=True, description="Build shared libraries")
     variant("double-precision", default=True, description="Treat REAL as double precision")
@@ -48,7 +48,7 @@ class Openfast(CMakePackage):
     depends_on("mpi", when="+cxx")
     depends_on("yaml-cpp", when="+cxx")
     depends_on("hdf5+mpi+cxx+hl", when="+cxx")
-    depends_on("zlib", when="+cxx")
+    depends_on("zlib-api", when="+cxx")
     depends_on("libxml2", when="+cxx")
     depends_on("netcdf-c", when="+cxx+netcdf")
 
diff --git a/var/spack/repos/builtin/packages/openfoam-org/package.py b/var/spack/repos/builtin/packages/openfoam-org/package.py
index 93ece7dc7420fe..01d57489ac534d 100644
--- a/var/spack/repos/builtin/packages/openfoam-org/package.py
+++ b/var/spack/repos/builtin/packages/openfoam-org/package.py
@@ -64,50 +64,18 @@ class OpenfoamOrg(Package):
     maintainers("kjrstory")
     homepage = "https://www.openfoam.org/"
     baseurl = "https://github.com/OpenFOAM"
-    url = "https://github.com/OpenFOAM/OpenFOAM-4.x/archive/version-4.1.tar.gz"
+    url = "https://github.com/OpenFOAM/OpenFOAM-6/archive/version-6.tar.gz"
     git = "https://github.com/OpenFOAM/OpenFOAM-dev.git"
 
     version("develop", branch="master")
-    version(
-        "10",
-        sha256="59d712ba798ca44b989b6ac50bcb7c534eeccb82bcf961e10ec19fc8d84000cf",
-        url=baseurl + "/OpenFOAM-10/archive/version-10.tar.gz",
-    )
-    version(
-        "9",
-        sha256="0c48fb56e2fbb4dd534112811364d3b2dc12106e670a6486b361e4f864b435ee",
-        url=baseurl + "/OpenFOAM-9/archive/version-9.tar.gz",
-    )
-    version(
-        "8",
-        sha256="94ba11cbaaa12fbb5b356e01758df403ac8832d69da309a5d79f76f42eb008fc",
-        url=baseurl + "/OpenFOAM-8/archive/version-8.tar.gz",
-    )
-    version(
-        "7",
-        sha256="12389cf092dc032372617785822a597aee434a50a62db2a520ab35ba5a7548b5",
-        url=baseurl + "/OpenFOAM-7/archive/version-7.tar.gz",
-    )
-    version(
-        "6",
-        sha256="32a6af4120e691ca2df29c5b9bd7bc7a3e11208947f9bccf6087cfff5492f025",
-        url=baseurl + "/OpenFOAM-6/archive/version-6.tar.gz",
-    )
-    version(
-        "5.0",
-        sha256="9057d6a8bb9fa18802881feba215215699065e0b3c5cdd0c0e84cb29c9916c89",
-        url=baseurl + "/OpenFOAM-5.x/archive/version-5.0.tar.gz",
-    )
-    version(
-        "4.1",
-        sha256="2de18de64e7abdb1b649ad8e9d2d58b77a2b188fb5bcb6f7c2a038282081fd31",
-        url=baseurl + "/OpenFOAM-4.x/archive/version-4.1.tar.gz",
-    )
-    version(
-        "2.4.0",
-        sha256="9529aa7441b64210c400c019dcb2e0410fcfd62a6f62d23b6c5994c4753c4465",
-        url=baseurl + "/OpenFOAM-2.4.x/archive/version-2.4.0.tar.gz",
-    )
+    version("10", sha256="59d712ba798ca44b989b6ac50bcb7c534eeccb82bcf961e10ec19fc8d84000cf")
+    version("9", sha256="0c48fb56e2fbb4dd534112811364d3b2dc12106e670a6486b361e4f864b435ee")
+    version("8", sha256="94ba11cbaaa12fbb5b356e01758df403ac8832d69da309a5d79f76f42eb008fc")
+    version("7", sha256="12389cf092dc032372617785822a597aee434a50a62db2a520ab35ba5a7548b5")
+    version("6", sha256="32a6af4120e691ca2df29c5b9bd7bc7a3e11208947f9bccf6087cfff5492f025")
+    version("5.0", sha256="9057d6a8bb9fa18802881feba215215699065e0b3c5cdd0c0e84cb29c9916c89")
+    version("4.1", sha256="2de18de64e7abdb1b649ad8e9d2d58b77a2b188fb5bcb6f7c2a038282081fd31")
+    version("2.4.0", sha256="9529aa7441b64210c400c019dcb2e0410fcfd62a6f62d23b6c5994c4753c4465")
     version(
         "2.3.1",
         sha256="2bbcf4d5932397c2087a9b6d7eeee6d2b1350c8ea4f455415f05e7cd94d9e5ba",
@@ -115,20 +83,29 @@ class OpenfoamOrg(Package):
     )
 
     variant("int64", default=False, description="Compile with 64-bit label")
-    variant("float32", default=False, description="Compile with 32-bit scalar (single-precision)")
     variant(
         "source", default=True, description="Install library/application sources and tutorials"
     )
     variant("metis", default=False, description="With metis decomposition")
+    variant("scotch", default=True, description="With scotch/ptscotch decomposition")
+    variant("zoltan", default=False, description="Enable Zoltan renumbering and decomposition")
+    variant(
+        "precision",
+        default="dp",
+        description="Precision option",
+        values=("sp", "dp", conditional("lp", when="@6:")),
+        multi=False,
+    )
 
     depends_on("mpi")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("flex")
     depends_on("cmake", type="build")
 
     # Require scotch with ptscotch - corresponds to standard OpenFOAM setup
-    depends_on("scotch~metis+mpi~int64", when="~int64")
-    depends_on("scotch~metis+mpi+int64", when="+int64")
+    depends_on("scotch~metis+mpi~int64", when="+scotch~int64")
+    depends_on("scotch~metis+mpi+int64", when="+scotch+int64")
+    depends_on("zoltan+shared", when="+zoltan")
 
     depends_on("metis@5:", when="+metis")
     depends_on("metis+int64", when="+metis+int64")
@@ -172,6 +149,25 @@ class OpenfoamOrg(Package):
     #
 
     # Some user config settings
+
+    def url_for_version(self, version):
+        """If the version number is 5.0 or lower, the returned URL includes
+        the ".x" suffix in the OpenFOAM directory name to reflect
+        the old directory naming convention for these versions.
+
+        """
+        if version == Version("2.3.1"):
+            return "http://downloads.sourceforge.net/foam/OpenFOAM-2.3.1.tgz"
+        elif version <= Version("5.0"):
+            version_prefix = str(version.up_to(-1)) + ".x"
+        else:
+            version_prefix = version
+
+        url = "https://github.com/OpenFOAM/OpenFOAM-{}/archive/version-{}.tar.gz".format(
+            version_prefix, version
+        )
+        return url
+
     @property
     def config(self):
         settings = {
@@ -266,6 +262,13 @@ def patch(self):
         rewrite_environ_files(  # Adjust etc/bashrc and etc/cshrc
             edits, posix=join_path("etc", "bashrc"), cshell=join_path("etc", "cshrc")
         )
+        if self.spec.satisfies("@10:") and "+zoltan" in self.spec:
+            filter_file("libzoltan.a", "libzoltan.so", join_path("src", "renumber", "Allwmake"))
+            filter_file(
+                "libzoltan.a",
+                "libzoltan.so",
+                join_path("src", "parallel", "decompose", "Allwmake"),
+            )
 
     def configure(self, spec, prefix):
         """Make adjustments to the OpenFOAM configuration files in their various
@@ -295,18 +298,29 @@ def configure(self, spec, prefix):
         self.etc_config = {
             "CGAL": {},
             "scotch": {},
+            "zoltan": {},
             "metis": {},
             "paraview": [],
             "gperftools": [],  # Currently unused
         }
 
-        if True:
+        if "+scotch" in spec:
             self.etc_config["scotch"] = {
                 "SCOTCH_ARCH_PATH": spec["scotch"].prefix,
                 # For src/parallel/decompose/Allwmake
                 "SCOTCH_VERSION": "scotch-{0}".format(spec["scotch"].version),
             }
 
+        if "+zoltan" in spec:
+            if spec.satisfies("@:9"):
+                self.etc_prefs["ZOLTAN_ARCH_PATH"] = spec["zoltan"].prefix
+                self.etc_prefs["ZOLTAN_VERSION"] = "Zoltan-{0}".format(spec["zoltan"].version)
+            else:
+                self.etc_config["zoltan"] = {
+                    "ZOLTAN_ARCH_PATH": spec["zoltan"].prefix,
+                    "ZOLTAN_VERSION": "Zoltan-{0}".format(spec["zoltan"].version),
+                }
+
         if "+metis" in spec:
             self.etc_config["metis"] = {"METIS_ARCH_PATH": spec["metis"].prefix}
 
@@ -428,6 +442,14 @@ def install_links(self):
 class OpenfoamOrgArch(OpenfoamArch):
     """An openfoam-org variant of OpenfoamArch"""
 
+    def __init__(self, spec, **kwargs):
+        super().__init__(spec, **kwargs)
+        if "precision=lp" in spec:
+            self.precision_option = "LP"
+        elif "precision=sp" in spec:
+            self.precision_option = "SP"
+        self.update_options()
+
     def update_arch(self, spec):
         """Handle differences in WM_ARCH naming"""
         OpenfoamArch.update_arch(self, spec)
diff --git a/var/spack/repos/builtin/packages/openfoam/package.py b/var/spack/repos/builtin/packages/openfoam/package.py
index 977f99fad576d2..997aeee3f2a4c7 100644
--- a/var/spack/repos/builtin/packages/openfoam/package.py
+++ b/var/spack/repos/builtin/packages/openfoam/package.py
@@ -267,6 +267,11 @@ class Openfoam(Package):
 
     version("develop", branch="develop", submodules="True")
     version("master", branch="master", submodules="True")
+    version("2306", sha256="d7fba773658c0f06ad17f90199565f32e9bf502b7bb03077503642064e1f5344")
+    version(
+        "2212_230612", sha256="604cd731173ec2a3645c838cf2468fae050a35c6340e2ca7c157699899d904c0"
+    )
+    version("2212", sha256="0a3ddbfea9abca04c3a811e72fcbb184c6b1f92c295461e63b231f1a97e96476")
     version("2206", sha256="db95eda4afb97ca870733b2d4201ef539099d0778e3f3eca9a075d4f1a0eea46")
     version(
         "2112_220610", sha256="e07fd7220520e4bcfd6c8100a7e027fba13eeca2b11085c9dd4642758422a63d"
@@ -353,7 +358,7 @@ class Openfoam(Package):
     # but particular mixes of mpi versions and InfiniBand may not work so well
     # conflicts('^openmpi~thread_multiple', when='@1712:')
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("fftw-api")
 
     # TODO: replace this with an explicit list of components of Boost,
diff --git a/var/spack/repos/builtin/packages/openimagedenoise/package.py b/var/spack/repos/builtin/packages/openimagedenoise/package.py
index 90eaa559ca00fc..9ccce30a86c266 100644
--- a/var/spack/repos/builtin/packages/openimagedenoise/package.py
+++ b/var/spack/repos/builtin/packages/openimagedenoise/package.py
@@ -17,6 +17,7 @@ class Openimagedenoise(CMakePackage):
 
     # maintainers("github_user1", "github_user2")
 
+    version("2.0.1", sha256="328eeb9809d18e835dca7203224af3748578794784c026940c02eea09c695b90")
     version("1.4.3", sha256="3276e252297ebad67a999298d8f0c30cfb221e166b166ae5c955d88b94ad062a")
     version("1.4.2", sha256="e70d27ce24b41364782376c1b3b4f074f77310ccfe5f8ffec4a13a347e48a0ea")
     version("1.4.1", sha256="9088966685a78adf24b8de075d66e4c0019bd7b2b9d29c6e45aaf35d294e3f6f")
diff --git a/var/spack/repos/builtin/packages/openjdk/package.py b/var/spack/repos/builtin/packages/openjdk/package.py
index c216a586aec0ac..bf614b6465f393 100644
--- a/var/spack/repos/builtin/packages/openjdk/package.py
+++ b/var/spack/repos/builtin/packages/openjdk/package.py
@@ -18,6 +18,24 @@
 #    format returned by platform.system() and 'arch' by platform.machine()
 
 _versions = {
+    "17.0.8.1_1": {
+        "Linux-x86_64": (
+            "c25dfbc334068a48c19c44ce39ad4b8427e309ae1cfa83f23c102e78b8a6dcc0",
+            "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.8.1%2B1/OpenJDK17U-jdk_x64_linux_hotspot_17.0.8.1_1.tar.gz",
+        ),
+        "Linux-aarch64": (
+            "eefd3cf3b3dd47ff269fa5b5c10b5e096b163f4e9c1810023abdbc00dc6cc304",
+            "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.8.1%2B1/OpenJDK17U-jdk_aarch64_linux_hotspot_17.0.8.1_1.tar.gz",
+        ),
+        "Darwin-x86_64": (
+            "18be56732c1692ef131625d814dcb02ee091a43fdd6f214a33d87cc14842fc3f",
+            "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.8.1%2B1/OpenJDK17U-jdk_x64_mac_hotspot_17.0.8.1_1.tar.gz",
+        ),
+        "Darwin-aarch64": (
+            "2e95eed48650f00650e963c8213b6c6ecda54458edf8d254ebc99d6a6966ffad",
+            "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.8.1%2B1/OpenJDK17U-jdk_aarch64_mac_hotspot_17.0.8.1_1.tar.gz",
+        ),
+    },
     "17.0.5_8": {
         "Linux-x86_64": (
             "482180725ceca472e12a8e6d1a4af23d608d78287a77d963335e2a0156a020af",
@@ -126,6 +144,24 @@
             "https://download.java.net/java/GA/jdk16.0.2/d4a915d82b4c4fbb9bde534da945d746/7/GPL/openjdk-16.0.2_linux-aarch64_bin.tar.gz",
         ),
     },
+    "11.0.20.1_1": {
+        "Linux-x86_64": (
+            "398a64bff002f0e3b0c01ecd24a1a32c83cb72a5255344219e9757d4ddd9f857",
+            "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.20.1%2B1/OpenJDK11U-jdk_x64_linux_hotspot_11.0.20.1_1.tar.gz",
+        ),
+        "Linux-aarch64": (
+            "69d39682c4a2fac294a9eaacbf62c26d3c8a2f9123f1b5d287498a5472c6b672",
+            "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.20.1%2B1/OpenJDK11U-jdk_aarch64_linux_hotspot_11.0.20.1_1.tar.gz",
+        ),
+        "Darwin-x86_64": (
+            "42fd1373ee3f7c24f13551be20c8a5ae7ade778f83c45476ea333b2e3e025267",
+            "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.20.1%2B1/OpenJDK11U-jdk_x64_mac_hotspot_11.0.20.1_1.tar.gz",
+        ),
+        "Darwin-aarch64": (
+            "d36abd2f8a8cd2c73a7893306d65a5ae03eaa73565c1fc197a69d1d6fb02405e",
+            "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.20.1%2B1/OpenJDK11U-jdk_aarch64_mac_hotspot_11.0.20.1_1.tar.gz",
+        ),
+    },
     "11.0.17_8": {
         "Linux-x86_64": (
             "b8d46ed08ef4859476fe6421a7690d899ed83dce63f13fd894f994043177ef3c",
diff --git a/var/spack/repos/builtin/packages/openjpeg/package.py b/var/spack/repos/builtin/packages/openjpeg/package.py
index e2bbfe233c4cc3..d420cc274cbd49 100644
--- a/var/spack/repos/builtin/packages/openjpeg/package.py
+++ b/var/spack/repos/builtin/packages/openjpeg/package.py
@@ -34,7 +34,7 @@ class Openjpeg(CMakePackage):
 
     variant("codec", default=False, description="Build the CODEC executables")
 
-    depends_on("zlib", when="+codec")
+    depends_on("zlib-api", when="+codec")
     depends_on("libpng", when="+codec")
     depends_on("libtiff", when="+codec")
     depends_on("lcms", when="+codec")
diff --git a/var/spack/repos/builtin/packages/openmm/package.py b/var/spack/repos/builtin/packages/openmm/package.py
index 3b8329fdea1683..6bdac8640351d2 100644
--- a/var/spack/repos/builtin/packages/openmm/package.py
+++ b/var/spack/repos/builtin/packages/openmm/package.py
@@ -17,6 +17,7 @@ class Openmm(CMakePackage, CudaPackage):
     homepage = "https://openmm.org/"
     url = "https://github.com/openmm/openmm/archive/7.4.1.tar.gz"
 
+    version("8.0.0", sha256="dc63d7b47c8bb7b169c409cfd63d909ed0ce1ae114d37c627bf7a4231acf488e")
     version("7.7.0", sha256="51970779b8dc639ea192e9c61c67f70189aa294575acb915e14be1670a586c25")
     version("7.6.0", sha256="5a99c491ded9ba83ecc3fb1d8d22fca550f45da92e14f64f25378fda0048a89d")
     version("7.5.1", sha256="c88d6946468a2bde2619acb834f57b859b5e114a93093cf562165612e10f4ff7")
@@ -32,7 +33,7 @@ class Openmm(CMakePackage, CudaPackage):
     depends_on("doxygen@:1.9.1", type="build", when="@:7.6.0")
     depends_on("doxygen", type="build", when="@7.7:")
     depends_on("swig", type="build")
-    depends_on("fftw")
+    depends_on("fftw", when="@:7")
     depends_on("py-cython", type="build")
     depends_on("py-numpy", type=("build", "run"))
     depends_on("cuda", when="+cuda", type=("build", "link", "run"))
diff --git a/var/spack/repos/builtin/packages/openmolcas/package.py b/var/spack/repos/builtin/packages/openmolcas/package.py
index c0d7d79bfe8b64..0db8aaeae5636c 100644
--- a/var/spack/repos/builtin/packages/openmolcas/package.py
+++ b/var/spack/repos/builtin/packages/openmolcas/package.py
@@ -16,6 +16,7 @@ class Openmolcas(CMakePackage):
     homepage = "https://gitlab.com/Molcas/OpenMolcas"
     url = "https://github.com/Molcas/OpenMolcas/archive/v19.11.tar.gz"
 
+    version("23.06", sha256="31727161c15ea588217c6511a3007792c74c35391849fa0296c2288d836cf951")
     version("21.02", sha256="d0b9731a011562ff4740c0e67e48d9af74bf2a266601a38b37640f72190519ca")
     version("19.11", sha256="8ebd1dcce98fc3f554f96e54e34f1e8ad566c601196ee68153763b6c0a04c7b9")
 
diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py
index 923e6237cab69b..3fcd24584782d0 100644
--- a/var/spack/repos/builtin/packages/openmpi/package.py
+++ b/var/spack/repos/builtin/packages/openmpi/package.py
@@ -507,6 +507,7 @@ def patch(self):
     # Variants to use internal packages
     variant("internal-hwloc", default=False, description="Use internal hwloc")
     variant("internal-pmix", default=False, description="Use internal pmix")
+    variant("openshmem", default=False, description="Enable building OpenSHMEM")
 
     variant(
         "two_level_namespace",
@@ -543,7 +544,7 @@ def patch(self):
     depends_on("hwloc +cuda", when="+cuda ~internal-hwloc")
     depends_on("java", when="+java")
     depends_on("sqlite", when="+sqlite3")
-    depends_on("zlib", when="@3:")
+    depends_on("zlib-api", when="@3:")
     depends_on("valgrind~mpi", when="+memchecker")
     # Singularity release 3 works better
     depends_on("singularity@3:", when="+singularity")
@@ -572,11 +573,14 @@ def patch(self):
 
     # PMIx is unavailable for @1, and required for @2:
     # OpenMPI @2: includes a vendored version:
-    # depends_on('pmix@1.1.2', when='@2.1.6')
-    # depends_on('pmix@3.2.3', when='@4.1.2')
-    depends_on("pmix@1.0:1", when="@2.0:2 ~internal-pmix")
-    depends_on("pmix@3.2:", when="@4.0:4 ~internal-pmix")
-    depends_on("pmix@4.2:", when="@5.0:5 ~internal-pmix")
+    with when("~internal-pmix"):
+        depends_on("pmix@1", when="@2")
+        depends_on("pmix@3.2:", when="@4:")
+        depends_on("pmix@4.2:", when="@5:")
+
+        # pmix@4.2.3 contains a breaking change, compat fixed in openmpi@4.1.6
+        # See https://www.mail-archive.com/announce@lists.open-mpi.org//msg00158.html
+        depends_on("pmix@:4.2.2", when="@:4.1.5")
 
     # Libevent is required when *vendored* PMIx is used
     depends_on("libevent@2:", when="@main")
@@ -917,7 +921,7 @@ def die_without_fortran(self):
         # Until we can pass variants such as +fortran through virtual
         # dependencies depends_on('mpi'), require Fortran compiler to
         # avoid delayed build errors in dependents.
-        if (self.compiler.f77 is None) or (self.compiler.fc is None):
+        if (self.compiler.f77 is None) and (self.compiler.fc is None):
             raise InstallError("OpenMPI requires both C and Fortran compilers!")
 
     @when("@main")
@@ -981,7 +985,7 @@ def configure_args(self):
             config_args.extend(["--enable-debug"])
 
         # Package dependencies
-        for dep in ["libevent", "lustre", "singularity", "valgrind", "zlib"]:
+        for dep in ["libevent", "lustre", "singularity", "valgrind"]:
             if "^" + dep in spec:
                 config_args.append("--with-{0}={1}".format(dep, spec[dep].prefix))
 
@@ -991,6 +995,9 @@ def configure_args(self):
         elif "^pmix" in spec:
             config_args.append("--with-pmix={0}".format(spec["pmix"].prefix))
 
+        if "^zlib-api" in spec:
+            config_args.append("--with-zlib={0}".format(spec["zlib-api"].prefix))
+
         # Hwloc support
         if spec.satisfies("+internal-hwloc"):
             config_args.append("--with-hwloc=internal")
@@ -1060,6 +1067,9 @@ def configure_args(self):
             # Workaround compiler issues
             config_args.append("CFLAGS=-O1")
 
+        if "+openshmem" in spec:
+            config_args.append("--enable-oshmem")
+
         if "+wrapper-rpath" in spec:
             config_args.append("--enable-wrapper-rpath")
 
diff --git a/var/spack/repos/builtin/packages/openpbs/package.py b/var/spack/repos/builtin/packages/openpbs/package.py
index 305723569d039c..7bc9e45ad07779 100644
--- a/var/spack/repos/builtin/packages/openpbs/package.py
+++ b/var/spack/repos/builtin/packages/openpbs/package.py
@@ -45,7 +45,7 @@ class Openpbs(AutotoolsPackage):
     depends_on("swig")
     depends_on("tcl")
     depends_on("tk")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # The configure script cannot properly handle dependencies in non-system
     # directories.
@@ -84,7 +84,7 @@ def configure_args(self):
             # The argument --with-tk is introduced with with_lib.patch
             "--with-tk=%s" % self.spec["tk"].prefix,
             "--with-xauth=xauth",
-            "--with-libz=%s" % self.spec["zlib"].prefix,
+            "--with-libz=%s" % self.spec["zlib-api"].prefix,
         ]
 
     @run_after("install")
diff --git a/var/spack/repos/builtin/packages/openpmd-api/package.py b/var/spack/repos/builtin/packages/openpmd-api/package.py
index 747b3c49d66227..bb9bdccc469c97 100644
--- a/var/spack/repos/builtin/packages/openpmd-api/package.py
+++ b/var/spack/repos/builtin/packages/openpmd-api/package.py
@@ -10,7 +10,7 @@ class OpenpmdApi(CMakePackage):
     """C++ & Python API for Scientific I/O"""
 
     homepage = "https://www.openPMD.org"
-    url = "https://github.com/openPMD/openPMD-api/archive/0.15.1.tar.gz"
+    url = "https://github.com/openPMD/openPMD-api/archive/0.15.2.tar.gz"
     git = "https://github.com/openPMD/openPMD-api.git"
 
     maintainers("ax3l", "franzpoeschel")
@@ -19,6 +19,7 @@ class OpenpmdApi(CMakePackage):
 
     # C++17 up until here
     version("develop", branch="dev")
+    version("0.15.2", sha256="fbe3b356fe6f4589c659027c8056844692c62382e3ec53b953bed1c87e58ba13")
     version("0.15.1", sha256="0e81652152391ba4d2b62cfac95238b11233a4f89ff45e1fcffcc7bcd79dabe1")
     version("0.15.0", sha256="290e3a3c5814204ea6527d53423bfacf7a8dc490713227c9e0eaa3abf4756177")
     # C++14 up until here
@@ -34,8 +35,8 @@ class OpenpmdApi(CMakePackage):
     version("0.13.1", sha256="81ff79419982eb1b0865d1736f73f950f5d4c356d3c78200ceeab7f54dc07fd7")
     version("0.13.0", sha256="97c2e43d80ee5c5288f278bd54f0dcb40e7f48a575b278fcef9660214b779bb0")
     # C++11 up until here
-    version("0.12.0", tag="0.12.0-alpha")
-    version("0.11.1", tag="0.11.1-alpha")
+    version("0.12.0", tag="0.12.0-alpha", commit="23be484dd2570b5277779eafcc5f1eb70c6d98f2")
+    version("0.11.1", tag="0.11.1-alpha", commit="c40292aafbf564807710424d106304f9670a8304")
 
     variant("shared", default=True, description="Build a shared version of the library")
     variant("mpi", default=True, description="Enable parallel I/O")
@@ -70,8 +71,8 @@ class OpenpmdApi(CMakePackage):
         depends_on("py-pybind11@2.6.2:", type="link")
         depends_on("py-numpy@1.15.1:", type=["test", "run"])
         depends_on("py-mpi4py@2.1.0:", when="+mpi", type=["test", "run"])
-        depends_on("python@3.6:", type=["link", "test", "run"])
-        depends_on("python@3.7:", when="@0.15.0:", type=["link", "test", "run"])
+        depends_on("python@3.7:", type=["link", "test", "run"])
+        depends_on("python@3.8:", when="@0.15.2:", type=["link", "test", "run"])
 
     conflicts("^hdf5 api=v16", msg="openPMD-api requires HDF5 APIs for 1.8+")
 
diff --git a/var/spack/repos/builtin/packages/openscenegraph/openscenegraph-3.6.5-openexr3.patch b/var/spack/repos/builtin/packages/openscenegraph/openscenegraph-3.6.5-openexr3.patch
new file mode 100644
index 00000000000000..6a6aa57950d60a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/openscenegraph/openscenegraph-3.6.5-openexr3.patch
@@ -0,0 +1,68 @@
+https://bugs.gentoo.org/833491
+
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -752,7 +752,6 @@ ELSE()
+-    FIND_PACKAGE(ilmbase)
+     FIND_PACKAGE(Inventor)
+     FIND_PACKAGE(Jasper)
+-    FIND_PACKAGE(OpenEXR)
++    FIND_PACKAGE(OpenEXR CONFIG)
+     FIND_PACKAGE(OpenCascade)
+     FIND_PACKAGE(COLLADA)
+     FIND_PACKAGE(FBX)
+--- a/src/osgPlugins/CMakeLists.txt
++++ b/src/osgPlugins/CMakeLists.txt
+@@ -105,7 +105,7 @@ ENDIF()
+ IF(JASPER_FOUND)
+     ADD_PLUGIN_DIRECTORY(jp2)
+ ENDIF()
+-IF(OPENEXR_FOUND AND ZLIB_FOUND AND OSG_CPP_EXCEPTIONS_AVAILABLE)
++IF(OpenEXR_FOUND AND ZLIB_FOUND AND OSG_CPP_EXCEPTIONS_AVAILABLE)
+     ADD_PLUGIN_DIRECTORY(exr)
+ ENDIF()
+ IF(GIFLIB_FOUND)
+--- a/src/osgPlugins/exr/CMakeLists.txt
++++ b/src/osgPlugins/exr/CMakeLists.txt
+@@ -1,9 +1,7 @@
+-INCLUDE_DIRECTORIES( ${ILMBASE_INCLUDE_DIR}/OpenEXR )
+-INCLUDE_DIRECTORIES( ${OPENEXR_INCLUDE_DIR}/OpenEXR )
+-
+ SET(TARGET_SRC ReaderWriterEXR.cpp )
+ 
+-SET(TARGET_LIBRARIES_VARS ${OPENEXR_LIBRARIES_VARS} ${ILMBASE_LIBRARIES_VARS} ZLIB_LIBRARIES)
++SET(OPENEXR_LIBRARIES_VARS OpenEXR::OpenEXR)
++SET(TARGET_LIBRARIES_VARS OPENEXR_LIBRARIES_VARS ZLIB_LIBRARIES)
+ 
+ IF(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+     REMOVE_CXX_FLAG(-Wshadow)
+--- a/src/osgPlugins/exr/ReaderWriterEXR.cpp
++++ b/src/osgPlugins/exr/ReaderWriterEXR.cpp
+@@ -41,11 +41,11 @@ public:
+       {
+         return _inStream->read(c,n).good();
+       };
+-      virtual Int64    tellg ()
++      virtual uint64_t    tellg ()
+       {
+           return _inStream->tellg();
+       };
+-      virtual void    seekg (Int64 pos)
++      virtual void    seekg (uint64_t pos)
+       {
+         _inStream->seekg(pos);
+       };
+@@ -69,11 +69,11 @@ public:
+       {
+         _outStream->write(c,n);
+       };
+-      virtual Int64    tellp ()
++      virtual uint64_t    tellp ()
+       {
+         return _outStream->tellp();
+       };
+-      virtual void seekp (Int64 pos)
++      virtual void seekp (uint64_t pos)
+       {
+         _outStream->seekp(pos);
+       };
diff --git a/var/spack/repos/builtin/packages/openscenegraph/package.py b/var/spack/repos/builtin/packages/openscenegraph/package.py
index 273c91fa2b30eb..bce48ff1c13868 100644
--- a/var/spack/repos/builtin/packages/openscenegraph/package.py
+++ b/var/spack/repos/builtin/packages/openscenegraph/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import sys
+
 from spack.package import *
 
 
@@ -14,6 +16,10 @@ class Openscenegraph(CMakePackage):
     git = "https://github.com/openscenegraph/OpenSceneGraph.git"
     url = "https://github.com/openscenegraph/OpenSceneGraph/archive/OpenSceneGraph-3.6.4.tar.gz"
 
+    maintainers("aumuell")
+
+    version("master", branch="master")
+    version("stable", branch="OpenSceneGraph-3.6")
     version("3.6.5", sha256="aea196550f02974d6d09291c5d83b51ca6a03b3767e234a8c0e21322927d1e12")
     version("3.6.4", sha256="81394d1b484c631028b85d21c5535280c21bbd911cb058e8746c87e93e7b9d33")
     version("3.4.1", sha256="930eb46f05781a76883ec16c5f49cfb29a059421db131005d75bec4d78401fd5")
@@ -22,11 +28,25 @@ class Openscenegraph(CMakePackage):
     version("3.1.5", sha256="dddecf2b33302076712100af59b880e7647bc595a9a7cc99186e98d6e0eaeb5c")
 
     variant("shared", default=True, description="Builds a shared version of the library")
+    variant("apps", default=False, description="Build OpenSceneGraph tools")
+    variant("dcmtk", default=False, description="Build support for DICOM files using DCMTK")
     variant(
         "ffmpeg", default=False, description="Builds ffmpeg plugin for audio encoding/decoding"
     )
+    variant("gdal", default=False, description="Build support for geospatial files using GDAL")
+    variant("gta", default=False, description="Build support for Generic Tagged Array (GTA) files")
+    variant(
+        "inventor", default=False, description="Build support for Open Inventor files using Coin3D"
+    )
+    variant(
+        "opencascade", default=False, description="Build support for CAD files using Open CASCADE"
+    )
+    variant("openexr", default=False, description="Build support for OpenEXR files")
+    variant("pdf", default=False, description="Build support for PDF files using Poppler")
+    variant("svg", default=False, description="Build support for SVG files using librsvg")
 
     depends_on("cmake@2.8.7:", type="build")
+    depends_on("pkgconfig", type="build")
     depends_on("gl")
     depends_on(
         "qt+opengl", when="@:3.5.4"
@@ -39,42 +59,61 @@ class Openscenegraph(CMakePackage):
     depends_on("jasper")
     depends_on("libtiff")
     depends_on("glib")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("fontconfig")
 
-    depends_on("ffmpeg+avresample", when="+ffmpeg")
+    depends_on("dcmtk+pic", when="+dcmtk")
+    depends_on("gdal", when="+gdal")
+    depends_on("libgta", when="+gta")
+    depends_on("coin3d", when="+inventor")
+    depends_on("opencascade@:7.5", when="+opencascade")
+    depends_on("openexr", when="+openexr")
+    depends_on("ilmbase", when="+openexr ^openexr@:2")
+    depends_on("poppler+glib", when="+pdf")
+    depends_on("librsvg", when="+svg")
+
+    depends_on("ffmpeg@:4", when="+ffmpeg")
+    depends_on("ffmpeg+avresample", when="^ffmpeg@:4")
     # https://github.com/openscenegraph/OpenSceneGraph/issues/167
     depends_on("ffmpeg@:2", when="@:3.4.0+ffmpeg")
 
     patch("glibc-jasper.patch", when="@3.4%gcc")
+    # from gentoo: https://raw.githubusercontent.com/gentoo/gentoo/9523b20c27d12dd72d1fd5ced3ba4995099925a2/dev-games/openscenegraph/files/openscenegraph-3.6.5-openexr3.patch
+    patch("openscenegraph-3.6.5-openexr3.patch", when="@3.6:")
+
+    def patch(self):
+        # pkgconfig does not work for GTA on macos
+        if sys.platform == "darwin":
+            filter_file("PKG_CHECK_MODULES\\(GTA gta\\)", "", "CMakeModules/FindGTA.cmake")
 
     def cmake_args(self):
         spec = self.spec
 
-        shared_status = "ON" if "+shared" in spec else "OFF"
-        opengl_profile = "GL{0}".format(spec["gl"].version.up_to(1))
-
         args = [
             # Variant Options #
-            "-DDYNAMIC_OPENSCENEGRAPH={0}".format(shared_status),
-            "-DDYNAMIC_OPENTHREADS={0}".format(shared_status),
-            "-DOPENGL_PROFILE={0}".format(opengl_profile),
+            self.define_from_variant("DYNAMIC_OPENSCENEGRAPH", "shared"),
+            self.define_from_variant("DYNAMIC_OPENTHREADS", "shared"),
+            self.define_from_variant("BUILD_OSG_APPLICATIONS", "apps"),
             # General Options #
-            "-DBUILD_OSG_APPLICATIONS=OFF",
-            "-DOSG_NOTIFY_DISABLED=ON",
-            "-DLIB_POSTFIX=",
-            "-DCMAKE_RELWITHDEBINFO_POSTFIX=",
-            "-DCMAKE_MINSIZEREL_POSTFIX=",
+            self.define("OPENGL_PROFILE", f"GL{spec['gl'].version.up_to(1)}"),
+            self.define("OSG_NOTIFY_DISABLED", True),
+            self.define("LIB_POSTFIX", ""),
+            self.define("CMAKE_RELWITHDEBINFO_POSTFIX", ""),
+            self.define("CMAKE_MINSIZEREL_POSTFIX", ""),
         ]
 
-        if spec.satisfies("~ffmpeg"):
-            for ffmpeg_lib in ["libavcodec", "libavformat", "libavutil"]:
-                args.extend(
-                    [
-                        "-DFFMPEG_{0}_INCLUDE_DIRS=".format(ffmpeg_lib.upper()),
-                        "-DFFMPEG_{0}_LIBRARIES=".format(ffmpeg_lib.upper()),
-                    ]
-                )
+        # explicitly disable or enable plugins depending on variants
+        # CMake will still search for the packages, but won't build the plugins requiring them
+        args.append(self.define_from_variant("BUILD_OSG_PLUGIN_DICOM", "dcmtk"))
+        args.append(self.define_from_variant("BUILD_OSG_PLUGIN_EXR", "openexr"))
+        args.append(self.define_from_variant("BUILD_OSG_PLUGIN_FFMPEG", "ffmpeg"))
+        args.append(self.define_from_variant("BUILD_OSG_PLUGIN_GDAL", "gdal"))
+        args.append(self.define_from_variant("BUILD_OSG_PLUGIN_OGR", "gdal"))
+        args.append(self.define_from_variant("BUILD_OSG_PLUGIN_GTA", "gta"))
+        args.append(self.define_from_variant("BUILD_OSG_PLUGIN_INVENTOR", "inventor"))
+        args.append(self.define_from_variant("BUILD_OSG_PLUGIN_OPENCASCADE", "opencascade"))
+        args.append(self.define_from_variant("BUILD_OSG_PLUGIN_PDF", "pdf"))
+        args.append(self.define_from_variant("BUILD_OSG_PLUGIN_SVG", "svg"))
 
         # NOTE: This is necessary in order to allow OpenSceneGraph to compile
         # despite containing a number of implicit bool to int conversions.
diff --git a/var/spack/repos/builtin/packages/openssh/package.py b/var/spack/repos/builtin/packages/openssh/package.py
index 3ced73b62d7afb..f7b010c8d1f077 100755
--- a/var/spack/repos/builtin/packages/openssh/package.py
+++ b/var/spack/repos/builtin/packages/openssh/package.py
@@ -23,6 +23,8 @@ class Openssh(AutotoolsPackage):
 
     tags = ["core-packages"]
 
+    version("9.5p1", sha256="f026e7b79ba7fb540f75182af96dc8a8f1db395f922bbc9f6ca603672686086b")
+    version("9.4p1", sha256="3608fd9088db2163ceb3e600c85ab79d0de3d221e59192ea1923e23263866a85")
     version("9.3p1", sha256="e9baba7701a76a51f3d85a62c383a3c9dcd97fa900b859bc7db114c1868af8a8")
     version("9.2p1", sha256="3f66dbf1655fb45f50e1c56da62ab01218c228807b21338d634ebcdf9d71cf46")
     version("9.1p1", sha256="19f85009c7e3e23787f0236fbb1578392ab4d4bf9f8ec5fe6bc1cd7e8bfdd288")
@@ -54,10 +56,13 @@ class Openssh(AutotoolsPackage):
 
     depends_on("krb5+shared", when="+gssapi")
     depends_on("openssl@:1.0", when="@:7.7p1")
+    depends_on("openssl@:1.1", when="@:7.9p1")
+    # 8.7 and earlier don't support openssl@3.1:
+    depends_on("openssl@:3.0", when="@:8.7p1")
     depends_on("openssl")
     depends_on("libedit")
     depends_on("ncurses")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("py-twisted", type="test")
     depends_on("libxcrypt", type="link")
 
@@ -72,6 +77,21 @@ class Openssh(AutotoolsPackage):
         "^ssh-keyscan$",
     ]
 
+    # Both these patches are applied by Apple.
+    # https://github.com/Homebrew/homebrew-core/blob/7aabdeb30506be9b01708793ae553502c115dfc8/Formula/o/openssh.rb#L40-L45
+    patch(
+        "https://raw.githubusercontent.com/Homebrew/patches/1860b0a745f1fe726900974845d1b0dd3c3398d6/openssh/patch-sandbox-darwin.c-apple-sandbox-named-external.diff",
+        sha256="d886b98f99fd27e3157b02b5b57f3fb49f43fd33806195970d4567f12be66e71",
+        when="platform=darwin",
+    )
+
+    # https://github.com/Homebrew/homebrew-core/blob/7aabdeb30506be9b01708793ae553502c115dfc8/Formula/o/openssh.rb#L48-L52C6
+    patch(
+        "https://raw.githubusercontent.com/Homebrew/patches/d8b2d8c2612fd251ac6de17bf0cc5174c3aab94c/openssh/patch-sshd.c-apple-sandbox-named-external.diff",
+        sha256="3505c58bf1e584c8af92d916fe5f3f1899a6b15cc64a00ddece1dc0874b2f78f",
+        when="platform=darwin",
+    )
+
     @classmethod
     def determine_version(cls, exe):
         output = Executable(exe)("-V", output=str, error=str).rstrip()
@@ -81,6 +101,12 @@ def determine_version(cls, exe):
     def patch(self):
         # #29938: skip set-suid (also see man ssh-key-sign: it's not enabled by default)
         filter_file(r"\$\(INSTALL\) -m 4711", "$(INSTALL) -m711", "Makefile.in")
+        # #39599: fix configure to parse zlib 1.3's version number to prevent build fail
+        filter_file(r"if \(n != 3 && n != 4\)", "if (n < 2)", "configure")
+
+        # https://github.com/Homebrew/homebrew-core/blob/7aabdeb30506be9b01708793ae553502c115dfc8/Formula/o/openssh.rb#L71-L77
+        if self.spec.target.family == "x86_64" and self.spec.platform == "darwin":
+            filter_file(r"-fzero-call-used-regs=all", "-fzero-call-used-regs=used", "configure")
 
     def configure_args(self):
         # OpenSSH's privilege separation path defaults to /var/empty. At
@@ -109,6 +135,14 @@ def setup_build_environment(self, env):
         if self.run_tests:
             self.setup_test_environment(env)
 
+        # https://github.com/Homebrew/homebrew-core/blob/7aabdeb30506be9b01708793ae553502c115dfc8/Formula/o/openssh.rb#L65C31-L65C65
+        # to use the MacOS patches
+        if self.spec.platform == "darwin":
+            env.append_flags("CPPFLAGS", "-D__APPLE_SANDBOX_NAMED_EXTERNAL__")
+        # For "@:7": Newer compilers use -fno-common by default and fail on tun_fwd_ifnames:
+        if self.spec.satisfies("@:7 %gcc@10:") or self.spec.satisfies("@:7 %clang@11:"):
+            env.append_flags("CFLAGS", "-fcommon")
+
     def setup_test_environment(self, env):
         """Configure the regression test suite like Debian's openssh-tests package"""
         p = self.prefix
diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py
index e0c46ce6428e2e..358a008088883a 100644
--- a/var/spack/repos/builtin/packages/openssl/package.py
+++ b/var/spack/repos/builtin/packages/openssl/package.py
@@ -24,17 +24,40 @@ class Openssl(Package):  # Uses Fake Autotools, should subclass Package
     list_url = "https://www.openssl.org/source/old/"
     list_depth = 1
 
+    maintainers("AlexanderRichert-NOAA")
+
     tags = ["core-packages", "windows"]
 
     executables = ["openssl"]
 
-    version("3.1.1", sha256="b3aa61334233b852b63ddb048df181177c2c659eb9d4376008118f9c08d07674")
+    version("3.1.3", sha256="f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6")
+    version("3.0.11", sha256="b3425d3bb4a2218d0697eb41f7fc0cdede016ed19ca49d168b78e8d947887f55")
+
+    version(
+        "3.1.2",
+        sha256="a0ce69b8b97ea6a35b96875235aa453b966ba3cba8af2de23657d8b6767d6539",
+        deprecated=True,
+    )
+    version(
+        "3.1.1",
+        sha256="b3aa61334233b852b63ddb048df181177c2c659eb9d4376008118f9c08d07674",
+        deprecated=True,
+    )
     version(
         "3.1.0",
         sha256="aaa925ad9828745c4cad9d9efeb273deca820f2cdcf2c3ac7d7c1212b7c497b4",
         deprecated=True,
     )
-    version("3.0.9", sha256="eb1ab04781474360f77c318ab89d8c5a03abc38e63d65a603cabbf1b00a1dc90")
+    version(
+        "3.0.10",
+        sha256="1761d4f5b13a1028b9b6f3d4b8e17feb0cedc9370f6afe61d7193d2cdce83323",
+        deprecated=True,
+    )
+    version(
+        "3.0.9",
+        sha256="eb1ab04781474360f77c318ab89d8c5a03abc38e63d65a603cabbf1b00a1dc90",
+        deprecated=True,
+    )
     version(
         "3.0.8",
         sha256="6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e",
@@ -70,10 +93,21 @@ class Openssl(Package):  # Uses Fake Autotools, should subclass Package
         sha256="59eedfcb46c25214c9bd37ed6078297b4df01d012267fe9e9eee31f61bc70536",
         deprecated=True,
     )
-
-    # The latest stable version is the 1.1.1 series. This is also our Long Term
-    # Support (LTS) version, supported until 11th September 2023.
-    version("1.1.1u", sha256="e2f8d84b523eecd06c7be7626830370300fbcc15386bf5142d72758f6963ebc6")
+    version(
+        "1.1.1w",
+        sha256="cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8",
+        deprecated=True,
+    )
+    version(
+        "1.1.1v",
+        sha256="d6697e2871e77238460402e9362d47d18382b15ef9f246aba6c7bd780d38a6b0",
+        deprecated=True,
+    )
+    version(
+        "1.1.1u",
+        sha256="e2f8d84b523eecd06c7be7626830370300fbcc15386bf5142d72758f6963ebc6",
+        deprecated=True,
+    )
     version(
         "1.1.1t",
         sha256="8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b",
@@ -174,8 +208,6 @@ class Openssl(Package):  # Uses Fake Autotools, should subclass Package
         sha256="2836875a0f89c03d0fdf483941512613a50cfb421d6fd94b9f41d7279d586a3d",
         deprecated=True,
     )
-
-    # The 1.1.0 series is out of support and should not be used.
     version(
         "1.1.0l",
         sha256="74a2f756c64fd7386a29184dc0344f4831192d61dc2481a93a4c5dd727f41148",
@@ -216,8 +248,6 @@ class Openssl(Package):  # Uses Fake Autotools, should subclass Package
         sha256="fc436441a2e05752d31b4e46115eb89709a28aef96d4fe786abe92409b2fd6f5",
         deprecated=True,
     )
-
-    # The 1.0.2 series is out of support and should not be used.
     version(
         "1.0.2u",
         sha256="ecd0c6ffb493dd06707d38b14bb4d8c2288bb7033735606569d8f90f89669d16",
@@ -298,8 +328,6 @@ class Openssl(Package):  # Uses Fake Autotools, should subclass Package
         sha256="671c36487785628a703374c652ad2cebea45fa920ae5681515df25d9f2c9a8c8",
         deprecated=True,
     )
-
-    # The 1.0.1 version is out of support and should not be used.
     version(
         "1.0.1u",
         sha256="4312b4ca1215b6f2c97007503d80db80d5157f76f8f7d3febbe6b4c56ff26739",
@@ -344,25 +372,11 @@ class Openssl(Package):  # Uses Fake Autotools, should subclass Package
         ),
     )
     variant("docs", default=False, description="Install docs and manpages")
-    variant(
-        "shared", default=True, description="Build shared library version", when="platform=linux"
-    )
-    variant(
-        "shared", default=True, description="Build shared library version", when="platform=darwin"
-    )
-    variant(
-        "shared", default=True, description="Build shared library version", when="platform=cray"
-    )
-    variant(
-        "shared",
-        default=False,
-        description="Build shared library version",
-        when="platform=windows",
-    )
+    variant("shared", default=True, description="Build shared library version")
     with when("platform=windows"):
         variant("dynamic", default=False, description="Link with MSVC's dynamic runtime library")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("perl@5.14.0:", type=("build", "test"))
     depends_on("ca-certificates-mozilla", type="build", when="certs=mozilla")
     depends_on("nasm", when="platform=windows")
@@ -440,8 +454,8 @@ def install(self, spec, prefix):
         else:
             base_args.extend(
                 [
-                    "-I{0}".format(self.spec["zlib"].prefix.include),
-                    "-L{0}".format(self.spec["zlib"].prefix.lib),
+                    "-I{0}".format(self.spec["zlib-api"].prefix.include),
+                    "-L{0}".format(self.spec["zlib-api"].prefix.lib),
                 ]
             )
             base_args.extend(options)
diff --git a/var/spack/repos/builtin/packages/openvdb/package.py b/var/spack/repos/builtin/packages/openvdb/package.py
index aabd5f5c5ead1d..c382eebcfe5125 100644
--- a/var/spack/repos/builtin/packages/openvdb/package.py
+++ b/var/spack/repos/builtin/packages/openvdb/package.py
@@ -41,7 +41,7 @@ class Openvdb(CMakePackage):
     depends_on("openexr@2.3:3.1", when="@10:")
     depends_on("intel-tbb@:2020.1", when="@:8.1")
     depends_on("intel-tbb@2021", when="@8.2:")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("c-blosc@1.17.0")  # depends_on('c-blosc@1.5:')
     depends_on("py-numpy", when="+python")
     depends_on("boost+iostreams+system+python+numpy", when="+python")
diff --git a/var/spack/repos/builtin/packages/openvkl/package.py b/var/spack/repos/builtin/packages/openvkl/package.py
index 51de419822ec99..32bbdcafe26c3b 100644
--- a/var/spack/repos/builtin/packages/openvkl/package.py
+++ b/var/spack/repos/builtin/packages/openvkl/package.py
@@ -16,6 +16,7 @@ class Openvkl(CMakePackage):
 
     # maintainers("github_user1", "github_user2")
 
+    version("1.3.2", sha256="7704736566bf17497a3e51c067bd575316895fda96eccc682dae4aac7fb07b28")
     version("1.3.1", sha256="c9cefb6c313f2b4c0331e9629931759a6bc204ec00deed6ec0becad1670a1933")
     version("1.3.0", sha256="c6d4d40e6d232839c278b53dee1e7bd3bd239c3ccac33f49b465fc65a0692be9")
     version("1.2.0", sha256="dc468c2f0a359aaa946e04a01c2a6634081f7b6ce31b3c212c74bf7b4b0c9ec2")
@@ -24,7 +25,8 @@ class Openvkl(CMakePackage):
     version("1.0.0", sha256="81ccae679bfa2feefc4d4b1ce72bcd242ba34d2618fbb418a1c2a05d640d16b4")
     version("0.13.0", sha256="974608259e3a5d8e29d2dfe81c6b2b1830aadeb9bbdc87127f3a7c8631e9f1bd")
 
-    depends_on("embree@3.13.0:")
+    depends_on("embree@4", when="@1.3.2:")
+    depends_on("embree@3.13.0:3", when="@:1.3.1")
     depends_on("embree@3.13.1:", when="@1.0.0:")
     depends_on("ispc@1.15.0:", type=("build"))
     depends_on("ispc@1.16.0:", when="@1.0.0:", type=("build"))
@@ -32,10 +34,14 @@ class Openvkl(CMakePackage):
     depends_on("rkcommon@1.6.1:")
     depends_on("rkcommon@1.7.0:", when="@1.0.0:")
     depends_on("rkcommon@1.8.0:", when="@1.1:")
+    depends_on("rkcommon@:1.10.0", when="@:1.3.1")
+    depends_on("rkcommon@1.11.0:", when="@1.3.2:")
     depends_on("tbb")
 
     def cmake_args(self):
         args = [
+            # otherwise, openvkl 1.3.2 tries to install its headers into /openvkl
+            self.define("CMAKE_INSTALL_INCLUDEDIR", f"{self.spec.prefix}/include"),
             self.define("BUILD_BENCHMARKS", False),
             self.define("BUILD_EXAMPLES", False),
             self.define("BUILD_TESTING", False),
diff --git a/var/spack/repos/builtin/packages/ophidia-primitives/package.py b/var/spack/repos/builtin/packages/ophidia-primitives/package.py
index a7728fbd132f8b..fc9486e72144c5 100644
--- a/var/spack/repos/builtin/packages/ophidia-primitives/package.py
+++ b/var/spack/repos/builtin/packages/ophidia-primitives/package.py
@@ -23,7 +23,7 @@ class OphidiaPrimitives(AutotoolsPackage):
     depends_on("boost@1.79.0")
     depends_on("mysql")
     depends_on("libmatheval")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("gsl")
 
     def autoreconf(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/orc/package.py b/var/spack/repos/builtin/packages/orc/package.py
index 194b13cf280852..e07111440b8684 100644
--- a/var/spack/repos/builtin/packages/orc/package.py
+++ b/var/spack/repos/builtin/packages/orc/package.py
@@ -17,7 +17,8 @@ class Orc(CMakePackage):
 
     depends_on("maven")
     depends_on("openssl")
-    depends_on("zlib@1.2.11:")
+    depends_on("zlib-api")
+    depends_on("zlib@1.2.11:", when="^zlib")
     depends_on("pcre")
     depends_on("protobuf@3.5.1:")
     depends_on("zstd@1.4.5:")
@@ -36,7 +37,9 @@ def cmake_args(self):
         args.append("-DBUILD_TOOLS:BOOL=OFF")
         args.append("-DBUILD_CPP_TESTS:BOOL=OFF")
 
-        for x in ("snappy", "zlib", "zstd", "lz4", "protobuf"):
+        for x in ("snappy", "zstd", "lz4", "protobuf"):
             args.append("-D{0}_HOME={1}".format(x.upper(), self.spec[x].prefix))
 
+        args.append(self.define("ZLIB_HOME", self.spec["zlib-api"].prefix))
+
         return args
diff --git a/var/spack/repos/builtin/packages/orfm/package.py b/var/spack/repos/builtin/packages/orfm/package.py
index 4e2a37de2678fd..b2175c6ffeb36b 100644
--- a/var/spack/repos/builtin/packages/orfm/package.py
+++ b/var/spack/repos/builtin/packages/orfm/package.py
@@ -17,4 +17,4 @@ class Orfm(AutotoolsPackage):
 
     version("0.7.1", sha256="19f39c72bcc48127b757613c5eef4abae95ee6c82dccf96b041db527b27f319a")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
diff --git a/var/spack/repos/builtin/packages/osmctools/package.py b/var/spack/repos/builtin/packages/osmctools/package.py
index f5097770b2d510..0a350a8f57374a 100644
--- a/var/spack/repos/builtin/packages/osmctools/package.py
+++ b/var/spack/repos/builtin/packages/osmctools/package.py
@@ -16,7 +16,7 @@ class Osmctools(AutotoolsPackage):
     version("0.9", sha256="2f5298be5b4ba840a04f360c163849b34a31386ccd287657885e21268665f413")
     version("0.8", sha256="54ae48717afd05707c9b1fd750dd56c33c3bae0755424ce8ca3795ee28e0ece8")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
     depends_on("libtool", type="build")
diff --git a/var/spack/repos/builtin/packages/ospray/package.py b/var/spack/repos/builtin/packages/ospray/package.py
index fcac3239a4d99c..85a79894bbf246 100644
--- a/var/spack/repos/builtin/packages/ospray/package.py
+++ b/var/spack/repos/builtin/packages/ospray/package.py
@@ -16,6 +16,8 @@ class Ospray(CMakePackage):
 
     # maintainers("aumuell")
 
+    version("2.12.0", sha256="268b16952b2dd44da2a1e40d2065c960bc2442dd09b63ace8b65d3408f596301")
+    version("2.11.0", sha256="55974e650d9b78989ee55adb81cffd8c6e39ce5d3cf0a3b3198c522bf36f6e81")
     version("2.10.0", sha256="bd478284f48d2cb775fc41a2855a9d9f5ea16c861abda0f8dc94e02ea7189cb8")
     version("2.9.0", sha256="0145e09c3618fb8152a32d5f5cff819eb065d90975ee4e35400d2db9eb9f6398")
     version("2.8.0", sha256="2dabc75446a0e2e970952d325f930853a51a9b4d1868c8135f05552a4ae04d39")
@@ -27,21 +29,35 @@ class Ospray(CMakePackage):
     variant("denoiser", default=True, description="Enable denoiser image operation")
     variant("glm", default=False, description="Build ospray_cpp GLM tests/tutorial")
     variant("mpi", default=True, description="Enable MPI support")
+    variant("volumes", default=True, description="Enable volumetric rendering with Open VKL")
+
+    conflicts("~volumes", when="@:2.10")
 
     depends_on("rkcommon@1.5:")
     depends_on("rkcommon@1.7:1.9", when="@2.7.0:2.8")
     depends_on("rkcommon@1.9", when="@2.9.0")
     depends_on("rkcommon@1.10:", when="@2.10.0:")
+    depends_on("rkcommon@1.11:", when="@2.11:")
     depends_on("embree@3.12: +ispc")
     depends_on("embree@3.13.1:", when="@2.7.0:")
-    depends_on("openvkl@0.13.0:")
-    depends_on("openvkl@1.0.1:", when="@2.7.0:")
-    depends_on("openvkl@1.2.0:", when="@2.9.0:")
-    depends_on("openvkl@1.3.0:", when="@2.10.0:")
-    depends_on("openimagedenoise@1.2.3:", when="+denoiser")
+    depends_on("embree@:3", when="@:2.10")
+    depends_on("embree@4:", when="@2.11:")
+    with when("+volumes"):
+        depends_on("openvkl@0.13.0:")
+        depends_on("openvkl@1.0.1:", when="@2.7.0:")
+        depends_on("openvkl@1.2.0:", when="@2.9.0:")
+        depends_on("openvkl@1.3.0:", when="@2.10.0:")
+        depends_on("openvkl@1.3.2:", when="@2.11:")
+    with when("+denoiser"):
+        depends_on("openimagedenoise@1.2.3:")
+        depends_on("openimagedenoise@1.3:", when="@2.5:")
+        depends_on("openimagedenoise@:1", when="@:2.11")
+        depends_on("openimagedenoise@2:", when="@2.12:")
     depends_on("ispc@1.14.1:", type=("build"))
     depends_on("ispc@1.16.0:", when="@2.7.0:", type=("build"))
     depends_on("ispc@1.18.0:", when="@2.10.0:", type=("build"))
+    depends_on("ispc@1.19.0:", when="@2.11.0:", type=("build"))
+    depends_on("ispc@1.20.0:", when="@2.12.0:", type=("build"))
     depends_on("tbb")
 
     depends_on("mpi", when="+mpi")
@@ -58,6 +74,10 @@ def cmake_args(self):
             self.define_from_variant("OSPRAY_APPS_ENABLE_GLM", "glm"),
         ]
 
+        # support for volumetric data
+        if self.spec.satisfies("@2.11:"):
+            args.append(self.define_from_variant("OSPRAY_ENABLE_VOLUMES", "volumes"))
+
         # Apps
         enable_apps_arg = "" if self.spec.satisfies("@2.9:") else "ENABLE_"
         args.extend(
diff --git a/var/spack/repos/builtin/packages/osu-micro-benchmarks/package.py b/var/spack/repos/builtin/packages/osu-micro-benchmarks/package.py
index 8ed8a6933b94a6..6a755fcbf11f89 100644
--- a/var/spack/repos/builtin/packages/osu-micro-benchmarks/package.py
+++ b/var/spack/repos/builtin/packages/osu-micro-benchmarks/package.py
@@ -20,6 +20,8 @@ class OsuMicroBenchmarks(AutotoolsPackage, CudaPackage, ROCmPackage):
 
     maintainers("natshineman", "harisubramoni", "MatthewLieber")
 
+    version("7.3", sha256="8fa25b8aaa34e4b07ab3a4f30b7690ab46b038b08d204a853a9b6aa7bdb02f2f")
+    version("7.2", sha256="1a4e1f2aab0e65404b3414e23bd46616184b69b6231ce9313d9c630bd6e633c1")
     version("7.1-1", sha256="85f4dd8be1df31255e232852769ae5b82e87a5fb14be2f8eba1ae9de8ffe391a")
     version("7.1", sha256="2c4c931ecaf19e8ab72a393ee732e25743208c9a58fa50023e3fac47064292cc")
     version("7.0.1", sha256="04954aea082ba1b90a461ffab82a3cee43fe2d5a60fed99f5cb4585ac7da8c66")
diff --git a/var/spack/repos/builtin/packages/otf/package.py b/var/spack/repos/builtin/packages/otf/package.py
index fd502ee10b4403..01ccec79ee6a7a 100644
--- a/var/spack/repos/builtin/packages/otf/package.py
+++ b/var/spack/repos/builtin/packages/otf/package.py
@@ -20,7 +20,7 @@ class Otf(AutotoolsPackage):
         "1.12.5salmon", sha256="0a8427360dedb38e8ddca30f14d95f826420c550337c5a79dbb754904e194088"
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def configure_args(self):
         args = []
@@ -30,6 +30,6 @@ def configure_args(self):
         args.append("--without-zoidfs")
 
         args.append("--with-zlib")
-        args.append("--with-zlib-dir={0}".format(self.spec["zlib"].prefix))
+        args.append("--with-zlib-dir={0}".format(self.spec["zlib-api"].prefix))
 
         return args
diff --git a/var/spack/repos/builtin/packages/p4est/package.py b/var/spack/repos/builtin/packages/p4est/package.py
index 224bb61b7ed95e..1068da372a4b6d 100644
--- a/var/spack/repos/builtin/packages/p4est/package.py
+++ b/var/spack/repos/builtin/packages/p4est/package.py
@@ -37,7 +37,7 @@ class P4est(AutotoolsPackage):
 
     # other dependencies
     depends_on("mpi", when="+mpi")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # from sc upstream, correct the default libraries
     patch(
diff --git a/var/spack/repos/builtin/packages/paintor/package.py b/var/spack/repos/builtin/packages/paintor/package.py
new file mode 100644
index 00000000000000..421a92cb1160c6
--- /dev/null
+++ b/var/spack/repos/builtin/packages/paintor/package.py
@@ -0,0 +1,37 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Paintor(MakefilePackage):
+    """Probabilistic Annotation integrator. Fast, integrative fine mapping with functional
+    data"""
+
+    homepage = "https://github.com/gkichaev/PAINTOR_V3.0"
+    url = "https://github.com/gkichaev/PAINTOR_V3.0/archive/refs/tags/3.0.tar.gz"
+
+    version("3.0", sha256="cc39d3c334cc6d787e4f04847192c9d0185025a2ca46910bd38901b6679d198f")
+
+    depends_on("nlopt")
+    depends_on("eigen")
+
+    def edit(self, spec, prefix):
+        makefile = FileFilter("Makefile")
+        makefile.filter("CC = g\\+\\+", f"CC = {spack_cxx}")
+        makefile.filter(
+            r"(.*)-I/\$\(curr\)/eigen/Eigen(.*)",
+            r"\1-I{}/eigen3/Eigen\2".format(spec["eigen"].prefix.include),
+        )
+        makefile.filter(r"(.*)-L/\$\{curr}/lib(.*)", r"\1-L{}\2".format(spec["nlopt"].prefix.lib))
+        makefile.filter(
+            r"(.*)-I/\${curr}/include(.*)", r"\1-I{}\2".format(spec["nlopt"].prefix.include)
+        )
+
+    @run_after("install")
+    def mv_binary(self):
+        mkdirp(self.prefix.bin)
+        with working_dir(self.build_directory):
+            install("PAINTOR", self.prefix.bin)
diff --git a/var/spack/repos/builtin/packages/palace/package.py b/var/spack/repos/builtin/packages/palace/package.py
index 9424e0d66caf7c..9da8c209674149 100644
--- a/var/spack/repos/builtin/packages/palace/package.py
+++ b/var/spack/repos/builtin/packages/palace/package.py
@@ -14,14 +14,12 @@ class Palace(CMakePackage):
     homepage = "https://github.com/awslabs/palace"
     git = "https://github.com/awslabs/palace.git"
 
-    root_cmakelists_dir = "palace"
-
     maintainers("sebastiangrimberg")
 
     version("develop", branch="main")
-    version("0.11.1", tag="v0.11.1")
-    version("0.11.0", tag="v0.11.0")
+    version("0.11.2", tag="v0.11.2", commit="6c3aa5f84a934a6ddd58022b2945a1bdb5fa329d")
 
+    variant("shared", default=True, description="Enables the build of shared libraries")
     variant("int64", default=False, description="Use 64 bit integers")
     variant("openmp", default=False, description="Use OpenMP")
     variant(
@@ -41,30 +39,28 @@ class Palace(CMakePackage):
     depends_on("cmake@3.13:", type="build")
     depends_on("pkgconfig", type="build")
     depends_on("mpi")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("nlohmann-json")
     depends_on("fmt")
+    depends_on("eigen")
 
-    depends_on("metis@5:+int64", when="+int64")
-    depends_on("metis@5:~int64", when="~int64")
+    depends_on("metis@5:")
+    depends_on("metis+shared", when="+shared")
+    depends_on("metis~shared", when="~shared")
+    depends_on("metis+int64", when="+int64")
+    depends_on("metis~int64", when="~int64")
 
-    depends_on("hypre+shared")
+    depends_on("hypre~complex")
+    depends_on("hypre+shared", when="+shared")
+    depends_on("hypre~shared", when="~shared")
     depends_on("hypre+mixedint", when="+int64")
     depends_on("hypre~mixedint", when="~int64")
     depends_on("hypre+openmp", when="+openmp")
     depends_on("hypre~openmp", when="~openmp")
 
-    depends_on("petsc+double+complex")
-    depends_on("petsc+int64", when="+int64")
-    depends_on("petsc~int64", when="~int64")
-    depends_on("petsc+openmp", when="+openmp")
-    depends_on("petsc~openmp", when="~openmp")
-
-    depends_on("slepc", when="+slepc")
-    depends_on("arpack-ng+icb@develop", when="+arpack")
-    depends_on("gslib", when="+gslib")
-
     with when("+superlu-dist"):
+        depends_on("superlu-dist+shared", when="+shared")
+        depends_on("superlu-dist~shared", when="~shared")
         depends_on("superlu-dist+int64", when="+int64")
         depends_on("superlu-dist~int64", when="~int64")
         depends_on("superlu-dist+openmp", when="+openmp")
@@ -72,29 +68,45 @@ class Palace(CMakePackage):
 
     with when("+strumpack"):
         depends_on("strumpack+butterflypack+zfp+parmetis")
+        depends_on("strumpack+shared", when="+shared")
+        depends_on("strumpack~shared", when="~shared")
         depends_on("strumpack+openmp", when="+openmp")
         depends_on("strumpack~openmp", when="~openmp")
 
     with when("+mumps"):
         depends_on("mumps+metis+parmetis")
+        depends_on("mumps+shared", when="+shared")
+        depends_on("mumps~shared", when="~shared")
         depends_on("mumps+openmp", when="+openmp")
         depends_on("mumps~openmp", when="~openmp")
 
-    # Conflicts: Palace always builds its own internal MFEM
-    conflicts("^mfem", msg="Palace builds its own internal MFEM")
+    with when("+slepc"):
+        depends_on("slepc ^petsc+mpi+double+complex")
+        depends_on("petsc+shared", when="+shared")
+        depends_on("petsc~shared", when="~shared")
+        depends_on("petsc+int64", when="+int64")
+        depends_on("petsc~int64", when="~int64")
+        depends_on("petsc+openmp", when="+openmp")
+        depends_on("petsc~openmp", when="~openmp")
+
+    with when("+arpack"):
+        depends_on("arpack-ng+mpi+icb@develop")
+        depends_on("arpack-ng+shared", when="+shared")
+        depends_on("arpack-ng~shared", when="~shared")
+
+    # Palace always builds its own internal MFEM, GSLIB
+    conflicts("mfem")
+    conflicts("gslib")
 
     # More dependency variant conflicts
     conflicts("^hypre+int64", msg="Palace uses HYPRE's mixedint option for 64 bit integers")
-    conflicts("^petsc~double", msg="Palace requires PETSc with double precision")
-    conflicts("^petsc~complex", msg="Palace requires PETSc with complex numbers")
-    conflicts("^petsc+metis", msg="Palace requires PETSc without METIS")
-    conflicts("^petsc+hypre", msg="Palace requires PETSc without HYPRE")
-    conflicts("^petsc+superlu-dist", msg="Palace requires PETSc without SuperLU_DIST")
-    conflicts("^slepc+arpack", msg="Palace requires SLEPc without ARPACK")
     conflicts("^mumps+int64", msg="Palace requires MUMPS without 64 bit integers")
+    conflicts("^slepc+arpack", msg="Palace requires SLEPc without ARPACK")
 
     def cmake_args(self):
         args = [
+            self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
+            self.define_from_variant("PALACE_WITH_64BIT_INT", "int64"),
             self.define_from_variant("PALACE_WITH_OPENMP", "openmp"),
             self.define_from_variant("PALACE_WITH_GSLIB", "gslib"),
             self.define_from_variant("PALACE_WITH_SUPERLU", "superlu-dist"),
@@ -102,26 +114,27 @@ def cmake_args(self):
             self.define_from_variant("PALACE_WITH_MUMPS", "mumps"),
             self.define_from_variant("PALACE_WITH_SLEPC", "slepc"),
             self.define_from_variant("PALACE_WITH_ARPACK", "arpack"),
-            "-DPALACE_WITH_INTERNAL_JSON=OFF",
-            "-DPALACE_WITH_INTERNAL_FMT=OFF",
+            self.define("PALACE_BUILD_EXTERNAL_DEPS", False),
+        ]
+
+        # HYPRE is always built with external BLAS/LAPACK
+        args += [
+            self.define("HYPRE_REQUIRED_PACKAGES", "LAPACK;BLAS"),
+            self.define("BLAS_LIBRARIES", "{0}".format(self.spec["blas"].libs.joined(";"))),
+            self.define("LAPACK_LIBRARIES", "{0}".format(self.spec["lapack"].libs.joined(";"))),
         ]
 
         # MPI compiler wrappers are not required, but MFEM test builds need to know to link
         # against MPI libraries
         if "+superlu-dist" in self.spec:
-            args += ["-DSuperLUDist_REQUIRED_PACKAGES=MPI"]
+            args += [self.define("SuperLUDist_REQUIRED_PACKAGES", "LAPACK;BLAS;MPI")]
         if "+strumpack" in self.spec:
-            args += ["-DSTRUMPACK_REQUIRED_PACKAGES=MPI;MPI_Fortran"]
+            args += [self.define("STRUMPACK_REQUIRED_PACKAGES", "LAPACK;BLAS;MPI;MPI_Fortran")]
         if "+mumps" in self.spec:
-            args += ["-DMUMPS_REQUIRED_PACKAGES=MPI;MPI_Fortran"]
-
-        # BLAS/LAPACK linkage
-        args += [
-            "-DBLAS_LIBRARIES={0}".format(self.spec["blas"].libs.joined(";")),
-            "-DLAPACK_LIBRARIES={0}".format(self.spec["lapack"].libs.joined(";")),
-        ]
-
-        # HYPRE is always built with external BLAS/LAPACK
-        args += ["-DHYPRE_REQUIRED_PACKAGES=LAPACK;BLAS"]
+            args += [self.define("MUMPS_REQUIRED_PACKAGES", "LAPACK;BLAS;MPI;MPI_Fortran")]
 
         return args
+
+    def install(self, spec, prefix):
+        # No install phase for Palace (always performed during build)
+        pass
diff --git a/var/spack/repos/builtin/packages/palisade-development/package.py b/var/spack/repos/builtin/packages/palisade-development/package.py
index b165debf7b38e6..4bd7f66c70f8a6 100644
--- a/var/spack/repos/builtin/packages/palisade-development/package.py
+++ b/var/spack/repos/builtin/packages/palisade-development/package.py
@@ -29,7 +29,12 @@ class PalisadeDevelopment(CMakePackage):
     maintainers("wohlbier")
 
     version("feature-fixed-point-encoding", branch="feature-fixed-point-encoding", submodules=True)
-    version("fppe-logreg-v1.0", tag="fppe-logreg-v1.0", submodules=True)
+    version(
+        "fppe-logreg-v1.0",
+        tag="fppe-logreg-v1.0",
+        commit="5ccb14ffffd65e70db048e143ec2350b97a20d40",
+        submodules=True,
+    )
     version("master", branch="master", preferred=True, submodules=True)
 
     variant("shared", default=True, description="Build shared library.")
diff --git a/var/spack/repos/builtin/packages/pandaseq/package.py b/var/spack/repos/builtin/packages/pandaseq/package.py
index 1594b5399d489a..edcdfaabc1aedf 100644
--- a/var/spack/repos/builtin/packages/pandaseq/package.py
+++ b/var/spack/repos/builtin/packages/pandaseq/package.py
@@ -21,7 +21,7 @@ class Pandaseq(AutotoolsPackage):
     depends_on("automake", type="build")
     depends_on("libtool", type=("build", "link"))
     depends_on("m4", type="build")
-    depends_on("zlib", type="build")
+    depends_on("zlib-api", type="build")
     depends_on("pkgconfig", type="build")
     depends_on("bzip2", type="link")
 
diff --git a/var/spack/repos/builtin/packages/pango/package.py b/var/spack/repos/builtin/packages/pango/package.py
index 8e423a762d34dd..06c68357b4149e 100644
--- a/var/spack/repos/builtin/packages/pango/package.py
+++ b/var/spack/repos/builtin/packages/pango/package.py
@@ -127,14 +127,10 @@ def configure_args(self):
         args.append("GTKDOC_MKPDF={0}".format(true))
         args.append("GTKDOC_REBASE={0}".format(true))
 
-        if self.spec["cairo"].satisfies("~shared"):
-            ldflags = [self.spec["pixman"].libs.search_flags]
-            libs = [self.spec["pixman"].libs.link_flags]
-            if self.spec["cairo"].satisfies("+png"):
-                ldflags.append(self.spec["libpng"].libs.search_flags)
-                libs.append(self.spec["libpng"].libs.link_flags)
-            args.append("LDFLAGS=%s" % " ".join(ldflags))
-            args.append("LIBS=%s" % " ".join(libs))
+        if self.spec.satisfies("^cairo ~shared"):
+            pkgconfig = which("pkg-config")
+            cairo_libs = pkgconfig("cairo", "--static", "--libs", output=str).strip()
+            args.append(f"LIBS={cairo_libs}")
 
         return args
 
diff --git a/var/spack/repos/builtin/packages/papi/package.py b/var/spack/repos/builtin/packages/papi/package.py
index 1adf089dcff059..feacc7e91657da 100644
--- a/var/spack/repos/builtin/packages/papi/package.py
+++ b/var/spack/repos/builtin/packages/papi/package.py
@@ -22,12 +22,12 @@ class Papi(AutotoolsPackage, ROCmPackage):
     components that expose performance measurement opportunities
     across the hardware and software stack."""
 
-    homepage = "https://icl.cs.utk.edu/papi/index.html"
+    homepage = "https://icl.utk.edu/papi/"
     maintainers("G-Ragghianti")
 
     tags = ["e4s"]
 
-    url = "https://icl.cs.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz"
+    url = "https://icl.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz"
     git = "https://github.com/icl-utk-edu/papi"
 
     version("master", branch="master")
@@ -54,6 +54,7 @@ class Papi(AutotoolsPackage, ROCmPackage):
     variant("shared", default=True, description="Build shared libraries")
     # PAPI requires building static libraries, so there is no "static" variant
     variant("static_tools", default=False, description="Statically link the PAPI tools")
+    variant("debug", default=False, description="Enable debug symbols in PAPI")
     # The PAPI configure option "--with-shlib-tools" is deprecated
     # and therefore not implemented here
 
@@ -90,6 +91,7 @@ def setup_build_environment(self, env):
             env.set("PAPI_CUDA_ROOT", spec["cuda"].prefix)
         if "+rocm" in spec:
             env.set("PAPI_ROCM_ROOT", spec["hsa-rocr-dev"].prefix)
+            env.set("HSA_TOOLS_LIB", "%s/librocprofiler64.so" % spec["rocprofiler-dev"].prefix.lib)
             env.append_flags("CFLAGS", "-I%s/rocprofiler/include" % spec["rocprofiler-dev"].prefix)
             env.set(
                 "ROCP_METRICS", "%s/rocprofiler/lib/metrics.xml" % spec["rocprofiler-dev"].prefix
@@ -97,9 +99,6 @@ def setup_build_environment(self, env):
             env.set("ROCPROFILER_LOG", "1")
             env.set("HSA_VEN_AMD_AQLPROFILE_LOG", "1")
             env.set("AQLPROFILE_READ_API", "1")
-            # Setting HSA_TOOLS_LIB=librocprofiler64.so (as recommended) doesn't work
-            # due to a conflict between the spack and system-installed versions.
-            env.set("HSA_TOOLS_LIB", "unset")
         if "+rocm_smi" in spec:
             env.append_flags("CFLAGS", "-I%s/rocm_smi" % spec["rocm-smi-lib"].prefix.include)
         #
@@ -146,6 +145,9 @@ def configure_args(self):
         if "+static_tools" in spec:
             options.append("--with-static-tools")
 
+        if "+debug" in spec:
+            options.append("--with-debug=yes")
+
         return options
 
     @run_before("configure")
@@ -181,3 +183,26 @@ def fix_darwin_install(self):
                 join_path(self.prefix.lib, "libpapi.dylib"),
             )
             fs.fix_darwin_install_name(self.prefix.lib)
+
+    test_src_dir = "src/smoke_tests"
+    test_requires_compiler = True
+
+    @run_after("install")
+    def cache_test_sources(self):
+        """Copy the example source files after the package is installed to an
+        install test subdirectory for use during `spack test run`."""
+        if os.path.exists(self.test_src_dir):
+            self.cache_extra_test_sources([self.test_src_dir])
+
+    def test_smoke(self):
+        """Compile and run simple code against the installed papi library."""
+        test_dir = join_path(self.test_suite.current_test_cache_dir, self.test_src_dir)
+        if not os.path.exists(test_dir):
+            raise SkipTest("Skipping smoke tests, directory doesn't exist")
+        with working_dir(test_dir, create=False):
+            with spack.util.environment.set_env(PAPIROOT=self.prefix):
+                make()
+                exe_simple = which("simple")
+                exe_simple()
+                exe_threads = which("threads")
+                exe_threads()
diff --git a/var/spack/repos/builtin/packages/parallel-hashmap/package.py b/var/spack/repos/builtin/packages/parallel-hashmap/package.py
new file mode 100644
index 00000000000000..349fafe52ce0e4
--- /dev/null
+++ b/var/spack/repos/builtin/packages/parallel-hashmap/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class ParallelHashmap(CMakePackage):
+    """A family of header-only, very fast and memory-friendly hashmap and btree
+    containers."""
+
+    homepage = "https://github.com/greg7mdp/parallel-hashmap"
+    url = "https://github.com/greg7mdp/parallel-hashmap/archive/refs/tags/v1.3.11.tar.gz"
+
+    version("1.3.11", sha256="0515a681bfb24207013786a7737e9d8561302e656689d8a65ea480bbabab460f")
+
+    depends_on("cmake@3.8:", type="build")
+
+    patch("pthread.patch")
diff --git a/var/spack/repos/builtin/packages/parallel-hashmap/pthread.patch b/var/spack/repos/builtin/packages/parallel-hashmap/pthread.patch
new file mode 100644
index 00000000000000..68dce0b6f52ebd
--- /dev/null
+++ b/var/spack/repos/builtin/packages/parallel-hashmap/pthread.patch
@@ -0,0 +1,11 @@
+diff -ur a/CMakeLists.txt b/CMakeLists.txt
+--- a/CMakeLists.txt	2023-08-01 13:46:16.726147070 -0400
++++ b/CMakeLists.txt	2023-08-01 13:46:36.819307049 -0400
+@@ -197,6 +197,7 @@
+     add_executable(ex_matt examples/matt.cc phmap.natvis)
+     add_executable(ex_mt_word_counter examples/mt_word_counter.cc phmap.natvis)
+ 
++    target_link_libraries(ex_mt_word_counter Threads::Threads)
+     target_link_libraries(ex_knucleotide Threads::Threads)
+     target_link_libraries(ex_bench Threads::Threads)
+ endif()
diff --git a/var/spack/repos/builtin/packages/parallel-netcdf/package.py b/var/spack/repos/builtin/packages/parallel-netcdf/package.py
index b2ee73ee4a2b9f..a753da24bd9e0f 100644
--- a/var/spack/repos/builtin/packages/parallel-netcdf/package.py
+++ b/var/spack/repos/builtin/packages/parallel-netcdf/package.py
@@ -111,12 +111,10 @@ def autoreconf(self, spec, prefix):
 
     def configure_args(self):
         if self.spec["mpi"].satisfies("intel-oneapi-mpi"):
-            prefix = os.path.join(self.spec["mpi"].prefix,
-                                  "mpi", str(self.spec["mpi"].version))
+            prefix = os.path.join(self.spec["mpi"].prefix, "mpi", str(self.spec["mpi"].version))
         else:
             prefix = self.spec["mpi"].prefix
-        args = ["--with-mpi=%s" % prefix,
-                "SEQ_CC=%s" % spack_cc]
+        args = ["--with-mpi=%s" % prefix, "SEQ_CC=%s" % spack_cc]
 
         args += self.enable_or_disable("cxx")
         args += self.enable_or_disable("fortran")
diff --git a/var/spack/repos/builtin/packages/parallelio/package.py b/var/spack/repos/builtin/packages/parallelio/package.py
index fa1761af113b4d..8930de7164dbc3 100644
--- a/var/spack/repos/builtin/packages/parallelio/package.py
+++ b/var/spack/repos/builtin/packages/parallelio/package.py
@@ -17,6 +17,8 @@ class Parallelio(CMakePackage):
 
     maintainers("jedwards4b")
 
+    version("2.6.2", sha256="c318894f0230197458917e932ec66301b4407a744df481e9c6a6d9d85f7e5ab1")
+    version("2.6.1", sha256="83d3108d2b9db8219aa6b6ee333cfc12b2a588bcfc781587df5f8b24a716a6eb")
     version("2.6.0", sha256="e56a980c71c7f57f396a88beae08f1670d4adf59be6411cd573fe85868ef98c0")
     version("2.5.10", sha256="fac694827c81434a7766976711ba7179940e361e8ed0c189c7b397fd44d401de")
     version("2.5.9", sha256="e5dbc153d8637111de3a51a9655660bf15367d55842de78240dcfc024380553d")
@@ -36,8 +38,14 @@ class Parallelio(CMakePackage):
     )
     variant("mpi", default=True, description="Use mpi to build, otherwise use mpi-serial")
 
+    # This patch addresses building pio2.6.1 with serial netcdf, the issue is netcdf filters
+    patch("serial261.patch", when="@2.6.1")
     patch("remove_redefinition_of_mpi_offset.patch", when="@:2.5.6")
 
+    # This patch addresses an issue when compiling pio2.6.0 with a serial netcdf library.
+    # netcdf4 filters are only available with the parallel build of netcdf.
+    patch("pio_260.patch", when="@2.6.0")
+
     depends_on("cmake@3.7:", type="build")
     depends_on("mpi", when="+mpi")
     depends_on("mpi-serial", when="~mpi")
@@ -47,6 +55,11 @@ class Parallelio(CMakePackage):
     depends_on("parallel-netcdf", type="link", when="+pnetcdf")
 
     resource(name="genf90", git="https://github.com/PARALLELIO/genf90.git", tag="genf90_200608")
+    resource(
+        name="CMake_Fortran_utils",
+        git="https://github.com/CESM-Development/CMake_Fortran_utils.git",
+        tag="CMake_Fortran_utils_150308",
+    )
 
     # Allow argument mismatch in gfortran versions > 10 for mpi library compatibility
     patch("gfortran.patch", when="@:2.5.8 +fortran %gcc@10:")
diff --git a/var/spack/repos/builtin/packages/parallelio/pio_260.patch b/var/spack/repos/builtin/packages/parallelio/pio_260.patch
new file mode 100644
index 00000000000000..1ada18b3063ef9
--- /dev/null
+++ b/var/spack/repos/builtin/packages/parallelio/pio_260.patch
@@ -0,0 +1,20 @@
+diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h
+index c360ae4e..79cc48eb 100644
+--- a/src/clib/pio_internal.h
++++ b/src/clib/pio_internal.h
+@@ -17,11 +17,13 @@
+ #include 
+ #include 
+ #include 
+-#ifdef _NETCDF4
+-#include 
++#ifdef PIO_HAS_PAR_FILTERS
+ #include 
+ #include 
+ #endif
++#ifdef _NETCDF4
++#include 
++#endif
+ #ifdef _PNETCDF
+ #include 
+ #endif
diff --git a/var/spack/repos/builtin/packages/parallelio/serial261.patch b/var/spack/repos/builtin/packages/parallelio/serial261.patch
new file mode 100644
index 00000000000000..a69db1f791a9a9
--- /dev/null
+++ b/var/spack/repos/builtin/packages/parallelio/serial261.patch
@@ -0,0 +1,36 @@
+diff --git a/src/clib/pio.h b/src/clib/pio.h
+index 0aea5a5f..767de18f 100644
+--- a/src/clib/pio.h
++++ b/src/clib/pio.h
+@@ -1267,9 +1267,8 @@ extern "C" {
+                                const long long *op);
+     int PIOc_put_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                                 const unsigned long long *op);
+-/* use this variable in the NETCDF library (introduced in v4.9.0) to determine if the following
+-   functions are available */
+-#ifdef NC_HAS_MULTIFILTERS
++
++#ifdef PIO_HAS_PAR_FILTERS
+   int PIOc_def_var_filter(int ncid, int varid,unsigned int id, size_t nparams, unsigned int *params);
+   int PIOc_inq_var_filter_ids(int ncid, int varid, size_t *nfiltersp, unsigned int *ids);
+   int PIOc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t *nparamsp, unsigned int *params );
+diff --git a/src/ncint/ncintdispatch.c b/src/ncint/ncintdispatch.c
+index a77396bd..3dce9d2c 100644
+--- a/src/ncint/ncintdispatch.c
++++ b/src/ncint/ncintdispatch.c
+@@ -127,6 +127,7 @@ NC_Dispatch NCINT_dispatcher = {
+     NC_NOTNC4_def_var_filter,
+     NC_NOTNC4_set_var_chunk_cache,
+     NC_NOTNC4_get_var_chunk_cache,
++#ifdef PIO_HAS_PAR_FILTERS
+ #if NC_DISPATCH_VERSION == 2
+     PIO_NCINT_filter_actions,
+ #endif
+@@ -141,6 +142,7 @@ NC_Dispatch NCINT_dispatcher = {
+ #if NC_DISPATCH_VERSION >= 5
+     PIOc_inq_filter_avail,
+ #endif
++#endif
+ };
+ 
+ /**
diff --git a/var/spack/repos/builtin/packages/parallelmergetree/package.py b/var/spack/repos/builtin/packages/parallelmergetree/package.py
index f39e709c06f0de..d9934e0542a8bf 100644
--- a/var/spack/repos/builtin/packages/parallelmergetree/package.py
+++ b/var/spack/repos/builtin/packages/parallelmergetree/package.py
@@ -21,6 +21,7 @@ class Parallelmergetree(CMakePackage):
         "1.1.2",
         git="https://bitbucket.org/cedmav/parallelmergetree.git",
         tag="v1.1.2",
+        commit="22ec85177a66b3850ec3aa8ae73da4ad187f6156",
         submodules=True,
     )
 
@@ -28,6 +29,7 @@ class Parallelmergetree(CMakePackage):
         "1.1.1",
         git="https://bitbucket.org/cedmav/parallelmergetree.git",
         tag="v1.1.1",
+        commit="d4b56978dd1b9c9d62d5dd3a0caadfc3102bdf42",
         submodules=True,
     )
 
@@ -35,6 +37,7 @@ class Parallelmergetree(CMakePackage):
         "1.1.0",
         git="https://bitbucket.org/cedmav/parallelmergetree.git",
         tag="v1.1.0",
+        commit="4a5a81b2c857eda9599f257de8719f68f31a5900",
         submodules=True,
     )
 
@@ -42,6 +45,7 @@ class Parallelmergetree(CMakePackage):
         "1.0.2",
         git="https://bitbucket.org/cedmav/parallelmergetree.git",
         tag="v1.0.2",
+        commit="c0b1f305d7f8e3d0bf7cffe22c1e2ac1c0faf05e",
         submodules=True,
     )
 
@@ -49,6 +53,7 @@ class Parallelmergetree(CMakePackage):
         "1.0.0",
         git="https://bitbucket.org/cedmav/parallelmergetree.git",
         tag="v1.0.0",
+        commit="9cfb68fdf0f8e881a4bfd94ae5d3ae25c9e01ea6",
         submodules=True,
     )
 
diff --git a/var/spack/repos/builtin/packages/paraver/package.py b/var/spack/repos/builtin/packages/paraver/package.py
index 7cb1942da31c31..7764dcfc7033c2 100644
--- a/var/spack/repos/builtin/packages/paraver/package.py
+++ b/var/spack/repos/builtin/packages/paraver/package.py
@@ -36,7 +36,7 @@ class Paraver(Package):
     depends_on("wxwidgets@2.8:")  # NOTE: using external for this one is usually simpler
     depends_on("wxpropgrid@1.4:")
     depends_on("libxml2")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         os.chdir("ptools_common_files")
diff --git a/var/spack/repos/builtin/packages/paraview/FindFreetype.cmake.patch b/var/spack/repos/builtin/packages/paraview/FindFreetype.cmake.patch
index ebb5f7c29c91db..7e20b0a57180b4 100644
--- a/var/spack/repos/builtin/packages/paraview/FindFreetype.cmake.patch
+++ b/var/spack/repos/builtin/packages/paraview/FindFreetype.cmake.patch
@@ -1,8 +1,9 @@
+Submodule VTK contains modified content
 diff --git a/VTK/CMake/FindFreetype.cmake b/VTK/CMake/FindFreetype.cmake
-index b4532735c2..f06d32327e 100644
+index b4532735c2..51671d4c3c 100644
 --- a/VTK/CMake/FindFreetype.cmake
 +++ b/VTK/CMake/FindFreetype.cmake
-@@ -63,6 +63,30 @@ directory of a Freetype installation.
+@@ -63,6 +63,32 @@ directory of a Freetype installation.
  # I'm going to attempt to cut out the middleman and hope
  # everything still works.
  
@@ -14,6 +15,8 @@ index b4532735c2..f06d32327e 100644
 +    get_target_property(freetype_location freetype LOCATION)
 +    if (freetype_library_type STREQUAL STATIC_LIBRARY)
 +      set(freetype_library_type STATIC)
++    elseif (freetype_library_type STREQUAL SHARED_LIBRARY)
++      set(freetype_library_type SHARED)
 +    endif()
 +    get_target_property(freetype_interface_include_directories freetype INTERFACE_INCLUDE_DIRECTORIES)
 +    get_target_property(freetype_interface_link_libraries freetype INTERFACE_LINK_LIBRARIES)
diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py
index b5c5cf7ca4f36f..5ca64f29c08610 100644
--- a/var/spack/repos/builtin/packages/paraview/package.py
+++ b/var/spack/repos/builtin/packages/paraview/package.py
@@ -29,10 +29,11 @@ class Paraview(CMakePackage, CudaPackage, ROCmPackage):
 
     version("master", branch="master", submodules=True)
     version(
-        "5.11.1",
-        sha256="5cc2209f7fa37cd3155d199ff6c3590620c12ca4da732ef7698dec37fa8dbb34",
+        "5.11.2",
+        sha256="5c5d2f922f30d91feefc43b4a729015dbb1459f54c938896c123d2ac289c7a1e",
         preferred=True,
     )
+    version("5.11.1", sha256="5cc2209f7fa37cd3155d199ff6c3590620c12ca4da732ef7698dec37fa8dbb34")
     version("5.11.0", sha256="9a0b8fe8b1a2cdfd0ace9a87fa87e0ec21ee0f6f0bcb1fdde050f4f585a25165")
     version("5.10.1", sha256="520e3cdfba4f8592be477314c2f6c37ec73fb1d5b25ac30bdbd1c5214758b9c2")
     version("5.10.0", sha256="86d85fcbec395cdbc8e1301208d7c76d8f48b15dc6b967ffbbaeee31242343a5")
@@ -71,6 +72,7 @@ class Paraview(CMakePackage, CudaPackage, ROCmPackage):
     variant("kits", default=True, description="Use module kits")
     variant("pagosa", default=False, description="Build the pagosa adaptor")
     variant("eyedomelighting", default=False, description="Enable Eye Dome Lighting feature")
+    variant("nvindex", default=False, description="Enable the pvNVIDIAIndeX plugin")
     variant("tbb", default=False, description="Enable multi-threaded parallelism with TBB")
     variant("adios2", default=False, description="Enable ADIOS2 support", when="@5.8:")
     variant("visitbridge", default=False, description="Enable VisItBridge support")
@@ -139,9 +141,6 @@ class Paraview(CMakePackage, CudaPackage, ROCmPackage):
         msg="Use paraview@5.9.0 with %xl_r. Earlier versions are not able to build with xl.",
     )
 
-    # Newer abseil-cpp requires C++14, but paraview uses C++11 by default
-    conflicts("^abseil-cpp@2023:")
-
     # We only support one single Architecture
     for _arch, _other_arch in itertools.permutations(CudaPackage.cuda_arch_values, 2):
         conflicts(
@@ -226,10 +225,15 @@ class Paraview(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("protobuf@3.4:3.18", when="@:5.10%intel@2021:")
     depends_on("protobuf@3.4:3.18", when="@:5.10%xl")
     depends_on("protobuf@3.4:3.18", when="@:5.10%xl_r")
+    # protobuf requires newer abseil-cpp, which in turn requires C++14,
+    # but paraview uses C++11 by default. Use for 5.11+ until ParaView updates
+    # its C++ standard level.
+    depends_on("protobuf@3.4:3.21", when="@5.11:")
+    depends_on("protobuf@3.4:3.21", when="@master")
     depends_on("libxml2")
     depends_on("lz4")
     depends_on("xz")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libcatalyst@2:", when="+libcatalyst")
     depends_on("hip@5.2:", when="+rocm")
     for target in ROCmPackage.amdgpu_targets:
@@ -252,6 +256,7 @@ class Paraview(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("nlohmann-json", when="@5.11:")
 
     # ParaView depends on proj@8.1.0 due to changes in MR
+    # v8.1.0 is required for VTK::GeoVis
     # https://gitlab.kitware.com/vtk/vtk/-/merge_requests/8474
     depends_on("proj@8.1.0", when="@5.11:")
 
@@ -277,7 +282,9 @@ class Paraview(CMakePackage, CudaPackage, ROCmPackage):
 
     # Fix IOADIOS2 module to work with kits
     # https://gitlab.kitware.com/vtk/vtk/-/merge_requests/8653
-    patch("vtk-adios2-module-no-kit.patch", when="@5.8:")
+    patch("vtk-adios2-module-no-kit.patch", when="@5.8:5.11")
+    # https://gitlab.kitware.com/vtk/vtk/-/merge_requests/8653
+    patch("vtk-adios2-module-no-kit-5.12.patch", when="@5.12:")
 
     # Patch for paraview 5.9.0%xl_r
     # https://gitlab.kitware.com/vtk/vtk/-/merge_requests/7591
@@ -423,6 +430,10 @@ def nvariant_bool(feature):
             self.define_from_variant("VISIT_BUILD_READER_Silo", "visitbridge"),
         ]
 
+        if spec.satisfies("@5.12:"):
+            cmake_args.append("-DVTK_MODULE_USE_EXTERNAL_VTK_fast_float:BOOL=OFF")
+            cmake_args.append("-DVTK_MODULE_USE_EXTERNAL_VTK_token:BOOL=OFF")
+
         if spec.satisfies("@5.11:"):
             cmake_args.append("-DVTK_MODULE_USE_EXTERNAL_VTK_verdict:BOOL=OFF")
 
@@ -607,6 +618,9 @@ def nvariant_bool(feature):
         if "+tbb" in spec:
             cmake_args.append("-DVTK_SMP_IMPLEMENTATION_TYPE=TBB")
 
+        if "+nvindex" in spec:
+            cmake_args.append("-DPARAVIEW_PLUGIN_ENABLE_pvNVIDIAIndeX:BOOL=ON")
+
         # Hide git from Paraview so it will not use `git describe`
         # to find its own version number
         if spec.satisfies("@5.4.0:5.4.1"):
diff --git a/var/spack/repos/builtin/packages/paraview/vtk-adios2-module-no-kit-5.12.patch b/var/spack/repos/builtin/packages/paraview/vtk-adios2-module-no-kit-5.12.patch
new file mode 100644
index 00000000000000..34a98eac474716
--- /dev/null
+++ b/var/spack/repos/builtin/packages/paraview/vtk-adios2-module-no-kit-5.12.patch
@@ -0,0 +1,230 @@
+diff --git a/VTK/IO/ADIOS2/CMakeLists.txt b/VTK/IO/ADIOS2/CMakeLists.txt
+index 86c6d49cc4f..07b1d4fe0ef 100644
+--- a/VTK/IO/ADIOS2/CMakeLists.txt
++++ b/VTK/IO/ADIOS2/CMakeLists.txt
+@@ -1,9 +1,9 @@
+ vtk_module_find_package(PRIVATE_IF_SHARED
+   PACKAGE ADIOS2
+   VERSION 2.4)
+-if (VTK_USE_MPI AND NOT ADIOS2_HAVE_MPI)
++if (TARGET VTK::ParallelMPI AND NOT ADIOS2_HAVE_MPI)
+   message(FATAL_ERROR "VTK built with MPI requires ADIOS2 built with MPI")
+-elseif(NOT VTK_USE_MPI AND ADIOS2_HAVE_MPI)
++elseif(NOT TARGET VTK::ParallelMPI AND ADIOS2_HAVE_MPI)
+   message(FATAL_ERROR "VTK built without MPI requires ADIOS2 built without MPI")
+ endif()
+ 
+@@ -18,38 +18,30 @@ set(classes_core vtkADIOS2CoreImageReader)
+ set(private_classes_core Core/vtkADIOS2CoreArraySelection)
+ set(private_headers_core Core/vtkADIOS2CoreTypeTraits.h)
+ set(private_templates_core)
+-set(vtk_io_adios2_core_enabled TRUE CACHE INTERNAL "" FORCE)
+ 
+-if (vtk_io_adios2_core_enabled)
+-  list(APPEND classes   ${classes_core})
+-  list(APPEND private_classes   ${private_classes_core})
+-  list(APPEND private_headers   ${private_headers_core})
+-  list(APPEND private_templates ${private_templates_core})
+-endif()
++list(APPEND classes   ${classes_core})
++list(APPEND private_classes   ${private_classes_core})
++list(APPEND private_headers   ${private_headers_core})
++list(APPEND private_templates ${private_templates_core})
++
++# Build VTX Schema for Parallel
++if (TARGET VTK::ParallelMPI)
++  set(classes_vtx vtkADIOS2VTXReader)
++  set(private_classes_vtx
++    VTX/VTXSchemaManager
++    VTX/common/VTXDataArray
++    VTX/common/VTXHelper
++    VTX/schema/VTXSchema
++    VTX/schema/vtk/VTXvtkBase
++    VTX/schema/vtk/VTXvtkVTI
++    VTX/schema/vtk/VTXvtkVTU)
++  set(private_headers_vtx VTX/common/VTXTypes.h)
++  set(private_templates_vtx
++    VTX/common/VTXHelper.txx
++    VTX/schema/VTXSchema.txx
++    VTX/schema/vtk/VTXvtkVTI.txx
++    VTX/schema/vtk/VTXvtkVTU.txx)
+ 
+-set(classes_vtx vtkADIOS2VTXReader)
+-set(private_classes_vtx
+-  VTX/VTXSchemaManager
+-  VTX/common/VTXDataArray
+-  VTX/common/VTXHelper
+-  VTX/schema/VTXSchema
+-  VTX/schema/vtk/VTXvtkBase
+-  VTX/schema/vtk/VTXvtkVTI
+-  VTX/schema/vtk/VTXvtkVTU)
+-set(private_headers_vtx VTX/common/VTXTypes.h)
+-set(private_templates_vtx
+-  VTX/common/VTXHelper.txx
+-  VTX/schema/VTXSchema.txx
+-  VTX/schema/vtk/VTXvtkVTI.txx
+-  VTX/schema/vtk/VTXvtkVTU.txx)
+-
+-if (VTK_USE_MPI)
+-  set(vtk_io_adios2_vtx_enabled TRUE CACHE INTERNAL "" FORCE)
+-else ()
+-  set(vtk_io_adios2_vtx_enabled FALSE CACHE INTERNAL "" FORCE)
+-endif()
+-
+-if (vtk_io_adios2_vtx_enabled)
+   list(APPEND classes   ${classes_vtx})
+   list(APPEND private_classes   ${private_classes_vtx})
+   list(APPEND private_headers   ${private_headers_vtx})
+@@ -63,10 +55,6 @@ vtk_module_add_module(VTK::IOADIOS2
+   PRIVATE_TEMPLATES ${private_templates})
+ vtk_module_link(VTK::IOADIOS2 PRIVATE adios2::adios2)
+ 
+-if (ADIOS2_HAVE_MPI)
+-  vtk_module_definitions(VTK::IOADIOS2 PRIVATE IOADIOS2_HAVE_MPI)
+-endif ()
+-
+ if (ADIOS2_VERSION VERSION_GREATER_EQUAL "2.8.0")
+   vtk_module_definitions(VTK::IOADIOS2 PRIVATE IOADIOS2_BP5_RANDOM_ACCESS)
+ endif ()
+diff --git a/VTK/IO/ADIOS2/Testing/Cxx/CMakeLists.txt b/VTK/IO/ADIOS2/Testing/Cxx/CMakeLists.txt
+index 1534a1e7271..29c51970daf 100644
+--- a/VTK/IO/ADIOS2/Testing/Cxx/CMakeLists.txt
++++ b/VTK/IO/ADIOS2/Testing/Cxx/CMakeLists.txt
+@@ -2,40 +2,34 @@ find_package(ADIOS2 2.4 REQUIRED
+   COMPONENTS CXX
+   OPTIONAL_COMPONENTS MPI)
+ 
+-if (ADIOS2_HAVE_MPI)
+-  if (vtk_io_adios2_core_enabled)
+-    set(TestADIOS2BPReaderSingleTimeStep_NUMPROCS 2)
++if (TARGET VTK::ParallelMPI)
++  set(TestADIOS2BPReaderSingleTimeStep_NUMPROCS 2)
+   # For now vtkMultiBlockVolumeMapper does not support rendering in parallel
+-    set(TestADIOS2BPReaderMultiTimeSteps_NUMPROCS 2)
+-    set(TestADIOS2BPReaderMultiTimeSteps2D_NUMPROCS 2)
+-    vtk_add_test_mpi(vtkIOADIOS2CxxTests-MPI mpiTests TESTING_DATA
+-      TestADIOS2BPReaderMPISingleTimeStep.cxx
+-      TestADIOS2BPReaderMPIMultiTimeSteps3D.cxx,NO_VALID
+-      TestADIOS2BPReaderMPIMultiTimeSteps2D.cxx)
+-    vtk_test_cxx_executable(vtkIOADIOS2CxxTests-MPI mpiTests)
+-  endif()
++  set(TestADIOS2BPReaderMultiTimeSteps_NUMPROCS 2)
++  set(TestADIOS2BPReaderMultiTimeSteps2D_NUMPROCS 2)
++  vtk_add_test_mpi(vtkIOADIOS2CxxTests-MPI mpiTests TESTING_DATA
++    TestADIOS2BPReaderMPISingleTimeStep.cxx
++    TestADIOS2BPReaderMPIMultiTimeSteps3D.cxx,NO_VALID
++    TestADIOS2BPReaderMPIMultiTimeSteps2D.cxx)
++  vtk_test_cxx_executable(vtkIOADIOS2CxxTests-MPI mpiTests)
+ 
+   # VTX tests
+-  if (vtk_io_adios2_vtx_enabled)
+-    vtk_add_test_cxx(vtkIOADIOS2VTXCxxTests tests TESTING_DATA NO_OUTPUT
+-      UnitTestIOADIOS2VTX.cxx,NO_VALID
+-      #TestIOADIOS2VTX_VTI3D.cxx,
+-      TestIOADIOS2VTX_VTI3DRendering.cxx,NO_VALID
+-      #TestIOADIOS2VTX_VTU3D.cxx,NO_VALID
+-      TestIOADIOS2VTX_VTU3DRendering.cxx,NO_VALID
+-      TestIOADIOS2VTX_VTU2DRendering.cxx,NO_VALID
+-      TestIOADIOS2VTX_VTU1DRendering.cxx,NO_VALID)
++  vtk_add_test_cxx(vtkIOADIOS2VTXCxxTests tests TESTING_DATA NO_OUTPUT
++    UnitTestIOADIOS2VTX.cxx,NO_VALID
++    #TestIOADIOS2VTX_VTI3D.cxx,
++    TestIOADIOS2VTX_VTI3DRendering.cxx,NO_VALID
++    #TestIOADIOS2VTX_VTU3D.cxx,NO_VALID
++    TestIOADIOS2VTX_VTU3DRendering.cxx,NO_VALID
++    TestIOADIOS2VTX_VTU2DRendering.cxx,NO_VALID
++    TestIOADIOS2VTX_VTU1DRendering.cxx,NO_VALID)
+ 
+-    vtk_test_cxx_executable(vtkIOADIOS2VTXCxxTests tests)
+-    target_link_libraries(vtkIOADIOS2VTXCxxTests PUBLIC adios2::adios2)
+-  endif ()
++  vtk_test_cxx_executable(vtkIOADIOS2VTXCxxTests tests)
++  target_link_libraries(vtkIOADIOS2VTXCxxTests PUBLIC adios2::adios2)
+ else ()
+-  if (vtk_io_adios2_core_enabled)
+-    vtk_add_test_cxx(vtkIOADIOS2CxxTests tests TESTING_DATA
+-      TestADIOS2BPReaderSingleTimeStep.cxx
+-      TestADIOS2BPReaderMultiTimeSteps3D.cxx
+-      TestADIOS2BPReaderMultiTimeSteps2D.cxx)
++  vtk_add_test_cxx(vtkIOADIOS2CxxTests tests TESTING_DATA
++    TestADIOS2BPReaderSingleTimeStep.cxx
++    TestADIOS2BPReaderMultiTimeSteps3D.cxx
++    TestADIOS2BPReaderMultiTimeSteps2D.cxx)
+ 
+-    vtk_test_cxx_executable(vtkIOADIOS2CxxTests tests)
+-  endif ()
++  vtk_test_cxx_executable(vtkIOADIOS2CxxTests tests)
+ endif ()
+diff --git a/VTK/IO/ADIOS2/vtk.module b/VTK/IO/ADIOS2/vtk.module
+index 5069bd828b0..fe37260eb6d 100644
+--- a/VTK/IO/ADIOS2/vtk.module
++++ b/VTK/IO/ADIOS2/vtk.module
+@@ -2,8 +2,6 @@ NAME
+   VTK::IOADIOS2
+ LIBRARY_NAME
+   vtkIOADIOS2
+-KIT
+-  VTK::IO
+ SPDX_LICENSE_IDENTIFIER
+   LicenseRef-BSD-3-Clause-Sandia-USGov
+ SPDX_COPYRIGHT_TEXT
+diff --git a/VTK/IO/ADIOS2/vtkADIOS2CoreImageReader.cxx b/VTK/IO/ADIOS2/vtkADIOS2CoreImageReader.cxx
+index 6ba4d25230d..c209fd905d5 100644
+--- a/VTK/IO/ADIOS2/vtkADIOS2CoreImageReader.cxx
++++ b/VTK/IO/ADIOS2/vtkADIOS2CoreImageReader.cxx
+@@ -28,7 +28,7 @@
+ #include "vtkLongLongArray.h"
+ #include "vtkMultiBlockDataSet.h"
+ #include "vtkMultiPieceDataSet.h"
+-#include "vtkMultiProcessController.h"
++#include "vtkMultiProcessController.h" // For the MPI controller member
+ #include "vtkNew.h"
+ #include "vtkObjectFactory.h"
+ #include "vtkPointData.h"
+@@ -46,7 +46,7 @@
+ #include "vtkUnstructuredGrid.h"
+ #include "vtksys/SystemTools.hxx"
+ 
+-#ifdef IOADIOS2_HAVE_MPI
++#if VTK_MODULE_ENABLE_VTK_ParallelMPI
+ #include "vtkMPI.h"
+ #include "vtkMPIController.h"
+ #endif
+@@ -126,7 +126,7 @@ vtkNew vtkADIOS2CoreImageReader::vtkADIOS2CoreImageReaderI
+   int myLen = static_cast(ibds->GetNumberOfBlocks());
+   int* allLens{ nullptr };
+   int procId{ 0 }, numProcess{ 0 };
+-#ifdef IOADIOS2_HAVE_MPI
++#if VTK_MODULE_ENABLE_VTK_ParallelMPI
+   auto ctrl = vtkMultiProcessController::GetGlobalController();
+   if (ctrl)
+   {
+@@ -286,7 +286,7 @@ const vtkADIOS2CoreImageReader::StringToParams& vtkADIOS2CoreImageReader::GetAva
+ //------------------------------------------------------------------------------
+ void vtkADIOS2CoreImageReader::SetController(vtkMultiProcessController* controller)
+ {
+-#ifdef IOADIOS2_HAVE_MPI
++#if VTK_MODULE_ENABLE_VTK_ParallelMPI
+   vtkMPIController* mpiController = vtkMPIController::SafeDownCast(controller);
+   if (controller && !mpiController)
+   {
+@@ -337,7 +337,7 @@ bool vtkADIOS2CoreImageReader::OpenAndReadMetaData()
+   // Initialize the ADIOS2 data structures
+   if (!this->Impl->Adios)
+   {
+-#ifdef IOADIOS2_HAVE_MPI
++#if VTK_MODULE_ENABLE_VTK_ParallelMPI
+     // Make sure the ADIOS subsystem is initialized before processing any
+     // sort of request.
+     if (!this->Controller)
+@@ -910,7 +910,7 @@ void vtkADIOS2CoreImageReader::CalculateWorkDistribution(const std::string& varN
+   auto var = this->Impl->AdiosIO.InquireVariable(varName);
+   size_t blockNum = this->Impl->BpReader.BlocksInfo(var, this->Impl->RequestStep).size();
+ 
+-#ifdef IOADIOS2_HAVE_MPI
++#if VTK_MODULE_ENABLE_VTK_ParallelMPI
+   size_t rank = static_cast(this->Controller->GetLocalProcessId());
+   size_t procs = static_cast(this->Controller->GetNumberOfProcesses());
+ #else
+-- 
+GitLab
diff --git a/var/spack/repos/builtin/packages/pastix/package.py b/var/spack/repos/builtin/packages/pastix/package.py
index 871e7cb61e09a1..ecd303784fdc0f 100644
--- a/var/spack/repos/builtin/packages/pastix/package.py
+++ b/var/spack/repos/builtin/packages/pastix/package.py
@@ -17,6 +17,7 @@ class Pastix(CMakePackage, CudaPackage):
     maintainers("fpruvost", "mfaverge", "ramet")
 
     version("master", branch="master", submodules=True)
+    version("6.3.0", sha256="a6bfec32a3279d7b24c5fc05885c6632d177e467f1584707c6fd7c42a8703c3e")
     version("6.2.2", sha256="cce9a1fe4678b5733c9f1a5a52f77b040eadc3e254418c6fb03d8ab37dede508")
     version("6.2.1", sha256="b680cbfc265df8cba18d3a7093fcc02e260198c4a2d6a86d1e684bb291e309dd")
 
diff --git a/var/spack/repos/builtin/packages/patchelf/513.patch b/var/spack/repos/builtin/packages/patchelf/513.patch
new file mode 100644
index 00000000000000..e5e8dd7174c33c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/patchelf/513.patch
@@ -0,0 +1,25 @@
+From 5fb5d82637c1b547b800b5994a1f5342b3224da4 Mon Sep 17 00:00:00 2001
+From: Rosen Penev 
+Date: Sat, 12 Aug 2023 11:46:14 -0700
+Subject: [PATCH] fix compilation with GCC7
+
+CTAD is not working here.
+
+Signed-off-by: Rosen Penev 
+---
+ src/patchelf.cc | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/patchelf.cc b/src/patchelf.cc
+index 82b4b46c..d6000160 100644
+--- a/src/patchelf.cc
++++ b/src/patchelf.cc
+@@ -2069,7 +2069,7 @@ void ElfFile::rebuildGnuHashTable(span strTab, span> tmp(dst.begin(), dst.end());
+         for (size_t i = 0; i < tmp.size(); ++i)
+             dst[old2new[i]] = tmp[i];
+     };
diff --git a/var/spack/repos/builtin/packages/patchelf/package.py b/var/spack/repos/builtin/packages/patchelf/package.py
index b021a07ac89453..2630085d4eeaf7 100644
--- a/var/spack/repos/builtin/packages/patchelf/package.py
+++ b/var/spack/repos/builtin/packages/patchelf/package.py
@@ -18,6 +18,7 @@ class Patchelf(AutotoolsPackage):
 
     maintainers("haampie")
 
+    version("0.18.0", sha256="64de10e4c6b8b8379db7e87f58030f336ea747c0515f381132e810dbf84a86e7")
     version("0.17.2", sha256="20427b718dd130e4b66d95072c2a2bd5e17232e20dad58c1bea9da81fae330e0")
     version("0.16.1", sha256="1a562ed28b16f8a00456b5f9ee573bb1af7c39c1beea01d94fc0c7b3256b0406")
     version("0.15.0", sha256="53a8d58ed4e060412b8fdcb6489562b3c62be6f65cee5af30eba60f4423bfa0f")
@@ -38,6 +39,9 @@ class Patchelf(AutotoolsPackage):
     conflicts("%gcc@:6", when="@0.14:", msg="Requires C++17 support")
     conflicts("%clang@:3", when="@0.14:", msg="Requires C++17 support")
 
+    # GCC 7.5 doesn't have __cpp_deduction_guides >= 201606
+    patch("513.patch", when="@0.18: %gcc@:7")
+
     def url_for_version(self, version):
         if version < Version("0.12"):
             return "https://nixos.org/releases/patchelf/patchelf-{0}/patchelf-{1}.tar.gz".format(
diff --git a/var/spack/repos/builtin/packages/pbbam/package.py b/var/spack/repos/builtin/packages/pbbam/package.py
index 5e76b223681746..83361112f78cf0 100644
--- a/var/spack/repos/builtin/packages/pbbam/package.py
+++ b/var/spack/repos/builtin/packages/pbbam/package.py
@@ -23,7 +23,7 @@ class Pbbam(MesonPackage):
     )
     version("0.18.0", sha256="45286e5f7deb7ff629e0643c8a416155915aec7b85d54c60b5cdc07f4d7b234a")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("boost@1.55.0:")
     depends_on("htslib@1.3.1:")
     # newer versions require C17
diff --git a/var/spack/repos/builtin/packages/pbmpi/package.py b/var/spack/repos/builtin/packages/pbmpi/package.py
index a2739c8a41251c..1fbb07177a0398 100644
--- a/var/spack/repos/builtin/packages/pbmpi/package.py
+++ b/var/spack/repos/builtin/packages/pbmpi/package.py
@@ -10,8 +10,13 @@ class Pbmpi(MakefilePackage):
     """A Bayesian software for phylogenetic reconstruction using mixture models"""
 
     homepage = "https://megasun.bch.umontreal.ca/People/lartillot/www/index.htm"
+    url = "https://github.com/bayesiancook/pbmpi/archive/refs/tags/v1.8c.tar.gz"
     git = "https://github.com/bayesiancook/pbmpi.git"
 
+    maintainers("snehring")
+
+    version("1.9", sha256="567d8db995f23b2b0109c1e6088a7e5621e38fec91d6b2f27abd886b90ea31ce")
+    version("1.8c", sha256="2a80ec4a98d92ace61c67ff9ba78249d45d03094b364959d490b1ad05797a279")
     version("partition", branch="partition")
 
     depends_on("mpi")
diff --git a/var/spack/repos/builtin/packages/pcl/package.py b/var/spack/repos/builtin/packages/pcl/package.py
index f57dfed3871772..e0dd4967aa44cc 100644
--- a/var/spack/repos/builtin/packages/pcl/package.py
+++ b/var/spack/repos/builtin/packages/pcl/package.py
@@ -14,12 +14,29 @@ class Pcl(CMakePackage):
     homepage = "https://pointclouds.org/"
     url = "https://github.com/PointCloudLibrary/pcl/releases/download/pcl-1.11.1/source.tar.gz"
 
+    version("1.13.1", sha256="be4d499c066203a3c296e2f7e823d6209be5983415f2279310ed1c9abb361d30")
+    version("1.13.0", sha256="bd110789f6a7416ed1c58da302afbdb80f8d297a9e23cc02fd78ab78b4762698")
+    version("1.12.1", sha256="a9573efad5e024c02f2cc9180bb8f82605c3772c62463efbe25c5d6e634b91dc")
+    version("1.12.0", sha256="606a2d5c7af304791731d6b8ea79365bc8f2cd75908006484d71ecee01d9b51c")
     version("1.11.1", sha256="19d1a0bee2bc153de47c05da54fc6feb23393f306ab2dea2e25419654000336e")
 
     depends_on("cmake@3.5:", type="build")
+    depends_on("cmake@3.10:", when="@1.12.1:", type="build")
     depends_on("eigen@3.1:")
+    depends_on("eigen@3.3:", when="@1.13:")
     depends_on("flann@1.7:")
-    depends_on("boost@1.55:+filesystem+date_time+iostreams+system")
+    depends_on("flann@1.9.1:", when="@1.12:")
+    depends_on("boost@1.55:")
+    depends_on("boost@1.65:", when="@1.12:")
+    depends_on("boost+filesystem+iostreams+system")
+    depends_on("boost+date_time", when="@:1.13.0")
+
+    # fix build with clang: #30653
+    with when("@:1.12"):
+        patch(
+            "https://github.com/PointCloudLibrary/pcl/commit/dff16af269fbd2c15772d53064882b2bf8c2ffe9.patch?full_index=1",
+            sha256="17a7a7aec8e63701294612cbb25d46ac1ce58f643dbc68e1517329ae0b68956d",
+        )
 
     # TODO: replace this with an explicit list of components of Boost,
     # for instance depends_on('boost +filesystem')
diff --git a/var/spack/repos/builtin/packages/pcre/package.py b/var/spack/repos/builtin/packages/pcre/package.py
index 4e6e04733508ab..4719f5ea379e0f 100644
--- a/var/spack/repos/builtin/packages/pcre/package.py
+++ b/var/spack/repos/builtin/packages/pcre/package.py
@@ -3,10 +3,12 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import spack.build_systems.autotools
+import spack.build_systems.cmake
 from spack.package import *
 
 
-class Pcre(AutotoolsPackage):
+class Pcre(AutotoolsPackage, CMakePackage):
     """The PCRE package contains Perl Compatible Regular Expression
     libraries. These are useful for implementing regular expression
     pattern matching using the same syntax and semantics as Perl 5."""
@@ -26,6 +28,8 @@ class Pcre(AutotoolsPackage):
     maintainers("drkennetz")
     patch("intel.patch", when="@8.38")
 
+    build_system("autotools", "cmake", default="autotools")
+
     variant("jit", default=False, description="Enable JIT support.")
 
     variant("multibyte", default=True, description="Enable support for 16 and 32 bit characters.")
@@ -36,6 +40,8 @@ class Pcre(AutotoolsPackage):
         description="Enable support for UTF-8/16/32, " "incompatible with EBCDIC.",
     )
 
+
+class AutotoolsBuilder(spack.build_systems.autotools.AutotoolsBuilder):
     def configure_args(self):
         args = []
 
@@ -51,3 +57,21 @@ def configure_args(self):
             args.append("--enable-unicode-properties")
 
         return args
+
+
+class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder):
+    def cmake_args(self):
+        args = []
+
+        if "+jit" in self.spec:
+            args.append("-DPCRE_SUPPORT_JIT:BOOL=ON")
+
+        if "+multibyte" in self.spec:
+            args.append("-DPCRE_BUILD_PCRE16:BOOL=ON")
+            args.append("-DPCRE_BUILD_PCRE32:BOOL=ON")
+
+        if "+utf" in self.spec:
+            args.append("-DPCRE_SUPPORT_UTF:BOOL=ON")
+            args.append("-DPCRE_SUPPORT_UNICODE_PROPERTIES:BOOL=ON")
+
+        return args
diff --git a/var/spack/repos/builtin/packages/pdt/package.py b/var/spack/repos/builtin/packages/pdt/package.py
index 4f3ccac0c40e73..c8665aaac27758 100644
--- a/var/spack/repos/builtin/packages/pdt/package.py
+++ b/var/spack/repos/builtin/packages/pdt/package.py
@@ -54,12 +54,12 @@ def configure(self, spec, prefix):
             options.append("-pgCC")
         elif self.compiler.name == "gcc":
             options.append("-GNU")
-        elif self.compiler.name == "clang":
+        elif self.compiler.name == "clang" or self.compiler.name == "apple-clang":
             options.append("-clang")
         elif self.compiler.name == "cce":
             options.append("-CC")
         else:
-            raise InstallError("Unknown/unsupported compiler family")
+            raise InstallError("Unknown/unsupported compiler family: " + self.compiler.name)
 
         if "+pic" in spec:
             options.append("-useropt=" + self.compiler.cxx_pic_flag)
diff --git a/var/spack/repos/builtin/packages/pegtl/package.py b/var/spack/repos/builtin/packages/pegtl/package.py
index 8384ed91281e66..72a4bd67ada63f 100644
--- a/var/spack/repos/builtin/packages/pegtl/package.py
+++ b/var/spack/repos/builtin/packages/pegtl/package.py
@@ -19,6 +19,7 @@ class Pegtl(CMakePackage):
     git = "https://github.com/taocpp/PEGTL.git"
 
     version("master", branch="master")
+    version("3.2.7", sha256="444c3c33686c6b2d8d45ad03af5041b7bc910ef44ac10216237d8e3e8d6e7025")
     version("3.2.0", sha256="91aa6529ef9e6b57368e7b5b1f04a3bd26a39419d30e35a3c5c66ef073926b56")
     version("2.8.3", sha256="370afd0fbe6d73c448a33c10fbe4a7254f92077f5a217317d0a32a9231293015")
     version("2.1.4", sha256="d990dccc07b4d9ba548326d11c5c5e34fa88b34fe113cb5377da03dda29f23f2")
diff --git a/var/spack/repos/builtin/packages/perl-alien-svn/package.py b/var/spack/repos/builtin/packages/perl-alien-svn/package.py
index 78d99ae45964e7..579dca61ad98b6 100644
--- a/var/spack/repos/builtin/packages/perl-alien-svn/package.py
+++ b/var/spack/repos/builtin/packages/perl-alien-svn/package.py
@@ -26,7 +26,7 @@ class PerlAlienSvn(PerlPackage):
     depends_on("apr@1.6.2", type="build")
     depends_on("apr-util", type=("build", "link"))
     depends_on("sqlite", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libbsd")
 
     def setup_build_environment(self, env):
diff --git a/var/spack/repos/builtin/packages/perl-compress-raw-zlib/package.py b/var/spack/repos/builtin/packages/perl-compress-raw-zlib/package.py
index 7deefb592cb4f3..365a6b7d9a82c7 100644
--- a/var/spack/repos/builtin/packages/perl-compress-raw-zlib/package.py
+++ b/var/spack/repos/builtin/packages/perl-compress-raw-zlib/package.py
@@ -15,5 +15,5 @@ class PerlCompressRawZlib(PerlPackage):
     version("2.204", sha256="f161f4297efadbed79c8b096a75951784fc5ccd3170bd32866a19e5c6876d13f")
     version("2.081", sha256="e156de345bd224bbdabfcab0eeb3f678a3099a4e86c9d1b6771d880b55aa3a1b")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("perl-extutils-makemaker", type="build")
diff --git a/var/spack/repos/builtin/packages/perl-dbd-mysql/package.py b/var/spack/repos/builtin/packages/perl-dbd-mysql/package.py
index 12b4a799a8437b..03b2f67747630f 100644
--- a/var/spack/repos/builtin/packages/perl-dbd-mysql/package.py
+++ b/var/spack/repos/builtin/packages/perl-dbd-mysql/package.py
@@ -12,8 +12,14 @@ class PerlDbdMysql(PerlPackage):
     homepage = "https://metacpan.org/pod/DBD::mysql"
     url = "https://search.cpan.org/CPAN/authors/id/M/MI/MICHIELB/DBD-mysql-4.043.tar.gz"
 
+    version(
+        "4.050",
+        sha256="4f48541ff15a0a7405f76adc10f81627c33996fbf56c95c26c094444c0928d78",
+        url="https://cpan.metacpan.org/authors/id/D/DV/DVEEDEN/DBD-mysql-4.050.tar.gz",
+    )
     version("4.043", sha256="629f865e8317f52602b2f2efd2b688002903d2e4bbcba5427cb6188b043d6f99")
 
+    depends_on("perl-devel-checklib", type="build", when="@4.050:")
     depends_on("perl-test-deep", type=("build", "run"))
     depends_on("perl-dbi", type=("build", "run"))
     depends_on("mysql-client")
diff --git a/var/spack/repos/builtin/packages/perl-devel-checklib/package.py b/var/spack/repos/builtin/packages/perl-devel-checklib/package.py
new file mode 100644
index 00000000000000..7bf5077554f05e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/perl-devel-checklib/package.py
@@ -0,0 +1,16 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PerlDevelChecklib(PerlPackage):
+    """Devel::CheckLib - check that a library is available"""
+
+    homepage = "https://metacpan.org/pod/Devel::CheckLib"
+    url = "https://cpan.metacpan.org/authors/id/M/MA/MATTN/Devel-CheckLib-1.16.tar.gz"
+    maintainers("snehring")
+
+    version("1.16", sha256="869d38c258e646dcef676609f0dd7ca90f085f56cf6fd7001b019a5d5b831fca")
diff --git a/var/spack/repos/builtin/packages/perl-http-message/package.py b/var/spack/repos/builtin/packages/perl-http-message/package.py
index b441b2593e60dc..a0bcd57cd8765d 100644
--- a/var/spack/repos/builtin/packages/perl-http-message/package.py
+++ b/var/spack/repos/builtin/packages/perl-http-message/package.py
@@ -21,3 +21,4 @@ class PerlHttpMessage(PerlPackage):
     depends_on("perl-try-tiny", type=("build", "run"))
     depends_on("perl-uri", type=("build", "run"))
     depends_on("perl-http-date", type=("build", "run"))
+    depends_on("perl-clone", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/perl-perlio-gzip/package.py b/var/spack/repos/builtin/packages/perl-perlio-gzip/package.py
index e1b08439ac53ba..fb6756342f4e7f 100644
--- a/var/spack/repos/builtin/packages/perl-perlio-gzip/package.py
+++ b/var/spack/repos/builtin/packages/perl-perlio-gzip/package.py
@@ -15,8 +15,8 @@ class PerlPerlioGzip(PerlPackage):
     version("0.20", sha256="4848679a3f201e3f3b0c5f6f9526e602af52923ffa471a2a3657db786bd3bdc5")
     version("0.19", sha256="d2e9351d58b8a93c86811e25a898ee651fc393a157413652bf42f9aada2eb284")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     def configure_args(self):
-        p = self.spec["zlib"].prefix.include
+        p = self.spec["zlib-api"].prefix.include
         return ["INC=-I{0}".format(p)]
diff --git a/var/spack/repos/builtin/packages/perl-star-fusion/package.py b/var/spack/repos/builtin/packages/perl-star-fusion/package.py
index 78464b004ba909..067e5a558ba10c 100644
--- a/var/spack/repos/builtin/packages/perl-star-fusion/package.py
+++ b/var/spack/repos/builtin/packages/perl-star-fusion/package.py
@@ -17,7 +17,7 @@ class PerlStarFusion(Package):
     homepage = "https://github.com/STAR-Fusion/STAR-Fusion"
     git = "https://github.com/STAR-Fusion/STAR-Fusion.git"
 
-    version("master", commit="8c5a541")
+    version("master", commit="8c5a541ab701debebb68e5c4ee9d6a03e03d65ca")
 
     extends("perl")
 
diff --git a/var/spack/repos/builtin/packages/perl/package.py b/var/spack/repos/builtin/packages/perl/package.py
index 599504b77ace62..299ae19436eae9 100644
--- a/var/spack/repos/builtin/packages/perl/package.py
+++ b/var/spack/repos/builtin/packages/perl/package.py
@@ -32,6 +32,8 @@ class Perl(Package):  # Perl doesn't use Autotools, it should subclass Package
     url = "http://www.cpan.org/src/5.0/perl-5.34.0.tar.gz"
     tags = ["windows"]
 
+    maintainers("LydDeb")
+
     executables = [r"^perl(-?\d+.*)?$"]
 
     # see https://www.cpan.org/src/README.html for
@@ -124,9 +126,9 @@ class Perl(Package):  # Perl doesn't use Autotools, it should subclass Package
         depends_on("gdbm@:1.14.1", when="@:5.28.0")
         depends_on("berkeley-db")
         depends_on("bzip2")
-        depends_on("zlib")
+        depends_on("zlib-api")
         # :5.24.1 needs zlib@:1.2.8: https://rt.cpan.org/Public/Bug/Display.html?id=120134
-        depends_on("zlib@:1.2.8", when="@5.20.3:5.24.1")
+        conflicts("^zlib@1.2.9:", when="@5.20.3:5.24.1")
 
     conflicts("@5.34.1:", when="%msvc@:19.29.30136")
     # there has been a long fixed issue with 5.22.0 with regard to the ccflags
@@ -134,6 +136,12 @@ class Perl(Package):  # Perl doesn't use Autotools, it should subclass Package
     # https://rt.perl.org/Public/Bug/Display.html?id=126468
     patch("protect-quotes-in-ccflags.patch", when="@5.22.0")
 
+    # Support zlib-ng 2.1.2 and above for recent Perl
+    # Restrict zlib-ng to older versions for older Perl
+    # See https://github.com/pmqs/Compress-Raw-Zlib/issues/24
+    patch("zlib-ng.patch", when="@5.38 ^zlib-ng@2.1.2:")
+    conflicts("^zlib-ng@2.1.2:", when="@:5.37")
+
     # Fix the Time-Local testase http://blogs.perl.org/users/tom_wyant/2020/01/my-y2020-bug.html
     patch(
         "https://rt.cpan.org/Public/Ticket/Attachment/1776857/956088/0001-Fix-Time-Local-tests.patch",
@@ -179,6 +187,7 @@ class Perl(Package):  # Perl doesn't use Autotools, it should subclass Package
     variant("shared", default=True, description="Build a shared libperl.so library")
     variant("threads", default=True, description="Build perl with threads support")
     variant("open", default=True, description="Support open.pm")
+    variant("opcode", default=True, description="Support Opcode.pm")
 
     resource(
         name="cpanm",
@@ -241,19 +250,33 @@ def determine_variants(cls, exes, version):
                 fail_on_error=False,
             )
             variants += "+open" if perl.returncode == 0 else "~open"
+            # this is just to detect incomplete installs
+            # normally perl installs Opcode.pm
+            perl("-e", "use Opcode", output=os.devnull, error=os.devnull, fail_on_error=False)
+            variants += "+opcode" if perl.returncode == 0 else "~opcode"
             return variants
 
     # On a lustre filesystem, patch may fail when files
     # aren't writeable so make pp.c user writeable
     # before patching. This should probably walk the
     # source and make everything writeable in the future.
+    # The patch "zlib-ng.patch" also fail. So, apply chmod
+    # to Makefile.PL and Zlib.xs too.
     def do_stage(self, mirror_only=False):
         # Do Spack's regular stage
         super().do_stage(mirror_only)
-        # Add write permissions on file to be patched
-        filename = join_path(self.stage.source_path, "pp.c")
-        perm = os.stat(filename).st_mode
-        os.chmod(filename, perm | 0o200)
+        # Add write permissions on files to be patched
+        files_to_chmod = [
+            join_path(self.stage.source_path, "pp.c"),
+            join_path(self.stage.source_path, "cpan/Compress-Raw-Zlib/Makefile.PL"),
+            join_path(self.stage.source_path, "cpan/Compress-Raw-Zlib/Zlib.xs"),
+        ]
+        for filename in files_to_chmod:
+            try:
+                perm = os.stat(filename).st_mode
+                os.chmod(filename, perm | 0o200)
+            except IOError:
+                continue
 
     def nmake_arguments(self):
         args = []
@@ -390,14 +413,13 @@ def install_cpanm(self):
                 maker()
                 maker("install")
 
-    def _setup_dependent_env(self, env, dependent_spec, deptype):
+    def _setup_dependent_env(self, env, dependent_spec):
         """Set PATH and PERL5LIB to include the extension and
         any other perl extensions it depends on,
         assuming they were installed with INSTALL_BASE defined."""
         perl_lib_dirs = []
-        for d in dependent_spec.traverse(deptype=deptype):
-            if d.package.extends(self.spec):
-                perl_lib_dirs.append(d.prefix.lib.perl5)
+        if dependent_spec.package.extends(self.spec):
+            perl_lib_dirs.append(dependent_spec.prefix.lib.perl5)
         if perl_lib_dirs:
             perl_lib_path = ":".join(perl_lib_dirs)
             env.prepend_path("PERL5LIB", perl_lib_path)
@@ -405,10 +427,10 @@ def _setup_dependent_env(self, env, dependent_spec, deptype):
             env.append_path("PATH", self.prefix.bin)
 
     def setup_dependent_build_environment(self, env, dependent_spec):
-        self._setup_dependent_env(env, dependent_spec, deptype=("build", "run", "test"))
+        self._setup_dependent_env(env, dependent_spec)
 
     def setup_dependent_run_environment(self, env, dependent_spec):
-        self._setup_dependent_env(env, dependent_spec, deptype=("run",))
+        self._setup_dependent_env(env, dependent_spec)
 
     def setup_dependent_package(self, module, dependent_spec):
         """Called before perl modules' install() methods.
@@ -448,8 +470,8 @@ def setup_build_environment(self, env):
         env.set("BZIP2_INCLUDE", spec["bzip2"].prefix.include)
         env.set("BZIP2_LIB", spec["bzip2"].libs.directories[0])
         env.set("BUILD_ZLIB", 0)
-        env.set("ZLIB_INCLUDE", spec["zlib"].prefix.include)
-        env.set("ZLIB_LIB", spec["zlib"].libs.directories[0])
+        env.set("ZLIB_INCLUDE", spec["zlib-api"].prefix.include)
+        env.set("ZLIB_LIB", spec["zlib-api"].libs.directories[0])
 
     @run_after("install")
     def filter_config_dot_pm(self):
diff --git a/var/spack/repos/builtin/packages/perl/zlib-ng.patch b/var/spack/repos/builtin/packages/perl/zlib-ng.patch
new file mode 100644
index 00000000000000..55f55bc59c2967
--- /dev/null
+++ b/var/spack/repos/builtin/packages/perl/zlib-ng.patch
@@ -0,0 +1,45 @@
+From 0a8fb7141a43b8e49609fb06f05fad5150a97c2a Mon Sep 17 00:00:00 2001
+From: pmqs 
+Date: Fri, 9 Jun 2023 14:30:36 +0100
+Subject: [PATCH] Change storage of ZLIBNG_VER_STATUS from IV to PV
+ https://github.com/pmqs/Compress-Raw-Zlib/issues/24
+
+---
+ Makefile.PL |  1 -
+ Zlib.xs     | 11 +++++++++++
+ 2 files changed, 11 insertions(+), 1 deletion(-)
+
+diff --git a/Makefile.PL b/Makefile.PL
+index 299093b..5944918 100644
+--- a/cpan/Compress-Raw-Zlib/Makefile.PL
++++ b/cpan/Compress-Raw-Zlib/Makefile.PL
+@@ -201,7 +201,6 @@ my @names = qw(
+     ZLIBNG_VER_MAJOR
+     ZLIBNG_VER_MINOR
+     ZLIBNG_VER_REVISION
+-    ZLIBNG_VER_STATUS
+     ZLIBNG_VER_MODIFIED
+ );
+ 
+diff --git a/Zlib.xs b/Zlib.xs
+index c47c66d..2d4fc58 100644
+--- a/cpan/Compress-Raw-Zlib/Zlib.xs
++++ b/cpan/Compress-Raw-Zlib/Zlib.xs
+@@ -951,6 +951,17 @@ ZLIB_VERNUM()
+ uLong
+ Zip_zlibCompileFlags()
+ 
++const char*
++ZLIBNG_VER_STATUS()
++    CODE:
++#ifdef ZLIBNG_VER_STATUS
++        RETVAL = STRINGIFY(ZLIBNG_VER_STATUS);
++#else
++        RETVAL = "0";
++#endif
++    OUTPUT:
++        RETVAL
++
+ MODULE = Compress::Raw::Zlib	PACKAGE = Compress::Raw::Zlib	PREFIX = Zip_
+ 
+ #define Zip_adler32(buf, adler) CRZ_adler32(adler, buf, (uInt)len)
diff --git a/var/spack/repos/builtin/packages/petaca/package.py b/var/spack/repos/builtin/packages/petaca/package.py
index 756b4e9a24f207..d137f5791e8434 100644
--- a/var/spack/repos/builtin/packages/petaca/package.py
+++ b/var/spack/repos/builtin/packages/petaca/package.py
@@ -23,7 +23,7 @@ class Petaca(CMakePackage):
     git = "https://github.com/nncarlson/petaca.git"
     url = "https://github.com/nncarlson/petaca/archive/refs/tags/v22.03.tar.gz"
 
-    maintainers("pbrady")
+    maintainers("pbrady", "zjibben")
 
     version("develop", branch="master")
     version("22.03", sha256="e6559e928c7cca6017ef0582c204eee775f6bb3f927f1c224c515c2ad574cc32")
diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py
index ed0fc0f5dde7db..5093961f7deffc 100644
--- a/var/spack/repos/builtin/packages/petsc/package.py
+++ b/var/spack/repos/builtin/packages/petsc/package.py
@@ -14,7 +14,7 @@ class Petsc(Package, CudaPackage, ROCmPackage):
     """
 
     homepage = "https://petsc.org"
-    url = "https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.15.0.tar.gz"
+    url = "https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-3.20.0.tar.gz"
     git = "https://gitlab.com/petsc/petsc.git"
     maintainers("balay", "barrysmith", "jedbrown")
 
@@ -22,6 +22,13 @@ class Petsc(Package, CudaPackage, ROCmPackage):
 
     version("main", branch="main")
 
+    version("3.20.1", sha256="3d54f13000c9c8ceb13ca4f24f93d838319019d29e6de5244551a3ec22704f32")
+    version("3.20.0", sha256="c152ccb12cb2353369d27a65470d4044a0c67e0b69814368249976f5bb232bd4")
+    version("3.19.6", sha256="6045e379464e91bb2ef776f22a08a1bc1ff5796ffd6825f15270159cbb2464ae")
+    version("3.19.5", sha256="511aa78cad36db2dfd298acf35e9f7afd2ecc1f089da5b0b5682507a31a5d6b2")
+    version("3.19.4", sha256="7c941b71be52c3b764214e492df60109d12f97f7d854c97a44df0c4d958b3906")
+    version("3.19.3", sha256="008239c016b869693ec8e81368a0b7638462e667d07f7d50ed5f9b75ccc58d17")
+    version("3.19.2", sha256="114f363f779bb16839b25c0e70f8b0ae0d947d50e72f7c6cddcb11b001079b16")
     version("3.19.1", sha256="74db60c53c80b48d5c39e07bc39a883ecced88b9f24a5de17cf6f485a903e120")
     version("3.19.0", sha256="8ced753e4d2fb6565662b2b1fbba75a426cbf8438203f82717ce270f0591322c")
     version("3.18.6", sha256="8b53c8b6652459ba0bbe6361b5baf8c4d17c1d04b6654a76e3b6a9ab4a576680")
@@ -81,6 +88,7 @@ class Petsc(Package, CudaPackage, ROCmPackage):
     variant("double", default=True, description="Switches between single and double precision")
     variant("complex", default=False, description="Build with complex numbers")
     variant("debug", default=False, description="Compile in debug mode")
+    variant("sycl", default=False, description="Enable sycl build")
 
     variant("metis", default=True, description="Activates support for metis and parmetis")
     variant(
@@ -136,6 +144,13 @@ class Petsc(Package, CudaPackage, ROCmPackage):
     )
     variant("cgns", default=False, description="Activates support for CGNS (only parallel)")
     variant("memkind", default=False, description="Activates support for Memkind")
+    variant(
+        "memalign",
+        default="none",
+        description="Specify alignment of allocated arrays",
+        values=("4", "8", "16", "32", "64", "none"),
+        multi=False,
+    )
     variant("p4est", default=False, description="Activates support for P4Est (only parallel)")
     variant("saws", default=False, description="Activates support for Saws")
     variant("libyaml", default=False, description="Activates support for YAML")
@@ -146,6 +161,10 @@ class Petsc(Package, CudaPackage, ROCmPackage):
 
     # https://github.com/spack/spack/issues/37416
     conflicts("^rocprim@5.3.0:5.3.2", when="+rocm")
+    # petsc 3.20 has workaround for breaking change in hipsparseSpSV_solve api,
+    # but it seems to misdetect hipsparse@5.6.1 as 5.6.0, so the workaround
+    # only makes things worse
+    conflicts("^hipsparse@5.6", when="+rocm @3.20.0")
 
     # 3.8.0 has a build issue with MKL - so list this conflict explicitly
     conflicts("^intel-mkl", when="@3.8.0")
@@ -190,6 +209,8 @@ def check_fortran_compiler(self):
     patch("revert-3.18.0-ver-format-for-dealii.patch", when="@3.18.0")
 
     depends_on("diffutils", type="build")
+    # not listed as a "build" dependency - so that slepc build gets the same dependency
+    depends_on("gmake")
 
     # Virtual dependencies
     # Git repository needs sowing to build Fortran interface
@@ -235,9 +256,9 @@ def check_fortran_compiler(self):
     depends_on("hdf5+mpi", when="@3.13:+hdf5+mpi")
     depends_on("hdf5+mpi", when="+exodusii+mpi")
     depends_on("hdf5+mpi", when="+cgns+mpi")
-    depends_on("zlib", when="+hdf5")
-    depends_on("zlib", when="+libpng")
-    depends_on("zlib", when="+p4est")
+    depends_on("zlib-api", when="+hdf5")
+    depends_on("zlib-api", when="+libpng")
+    depends_on("zlib-api", when="+p4est")
     depends_on("parmetis+int64", when="+metis+mpi+int64")
     depends_on("parmetis~int64", when="+metis+mpi~int64")
     depends_on("valgrind", when="+valgrind")
@@ -245,22 +266,19 @@ def check_fortran_compiler(self):
     depends_on("mmg", when="+parmmg")
     depends_on("parmmg", when="+parmmg")
     depends_on("tetgen+pic", when="+tetgen")
-    # hypre+/~fortran based on wheter fortran is enabled/disabled
+
     depends_on("hypre+fortran", when="+hypre+fortran")
     depends_on("hypre~fortran", when="+hypre~fortran")
-    # Hypre does not support complex numbers.
-    # Also PETSc prefer to build it without internal superlu, likely due to
-    # conflict in headers see
-    # https://bitbucket.org/petsc/petsc/src/90564b43f6b05485163c147b464b5d6d28cde3ef/config/BuildSystem/config/packages/hypre.py
-    depends_on("hypre@2.14:2.18.2~internal-superlu", when="@3.11:3.13+hypre")
-    depends_on("hypre@2.14:2.22.0~internal-superlu", when="@3.14:3.15+hypre")
-    depends_on("hypre@2.14:~internal-superlu", when="@3.16:+hypre")
-    depends_on("hypre@develop~internal-superlu", when="@main+hypre")
     depends_on("hypre+complex", when="+hypre+complex")
     depends_on("hypre~complex", when="+hypre~complex")
     depends_on("hypre+int64", when="+hypre+int64")
     depends_on("hypre~int64", when="+hypre~int64")
-    depends_on("hypre+mpi", when="+hypre+mpi")
+    depends_on("hypre+mpi~internal-superlu", when="+hypre")
+    depends_on("hypre@2.14:2.18.2", when="@3.11:3.13+hypre")
+    depends_on("hypre@2.14:2.22.0", when="@3.14:3.15+hypre")
+    depends_on("hypre@2.14:2.28.0", when="@3.16:3.19+hypre")
+    depends_on("hypre@2.14:", when="@3.20+hypre")
+    depends_on("hypre@develop", when="@main+hypre")
 
     depends_on("superlu-dist@:4.3~int64", when="@3.4.4:3.6.4+superlu-dist+mpi~int64")
     depends_on("superlu-dist@:4.3+int64", when="@3.4.4:3.6.4+superlu-dist+mpi+int64")
@@ -324,6 +342,9 @@ def check_fortran_compiler(self):
             when="+kokkos +rocm amdgpu_target=%s" % rocm_arch,
         )
 
+    conflicts("~kokkos", when="+sycl", msg="+sycl requires +kokkos")
+    depends_on("kokkos+sycl", when="+sycl +kokkos")
+
     phases = ["configure", "build", "install"]
 
     # Using the following tarballs
@@ -332,13 +353,11 @@ def check_fortran_compiler(self):
     # * petsc-3.15 and newer (without docs)
     def url_for_version(self, version):
         if self.spec.satisfies("@3.13.0:3.14.6"):
-            return (
-                "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-lite-{0}.tar.gz".format(
-                    version
-                )
+            return "http://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-lite-{0}.tar.gz".format(
+                version
             )
         else:
-            return "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-{0}.tar.gz".format(
+            return "http://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-{0}.tar.gz".format(
                 version
             )
 
@@ -413,11 +432,25 @@ def configure_options(self):
         if "+knl" in spec:
             options.append("--with-avx-512-kernels")
             options.append("--with-memalign=64")
+        elif self.spec.variants["memalign"].value != "none":
+            alignement = self.spec.variants["memalign"].value
+            options.append(f"--with-memalign={alignement}")
+
         if "+X" in spec:
             options.append("--with-x=1")
         else:
             options.append("--with-x=0")
 
+        if "+sycl" in spec:
+            sycl_compatible_compilers = ["icpx"]
+            if not (os.path.basename(self.compiler.cxx) in sycl_compatible_compilers):
+                raise InstallError("PETSc's SYCL GPU Backend requires oneAPI CXX (icpx) compiler.")
+            options.append("--with-sycl=1")
+            options.append("--with-syclc=" + self.compiler.cxx)
+            options.append("SYCLPPFLAGS=-Wno-tautological-constant-compare")
+        else:
+            options.append("--with-sycl=0")
+
         if "trilinos" in spec:
             if spec.satisfies("^trilinos+boost"):
                 options.append("--with-boost=1")
diff --git a/var/spack/repos/builtin/packages/pflotran/package.py b/var/spack/repos/builtin/packages/pflotran/package.py
index 0045f2758837dc..57a409b150c858 100644
--- a/var/spack/repos/builtin/packages/pflotran/package.py
+++ b/var/spack/repos/builtin/packages/pflotran/package.py
@@ -18,6 +18,7 @@ class Pflotran(AutotoolsPackage):
     maintainers("ghammond86", "balay")
 
     version("develop")
+    version("5.0.0", commit="f0fe931c72c03580e489724afeb8c5451406b942")  # tag v5.0.0
     version("4.0.1", commit="fd351a49b687e27f46eae92e9259156eea74897d")  # tag v4.0.1
     version("3.0.2", commit="9e07f416a66b0ad304c720b61aa41cba9a0929d5")  # tag v3.0.2
     version("xsdk-0.6.0", commit="46e14355c1827c057f2e1b3e3ae934119ab023b2")
@@ -30,6 +31,7 @@ class Pflotran(AutotoolsPackage):
     depends_on("mpi")
     depends_on("hdf5@1.8.12:+mpi+fortran+hl")
     depends_on("petsc@main:+hdf5+metis", when="@develop")
+    depends_on("petsc@3.20:+hdf5+metis", when="@5.0.0")
     depends_on("petsc@3.18:+hdf5+metis", when="@4.0.1")
     depends_on("petsc@3.16:+hdf5+metis", when="@3.0.2")
     depends_on("petsc@3.14:+hdf5+metis", when="@xsdk-0.6.0")
diff --git a/var/spack/repos/builtin/packages/pfunit/package.py b/var/spack/repos/builtin/packages/pfunit/package.py
index 0d7051d26c6459..0fb3af60eb33e6 100644
--- a/var/spack/repos/builtin/packages/pfunit/package.py
+++ b/var/spack/repos/builtin/packages/pfunit/package.py
@@ -19,6 +19,9 @@ class Pfunit(CMakePackage):
 
     maintainers("mathomp4", "tclune")
 
+    version("4.7.3", sha256="247239298b55e847417b7830183d7fc62cca93dc92c8ec7c0067784b7ce34544")
+    version("4.7.2", sha256="3142a1e56b7d127fdc9589cf6deff8505174129834a6a268d0ce7e296f51ab02")
+    version("4.7.1", sha256="64de3eb9f364b57ef6df81ba33400dfd4dcebca6eb5d0e9b7955ed8156e29165")
     version("4.7.0", sha256="5faf52d0ab8589b3cd3ea488b34a65dc931f70c07aaa7bf4f209b18af2b38e4e")
     version("4.6.3", sha256="a43a64c4338be57fdbe1cae1a89e277196f10931bc1f73418a463e05e5e7b2d1")
     version("4.6.2", sha256="fd302a1f7a131b38e18bc31ede69a216e580c640152e5e313f5a1e084669a950")
diff --git a/var/spack/repos/builtin/packages/phist/avoid-sse.patch b/var/spack/repos/builtin/packages/phist/avoid-sse.patch
new file mode 100644
index 00000000000000..44ae90ec8b6e28
--- /dev/null
+++ b/var/spack/repos/builtin/packages/phist/avoid-sse.patch
@@ -0,0 +1,346 @@
+commit eaef462cc07509fe8f380fbf520a2617b910b139
+Author: Jonas Thies <16190001+jthies@users.noreply.github.com>
+Date:   Sun Jul 9 21:33:30 2023 +0200
+
+    exit early from builtin kernels requiring SSE so that they are not compiled if it is not available
+    (this broke phist compilation on ARM systems, even though we never called these kernels if SSE was disabled)
+
+diff --git a/src/kernels/builtin/axpy_kernels_nt.c b/src/kernels/builtin/axpy_kernels_nt.c
+index 64d5fbd0..17c5024a 100644
+--- a/src/kernels/builtin/axpy_kernels_nt.c
++++ b/src/kernels/builtin/axpy_kernels_nt.c
+@@ -19,7 +19,9 @@
+ #endif
+ #include 
+ #include 
++#ifdef PHIST_HAVE_SSE
+ #include 
++#endif
+ #include 
+ 
+ static inline _Bool is_aligned(const void *restrict pointer, size_t byte_count)
+@@ -30,6 +32,10 @@ static inline _Bool is_aligned(const void *restrict pointer, size_t byte_count)
+ 
+ void daxpy_nt_2_c(int nrows, const double *restrict alpha, const double *restrict x, double *restrict y)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(y,16) )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)y);
+@@ -54,11 +60,16 @@ void daxpy_nt_2_c(int nrows, const double *restrict alpha, const double *restric
+     // non-temporal store
+     _mm_stream_pd(y+2*i, y_);
+   }
++#endif
+ }
+ 
+ 
+ void daxpy_nt_4_c(int nrows, const double *restrict alpha, const double *restrict x, double *restrict y)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(y,16) )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)y);
+@@ -86,11 +97,16 @@ void daxpy_nt_4_c(int nrows, const double *restrict alpha, const double *restric
+       _mm_stream_pd(y+4*i+2*k, y_);
+     }
+   }
++#endif
+ }
+ 
+ 
+ void daxpy_nt_8_c(int nrows, const double *restrict alpha, const double *restrict x, double *restrict y)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(y,16) )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)y);
+@@ -118,11 +134,16 @@ void daxpy_nt_8_c(int nrows, const double *restrict alpha, const double *restric
+       _mm_stream_pd(y+8*i+2*k, y_);
+     }
+   }
++#endif
+ }
+ 
+ 
+ void daxpy_nt_strided_2_c(int nrows, const double *restrict alpha, const double *restrict x, int ldx, double *restrict y, int ldy)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(y,16) )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)y);
+@@ -140,11 +161,16 @@ void daxpy_nt_strided_2_c(int nrows, const double *restrict alpha, const double
+     // non-temporal store
+     _mm_stream_pd(y+ldy*i, y_);
+   }
++#endif
+ }
+ 
+ 
+ void daxpy_nt_strided_4_c(int nrows, const double *restrict alpha, const double *restrict x, int ldx, double *restrict y, int ldy)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(y,16) || ldy % 2 != 0 )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)y);
+@@ -165,11 +191,16 @@ void daxpy_nt_strided_4_c(int nrows, const double *restrict alpha, const double
+       _mm_stream_pd(y+ldy*i+2*k, y_);
+     }
+   }
++#endif
+ }
+ 
+ 
+ void daxpy_nt_strided_8_c(int nrows, const double *restrict alpha, const double *restrict x, int ldx, double *restrict y, int ldy)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(y,16) || ldy % 2 != 0 )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)y);
+@@ -190,11 +221,16 @@ void daxpy_nt_strided_8_c(int nrows, const double *restrict alpha, const double
+       _mm_stream_pd(y+ldy*i+2*k, y_);
+     }
+   }
++#endif
+ }
+ 
+ 
+ void dcopy_general_nt_c(int nrows, int nvec, const double *restrict x, int ldx, double *restrict y, int ldy)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( nvec % 2 != 0 )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)x);
+@@ -217,5 +253,6 @@ void dcopy_general_nt_c(int nrows, int nvec, const double *restrict x, int ldx,
+       _mm_stream_pd(y+i*ldy+2*j, tmp);
+     }
+   }
++#endif
+ }
+ 
+diff --git a/src/kernels/builtin/spmvm_kernels_nt.c b/src/kernels/builtin/spmvm_kernels_nt.c
+index d4d30bff..5d858878 100644
+--- a/src/kernels/builtin/spmvm_kernels_nt.c
++++ b/src/kernels/builtin/spmvm_kernels_nt.c
+@@ -19,7 +19,9 @@
+ #endif
+ #include 
+ #include 
++#ifdef PHIST_HAVE_SSE
+ #include 
++#endif
+ #include 
+ 
+ #ifdef PHIST_HIGH_PRECISION_KERNELS
+@@ -35,6 +37,10 @@ static inline _Bool is_aligned(const void *restrict pointer, size_t byte_count)
+ void dspmvm_nt_1_c(int nrows, double alpha, const long *restrict row_ptr, const long *restrict halo_ptr, const int *restrict col_idx, const double *restrict val,
+                  const double *restrict shifts, const double *restrict rhsv, const double *restrict halo, double *restrict lhsv)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(lhsv,16) )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)lhsv);
+@@ -123,7 +129,7 @@ void dspmvm_nt_1_c(int nrows, double alpha, const long *restrict row_ptr, const
+ #endif
+ 
+   // last row
+-#ifdef PHIST_HIGH_PRECISION_KERNELS
++# ifdef PHIST_HIGH_PRECISION_KERNELS
+   if( nrows % 2 != 0 )
+   {
+     double lhs, lhsC;
+@@ -136,7 +142,7 @@ void dspmvm_nt_1_c(int nrows, double alpha, const long *restrict row_ptr, const
+ 
+     lhsv[nrows-1] = alpha*(lhs+lhsC);
+   }
+-#else
++# else
+   if( nrows % 2 != 0 )
+   {
+     lhsv[nrows-1] = shifts[0]*rhsv[nrows-1];
+@@ -146,6 +152,7 @@ void dspmvm_nt_1_c(int nrows, double alpha, const long *restrict row_ptr, const
+       lhsv[nrows-1] += val[j]*halo[ (col_idx[j]-1) ];
+     lhsv[nrows-1] *= alpha;
+   }
++# endif
+ #endif
+ }
+ 
+@@ -153,6 +160,10 @@ void dspmvm_nt_1_c(int nrows, double alpha, const long *restrict row_ptr, const
+ void dspmvm_nt_2_c(int nrows, double alpha, const long *restrict row_ptr, const long *restrict halo_ptr, const int *restrict col_idx, const double *restrict val,
+                  const double *restrict shifts, const double *restrict rhsv, const double *restrict halo, double *restrict lhsv, int ldl)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(lhsv,32) || ldl % 2 != 0 )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)lhsv);
+@@ -176,7 +187,7 @@ void dspmvm_nt_2_c(int nrows, double alpha, const long *restrict row_ptr, const
+   __m128d shifts_ = _mm_loadu_pd(shifts);
+   __m128d alpha_ = _mm_set1_pd(alpha);
+ 
+-#ifdef PHIST_HIGH_PRECISION_KERNELS
++# ifdef PHIST_HIGH_PRECISION_KERNELS
+ #pragma omp parallel for schedule(static)
+   for(int i = 0; i < nrows; i++)
+   {
+@@ -204,7 +215,7 @@ void dspmvm_nt_2_c(int nrows, double alpha, const long *restrict row_ptr, const
+     // non-temporal store
+     _mm_stream_pd(lhsv+i*ldl, lhs);
+   }
+-#else
++# else
+ #pragma omp parallel for schedule(static)
+   for(int i = 0; i < nrows; i++)
+   {
+@@ -232,16 +243,21 @@ void dspmvm_nt_2_c(int nrows, double alpha, const long *restrict row_ptr, const
+     // multiply with alpha
+     __m128d alpha_ = _mm_set1_pd(alpha);
+     lhs_ = _mm_mul_pd(alpha_,lhs_);
+- 
++
+     // non-temporal store
+     _mm_stream_pd(lhsv+i*ldl, lhs_);
+   }
++# endif
+ #endif
+ }
+ 
+ void dspmvm_nt_4_c(int nrows, double alpha, const long *restrict row_ptr, const long *restrict halo_ptr, const int *restrict col_idx, const double *restrict val,
+                  const double *restrict shifts, const double *restrict rhsv, const double *restrict halo, double *restrict lhsv, int ldl)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(lhsv,32) || ldl % 4 != 0 )
+   {
+     printf("%s: lhsv not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)lhsv);
+@@ -261,7 +277,7 @@ void dspmvm_nt_4_c(int nrows, double alpha, const long *restrict row_ptr, const
+   }
+ 
+ 
+-#ifdef PHIST_HIGH_PRECISION_KERNELS
++# ifdef PHIST_HIGH_PRECISION_KERNELS
+ 
+   __m256d shifts_ = _mm256_loadu_pd(shifts);
+   __m256d alpha_ = _mm256_set1_pd(alpha);
+@@ -294,7 +310,7 @@ void dspmvm_nt_4_c(int nrows, double alpha, const long *restrict row_ptr, const
+     _mm256_stream_pd(lhsv+i*ldl, lhs);
+   }
+ 
+-#else
++# else
+ 
+   __m128d shifts_[2];
+   shifts_[0] = _mm_loadu_pd(shifts);
+@@ -341,6 +357,7 @@ void dspmvm_nt_4_c(int nrows, double alpha, const long *restrict row_ptr, const
+     }
+   }
+ 
++# endif
+ #endif
+ }
+ 
+@@ -348,6 +365,10 @@ void dspmvm_nt_4_c(int nrows, double alpha, const long *restrict row_ptr, const
+ void dspmvm_nt_8_c(int nrows, double alpha, const long *restrict row_ptr, const long *restrict halo_ptr, const int *restrict col_idx, const double *restrict val,
+                  const double *restrict shifts, const double *restrict rhsv, const double *restrict halo, double *restrict lhsv, int ldl)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(lhsv,16) || ldl % 2 != 0 )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)lhsv);
+@@ -412,12 +433,17 @@ void dspmvm_nt_8_c(int nrows, double alpha, const long *restrict row_ptr, const
+       _mm_stream_pd(lhsv+i*ldl+2*k, lhs_[k]);
+     }
+   }
++#endif
+ }
+ 
+ 
+ void dspmvm_nt_strided_2_c(int nrows, double alpha, const long *restrict row_ptr, const long *restrict halo_ptr, const int *restrict col_idx, const double *restrict val,
+                          const double *restrict shifts, const double *restrict rhsv, int ldr, const double *restrict halo, double *restrict lhsv, int ldl)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(lhsv,16) || ldl % 2 != 0 )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)lhsv);
+@@ -460,15 +486,20 @@ void dspmvm_nt_strided_2_c(int nrows, double alpha, const long *restrict row_ptr
+     // multiply with alpha
+     __m128d alpha_ = _mm_set1_pd(alpha);
+     lhs_ = _mm_mul_pd(alpha_,lhs_);
+- 
++
+     // non-temporal store
+     _mm_stream_pd(lhsv+i*ldl, lhs_);
+   }
++#endif
+ }
+ 
+ void dspmvm_nt_strided_4_c(int nrows, double alpha, const long *restrict row_ptr, const long *restrict halo_ptr, const int *restrict col_idx, const double *restrict val,
+                          const double *restrict shifts, const double *restrict rhsv, int ldr, const double *restrict halo, double *restrict lhsv, int ldl)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(lhsv,16) || ldl % 2 != 0 )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)lhsv);
+@@ -526,11 +557,16 @@ void dspmvm_nt_strided_4_c(int nrows, double alpha, const long *restrict row_ptr
+       _mm_stream_pd(lhsv+i*ldl+2*k, lhs_[k]);
+     }
+   }
++#endif
+ }
+ 
+ void dspmvm_nt_strided_8_c(int nrows, double alpha, const long *restrict row_ptr, const long *restrict halo_ptr, const int *restrict col_idx, const double *restrict val,
+                          const double *restrict shifts, const double *restrict rhsv, int ldr, const double *restrict halo, double *restrict lhsv, int ldl)
+ {
++#ifndef PHIST_HAVE_SSE
++  printf("%s: must not be called on platforms without SSE.", __FUNCTION__);
++  exit(1);
++#else
+   if( !is_aligned(lhsv,16) || ldl % 2 != 0 )
+   {
+     printf("%s: not aligned %lx\n", __FUNCTION__, (uintptr_t)(void*)lhsv);
+@@ -589,6 +625,7 @@ void dspmvm_nt_strided_8_c(int nrows, double alpha, const long *restrict row_ptr
+       _mm_stream_pd(lhsv+i*ldl+2*k, lhs_[k]);
+     }
+   }
++#endif
+ }
+ 
+ 
diff --git a/var/spack/repos/builtin/packages/phist/package.py b/var/spack/repos/builtin/packages/phist/package.py
index fa055e47f2ca00..a67deddebd3557 100644
--- a/var/spack/repos/builtin/packages/phist/package.py
+++ b/var/spack/repos/builtin/packages/phist/package.py
@@ -34,6 +34,9 @@ class Phist(CMakePackage):
     version("develop", branch="devel")
     version("master", branch="master")
 
+    # compatible with trilinos@14:
+    version("1.12.0", sha256="0f02e39b16d14cf7c47a3c468e788c7c0e71857eb1c0a4edb601e1e5b67e8668")
+
     # compatible with python@3.11: and cray-libsci as BLAS/LAPACK provider
     version("1.11.2", sha256="e23f76307c26b930f7331a734b0a864ea6d7fb4a13c12f3c5d70c2c41481747b")
 
@@ -131,9 +134,11 @@ class Phist(CMakePackage):
         description="generate Fortran 2003 bindings (requires Python3 and " "a Fortran compiler)",
     )
 
+    # Trilinos 14 had some tpetra/kokkos API changes that are reflected in the phist 1.12 tag
+    conflicts("^trilinos@14:", when="@:1.11.2")
     # Build error with cray-libsci because they define macro 'I', workaround in phist-1.11.2
     conflicts("^cray-libsci", when="@:1.11.1")
-    # phist@1.11.2 got rid of some deprecated python code
+    # phist@1.11.2 got rid of some deprecated python code + a patch below
     conflicts("^python@3.11:", when="@:1.11.1")
     # The builtin kernels switched from the 'mpi' to the 'mpi_f08' module in
     # phist 1.9.6, which causes compile-time errors with mpich and older
@@ -150,6 +155,12 @@ class Phist(CMakePackage):
 
     # ###################### Patches ##########################
 
+    # remove 'rU' file mode in a python script
+    patch("remove_rU_mode_in_python_script.patch", when="@:1.12.0 +fortran ^python@3.11:")
+    # Avoid trying to compile some SSE code if SSE is not available
+    # This patch will be part of phist 1.11.3 and greater and only affects
+    # the 'builtin' kernel_lib.
+    patch("avoid-sse.patch", when="@:1.11.2 kernel_lib=builtin")
     # Only applies to 1.9.4: While SSE instructions are handled correctly,
     # build fails on ppc64le unless -DNO_WARN_X86_INTRINSICS is defined.
     patch("ppc64_sse.patch", when="@1.9.4")
@@ -244,6 +255,7 @@ def cmake_args(self):
         lapacke_include_dir = spec["lapack:c"].headers.directories[0]
 
         args = [
+            "-DCMAKE_FIND_DEBUG_MODE=On",
             "-DPHIST_USE_CCACHE=OFF",
             "-DPHIST_KERNEL_LIB=%s" % kernel_lib,
             "-DPHIST_OUTLEV=%s" % outlev,
diff --git a/var/spack/repos/builtin/packages/phist/remove_rU_mode_in_python_script.patch b/var/spack/repos/builtin/packages/phist/remove_rU_mode_in_python_script.patch
new file mode 100644
index 00000000000000..8076fd8189459e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/phist/remove_rU_mode_in_python_script.patch
@@ -0,0 +1,28 @@
+commit abef54185d43c0900bcde289a4436fea0790b9a7
+Author: Jonas Thies <16190001+jthies@users.noreply.github.com>
+Date:   Tue Sep 19 17:42:57 2023 +0200
+
+    cfwrapper script: do not use 'rU' mode when opening a file (fixes #246)
+
+diff --git a/fortran_bindings/cfwrapper.py.in b/fortran_bindings/cfwrapper.py.in
+index 9a2539d3..60808a5d 100755
+--- a/fortran_bindings/cfwrapper.py.in
++++ b/fortran_bindings/cfwrapper.py.in
+@@ -264,7 +264,7 @@ for library_path in PATH_DICT:
+         for c_file_name in directory[2]:
+             if (c_file_name.find(".CFWRAPPER_SKIP.") != -1):
+                 continue
+-            whole_file = open(directory[0] + "/" + c_file_name, 'rU', 
++            whole_file = open(directory[0] + "/" + c_file_name, 'r', 
+                               errors='replace').read()
+             enum_types = re.findall("(?ms)^typedef enum.*?}\s?(\w+);", whole_file)
+             phist_enums += enum_types
+@@ -338,7 +338,7 @@ for library_path in PATH_DICT:
+             if c_file_name in ["gstdio.h", "giochannel.h"]:
+                 continue    # Go to next file
+ 
+-            whole_file_original = open(directory[0] + "/" + c_file_name, 'rU', 
++            whole_file_original = open(directory[0] + "/" + c_file_name, 'r', 
+                                        errors='replace').read()
+             whole_file = whole_file_original
+             nb_files += 1
diff --git a/var/spack/repos/builtin/packages/picard/package.py b/var/spack/repos/builtin/packages/picard/package.py
index 923c64b357842d..f9e2acb6fa15e0 100644
--- a/var/spack/repos/builtin/packages/picard/package.py
+++ b/var/spack/repos/builtin/packages/picard/package.py
@@ -156,6 +156,8 @@ class Picard(Package):
     )
     version("1.140", sha256="0d27287217413db6b846284c617d502eaa578662dcb054a7017083eab9c54438")
 
+    variant("parameters", default=False, description="get java parameters in the adapter script")
+
     depends_on("java@17:", type="run", when="@3.0.0:")
     depends_on("java@8:", type="run", when="@:2.27.5")
 
@@ -169,7 +171,12 @@ def install(self, spec, prefix):
 
         # Set up a helper script to call java on the jar file,
         # explicitly codes the path for java and the jar file.
-        script_sh = join_path(os.path.dirname(__file__), "picard.sh")
+
+        script_sh = join_path(
+            os.path.dirname(__file__),
+            "picard_with_parameters.sh" if "+parameters" in spec else "picard.sh",
+        )
+
         script = prefix.bin.picard
         install(script_sh, script)
         set_executable(script)
diff --git a/var/spack/repos/builtin/packages/picard/picard_with_parameters.sh b/var/spack/repos/builtin/packages/picard/picard_with_parameters.sh
new file mode 100644
index 00000000000000..98a32a9f6019dd
--- /dev/null
+++ b/var/spack/repos/builtin/packages/picard/picard_with_parameters.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+# Picard executable shell script
+set -eu -o pipefail
+
+export LC_ALL=en_US.UTF-8
+
+# Find original directory of bash script, resolving symlinks
+# http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in/246128#246128
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+    DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+    SOURCE="$(readlink "$SOURCE")"
+    [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+JAR_DIR=$DIR
+ENV_PREFIX="$(dirname $(dirname $DIR))"
+# Use Java installed with Anaconda to ensure correct version
+java="$ENV_PREFIX/bin/java"
+
+# if JAVA_HOME is set (non-empty), use it. Otherwise keep "java"
+if [ -n "${JAVA_HOME:=}" ]; then
+  if [ -e "$JAVA_HOME/bin/java" ]; then
+      java="$JAVA_HOME/bin/java"
+  fi
+fi
+
+# extract memory and system property Java arguments from the list of provided arguments
+# http://java.dzone.com/articles/better-java-shell-script
+default_jvm_mem_opts="-Xms512m -Xmx2g"
+jvm_mem_opts=""
+jvm_prop_opts=""
+pass_args=""
+for arg in "$@"; do
+    case $arg in
+        '-D'*)
+            jvm_prop_opts="$jvm_prop_opts $arg"
+            ;;
+        '-XX'*)
+            jvm_prop_opts="$jvm_prop_opts $arg"
+            ;;
+         '-Xm'*)
+            jvm_mem_opts="$jvm_mem_opts $arg"
+            ;;
+         *)
+            if [[ ${pass_args} == '' ]] #needed to avoid preceeding space on first arg e.g. ' MarkDuplicates'
+                then
+                    pass_args="$arg"
+            else
+                    pass_args="$pass_args \"$arg\"" #quotes later arguments to avoid problem with ()s in MarkDuplicates regex arg
+            fi
+            ;;
+    esac
+done
+
+if [ "$jvm_mem_opts" == "" ] && [ -z ${_JAVA_OPTIONS+x} ]; then
+    jvm_mem_opts="$default_jvm_mem_opts"
+fi
+
+pass_arr=($pass_args)
+if [[ ${pass_arr[0]:=} == org* ]]
+then
+    eval "$java" $jvm_mem_opts $jvm_prop_opts -cp "$JAR_DIR/picard.jar" $pass_args
+else
+    eval "$java" $jvm_mem_opts $jvm_prop_opts -jar "$JAR_DIR/picard.jar" $pass_args
+fi
+exit
diff --git a/var/spack/repos/builtin/packages/picsarlite/package.py b/var/spack/repos/builtin/packages/picsarlite/package.py
index 53b18487bb27ea..8c8825093c1218 100644
--- a/var/spack/repos/builtin/packages/picsarlite/package.py
+++ b/var/spack/repos/builtin/packages/picsarlite/package.py
@@ -17,7 +17,7 @@ class Picsarlite(MakefilePackage):
     git = "https://bitbucket.org/berkeleylab/picsar.git"
 
     version("develop", branch="PICSARlite")
-    version("0.1", tag="PICSARlite-0.1")
+    version("0.1", tag="PICSARlite-0.1", commit="3c9cee9bdf32da0998f504bff7af31fcae2f0452")
 
     variant("prod", default=True, description="Production mode (without FFTW)")
     variant(
diff --git a/var/spack/repos/builtin/packages/pigz/package.py b/var/spack/repos/builtin/packages/pigz/package.py
index bd4d62235f1b47..2b5dd8a2c41eea 100644
--- a/var/spack/repos/builtin/packages/pigz/package.py
+++ b/var/spack/repos/builtin/packages/pigz/package.py
@@ -18,7 +18,7 @@ class Pigz(MakefilePackage):
     version("2.4", sha256="e228e7d18b34c4ece8d596eb6eee97bde533c6beedbb728d07d3abe90b4b1b52")
     version("2.3.4", sha256="763f2fdb203aa0b7b640e63385e38e5dd4e5aaa041bc8e42aa96f2ef156b06e8")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def build(self, spec, prefix):
         # force makefile to use cc as C compiler which is set by
diff --git a/var/spack/repos/builtin/packages/pika-algorithms/package.py b/var/spack/repos/builtin/packages/pika-algorithms/package.py
index a42614d54a625b..3387dfdb2736cb 100644
--- a/var/spack/repos/builtin/packages/pika-algorithms/package.py
+++ b/var/spack/repos/builtin/packages/pika-algorithms/package.py
@@ -15,6 +15,7 @@ class PikaAlgorithms(CMakePackage):
     git = "https://github.com/pika-org/pika-algorithms.git"
     maintainers("msimberg", "albestro", "teonnik", "aurianer")
 
+    version("0.1.4", sha256="67ea5e8545b234f82dcc75612a774f2e3df8425a283f2034c2d1e2e5ac74f945")
     version("0.1.3", sha256="53b79fcc0e5decc0a4d70abf0897a4f66141b85eea6d65013f51eec02ad123b7")
     version("0.1.2", sha256="286cf5c4db06717fa66c681cec8c99207154dd07e72d72f2b5b4a3cb9ff698bf")
     version("0.1.1", sha256="4aae88ac19864fd278bbdb49ae56348014c3d0d4b49a46ab3b9ba8a180f745f6")
@@ -24,7 +25,7 @@ class PikaAlgorithms(CMakePackage):
     generator("ninja")
 
     map_cxxstd = lambda cxxstd: "2a" if cxxstd == "20" else cxxstd
-    cxxstds = ("17", "20")
+    cxxstds = ("17", "20", "23")
     variant(
         "cxxstd",
         default="17",
@@ -38,6 +39,7 @@ class PikaAlgorithms(CMakePackage):
 
     conflicts("%gcc@:8")
     conflicts("%clang@:8")
+    conflicts("cxxstd=23", when="^cmake@:3.20.2")
 
     # Other dependencies
     depends_on("boost@1.71:")
@@ -46,6 +48,7 @@ class PikaAlgorithms(CMakePackage):
     depends_on("pika@0.11", when="@0.1.0")
     depends_on("pika@0.11:0.12", when="@0.1.1")
     depends_on("pika@0.11:0.15", when="@0.1.2")
+    depends_on("pika@0.11:0.16", when="@0.1.3")
 
     for cxxstd in cxxstds:
         depends_on("boost cxxstd={0}".format(map_cxxstd(cxxstd)), when="cxxstd={0}".format(cxxstd))
diff --git a/var/spack/repos/builtin/packages/pika/package.py b/var/spack/repos/builtin/packages/pika/package.py
index ed5d41118822a8..949475650240dc 100644
--- a/var/spack/repos/builtin/packages/pika/package.py
+++ b/var/spack/repos/builtin/packages/pika/package.py
@@ -17,6 +17,11 @@ class Pika(CMakePackage, CudaPackage, ROCmPackage):
     git = "https://github.com/pika-org/pika.git"
     maintainers("msimberg", "albestro", "teonnik", "aurianer")
 
+    version("0.20.0", sha256="f338cceea66a0e3954806b2aca08f6560bba524ecea222f04bc18b483851c877")
+    version("0.19.1", sha256="674675abf0dd4c6f5a0b2fa3db944b277ed65c62f654029d938a8cab608a9c1d")
+    version("0.19.0", sha256="f45cc16e4e50cbb183ed743bdc8b775d49776ee33c13ea39a650f4230a5744cb")
+    version("0.18.0", sha256="f34890e0594eeca6ac57f2b988d0807b502782817e53a7f7043c3f921b08c99f")
+    version("0.17.0", sha256="717429fc1bc986d62cbec190a69939e91608122d09d54bda1b028871c9ca9ad4")
     version("0.16.0", sha256="59f2baec91cc9bf71ca96d21d0da1ec0092bf59da106efa51789089e0d7adcbb")
     version("0.15.1", sha256="b68b87cf956ad1448f5c2327a72ba4d9fb339ecabeabb0a87b8ea819457e293b")
     version("0.15.0", sha256="4ecd5b64bd8067283a161e1aeacfbab7658d89fe1504b788fd3236298fe66b00")
@@ -38,8 +43,7 @@ class Pika(CMakePackage, CudaPackage, ROCmPackage):
 
     generator("ninja")
 
-    map_cxxstd = lambda cxxstd: "2a" if cxxstd == "20" else cxxstd
-    cxxstds = ("17", "20")
+    cxxstds = ("17", "20", "23")
     variant(
         "cxxstd",
         default="17",
@@ -86,13 +90,17 @@ class Pika(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("%gcc@:8", when="@0.2:")
     conflicts("%clang@:8", when="@0.2:")
     conflicts("+stdexec", when="cxxstd=17")
+    conflicts("cxxstd=23", when="^cmake@:3.20.2")
+    # CUDA version <= 11 does not support C++20 and newer
+    for cxxstd in filter(lambda x: x != "17", cxxstds):
+        conflicts(f"cxxstd={cxxstd}", when="^cuda@:11")
 
     # Other dependencies
     depends_on("boost@1.71:")
     depends_on("fmt@9:", when="@0.11:")
     # https://github.com/pika-org/pika/issues/686
-    conflicts("fmt@10:", when="@:0.15 +cuda")
-    conflicts("fmt@10:", when="@:0.15 +rocm")
+    conflicts("^fmt@10:", when="@:0.15 +cuda")
+    conflicts("^fmt@10:", when="@:0.15 +rocm")
     depends_on("hwloc@1.11.5:")
 
     depends_on("gperftools", when="malloc=tcmalloc")
@@ -109,7 +117,7 @@ class Pika(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("rocblas", when="+rocm")
     depends_on("rocsolver", when="@0.5: +rocm")
     depends_on("tracy-client", when="+tracy")
-    conflicts("tracy-client@0.9:", when="@:0.9")
+    conflicts("^tracy-client@0.9:", when="@:0.9")
     depends_on("whip@0.1: +rocm", when="@0.9: +rocm")
     depends_on("whip@0.1: +cuda", when="@0.9: +cuda")
 
@@ -134,7 +142,7 @@ class Pika(CMakePackage, CudaPackage, ROCmPackage):
             )
 
     for cxxstd in cxxstds:
-        depends_on("boost cxxstd={0}".format(map_cxxstd(cxxstd)), when="cxxstd={0}".format(cxxstd))
+        depends_on("boost cxxstd={0}".format(cxxstd), when="cxxstd={0}".format(cxxstd))
         depends_on("fmt cxxstd={0}".format(cxxstd), when="@0.11: cxxstd={0}".format(cxxstd))
 
     # COROUTINES
diff --git a/var/spack/repos/builtin/packages/plasma/package.py b/var/spack/repos/builtin/packages/plasma/package.py
index 98be783b3931d1..5cccfe3ff4e6b5 100644
--- a/var/spack/repos/builtin/packages/plasma/package.py
+++ b/var/spack/repos/builtin/packages/plasma/package.py
@@ -19,11 +19,13 @@ class Plasma(CMakePackage):
     homepage = "https://github.com/icl-utk-edu/plasma/"
     url = "https://github.com/icl-utk-edu/plasma/releases/download/21.8.29/plasma-21.8.29.tar.gz"
     git = "https://github.com/icl-utk-edu/plasma"
+
     maintainers("luszczek")
 
     tags = ["e4s"]
 
     version("develop", git=git)
+    version("23.8.2", sha256="2db34de0575f3e3d16531bdcf1caddef146f68e71335977a3e8ec193003ab943")
     version("22.9.29", sha256="78827898b7e3830eee2e388823b9180858279f77c5eda5aa1be173765c53ade5")
     version("21.8.29", sha256="e0bb4d9143c8540f9f46cbccac9ed0cbea12500a864e6954fce2fe94ea057a10")
     version("20.9.20", sha256="2144a77b739f8dd2f0dbe5b64d94cde0e916f55c4eb170facd168c0db7fc7970")
@@ -62,9 +64,6 @@ class Plasma(CMakePackage):
     conflicts("^netlib-lapack@:3.5")
 
     # clashes with OpenBLAS declarations and has a problem compiling on its own
-    conflicts("^cblas")
-
-    conflicts("^openblas-with-lapack")  # incomplete LAPACK implementation
     conflicts("^veclibfort")
 
     # only GCC 4.9+ and higher have sufficient support for OpenMP 4+ tasks+deps
diff --git a/var/spack/repos/builtin/packages/plink-ng/package.py b/var/spack/repos/builtin/packages/plink-ng/package.py
index afd59a9a230bda..a5b9a4c98935c7 100644
--- a/var/spack/repos/builtin/packages/plink-ng/package.py
+++ b/var/spack/repos/builtin/packages/plink-ng/package.py
@@ -14,7 +14,7 @@ class PlinkNg(Package):
 
     version("200511", sha256="00cff19bece88acb7a21ba098501cb677b78d22c9f3ca5bcdc869139a40db816")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("zstd@1.4.4:")
     depends_on("cblas")
     depends_on("blas")
@@ -27,7 +27,7 @@ def url_for_version(self, ver):
         return template.format(ver)
 
     def setup_build_environment(self, env):
-        zlib = join_path(self.spec["zlib"].prefix.lib, "libz.a")
+        zlib = join_path(self.spec["zlib-api"].prefix.lib, "libz.a")
         env.set("ZLIB", zlib)
 
     def install(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/plink/package.py b/var/spack/repos/builtin/packages/plink/package.py
index 9de14ebf209230..3e44f039bee06b 100644
--- a/var/spack/repos/builtin/packages/plink/package.py
+++ b/var/spack/repos/builtin/packages/plink/package.py
@@ -39,7 +39,7 @@ class Plink(Package):
     depends_on("netlib-lapack", when="@1.9-beta5:1.9-beta6.10")
 
     with when("@1.9-beta-6.27:"):
-        depends_on("zlib", when="@1.9-beta6.27:")
+        depends_on("zlib-api", when="@1.9-beta6.27:")
         depends_on("blas", when="@1.9-beta6.27:")
         depends_on("lapack", when="@1.9-beta6.27:")
 
@@ -61,7 +61,7 @@ def install(self, spec, prefix):
     @when("@1.9-beta6.27:")
     def setup_build_environment(self, env):
         env.set("BLASFLAGS", self.spec["blas"].libs.ld_flags)
-        env.set("ZLIB", self.spec["zlib"].libs.ld_flags)
+        env.set("ZLIB", self.spec["zlib-api"].libs.ld_flags)
 
     @when("@1.9-beta6.27:")
     def install(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/plink2/package.py b/var/spack/repos/builtin/packages/plink2/package.py
index 7846f5ae4d9aa1..d11c02730b7f3c 100644
--- a/var/spack/repos/builtin/packages/plink2/package.py
+++ b/var/spack/repos/builtin/packages/plink2/package.py
@@ -13,9 +13,10 @@ class Plink2(MakefilePackage):
     homepage = "https://www.cog-genomics.org/plink/2.0/"
     git = "https://github.com/chrchang/plink-ng.git"
 
-    version("2.00a4.3", tag="v2.00a4.3")
+    version("2.00a4.3", tag="v2.00a4.3", commit="59fca48f6f8135886ff68962fbe31ae0c6413228")
 
-    depends_on("zlib@1.2.12:")
+    depends_on("zlib-api")
+    depends_on("zlib@1.2.12:", when="^zlib")
     depends_on("zstd@1.5.2:")
     depends_on("libdeflate@1.10:")
     depends_on("blas")
diff --git a/var/spack/repos/builtin/packages/ploticus/package.py b/var/spack/repos/builtin/packages/ploticus/package.py
new file mode 100644
index 00000000000000..cff43fae29ab1e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/ploticus/package.py
@@ -0,0 +1,41 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Ploticus(MakefilePackage):
+    """Ploticus can produce various types of plots and graphs."""
+
+    homepage = "http://ploticus.sourceforge.net/doc/welcome.html"
+
+    maintainers("Christoph-TU")
+
+    version("2.42", sha256="3f29e4b9f405203a93efec900e5816d9e1b4381821881e241c08cab7dd66e0b0")
+
+    depends_on("zlib-api")
+    depends_on("libpng")
+
+    build_directory = "src"
+
+    def url_for_version(self, version):
+        # spack's default url_for_version may replace "242_src" with 2.42_src, causing a 404.
+        # Returning the correct url here instead of as 'url =' fixes this issue:
+        return (
+            "https://sourceforge.net/projects/ploticus/files/ploticus/2.42/ploticus242_src.tar.gz"
+        )
+
+    def setup_run_environment(self, env):
+        env.set("PLOTICUS_PREFABS", self.prefix.prefabs)
+
+    def edit(self, spec, prefix):
+        makefile = FileFilter("src/Makefile")
+        makefile.filter("CC = .*", "CC = {0}".format(spack_cc))
+
+    def install(self, spec, prefix):
+        mkdir(prefix.bin)
+        mkdir(prefix.prefabs)
+        install("src/pl", prefix.bin)
+        install_tree("prefabs", prefix.prefabs)
diff --git a/var/spack/repos/builtin/packages/plumed/package.py b/var/spack/repos/builtin/packages/plumed/package.py
index 836f7e485021dd..6baaa7e9cf3c51 100644
--- a/var/spack/repos/builtin/packages/plumed/package.py
+++ b/var/spack/repos/builtin/packages/plumed/package.py
@@ -30,6 +30,9 @@ class Plumed(AutotoolsPackage):
 
     version("master", branch="master")
 
+    version("2.9.0", sha256="612d2387416b5f82dd8545709921440370e144fd46cef633654cf0ee43bac5f8")
+
+    version("2.8.3", sha256="e98da486e252cdf290b0b5b2f3f021409ea0d2d775ab609a6ad68fc1ab143a3b")
     version("2.8.2", sha256="a2064bacba1dde36b05aaf351ba4b7e6d30a165b332b0a83b7a6db04b477be9f")
     version("2.8.1", sha256="f56bc9266c8a47241385c595717c2734a9b67148a7f4122b808bc0733710173e")
     version("2.8.0", sha256="8357eca6f280125037ad4e7c427f96f2af2f60ddfedce1a2e36e1e1cc3bff32b")
@@ -182,7 +185,7 @@ class Plumed(AutotoolsPackage):
     )
 
     # Dependencies. LAPACK and BLAS are recommended but not essential.
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("blas")
     depends_on("lapack")
     # For libmatheval support through the 'function' module
diff --git a/var/spack/repos/builtin/packages/pmix/package.py b/var/spack/repos/builtin/packages/pmix/package.py
index 651c41689f7a1c..97c389d70dd82f 100644
--- a/var/spack/repos/builtin/packages/pmix/package.py
+++ b/var/spack/repos/builtin/packages/pmix/package.py
@@ -37,6 +37,11 @@ class Pmix(AutotoolsPackage):
 
     # Branches 4.2 & 5.0 will also need submodules
     version("master", branch="master", submodules=True)
+    version("5.0.1", sha256="d4371792d4ba4c791e1010100b4bf9a65500ababaf5ff25d681f938527a67d4a")
+    version("5.0.0", sha256="92a85c4946346816c297ac244fbaf4f723bba87fb7e4424a057c2dabd569928d")
+    version("4.2.6", sha256="10b0d5a7fca70272e9427c677557578ac452cea02aeb00e30dec2116d20c3cd0")
+    version("4.2.5", sha256="a89c2c5dc69715a4df1e76fdc4318299386c184623a1d0d5eb1fb062e14b0d2b")
+    version("4.2.4", sha256="c4699543f2278d3a78bdac72b4b2da9cd92d11d18478d66522b8991764b021c8")
     version("4.2.3", sha256="c3d9d6885ae39c15627a86dc4718e050baf604acda71b8b9e2ee3b12ad5c2d2a")
     version("4.2.2", sha256="935b2f492e4bc409017f1425a83366aa72a7039605ea187c9fac7bb1371cd73c")
     version("4.2.1", sha256="3c992fa0d653b56e0e409bbaec9de8fc1b82c948364dbb28545442315ed2a179")
@@ -118,7 +123,7 @@ def find_external_lib_path(self, pkg_name, path_match_str=""):
     def configure_args(self):
         spec = self.spec
 
-        config_args = ["--enable-shared", "--enable-static", "--disable-sphinx"]
+        config_args = ["--enable-shared", "--enable-static", "--disable-sphinx", "--without-munge"]
 
         config_args.append("--with-libevent=" + spec["libevent"].prefix)
         config_args.append("--with-hwloc=" + spec["hwloc"].prefix)
diff --git a/var/spack/repos/builtin/packages/pngwriter/package.py b/var/spack/repos/builtin/packages/pngwriter/package.py
index 376cd3746e5ede..a7aea9ea9466f9 100644
--- a/var/spack/repos/builtin/packages/pngwriter/package.py
+++ b/var/spack/repos/builtin/packages/pngwriter/package.py
@@ -29,7 +29,7 @@ class Pngwriter(CMakePackage):
     version("0.5.6", sha256="0c5f3c1fd6f2470e88951f4b8add64cf5f5a7e7038115dba69604139359b08f1")
 
     depends_on("libpng")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("freetype")
 
     def cmake_args(self):
diff --git a/var/spack/repos/builtin/packages/pocl/package.py b/var/spack/repos/builtin/packages/pocl/package.py
index 850b8ce5045df0..50e048d9b3ad11 100644
--- a/var/spack/repos/builtin/packages/pocl/package.py
+++ b/var/spack/repos/builtin/packages/pocl/package.py
@@ -20,7 +20,7 @@ class Pocl(CMakePackage):
     git = "https://github.com/pocl/pocl.git"
 
     version("master", branch="master")
-    version("3.0", sha256="5f6bbc391ba144bc7becc3b90888b25468460d5aa6830f63a3b066137e7bfac3")
+    version("3.0", sha256="a3fd3889ef7854b90b8e4c7899c5de48b7494bf770e39fba5ad268a5cbcc719d")
     version("1.8", sha256="0f63377ae1826e16e90038fc8e7f65029be4ff6f9b059f6907174b5c0d1f8ab2")
     version("1.7", sha256="5f6bbc391ba144bc7becc3b90888b25468460d5aa6830f63a3b066137e7bfac3")
     version("1.6", sha256="b0a4c0c056371b6f0db726b88fbb76bbee94948fb2abd4dbc8d958f7c42f766c")
diff --git a/var/spack/repos/builtin/packages/podio/package.py b/var/spack/repos/builtin/packages/podio/package.py
index 5666b268ec1ea3..5f0b94ba3770eb 100644
--- a/var/spack/repos/builtin/packages/podio/package.py
+++ b/var/spack/repos/builtin/packages/podio/package.py
@@ -20,13 +20,49 @@ class Podio(CMakePackage):
     tags = ["hep", "key4hep"]
 
     version("master", branch="master")
-    version("0.16.5", sha256="42135e4d0e11be6f0d88748799fa2ec985514d4b4c979a10a56a00a378f65ee0")
-    version("0.16.3", sha256="d8208f98496af68ca8d02d302f428aab510e50d07575b90c3477fff7e499335b")
-    version("0.16.2", sha256="faf7167290faf322f23c734adff19904b10793b5ab14e1dfe90ce257c225114b")
-    version("0.16.1", sha256="23cd8dfd00f9cd5ae0b473ae3279fa2c22a2d90fb6c07b37d56e63a80dd76ab2")
-    version("0.16", sha256="4e149c2c9be9f9ca3a6d863498bb0f642dda1a43a19ac1afe7f99854ded5c510")
-    version("0.15", sha256="6c1520877ba1bce250e35a2a56c0a3da89fae0916c5ed7d5548d658237e067d9")
-    version("0.14.3", sha256="2a7a405dedc7f6980a0aad7df87b427a1f43bcf6d923a9bcce1698fd296359f7")
+    version("0.17.1", sha256="97d6c5f81d50ee42bf7c01f041af2fd333c806f1bbf0a4828ca961a24cea6bb2")
+    version("0.17", sha256="0c19f69970a891459cab227ab009514f1c1ce102b70e8c4b7d204eb6a0c643c1")
+    version("0.16.7", sha256="8af7c947e2637f508b7af053412bacd9218d41a455d69addd7492f05b7a4338d")
+    version(
+        "0.16.6",
+        sha256="859f7cd16bd2b833bee9c1f33eb4cdbc2a0c2b1a48a853f67c30e8a0301d16df",
+        deprecated=True,
+    )
+    version(
+        "0.16.5",
+        sha256="42135e4d0e11be6f0d88748799fa2ec985514d4b4c979a10a56a00a378f65ee0",
+        deprecated=True,
+    )
+    version(
+        "0.16.3",
+        sha256="d8208f98496af68ca8d02d302f428aab510e50d07575b90c3477fff7e499335b",
+        deprecated=True,
+    )
+    version(
+        "0.16.2",
+        sha256="faf7167290faf322f23c734adff19904b10793b5ab14e1dfe90ce257c225114b",
+        deprecated=True,
+    )
+    version(
+        "0.16.1",
+        sha256="23cd8dfd00f9cd5ae0b473ae3279fa2c22a2d90fb6c07b37d56e63a80dd76ab2",
+        deprecated=True,
+    )
+    version(
+        "0.16",
+        sha256="4e149c2c9be9f9ca3a6d863498bb0f642dda1a43a19ac1afe7f99854ded5c510",
+        deprecated=True,
+    )
+    version(
+        "0.15",
+        sha256="6c1520877ba1bce250e35a2a56c0a3da89fae0916c5ed7d5548d658237e067d9",
+        deprecated=True,
+    )
+    version(
+        "0.14.3",
+        sha256="2a7a405dedc7f6980a0aad7df87b427a1f43bcf6d923a9bcce1698fd296359f7",
+        deprecated=True,
+    )
     version(
         "0.14.1",
         sha256="361ac3f3ec6f5a4830729ab45f96c19f0f62e9415ff681f7c6cdb4ebdb796f72",
@@ -91,6 +127,7 @@ class Podio(CMakePackage):
         description="Use the specified C++ standard when building.",
     )
     variant("sio", default=False, description="Build the SIO I/O backend")
+    variant("rntuple", default=False, description="Build the RNTuple backend")
 
     # cpack config throws an error on some systems
     patch("cpack.patch", when="@:0.10.0")
@@ -98,9 +135,12 @@ class Podio(CMakePackage):
     patch("python-tests.patch", when="@:0.14.0")
 
     depends_on("root@6.08.06: cxxstd=17", when="cxxstd=17")
-    depends_on("root@6.25.02: cxxstd=20", when="cxxstd=20")
+    depends_on("root@6.28.04:", when="+rntuple")
+    depends_on("root@6.28:", when="@0.17:")
+    for cxxstd in ("17", "20"):
+        depends_on("root cxxstd={}".format(cxxstd), when="cxxstd={}".format(cxxstd))
 
-    depends_on("cmake@3.8:", type="build")
+    depends_on("cmake@3.12:", type="build")
     depends_on("python", type=("build", "run"))
     depends_on("py-pyyaml", type=("build", "run"))
     depends_on("py-jinja2@2.10.1:", type=("build", "run"), when="@0.12.0:")
@@ -110,10 +150,12 @@ class Podio(CMakePackage):
     depends_on("py-tabulate", type=("run", "test"), when="@0.16.6:")
 
     conflicts("+sio", when="@:0.12", msg="sio support requires at least podio@0.13")
+    conflicts("+rntuple", when="@:0.16", msg="rntuple support requires at least podio@0.17")
 
     def cmake_args(self):
         args = [
             self.define_from_variant("ENABLE_SIO", "sio"),
+            self.define_from_variant("ENABLE_RNTUPLE", "rntuple"),
             self.define("CMAKE_CXX_STANDARD", self.spec.variants["cxxstd"].value),
             self.define("BUILD_TESTING", self.run_tests),
         ]
@@ -135,6 +177,12 @@ def setup_dependent_build_environment(self, env, dependent_spec):
         env.prepend_path("PYTHONPATH", self.prefix.python)
         env.prepend_path("LD_LIBRARY_PATH", self.spec["podio"].libs.directories[0])
         env.prepend_path("ROOT_INCLUDE_PATH", self.prefix.include)
+        if self.spec.satisfies("+sio @0.17:"):
+            # sio needs to be on LD_LIBRARY_PATH for ROOT to be able to
+            # dynamicaly load the python libraries also in dependent build
+            # environments since the import structure has changed with
+            # podio@0.17
+            env.prepend_path("LD_LIBRARY_PATH", self.spec["sio"].libs.directories[0])
 
     def url_for_version(self, version):
         """Translate version numbers to ilcsoft conventions.
diff --git a/var/spack/repos/builtin/packages/poppler/package.py b/var/spack/repos/builtin/packages/poppler/package.py
index 42bcf249d72111..7b04a7639d8658 100644
--- a/var/spack/repos/builtin/packages/poppler/package.py
+++ b/var/spack/repos/builtin/packages/poppler/package.py
@@ -54,7 +54,7 @@ class Poppler(CMakePackage):
     depends_on("curl", when="+libcurl")
     depends_on("openjpeg", when="+openjpeg")
     depends_on("qt@4.0:", when="+qt")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("cairo+ft@1.10.0:", when="+glib")
     depends_on("iconv", when="+iconv")
     depends_on("jpeg", when="+jpeg")
diff --git a/var/spack/repos/builtin/packages/portcullis/package.py b/var/spack/repos/builtin/packages/portcullis/package.py
index 68b4eaad4d1d7d..09e9c6224769dc 100644
--- a/var/spack/repos/builtin/packages/portcullis/package.py
+++ b/var/spack/repos/builtin/packages/portcullis/package.py
@@ -22,7 +22,7 @@ class Portcullis(AutotoolsPackage):
     depends_on("boost")
     depends_on("m4", type="build")
 
-    depends_on("zlib", type="build")
+    depends_on("zlib-api", type="build")
     depends_on("samtools", type="build")
 
     depends_on("python@3.4:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/ports-of-call/package.py b/var/spack/repos/builtin/packages/ports-of-call/package.py
index 59974479e4d0ea..5cdbc95d13f7d5 100644
--- a/var/spack/repos/builtin/packages/ports-of-call/package.py
+++ b/var/spack/repos/builtin/packages/ports-of-call/package.py
@@ -16,6 +16,7 @@ class PortsOfCall(CMakePackage):
     maintainers("rbberger")
 
     version("main", branch="main")
+    version("1.5.2", sha256="73d16fe9236a9475010dbb01bf751c15bef01eb2e15bf92c8d9be2c0a606329f")
     version("1.5.1", sha256="b1f0232cd6d2aac65385d77cc061ec5035283ea50d0f167e7003eae034effb78")
     version("1.4.1", sha256="82d2c75fcca8bd613273fd4126749df68ccc22fbe4134ba673b4275f9972b78d")
     version("1.4.0", sha256="e08ae556b7c30d14d77147d248d118cf5343a2e8c0847943385c602394bda0fa")
diff --git a/var/spack/repos/builtin/packages/postgresql/package.py b/var/spack/repos/builtin/packages/postgresql/package.py
index ed13a9dea841ed..f1e904c560a4e5 100644
--- a/var/spack/repos/builtin/packages/postgresql/package.py
+++ b/var/spack/repos/builtin/packages/postgresql/package.py
@@ -57,7 +57,7 @@ class Postgresql(AutotoolsPackage):
     depends_on("libedit", when="lineedit=libedit")
     depends_on("openssl")
     depends_on("tcl", when="+tcl")
-    depends_on("perl", when="+perl")
+    depends_on("perl+opcode", when="+perl")
     depends_on("python", when="+python")
     depends_on("libxml2", when="+xml")
 
diff --git a/var/spack/repos/builtin/packages/povray/package.py b/var/spack/repos/builtin/packages/povray/package.py
index 632af24a0917e7..6f050f8cb28c8d 100644
--- a/var/spack/repos/builtin/packages/povray/package.py
+++ b/var/spack/repos/builtin/packages/povray/package.py
@@ -66,7 +66,7 @@ class Povray(AutotoolsPackage):
     # for instance depends_on('boost +filesystem')
     # See https://github.com/spack/spack/pull/22303 for reference
     depends_on(Boost.with_default_variants, when="+boost")
-    depends_on("zlib@1.2.1:", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("libpng@1.2.5:", when="+libpng")
     depends_on("jpeg", when="+jpeg")
     depends_on("libtiff@3.6.1:", when="+libtiff")
diff --git a/var/spack/repos/builtin/packages/precice/package.py b/var/spack/repos/builtin/packages/precice/package.py
index 7c47b7b32a252d..2c4eb9665b45af 100644
--- a/var/spack/repos/builtin/packages/precice/package.py
+++ b/var/spack/repos/builtin/packages/precice/package.py
@@ -47,6 +47,20 @@ class Precice(CMakePackage):
     variant("python", default=False, description="Enable Python support", when="@2:")
     variant("shared", default=True, description="Build shared libraries")
 
+    for build_type in ("Release", "RelWithDebInfo", "MinSizeRel"):
+        variant(
+            "debug_log",
+            default=False,
+            description="Enable debug log in non-debug builds",
+            when=f"@2.4: build_type={build_type}",
+        )
+        variant(
+            "checked",
+            default=False,
+            description="Enable assertions in non-debug builds",
+            when=f"@2.4: build_type={build_type}",
+        )
+
     depends_on("cmake@3.5:", type="build")
     depends_on("cmake@3.10.2:", type="build", when="@1.4:")
     depends_on("cmake@3.16.3:", type="build", when="@2.4:")
@@ -83,47 +97,66 @@ class Precice(CMakePackage):
     conflicts("%intel@:16")
     conflicts("%pgi@:17.3")
 
+    def xsdk_tpl_args(self):
+        return [
+            "-DTPL_ENABLE_BOOST:BOOL=ON",
+            "-DTPL_ENABLE_EIGEN3:BOOL=ON",
+            "-DTPL_ENABLE_LIBXML2:BOOL=ON",
+            self.define_from_variant("TPL_ENABLE_PETSC", "petsc"),
+            self.define_from_variant("TPL_ENABLE_PYTHON", "python"),
+        ]
+
     def cmake_args(self):
         """Populate cmake arguments for precice."""
         spec = self.spec
 
-        # The xSDK installation policies were implemented after 1.5.2
-        xsdk_mode = spec.satisfies("@1.6:")
-
         # Select the correct CMake variables by version
         mpi_option = "MPI"
-        if spec.satisfies("@2:"):
-            mpi_option = "PRECICE_MPICommunication"
         petsc_option = "PETSC"
-        if spec.satisfies("@2:"):
-            petsc_option = "PRECICE_PETScMapping"
         python_option = "PYTHON"
         if spec.satisfies("@2:"):
+            mpi_option = "PRECICE_MPICommunication"
+            petsc_option = "PRECICE_PETScMapping"
             python_option = "PRECICE_PythonActions"
+        if spec.satisfies("@3:"):
+            mpi_option = "PRECICE_FEATURE_MPI_COMMUNICATION"
+            petsc_option = "PRECICE_FEATURE_PETSC_MAPPING"
+            python_option = "PRECICE_FEATURE_PYTHON_ACTIONS"
+
+        cmake_args = [
+            self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
+            self.define_from_variant(mpi_option, "mpi"),
+            self.define_from_variant(petsc_option, "petsc"),
+            self.define_from_variant(python_option, "python"),
+        ]
+
+        # The xSDK installation policies were implemented after 1.5.2.
+        # The TPL arguments were removed in 3.0.0.
+        if spec.satisfies("@1.6:3"):
+            cmake_args.extend(self.xsdk_tpl_args())
+
+        # Release options
+        if spec.satisfies("@2.4:"):
+            cmake_args.extend(
+                [
+                    self.define_from_variant("PRECICE_RELEASE_WITH_DEBUG_LOG", "debug_log"),
+                    self.define_from_variant("PRECICE_RELEASE_WITH_ASSERTIONS", "checked"),
+                ]
+            )
 
-        def variant_bool(feature, on="ON", off="OFF"):
-            """Ternary for spec variant to ON/OFF string"""
-            if feature in spec:
-                return on
-            return off
-
-        cmake_args = ["-DBUILD_SHARED_LIBS:BOOL=%s" % variant_bool("+shared")]
+        # Disable CPack
+        if spec.satisfies("@3:"):
+            cmake_args.append("-DPRECICE_CONFIGURE_PACKAGE_GENERATION:BOOL=OFF")
 
-        cmake_args.append("-D%s:BOOL=%s" % (mpi_option, variant_bool("+mpi")))
+        # Dependencies
 
         # Boost
-        if xsdk_mode:
-            cmake_args.append("-DTPL_ENABLE_BOOST=ON")
         cmake_args.append("-DBOOST_ROOT=%s" % spec["boost"].prefix)
 
         # Eigen3
-        if xsdk_mode:
-            cmake_args.append("-DTPL_ENABLE_EIGEN3=ON")
         cmake_args.append("-DEIGEN3_INCLUDE_DIR=%s" % spec["eigen"].headers.directories[0])
 
         # LibXML2
-        if xsdk_mode:
-            cmake_args.append("-DTPL_ENABLE_LIBXML2=ON")
         libxml2_includes = spec["libxml2"].headers.directories[0]
         cmake_args.extend(
             [
@@ -134,13 +167,7 @@ def variant_bool(feature, on="ON", off="OFF"):
 
         # PETSc
         if "+petsc" in spec:
-            if xsdk_mode:
-                cmake_args.append("-DTPL_ENABLE_PETSC:BOOL=ON")
-            else:
-                cmake_args.append("-D%s:BOOL=ON" % petsc_option)
             cmake_args.extend(["-DPETSC_DIR=%s" % spec["petsc"].prefix, "-DPETSC_ARCH=."])
-        else:
-            cmake_args.append("-D%s:BOOL=OFF" % petsc_option)
 
         # Python
         if "+python" in spec:
@@ -149,10 +176,6 @@ def variant_bool(feature, on="ON", off="OFF"):
             numpy_include = join_path(
                 spec["py-numpy"].prefix, spec["python"].package.platlib, "numpy", "core", "include"
             )
-            if xsdk_mode:
-                cmake_args.append("-DTPL_ENABLE_PYTHON:BOOL=ON")
-            else:
-                cmake_args.append("-D%s:BOOL=ON" % python_option)
             cmake_args.extend(
                 [
                     "-DPYTHON_INCLUDE_DIR=%s" % python_include,
@@ -160,7 +183,5 @@ def variant_bool(feature, on="ON", off="OFF"):
                     "-DPYTHON_LIBRARY=%s" % python_library,
                 ]
             )
-        else:
-            cmake_args.append("-D%s:BOOL=OFF" % python_option)
 
         return cmake_args
diff --git a/var/spack/repos/builtin/packages/procps/libintl-3.3.14.patch b/var/spack/repos/builtin/packages/procps/libintl-3.3.14.patch
new file mode 100644
index 00000000000000..923679e6c04544
--- /dev/null
+++ b/var/spack/repos/builtin/packages/procps/libintl-3.3.14.patch
@@ -0,0 +1,10 @@
+--- a/Makefile.am	2023-06-29 14:12:25.671539354 -0500
++++ b/Makefile.am	2023-06-29 14:26:58.852425405 -0500
+@@ -1,6 +1,5 @@
+-CYGWINFLAGS =
++CYGWINFLAGS = $(LTLIBINTL)
+ if CYGWIN
+-CYGWINFLAGS += -lintl
+ usrbin_exec_PROGRAMS =
+ endif
+ 
diff --git a/var/spack/repos/builtin/packages/procps/libintl-4.0.0.patch b/var/spack/repos/builtin/packages/procps/libintl-4.0.0.patch
new file mode 100644
index 00000000000000..4fed0e0a442ddf
--- /dev/null
+++ b/var/spack/repos/builtin/packages/procps/libintl-4.0.0.patch
@@ -0,0 +1,29 @@
+--- a/Makefile.am	2022-03-22 05:40:10.000000000 -0500
++++ b/Makefile.am	2023-06-29 16:26:37.759057671 -0500
+@@ -1,7 +1,6 @@
+ 
+-CYGWINFLAGS =
++CYGWINFLAGS = $(LTLIBINTL)
+ if CYGWIN
+-CYGWINFLAGS += -lintl
+ usrbin_exec_PROGRAMS =
+ endif
+ 
+@@ -137,7 +136,7 @@
+ endif
+ dist_man_MANS += kill.1
+ kill_SOURCES = kill.c lib/strutils.c lib/fileutils.c lib/signals.c
+-kill_LDADD =
++kill_LDADD = $(LTLIBINTL)
+ else
+   EXTRA_DIST += kill.1
+ endif
+@@ -228,7 +227,7 @@
+ 	sysctl.c \
+ 	lib/fileutils.c \
+ 	lib/procio.c
+-sysctl_LDADD=
++sysctl_LDADD= $(LTLIBINTL)
+ endif
+ tload_SOURCES = tload.c lib/strutils.c lib/fileutils.c
+ uptime_SOURCES = uptime.c lib/fileutils.c
diff --git a/var/spack/repos/builtin/packages/procps/libintl-4.0.1.patch b/var/spack/repos/builtin/packages/procps/libintl-4.0.1.patch
new file mode 100644
index 00000000000000..a2c6f3ebcb47ed
--- /dev/null
+++ b/var/spack/repos/builtin/packages/procps/libintl-4.0.1.patch
@@ -0,0 +1,29 @@
+--- a/Makefile.am	2022-10-20 04:58:25.000000000 -0500
++++ b/Makefile.am	2023-06-29 16:38:47.941981053 -0500
+@@ -1,7 +1,6 @@
+ 
+-CYGWINFLAGS =
++CYGWINFLAGS = $(LTLIBINTL)
+ if CYGWIN
+-CYGWINFLAGS += -lintl
+ usrbin_exec_PROGRAMS =
+ endif
+ 
+@@ -136,7 +135,7 @@
+ endif
+ dist_man_MANS += man/kill.1
+ src_kill_SOURCES = src/kill.c local/strutils.c local/fileutils.c local/signals.c
+-src_kill_LDADD =
++src_kill_LDADD = $(LTLIBINTL)
+ else
+   EXTRA_DIST += man/kill.1
+ endif
+@@ -227,7 +226,7 @@
+ 	src/sysctl.c \
+ 	local/fileutils.c \
+ 	local/procio.c
+-src_sysctl_LDADD=
++src_sysctl_LDADD= $(LTLIBINTL)
+ endif
+ src_tload_SOURCES = src/tload.c local/strutils.c local/fileutils.c
+ src_uptime_SOURCES = src/uptime.c local/fileutils.c
diff --git a/var/spack/repos/builtin/packages/procps/package.py b/var/spack/repos/builtin/packages/procps/package.py
index 5c135e250564fb..791625102bc809 100644
--- a/var/spack/repos/builtin/packages/procps/package.py
+++ b/var/spack/repos/builtin/packages/procps/package.py
@@ -14,9 +14,18 @@ class Procps(AutotoolsPackage):
 
     homepage = "https://gitlab.com/procps-ng/procps"
     git = "https://gitlab.com/procps-ng/procps.git"
+    url = "https://gitlab.com/procps-ng/procps/-/archive/v4.0.3/procps-v4.0.3.tar.gz"
 
     version("master", branch="master")
-    version("3.3.15", tag="v3.3.15")
+    version("4.0.4", sha256="3214fab0f817d169f2c117842ba635bafb1cd6090273e311a8b5c6fc393ddb9d")
+    version("4.0.3", sha256="14cc21219c45d196772274ea3f194f6d668b6cc667fbde9ee6d8039121b73fa6")
+    version("4.0.2", sha256="b03e4b55eaa5661e726acb714e689356d80bc056b09965c2284d039ba8dc21e8")
+    version("4.0.1", sha256="1eaff353306aba12816d14881f2b88c7c9d06023825f7224700f0c01f66c65cd")
+    version("4.0.0", sha256="dea39e0e7b1367e28c887d736d1a9783df617497538603cdff432811a1016945")
+    version("3.3.17", sha256="efa6f6b4625a795f5c8a3d5bd630a121d270bc8573c5a0b6a6068e73611d6cd5")
+    version("3.3.16", sha256="7f09945e73beac5b12e163a7ee4cae98bcdd9a505163b6a060756f462907ebbc")
+    version("3.3.15", sha256="14dfa751517dd844efa9f492e3ad8071f908a269c6aea643b9a1759235fa2053")
+    version("3.3.14", sha256="1ff716e7bde6b3841b8519831690b10b644ed344490369c55e410edc8db2fe18")
 
     variant("nls", default=True, description="Enable Native Language Support.")
 
@@ -27,22 +36,25 @@ class Procps(AutotoolsPackage):
     depends_on("pkgconfig@0.9.0:", type="build")
     depends_on("dejagnu", type="test")
     depends_on("iconv")
-    depends_on("gettext", when="+nls")
+    depends_on("gettext", type="build")  # required by autogen.sh
+    with when("+nls"):
+        depends_on("gettext")
+        # msgfmt 0.22 gives parsing errors
+        depends_on("gettext@:0.21", when="@:4.0.3")
     depends_on("ncurses")
 
     conflicts("platform=darwin", msg="procps is linux-only")
 
+    # Need to tell the build to use the tools it already has to find
+    # libintl (if appropriate).
+    patch("libintl-3.3.14.patch", when="@3.3.14:3.3")
+    patch("libintl-4.0.0.patch", when="@=4.0.0")
+    patch("libintl-4.0.1.patch", when="@4.0.1:4.0.3")
+
     def autoreconf(self, spec, prefix):
         sh = which("sh")
         sh("autogen.sh")
 
-    def flag_handler(self, name, flags):
-        if name == "ldlibs":
-            spec = self.spec
-            if "+nls" in spec and "intl" in spec["gettext"].libs.names:
-                flags.append("-lintl")
-        return self.build_system_flags(name, flags)
-
     def configure_args(self):
         spec = self.spec
         args = ["--with-ncurses"]
diff --git a/var/spack/repos/builtin/packages/prod-util/darwin/apple-clang-13.0.0-stdlib.patch b/var/spack/repos/builtin/packages/prod-util/darwin/apple-clang-13.0.0-stdlib.patch
deleted file mode 100644
index 1f4080114ce41f..00000000000000
--- a/var/spack/repos/builtin/packages/prod-util/darwin/apple-clang-13.0.0-stdlib.patch
+++ /dev/null
@@ -1,7 +0,0 @@
---- a/sorc/fsync_file.cd/fsync_file.c	2021-12-23 15:17:16.000000000 -0700
-+++ b/sorc/fsync_file.cd/fsync_file.c	2021-12-23 15:17:37.000000000 -0700
-@@ -1,3 +1,4 @@
-+#include 
- #include 
- #include 
- #include 
diff --git a/var/spack/repos/builtin/packages/prod-util/package.py b/var/spack/repos/builtin/packages/prod-util/package.py
index a997aba9668918..885fddee43dcf8 100644
--- a/var/spack/repos/builtin/packages/prod-util/package.py
+++ b/var/spack/repos/builtin/packages/prod-util/package.py
@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import sys
-
 from spack.package import *
 
 
@@ -16,12 +14,15 @@ class ProdUtil(CMakePackage):
 
     homepage = "https://github.com/NOAA-EMC/NCEPLIBS-prod_util"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-prod_util/archive/refs/tags/v1.2.2.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-prod_util"
 
     maintainers("AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
 
-    version("1.2.2", sha256="c51b903ea5a046cb9b545b5c04fd28647c58b4ab6182e61710f0287846350ef8")
+    version("develop", branch="develop")
+    version("2.1.0", sha256="fa7df4a82dae269ffb347b9007376fb0d9979c17c4974814ea82204b12d70ea5")
 
-    depends_on("w3nco")
+    depends_on("w3emc")
 
-    if sys.platform == "darwin":
-        patch("darwin/apple-clang-13.0.0-stdlib.patch")
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/protobuf/package.py b/var/spack/repos/builtin/packages/protobuf/package.py
index 295e57850e0f5a..47880d98e86568 100644
--- a/var/spack/repos/builtin/packages/protobuf/package.py
+++ b/var/spack/repos/builtin/packages/protobuf/package.py
@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import spack.util.web
+import spack.url
 from spack.package import *
 
 
@@ -14,6 +14,8 @@ class Protobuf(CMakePackage):
     url = "https://github.com/protocolbuffers/protobuf/archive/v3.18.0.tar.gz"
     maintainers("hyoklee")
 
+    version("3.24.3", sha256="2c23dee0bdbc36bd43ee457083f8f5560265d0815cc1c56033de3932843262fe")
+    version("3.23.3", sha256="5e4b555f72a7e3f143a7aff7262292500bb02c49b174351684bb70fc7f2a6d33")
     version("3.22.2", sha256="2118051b4fb3814d59d258533a4e35452934b1ddb41230261c9543384cbb4dfc")
     version("3.21.12", sha256="930c2c3b5ecc6c9c12615cf5ad93f1cd6e12d0aba862b572e076259970ac3a53")
     version("3.21.9", sha256="1add10f9bd92775b91f326da259f243881e904dd509367d5031d4c782ba82810")
@@ -80,9 +82,10 @@ class Protobuf(CMakePackage):
         values=("Debug", "Release", "RelWithDebInfo"),
     )
 
+    depends_on("abseil-cpp@20230125.3:", when="@3.22.5:")
     # https://github.com/protocolbuffers/protobuf/issues/11828#issuecomment-1433557509
     depends_on("abseil-cpp@20230125:", when="@3.22:")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     conflicts("%gcc@:4.6", when="@3.6.0:")  # Requires c++11
     conflicts("%gcc@:4.6", when="@3.2.0:3.3.0")  # Breaks
@@ -102,6 +105,13 @@ class Protobuf(CMakePackage):
         sha256="fa1abf042eddc1b3b43875dc018c651c90cd1c0c5299975a818a1610bee54ab8",
     )
 
+    # fix build on Centos 8, see also https://github.com/protocolbuffers/protobuf/issues/5144
+    patch(
+        "https://github.com/protocolbuffers/protobuf/pull/11032/commits/3039f932aaf212bcf2f14a3f2fd00dbfb881e46b.patch?full_index=1",
+        when="@:3.21",
+        sha256="cefc4bf4aadf9ca33a336b2aa6d0d82006b6563e85122ae8cfb70345f85321dd",
+    )
+
     patch("msvc-abseil-target-namespace.patch", when="@3.22 %msvc")
 
     def fetch_remote_versions(self, *args, **kwargs):
@@ -111,9 +121,7 @@ def fetch_remote_versions(self, *args, **kwargs):
         return dict(
             map(
                 lambda u: (u, self.url_for_version(u)),
-                spack.util.web.find_versions_of_archive(
-                    self.all_urls, self.list_url, self.list_depth
-                ),
+                spack.url.find_versions_of_archive(self.all_urls, self.list_url, self.list_depth),
             )
         )
 
@@ -125,10 +133,11 @@ def cmake_args(self):
         ]
 
         if self.spec.satisfies("@3.22:"):
+            cxxstd = self.spec["abseil-cpp"].variants["cxxstd"].value
             args.extend(
                 [
                     self.define("protobuf_ABSL_PROVIDER", "package"),
-                    self.define("CMAKE_CXX_STANDARD", 14),
+                    self.define("CMAKE_CXX_STANDARD", cxxstd),
                 ]
             )
 
diff --git a/var/spack/repos/builtin/packages/pruners-ninja/package.py b/var/spack/repos/builtin/packages/pruners-ninja/package.py
index 7ec3f33243c4a6..9a74da5b3b4eb1 100644
--- a/var/spack/repos/builtin/packages/pruners-ninja/package.py
+++ b/var/spack/repos/builtin/packages/pruners-ninja/package.py
@@ -24,3 +24,9 @@ class PrunersNinja(AutotoolsPackage):
     depends_on("libtool", type="build")
 
     patch("pruners-mutli-def-a-pr3-fix.patch")
+
+    def flag_handler(self, name, flags):
+        if name == "cflags":
+            if self.spec.satisfies("%oneapi"):
+                flags.append("-Wno-error=implicit-function-declaration")
+        return (flags, None, None)
diff --git a/var/spack/repos/builtin/packages/psm/package.py b/var/spack/repos/builtin/packages/psm/package.py
index de956356c5f2e4..28beadccb69f34 100644
--- a/var/spack/repos/builtin/packages/psm/package.py
+++ b/var/spack/repos/builtin/packages/psm/package.py
@@ -18,7 +18,7 @@ class Psm(MakefilePackage):
         sha256="034b10e24d9f2967ef0f8d0f828572295e89cdfa1ba30c35e288b9b23c3dab8f",
         preferred=True,
     )
-    version("2017-04-28", commit="604758e")
+    version("2017-04-28", commit="604758e76dc31e68d1de736ccf5ddf16cb22355b")
 
     conflicts("%gcc@6:", when="@3.3")
 
diff --git a/var/spack/repos/builtin/packages/psmc/package.py b/var/spack/repos/builtin/packages/psmc/package.py
index d2ba830df700b7..d027c896274f97 100644
--- a/var/spack/repos/builtin/packages/psmc/package.py
+++ b/var/spack/repos/builtin/packages/psmc/package.py
@@ -15,7 +15,7 @@ class Psmc(MakefilePackage):
 
     version("2016-1-21", commit="e5f7df5d00bb75ec603ae0beff62c0d7e37640b9")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     def setup_run_environment(self, env):
         env.prepend_path("PATH", self.prefix.bin.utils)
diff --git a/var/spack/repos/builtin/packages/pumi/package.py b/var/spack/repos/builtin/packages/pumi/package.py
index 298041452b774d..5e99fbfac9e9d0 100644
--- a/var/spack/repos/builtin/packages/pumi/package.py
+++ b/var/spack/repos/builtin/packages/pumi/package.py
@@ -28,6 +28,9 @@ class Pumi(CMakePackage):
     # scorec/core develop branch and we prefer not to expose spack users
     # to the added instability.
     version("master", submodules=True, branch="master")
+    version(
+        "2.2.8", submodules=True, commit="736bb87ccd8db51fc499a1b91e53717a88841b1f"
+    )  # tag 2.2.8
     version(
         "2.2.7", submodules=True, commit="a295720d7b4828282484f2b78bac1f6504512de4"
     )  # tag 2.2.7
diff --git a/var/spack/repos/builtin/packages/py-accelerate/package.py b/var/spack/repos/builtin/packages/py-accelerate/package.py
index 3456c55ac2b326..564def61b8754c 100644
--- a/var/spack/repos/builtin/packages/py-accelerate/package.py
+++ b/var/spack/repos/builtin/packages/py-accelerate/package.py
@@ -12,11 +12,16 @@ class PyAccelerate(PythonPackage):
     homepage = "https://github.com/huggingface/accelerate"
     pypi = "accelerate/accelerate-0.16.0.tar.gz"
 
+    maintainers("meyersbs")
+
+    version("0.21.0", sha256="e2959a0bf74d97c0b3c0e036ed96065142a060242281d27970d4c4e34f11ca59")
     version("0.16.0", sha256="d13e30f3e6debfb46cada7b931af85560619b6a6a839d0cafeeab6ed7c6a498d")
 
+    depends_on("python@3.8.0:", when="@0.21.0:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("py-numpy@1.17:", type=("build", "run"))
     depends_on("py-packaging@20:", type=("build", "run"))
     depends_on("py-psutil", type=("build", "run"))
     depends_on("py-pyyaml", type=("build", "run"))
+    depends_on("py-torch@1.10.0:", when="@0.21.0:", type=("build", "run"))
     depends_on("py-torch@1.4:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-accessible-pygments/package.py b/var/spack/repos/builtin/packages/py-accessible-pygments/package.py
new file mode 100644
index 00000000000000..e2254161c79dda
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-accessible-pygments/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyAccessiblePygments(PythonPackage):
+    """This package includes a collection of accessible themes for pygments based on
+    different sources."""
+
+    homepage = "https://github.com/Quansight-Labs/accessible-pygments"
+    pypi = "accessible-pygments/accessible-pygments-0.0.4.tar.gz"
+
+    version("0.0.4", sha256="e7b57a9b15958e9601c7e9eb07a440c813283545a20973f2574a5f453d0e953e")
+
+    depends_on("py-pygments@1.5:", type=("build", "run"))
+    depends_on("py-setuptools", type=("build"))
diff --git a/var/spack/repos/builtin/packages/py-aiofiles/package.py b/var/spack/repos/builtin/packages/py-aiofiles/package.py
index b92bb4b26b2b84..ed1e2dc3e6fbda 100644
--- a/var/spack/repos/builtin/packages/py-aiofiles/package.py
+++ b/var/spack/repos/builtin/packages/py-aiofiles/package.py
@@ -13,7 +13,11 @@ class PyAiofiles(PythonPackage):
     homepage = "https://github.com/Tinche/aiofiles"
     pypi = "aiofiles/aiofiles-0.5.0.tar.gz"
 
+    version("0.7.0", sha256="a1c4fc9b2ff81568c83e21392a82f344ea9d23da906e4f6a52662764545e19d4")
     version("0.5.0", sha256="98e6bcfd1b50f97db4980e182ddd509b7cc35909e903a8fe50d8849e02d815af")
 
-    depends_on("py-setuptools", type="build")
-    depends_on("py-wheel", type="build")
+    depends_on("python@3.6:3", when="@0.7:", type=("build", "run"))
+    depends_on("py-poetry-core@1:", when="@0.7:", type="build")
+
+    # Historical dependencies
+    depends_on("py-setuptools", when="@:0.6", type="build")
diff --git a/var/spack/repos/builtin/packages/py-alpaca-eval/package.py b/var/spack/repos/builtin/packages/py-alpaca-eval/package.py
new file mode 100644
index 00000000000000..9ef8a2d14c4ea6
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-alpaca-eval/package.py
@@ -0,0 +1,28 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyAlpacaEval(PythonPackage):
+    """An automatic evaluator for instruction-following language models.
+    Human-validated, high-quality, cheap, and fast."""
+
+    homepage = "https://github.com/tatsu-lab/alpaca_eval"
+    pypi = "alpaca_eval/alpaca_eval-0.2.8.tar.gz"
+
+    maintainers("meyersbs")
+
+    version("0.2.8", sha256="5b21b74d7362ee229481b6a6d826dd620b2ef6b82e4f5470645e0a4b696a31e6")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("python@3.10:", type=("build", "run"))
+    depends_on("py-python-dotenv", type=("build", "run"))
+    depends_on("py-datasets", type=("build", "run"))
+    depends_on("py-openai", type=("build", "run"))
+    depends_on("py-pandas", type=("build", "run"))
+    depends_on("py-tiktoken@0.3.2:", type=("build", "run"))
+    depends_on("py-fire", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-alpaca-farm/package.py b/var/spack/repos/builtin/packages/py-alpaca-farm/package.py
new file mode 100644
index 00000000000000..8e278cb00ffd36
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-alpaca-farm/package.py
@@ -0,0 +1,39 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyAlpacaFarm(PythonPackage):
+    """AlpacaFarm is a simulator that enables research and development on learning
+    from feedback at a fraction of the usual cost, promoting accessible research on
+    instruction following and alignment."""
+
+    homepage = "https://github.com/tatsu-lab/alpaca_farm"
+    pypi = "alpaca_farm/alpaca_farm-0.1.9.tar.gz"
+
+    maintainers("meyersbs")
+
+    version("0.1.9", sha256="1039d33c814d0bbbcab6a0e77ed8e897992ad7107d5c4999d56bdad7e0b0a59f")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("python@3.10:", type=("build", "run"))
+    depends_on("py-datasets", type=("build", "run"))
+    depends_on("py-einops", type=("build", "run"))
+    depends_on("py-nltk", type=("build", "run"))
+    depends_on("py-accelerate@0.18.0:", type=("build", "run"))
+    depends_on("py-tabulate", type=("build", "run"))
+    depends_on("py-transformers@4.26.0:", type=("build", "run"))
+    depends_on("py-statsmodels", type=("build", "run"))
+    depends_on("py-tiktoken@0.3.2:", type=("build", "run"))
+    depends_on("py-markdown", type=("build", "run"))
+    depends_on("py-scikit-learn", type=("build", "run"))
+    depends_on("py-sentencepiece", type=("build", "run"))
+    depends_on("py-pandas", type=("build", "run"))
+    depends_on("py-wandb", type=("build", "run"))
+    depends_on("py-torch@1.13.1:", type=("build", "run"))
+    depends_on("py-fire", type=("build", "run"))
+    depends_on("py-openai", type=("build", "run"))
+    depends_on("py-alpaca-eval@0.2.8:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-aniso8601/package.py b/var/spack/repos/builtin/packages/py-aniso8601/package.py
index f5659a200847e9..17946429114297 100644
--- a/var/spack/repos/builtin/packages/py-aniso8601/package.py
+++ b/var/spack/repos/builtin/packages/py-aniso8601/package.py
@@ -13,5 +13,6 @@ class PyAniso8601(PythonPackage):
     pypi = "aniso8601/aniso8601-9.0.1.tar.gz"
 
     version("9.0.1", sha256="72e3117667eedf66951bb2d93f4296a56b94b078a8a95905a052611fb3f1b973")
+    version("7.0.0", sha256="513d2b6637b7853806ae79ffaca6f3e8754bdd547048f5ccc1420aec4b714f1e")
 
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-ansimarkup/package.py b/var/spack/repos/builtin/packages/py-ansimarkup/package.py
new file mode 100644
index 00000000000000..deedb8e44b6042
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-ansimarkup/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyAnsimarkup(PythonPackage):
+    """Produce colored terminal text with an xml-like markup."""
+
+    homepage = "https://github.com/gvalkov/python-ansimarkup"
+    pypi = "ansimarkup/ansimarkup-1.5.0.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("2.1.0", sha256="7b3e3d93fecc5b64d23a6e8eb96dbc8b0b576a211829d948afb397d241a8c51b")
+    version("1.5.0", sha256="96c65d75bbed07d3dcbda8dbede8c2252c984f90d0ca07434b88a6bbf345fad3")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-setuptools@61:", type="build", when="@2.1.0")
+    depends_on("py-colorama", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-anyio/package.py b/var/spack/repos/builtin/packages/py-anyio/package.py
index 09b8581b62670a..f5b74226a2ecfa 100644
--- a/var/spack/repos/builtin/packages/py-anyio/package.py
+++ b/var/spack/repos/builtin/packages/py-anyio/package.py
@@ -13,17 +13,24 @@ class PyAnyio(PythonPackage):
     homepage = "https://github.com/agronholm/anyio"
     pypi = "anyio/anyio-3.2.1.tar.gz"
 
+    version("4.0.0", sha256="f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a")
     version("3.6.2", sha256="25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421")
     version("3.6.1", sha256="413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b")
     version("3.5.0", sha256="a0aeffe2fb1fdf374a8e4b471444f0f3ac4fb9f5a5b542b48824475e0042a5a6")
     version("3.3.4", sha256="67da67b5b21f96b9d3d65daa6ea99f5d5282cb09f50eb4456f8fb51dffefc3ff")
     version("3.2.1", sha256="07968db9fa7c1ca5435a133dc62f988d84ef78e1d9b22814a59d1c62618afbc5")
 
+    depends_on("python@3.8:", when="@4:", type=("build", "run"))
     depends_on("python@3.6.2:", type=("build", "run"))
+    depends_on("py-setuptools@64:", when="@3.7:", type="build")
     depends_on("py-setuptools@42:", type="build")
-    depends_on("py-wheel@0.29:", type="build")
-    depends_on("py-setuptools-scm+toml@3.4:", type="build")
+    depends_on("py-setuptools-scm@6.4:", when="@3.7:", type="build")
+    depends_on("py-setuptools-scm+toml@3.4:", when="@:3.6", type="build")
 
+    depends_on("py-exceptiongroup@1.0.2:", when="@4: ^python@:3.10", type=("build", "run"))
     depends_on("py-idna@2.8:", type=("build", "run"))
     depends_on("py-sniffio@1.1:", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-wheel@0.29:", when="@:3.6", type="build")
     depends_on("py-typing-extensions", when="^python@:3.7", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-apeye-core/package.py b/var/spack/repos/builtin/packages/py-apeye-core/package.py
new file mode 100644
index 00000000000000..2939f6bd099e0d
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-apeye-core/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyApeyeCore(PythonPackage):
+    """Core (offline) functionality for the apeye library."""
+
+    homepage = "https://github.com/domdfcoding/apeye-core"
+    pypi = "apeye_core/apeye_core-1.1.4.tar.gz"
+
+    version("1.1.4", sha256="72bb89fed3baa647cb81aa28e1d851787edcbf9573853b5d2b5f87c02f50eaf5")
+
+    depends_on("py-hatch-requirements-txt", type="build")
+    depends_on("py-hatchling", type="build")
+    depends_on("py-domdf-python-tools@2.6:", type=("build", "run"))
+    depends_on("py-idna@2.5:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-apeye/package.py b/var/spack/repos/builtin/packages/py-apeye/package.py
new file mode 100644
index 00000000000000..fddd51ec1ea1af
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-apeye/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyApeye(PythonPackage):
+    """Handy tools for working with URLs and APIs."""
+
+    homepage = "https://github.com/domdfcoding/apeye"
+    pypi = "apeye/apeye-1.4.1.tar.gz"
+
+    version("1.4.1", sha256="14ea542fad689e3bfdbda2189a354a4908e90aee4bf84c15ab75d68453d76a36")
+
+    depends_on("py-flit-core@3.2:3", type="build")
+    depends_on("py-apeye-core@1:", type=("build", "run"))
+    depends_on("py-domdf-python-tools@2.6:", type=("build", "run"))
+    depends_on("py-platformdirs@2.3:", type=("build", "run"))
+    depends_on("py-requests@2.24:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-argcomplete/package.py b/var/spack/repos/builtin/packages/py-argcomplete/package.py
index 92e1319b2e049e..146562369311ae 100644
--- a/var/spack/repos/builtin/packages/py-argcomplete/package.py
+++ b/var/spack/repos/builtin/packages/py-argcomplete/package.py
@@ -12,13 +12,17 @@ class PyArgcomplete(PythonPackage):
     homepage = "https://github.com/kislyuk/argcomplete"
     pypi = "argcomplete/argcomplete-1.12.0.tar.gz"
 
+    version("3.1.2", sha256="d5d1e5efd41435260b8f85673b74ea2e883affcbec9f4230c582689e8e78251b")
     version("3.0.8", sha256="b9ca96448e14fa459d7450a4ab5a22bbf9cee4ba7adddf03e65c398b5daeea28")
     version("2.0.0", sha256="6372ad78c89d662035101418ae253668445b391755cfe94ea52f1b9d22425b20")
     version("1.12.3", sha256="2c7dbffd8c045ea534921e63b0be6fe65e88599990d8dc408ac8c542b72a5445")
     version("1.12.0", sha256="2fbe5ed09fd2c1d727d4199feca96569a5b50d44c71b16da9c742201f7cc295c")
     version("1.1.1", sha256="cca45b5fe07000994f4f06a0b95bd71f7b51b04f81c3be0b4ea7b666e4f1f084")
 
+    depends_on("py-setuptools@67.7.2:", when="@3.1:", type="build")
     depends_on("py-setuptools", type="build")
+    depends_on("py-setuptools-scm+toml@6.2:", when="@3.1:", type="build")
+
     depends_on("py-importlib-metadata@0.23:6", when="@3.0.6: ^python@:3.7", type=("build", "run"))
     depends_on(
         "py-importlib-metadata@0.23:4", when="@1.12.3:2 ^python@:3.7", type=("build", "run")
diff --git a/var/spack/repos/builtin/packages/py-argparse-manpage/package.py b/var/spack/repos/builtin/packages/py-argparse-manpage/package.py
new file mode 100644
index 00000000000000..74108bfbdfbaaf
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-argparse-manpage/package.py
@@ -0,0 +1,24 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyArgparseManpage(PythonPackage):
+    """Build manual page from python's ArgumentParser object."""
+
+    homepage = "https://github.com/praiskup/argparse-manpage"
+    pypi = "argparse-manpage/argparse-manpage-4.5.tar.gz"
+
+    version("4.5", sha256="213c061878a10bf0e40f6a293382f6e82409e5110d0683b16ebf87f903d604db")
+
+    variant("setuptools", default=False, description="Enable the setuptools.builds_meta backend")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-packaging", type="build")
+
+    depends_on("py-tomli", when="^python@:3.10", type=("build", "run"))
+
+    depends_on("py-setuptools", when="+setuptools", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-asttokens/package.py b/var/spack/repos/builtin/packages/py-asttokens/package.py
index 7bad57ade670b6..9f42ec37acec6e 100644
--- a/var/spack/repos/builtin/packages/py-asttokens/package.py
+++ b/var/spack/repos/builtin/packages/py-asttokens/package.py
@@ -12,6 +12,7 @@ class PyAsttokens(PythonPackage):
     homepage = "https://github.com/gristlabs/asttokens"
     pypi = "asttokens/asttokens-2.0.5.tar.gz"
 
+    version("2.4.0", sha256="2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e")
     version("2.2.1", sha256="4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3")
     version("2.0.8", sha256="c61e16246ecfb2cde2958406b4c8ebc043c9e6d73aaa83c941673b35e5d3a76b")
     version("2.0.5", sha256="9a54c114f02c7a9480d56550932546a3f1fe71d8a02f1bc7ccd0ee3ee35cf4d5")
@@ -19,4 +20,5 @@ class PyAsttokens(PythonPackage):
     depends_on("py-setuptools@44:", type="build")
     depends_on("py-setuptools-scm+toml@3.4.3:", type="build")
 
+    depends_on("py-six@1.12:", when="@2.3:", type=("build", "run"))
     depends_on("py-six", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-ats/package.py b/var/spack/repos/builtin/packages/py-ats/package.py
index 38d30c9a291ccb..7348ced38d81a6 100644
--- a/var/spack/repos/builtin/packages/py-ats/package.py
+++ b/var/spack/repos/builtin/packages/py-ats/package.py
@@ -17,9 +17,9 @@ class PyAts(PythonPackage):
     maintainers("white238")
 
     version("main", branch="main")
-    version("7.0.105", tag="7.0.105")
-    version("7.0.100", tag="7.0.100")
-    version("7.0.5", tag="7.0.5")
+    version("7.0.105", tag="7.0.105", commit="3a3461061d4493a002018f5bb3715db702212f72")
+    version("7.0.100", tag="7.0.100", commit="202c18d11b8f1c14f1a3361a6e45c9e4f83a3fa1")
+    version("7.0.5", tag="7.0.5", commit="86b0b18b96b179f97008393170f5e5bc95118867")
 
     # TODO: Add flux variant when Flux functionality works in ATS
 
diff --git a/var/spack/repos/builtin/packages/py-autocfg/package.py b/var/spack/repos/builtin/packages/py-autocfg/package.py
new file mode 100644
index 00000000000000..d941c89b9300b5
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-autocfg/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyAutocfg(PythonPackage):
+    """Deep learning configuration."""
+
+    homepage = "https://github.com/zhreshold/autocfg"
+    pypi = "autocfg/autocfg-0.0.8.tar.gz"
+
+    version("0.0.8", sha256="749986b4f3b3bd85b15298734bf8fa4a590e6c34a314ac515025e058ed76c319")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-pyyaml", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-backoff/package.py b/var/spack/repos/builtin/packages/py-backoff/package.py
new file mode 100644
index 00000000000000..8ccfcf6811ae91
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-backoff/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyBackoff(PythonPackage):
+    """Function decoration for backoff and retry."""
+
+    homepage = "https://github.com/litl/backoff"
+    pypi = "backoff/backoff-2.2.1.tar.gz"
+
+    version("2.2.1", sha256="03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba")
+
+    depends_on("python@3.7:3", type=("build", "run"))
+    depends_on("py-poetry-core@1:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-barectf/package.py b/var/spack/repos/builtin/packages/py-barectf/package.py
new file mode 100644
index 00000000000000..7975a3fff8f1af
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-barectf/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyBarectf(PythonPackage):
+    """barectf (from bare metal and CTF) is a generator of
+    tracer which produces CTF data streams."""
+
+    pypi = "barectf/barectf-3.1.2.tar.gz"
+
+    version("3.1.2", sha256="d4d626b22a33b7d9bc9ac033bba8893890aba0ee1011c9e78429a67296c09e1c")
+
+    depends_on("py-poetry-core", type="build")
diff --git a/var/spack/repos/builtin/packages/py-bids-validator/package.py b/var/spack/repos/builtin/packages/py-bids-validator/package.py
index a91047d9726f92..5ba74c229de73f 100644
--- a/var/spack/repos/builtin/packages/py-bids-validator/package.py
+++ b/var/spack/repos/builtin/packages/py-bids-validator/package.py
@@ -12,6 +12,7 @@ class PyBidsValidator(PythonPackage):
     homepage = "https://github.com/bids-standard/bids-validator"
     pypi = "bids-validator/bids-validator-1.7.2.tar.gz"
 
+    version("1.13.1", sha256="7205ce4e68fba172215332c786f1ac1665025b702b6dff2b1e158f00a2df9890")
     version("1.11.0", sha256="408c56748b7cf98cf7c31822f33a8d89c5e6e7db5254c345107e8d527576ff53")
     version("1.9.8", sha256="ff39799bb205f92d6f2c322f0b8eff0d1c0288f4291a0b18fce61afa4dfd7f3e")
     version("1.9.4", sha256="4bf07d375f231a2ad2f450beeb3ef6c54f93194fd993aa5157d57a8fba48ed50")
@@ -19,4 +20,5 @@ class PyBidsValidator(PythonPackage):
     version("1.8.4", sha256="63e7a02c9ddb5505a345e178f4e436b82c35ec0a177d7047b67ea10ea3029a68")
     version("1.7.2", sha256="12398831a3a3a2ed7c67e693cf596610c23dd23e0889bfeae0830bbd1d41e5b9")
 
+    depends_on("python@3.8:", when="@1.12:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-bidscoin/package.py b/var/spack/repos/builtin/packages/py-bidscoin/package.py
index 3c840f69716f9c..a5b7dd83024578 100644
--- a/var/spack/repos/builtin/packages/py-bidscoin/package.py
+++ b/var/spack/repos/builtin/packages/py-bidscoin/package.py
@@ -13,20 +13,23 @@ class PyBidscoin(PythonPackage):
     homepage = "https://github.com/Donders-Institute/bidscoin"
     pypi = "bidscoin/bidscoin-3.7.4.tar.gz"
 
+    version("4.1.1", sha256="28730e9202d3c44d77c0bbdea9565e00adfdd23e85a6f3f121c1bfce1a7b462b")
     version("4.0.0", sha256="3b0c26f2e250e06b6f526cdbee09517e1f339da8035c0a316609b4463d75824d")
     version("3.7.4", sha256="efa32238fb7b75e533e7f5cc318ad5a703716d291985435d43f1de4f18402517")
 
     depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-setuptools@62.2:", when="@4.1:", type="build")
     depends_on("py-setuptools@61:", when="@4:", type="build")
     depends_on("py-setuptools", type="build")
-    depends_on("py-pytest-runner", type="build")
+    depends_on("py-argparse-manpage+setuptools", when="@4.1:", type="build")
 
     depends_on("py-pandas", type=("build", "run"))
     depends_on("py-matplotlib", type=("build", "run"))
     depends_on("py-numpy", type=("build", "run"))
     depends_on("py-pydicom@2:", type=("build", "run"))
-    depends_on("py-pyqt5@5.12.1:", type=("build", "run"))
+    depends_on("py-pyqt6", when="@4.1:", type=("build", "run"))
     depends_on("py-ruamel-yaml@0.15.35:", type=("build", "run"))
+    depends_on("py-tomli@1.1:", when="@4.1: ^python@:3.10", type=("build", "run"))
     depends_on("py-coloredlogs", type=("build", "run"))
     depends_on("py-tqdm@4.60:", when="@4:", type=("build", "run"))
     depends_on("py-tqdm", type=("build", "run"))
@@ -34,6 +37,10 @@ class PyBidscoin(PythonPackage):
     depends_on("py-python-dateutil", type=("build", "run"))
     depends_on("py-nibabel", type=("build", "run"))
     depends_on("py-bids-validator", when="@4:", type=("build", "run"))
-    depends_on("py-pydeface", when="@4:", type=("build", "run"))
-    depends_on("py-pytest", when="@4:", type=("build", "run"))
     depends_on("dcm2niix", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-pytest-runner", when="@:3", type="build")
+    depends_on("py-pyqt5@5.12.1:", when="@:4.0", type=("build", "run"))
+    depends_on("py-pydeface", when="@4.0", type=("build", "run"))
+    depends_on("py-pytest", when="@4.0", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-bidskit/package.py b/var/spack/repos/builtin/packages/py-bidskit/package.py
index d54a2a778eecbd..25e38c5127ef56 100644
--- a/var/spack/repos/builtin/packages/py-bidskit/package.py
+++ b/var/spack/repos/builtin/packages/py-bidskit/package.py
@@ -12,6 +12,7 @@ class PyBidskit(PythonPackage):
     homepage = "https://github.com/jmtyszka/bidskit"
     pypi = "bidskit/bidskit-2022.10.13.tar.gz"
 
+    version("2023.9.7", sha256="029d9aecbbcb2df733858ceb3e6d5dd5013c36e431e40fb522a580adc7b667a5")
     version("2023.2.16", sha256="b2e4e3246d43a6f00af6c0391ec8fecc59405241de1ea9ca68eb4d8128d62c7b")
     version(
         "2022.10.13", sha256="576b92cef187032c73f64e2e6a5b0be0c06771442048a33c55e224b3df0aae3a"
diff --git a/var/spack/repos/builtin/packages/py-biom-format/package.py b/var/spack/repos/builtin/packages/py-biom-format/package.py
index e96614165eba51..79175dbfcb2a50 100644
--- a/var/spack/repos/builtin/packages/py-biom-format/package.py
+++ b/var/spack/repos/builtin/packages/py-biom-format/package.py
@@ -13,6 +13,10 @@ class PyBiomFormat(PythonPackage):
 
     pypi = "biom-format/biom-format-2.1.6.tar.gz"
 
+    version("2.1.15", sha256="3bda2096e663dc1cb6f90f51b394da0838b9be5164a44370c134ce5b3b2a4dd3")
+    version("2.1.14", sha256="c8bac94ab6aa8226c0d38af7a3341d65e5f3664b9f45ec44fdf8b5275b2f92c1")
+    version("2.1.13", sha256="c48ed8fe978adaff5832f9d65ffcf8b735298bb2175b0360251d556baac5d4dc")
+    version("2.1.12", sha256="a4460e803b2abfcabe76d5d8fec0f3f7e76a8cd0e09bf22bb38dea9fca224ac2")
     version("2.1.10", sha256="f5a277a8144f0b114606852c42f657b9cfde44b3cefa0b2638ab1c1d5e1d0488")
     version("2.1.9", sha256="18a6e4d4b4b2a6bf2d5544fa357ad168bedeac93f0837015ef9c72f41fa89491")
     version("2.1.7", sha256="b47e54282ef13cddffdb00aea9183a87175a2372c91a915259086a3f444c42f4")
@@ -20,12 +24,18 @@ class PyBiomFormat(PythonPackage):
 
     depends_on("python@2.7:", type=("build", "run"))
     depends_on("python@3:", type=("build", "run"), when="@2.1.9:")
-    depends_on("py-setuptools", type=("build", "run"))
-    depends_on("py-cython@0.29:", type="build")
-    depends_on("py-h5py", type=("build", "run"))
-    depends_on("py-click", type=("build", "run"), when="@2.1.5:")
-    depends_on("py-numpy@1.9.2:", type=("build", "run"))
-    depends_on("py-future@0.16.0:", type=("build", "run"))
-    depends_on("py-scipy@1.3.1:", type=("build", "run"))
-    depends_on("py-pandas@0.20.0:", type=("build", "run"))
-    depends_on("py-six@1.10.0:", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
+    depends_on("py-cython", type="build")
+    depends_on("py-h5py@2.2.0:", type=("build", "run"))
+    depends_on("py-click", type=("build", "run"))
+    depends_on("py-numpy@1.3.0:", type=("build", "run"))
+    depends_on("py-numpy@1.9.2:", type=("build", "run"), when="@2.1.7:")
+    depends_on("py-future@0.16.0:", type=("build", "run"), when="@:2.1.10")
+    depends_on("py-scipy@0.13.0:", type=("build", "run"))
+    depends_on("py-scipy@1.3.1:", type=("build", "run"), when="@2.1.8:")
+    depends_on("py-pandas@0.19.2:", type=("build", "run"))
+    depends_on("py-pandas@0.20.0:", type=("build", "run"), when="@2.1.7:")
+    depends_on("py-six@1.10.0:", type=("build", "run"), when="@:2.1.10")
+
+    # https://github.com/biocore/biom-format/pull/865
+    conflicts("^python@3.10:", when="@:2.1.10")
diff --git a/var/spack/repos/builtin/packages/py-bitarray/package.py b/var/spack/repos/builtin/packages/py-bitarray/package.py
index 5de17c22f64c6a..bb9b9cbc190799 100644
--- a/var/spack/repos/builtin/packages/py-bitarray/package.py
+++ b/var/spack/repos/builtin/packages/py-bitarray/package.py
@@ -11,8 +11,9 @@ class PyBitarray(PythonPackage):
 
     pypi = "bitarray/bitarray-0.8.1.tar.gz"
 
+    version("2.7.6", sha256="3807f9323c308bc3f9b58cbe5d04dc28f34ac32d852999334da96b42f64b5356")
+    version("2.7.4", sha256="143d4f65e1f45a533e13521be1dc557a782317ecf76520eabd5a903b26ecb187")
     version("2.6.0", sha256="56d3f16dd807b1c56732a244ce071c135ee973d3edc9929418c1b24c5439a0fd")
     version("0.8.1", sha256="7da501356e48a83c61f479393681c1bc4b94e5a34ace7e08cb29e7dd9290ab18")
 
-    depends_on("python")
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-bitstring/package.py b/var/spack/repos/builtin/packages/py-bitstring/package.py
index 461eb8d8c8fe68..61daaaefce749a 100644
--- a/var/spack/repos/builtin/packages/py-bitstring/package.py
+++ b/var/spack/repos/builtin/packages/py-bitstring/package.py
@@ -10,9 +10,19 @@ class PyBitstring(PythonPackage):
     """Simple construction, analysis and modification of binary data."""
 
     homepage = "http://pythonhosted.org/bitstring"
-    pypi = "bitstring/bitstring-3.1.5.zip"
+    pypi = "bitstring/bitstring-3.1.6.tar.gz"
 
+    version("4.0.2", sha256="a391db8828ac4485dd5ce72c80b27ebac3e7b989631359959e310cd9729723b2")
     version("3.1.5", sha256="c163a86fcef377c314690051885d86b47419e3e1770990c212e16723c1c08faa")
 
-    # pip silently replaces distutils with setuptools
-    depends_on("py-setuptools", type="build")
+    depends_on("py-setuptools", when="@:4.0.1", type="build")
+    depends_on("py-setuptools@61:", when="@4.0.2:", type="build")
+    depends_on("py-bitarray@2.7.4", when="@4.1:")
+
+    def url_for_version(self, version):
+        url = "https://pypi.org/packages/source/s/bitstring/bitstring-{}.{}"
+        if version < Version("3.1.6"):
+            suffix = "zip"
+        else:
+            suffix = "tar.gz"
+        return url.format(version, suffix)
diff --git a/var/spack/repos/builtin/packages/py-black/package.py b/var/spack/repos/builtin/packages/py-black/package.py
index e9813273f599f1..a2cba61bc88951 100644
--- a/var/spack/repos/builtin/packages/py-black/package.py
+++ b/var/spack/repos/builtin/packages/py-black/package.py
@@ -17,6 +17,12 @@ class PyBlack(PythonPackage):
 
     maintainers("adamjstewart")
 
+    version("23.11.0", sha256="4c68855825ff432d197229846f971bc4d6666ce90492e5b02013bcaca4d9ab05")
+    version("23.10.1", sha256="1f8ce316753428ff68749c65a5f7844631aa18c8679dfd3ca9dc1a289979c258")
+    version("23.10.0", sha256="31b9f87b277a68d0e99d2905edae08807c007973eaa609da5f0c62def6b7c0bd")
+    version("23.9.1", sha256="24b6b3ff5c6d9ea08a8888f6977eae858e1f340d7260cf56d70a49823236b62d")
+    version("23.9.0", sha256="3511c8a7e22ce653f89ae90dfddaf94f3bb7e2587a245246572d3b9c92adf066")
+    version("23.7.0", sha256="022a582720b0d9480ed82576c920a8c1dde97cc38ff11d8d8859b3bd6ca9eedb")
     version("23.3.0", sha256="1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940")
     version("23.1.0", sha256="b0bd97bea8903f5a2ba7219257a44e3f1f9d00073d6cc1add68f0beec69692ac")
     version("22.12.0", sha256="229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f")
@@ -34,24 +40,30 @@ class PyBlack(PythonPackage):
     depends_on("py-hatchling@1.8:", when="@22.10:", type="build")
     depends_on("py-hatch-vcs", when="@22.10:", type="build")
     depends_on("py-hatch-fancy-pypi-readme", when="@22.10:", type="build")
+
+    with default_args(type=("build", "run")):
+        depends_on("python@3.8:", when="@23.7:")
+        depends_on("python@3.7:", when="@22.10:")
+        depends_on("py-click@8:")
+        depends_on("py-mypy-extensions@0.4.3:")
+        depends_on("py-packaging@22:", when="@23.1:")
+        depends_on("py-pathspec@0.9:")
+        depends_on("py-platformdirs@2:")
+        depends_on("py-tomli@1.1:", when="@22.8: ^python@:3.10")
+        depends_on("py-tomli@1.1:", when="@21.7:22.6")
+        depends_on("py-typing-extensions@4.0.1:", when="@23.9: ^python@:3.10")
+        depends_on("py-typing-extensions@3.10:", when="@:23.7 ^python@:3.9")
+
+        depends_on("py-colorama@0.4.3:", when="+colorama")
+        depends_on("py-uvloop@0.15.2:", when="+uvloop")
+        depends_on("py-aiohttp@3.7.4:", when="+d")
+        depends_on("py-ipython@7.8:", when="+jupyter")
+        depends_on("py-tokenize-rt@3.2:", when="+jupyter")
+
+    # Historical dependencies
     depends_on("py-setuptools@45:", when="@:22.8", type=("build", "run"))
     depends_on("py-setuptools-scm@6.3.1:+toml", when="@:22.8", type="build")
-    # Needed to ensure that Spack can bootstrap black with Python 3.6
-    depends_on("python@3.7:", when="@22.10:", type=("build", "run"))
-    depends_on("py-click@8:", type=("build", "run"))
-    depends_on("py-mypy-extensions@0.4.3:", type=("build", "run"))
-    depends_on("py-packaging@22:", when="@23.1:", type=("build", "run"))
-    depends_on("py-pathspec@0.9:", type=("build", "run"))
-    depends_on("py-platformdirs@2:", type=("build", "run"))
-    depends_on("py-tomli@1.1:", when="@22.8: ^python@:3.10", type=("build", "run"))
-    depends_on("py-tomli@1.1:", when="@21.7:22.6", type=("build", "run"))
     depends_on("py-typed-ast@1.4.2:", when="^python@:3.7", type=("build", "run"))
-    depends_on("py-typing-extensions@3.10:", when="^python@:3.9", type=("build", "run"))
-    depends_on("py-colorama@0.4.3:", when="+colorama", type=("build", "run"))
-    depends_on("py-uvloop@0.15.2:", when="+uvloop", type=("build", "run"))
-    depends_on("py-aiohttp@3.7.4:", when="+d", type=("build", "run"))
-    depends_on("py-ipython@7.8:", when="+jupyter", type=("build", "run"))
-    depends_on("py-tokenize-rt@3.2:", when="+jupyter", type=("build", "run"))
 
     # Needed because this package is used to bootstrap Spack (Spack supports Python 3.6+)
     depends_on("py-dataclasses@0.6:", when="^python@:3.6", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-blosc2/package.py b/var/spack/repos/builtin/packages/py-blosc2/package.py
new file mode 100644
index 00000000000000..983ed0273b27e2
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-blosc2/package.py
@@ -0,0 +1,30 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyBlosc2(PythonPackage):
+    """Python wrapper for the C-Blosc2 library."""
+
+    homepage = "https://github.com/Blosc/python-blosc2"
+    pypi = "blosc2/blosc2-2.2.8.tar.gz"
+
+    version("2.2.8", sha256="59065aac5e9b01b0e9f3825d8e7f69f64b59bbfab148a47c54e4115f62a97474")
+    version("2.0.0", sha256="f19b0b3674f6c825b490f00d8264b0c540c2cdc11ec7e81178d38b83c57790a1")
+
+    depends_on("python@3.9:3", when="@2.2:", type=("build", "link", "run"))
+    depends_on("python@3.8:3", when="@2.0", type=("build", "link", "run"))
+    depends_on("py-setuptools", type="build")
+    depends_on("py-scikit-build", type="build")
+    depends_on("py-cython", type="build")
+    # FIXME: why doesn't this work?
+    # depends_on("py-cmake", type="build")
+    depends_on("cmake@3.11:", type="build")
+    depends_on("py-ninja", type="build")
+    depends_on("py-numpy@1.20.3:", type=("build", "link", "run"))
+    depends_on("py-ndindex@1.4:", when="@2.2:", type=("build", "run"))
+    depends_on("py-msgpack", type=("build", "run"))
+    depends_on("py-py-cpuinfo", when="@2.2:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-bluepyefe/package.py b/var/spack/repos/builtin/packages/py-bluepyefe/package.py
new file mode 100644
index 00000000000000..8a15e4edf9e2c5
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-bluepyefe/package.py
@@ -0,0 +1,25 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class PyBluepyefe(PythonPackage):
+    """Blue Brain Python E-feature extraction"""
+
+    homepage = "https://github.com/BlueBrain/BluePyEfe"
+    pypi = "bluepyefe/bluepyefe-2.2.18.tar.gz"
+    git = "https://github.com/BlueBrain/BluePyEfe.git"
+
+    version("2.2.18", sha256="bfb50c6482433ec2ffb4b65b072d2778bd89ae50d92dd6830969222aabb30275")
+
+    depends_on("py-setuptools", type="build")
+
+    depends_on("py-numpy@:1.23", type=("build", "run"))
+    depends_on("py-neo", type=("build", "run"))
+    depends_on("py-matplotlib", type=("build", "run"))
+    depends_on("py-efel", type=("build", "run"))
+    depends_on("py-scipy", type=("build", "run"))
+    depends_on("py-h5py", type=("build", "run"))
+    depends_on("py-igor", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-bluepyemodel/package.py b/var/spack/repos/builtin/packages/py-bluepyemodel/package.py
new file mode 100644
index 00000000000000..f865b9791b622b
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-bluepyemodel/package.py
@@ -0,0 +1,36 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyBluepyemodel(PythonPackage):
+    """Python library to optimize and evaluate electrical models."""
+
+    homepage = "https://github.com/BlueBrain/BluePyEModel"
+    pypi = "bluepyemodel/bluepyemodel-0.0.46.tar.gz"
+
+    version("0.0.46", sha256="ad4c125e491f3337fcc341a4f389b8a616d883ce50fd77d9fb0ea6e13be5da61")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-setuptools-scm", type="build")
+
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-scipy", type=("build", "run"))
+    depends_on("py-pandas", type=("build", "run"))
+    depends_on("py-ipyparallel@6.3:", type=("build", "run"))
+    depends_on("py-tqdm", type=("build", "run"))
+    depends_on("py-pyyaml", type=("build", "run"))
+    depends_on("py-gitpython", type=("build", "run"))
+    depends_on("py-bluepyopt@1.12.12:", type=("build", "run"))
+    depends_on("py-bluepyefe@2.2.0:", type=("build", "run"))
+    depends_on("py-neurom@3.0:3", type=("build", "run"))
+    depends_on("py-efel@3.1:", type=("build", "run"))
+    depends_on("py-configparser", type=("build", "run"))
+    depends_on("py-morph-tool@2.8:", type=("build", "run"))
+    depends_on("py-fasteners@0.16:", type=("build", "run"))
+    depends_on("neuron+python@8.0:", type=("build", "run"))
+    depends_on("py-jinja2@3.0.3", when="@0.0.11:", type=("build", "run"))
+    depends_on("py-currentscape@0.0.11:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-bluepyopt/package.py b/var/spack/repos/builtin/packages/py-bluepyopt/package.py
new file mode 100644
index 00000000000000..ccc39f913558fe
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-bluepyopt/package.py
@@ -0,0 +1,37 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class PyBluepyopt(PythonPackage):
+    """Bluebrain Python Optimisation Library"""
+
+    homepage = "https://github.com/BlueBrain/BluePyOpt"
+    pypi = "bluepyopt/bluepyopt-1.9.27.tar.gz"
+
+    # NOTE : while adding new release check pmi_rank.patch compatibility
+    version("1.14.4", sha256="7567fd736053250ca06030f67ad93c607b100c2b98df8dc588c26b64cb3e171c")
+
+    # patch required to avoid hpe-mpi linked mechanism library
+    patch("pmi_rank.patch")
+
+    variant("scoop", default=False, description="Use BluePyOpt together with py-scoop")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-numpy@1.6:", type=("build", "run"))
+    depends_on("py-pandas@0.18:", type=("build", "run"))
+    depends_on("py-deap@1.3.3:", type=("build", "run"))
+    depends_on("py-efel@2.13:", type=("build", "run"))
+    depends_on("py-ipyparallel", type=("build", "run"))
+    depends_on("py-pickleshare@0.7.3:", type=("build", "run"))
+    depends_on("py-jinja2@2.8:", type=("build", "run"))
+    depends_on("py-future", type=("build", "run"))
+    depends_on("py-pebble@4.6:", type=("build", "run"))
+    depends_on("py-scoop@0.7:", type=("build", "run"), when="+scoop")
+    depends_on("neuron@7.4:", type=("build", "run"))
+
+    def setup_run_environment(self, env):
+        env.unset("PMI_RANK")
+        env.set("NEURON_INIT_MPI", "0")
diff --git a/var/spack/repos/builtin/packages/py-bluepyopt/pmi_rank.patch b/var/spack/repos/builtin/packages/py-bluepyopt/pmi_rank.patch
new file mode 100644
index 00000000000000..21a73849b28683
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-bluepyopt/pmi_rank.patch
@@ -0,0 +1,17 @@
+diff --git a/bluepyopt/ephys/simulators.py b/bluepyopt/ephys/simulators.py
+index e71ad8b..3c93237 100644
+--- a/bluepyopt/ephys/simulators.py
++++ b/bluepyopt/ephys/simulators.py
+@@ -89,6 +89,12 @@ class NrnSimulator(object):
+             NrnSimulator._nrn_disable_banner()
+             self.banner_disabled = True
+ 
++        # certain mpi libraries (hpe-mpt) use PMI_RANK env variable to initialize
++        # MPI before calling MPI_Init (which is undesirable). Unset this variable
++        # if exist to avoid issue with loading neuron and mechanism library.
++        if 'PMI_RANK' in os.environ:
++            os.environ.pop("PMI_RANK")
++
+         import neuron  # NOQA
+ 
+         return neuron
diff --git a/var/spack/repos/builtin/packages/py-boost-histogram/package.py b/var/spack/repos/builtin/packages/py-boost-histogram/package.py
index 4780e20c0a7301..e14dadae5350fd 100644
--- a/var/spack/repos/builtin/packages/py-boost-histogram/package.py
+++ b/var/spack/repos/builtin/packages/py-boost-histogram/package.py
@@ -12,6 +12,7 @@ class PyBoostHistogram(PythonPackage):
     homepage = "https://github.com/scikit-hep/boost-histogram"
     pypi = "boost_histogram/boost_histogram-1.2.1.tar.gz"
 
+    version("1.3.2", sha256="e175efbc1054a27bc53fbbe95472cac9ea93999c91d0611840d776b99588d51a")
     version("1.3.1", sha256="31cd396656f3a37834e07d304cdb84d9906bc2172626a3d92fe577d08bcf410f")
     version("1.2.1", sha256="a27842b2f1cfecc509382da2b25b03056354696482b38ec3c0220af0fc9b7579")
 
diff --git a/var/spack/repos/builtin/packages/py-brotli/package.py b/var/spack/repos/builtin/packages/py-brotli/package.py
new file mode 100644
index 00000000000000..44fcbda75a23b3
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-brotli/package.py
@@ -0,0 +1,17 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyBrotli(PythonPackage):
+    """Python bindings for the Brotli compression library."""
+
+    homepage = "https://github.com/google/brotli"
+    pypi = "Brotli/Brotli-1.1.0.tar.gz"
+
+    version("1.1.0", sha256="81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724")
+
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-build/package.py b/var/spack/repos/builtin/packages/py-build/package.py
index 046955dac12e18..0812041e20ce1a 100644
--- a/var/spack/repos/builtin/packages/py-build/package.py
+++ b/var/spack/repos/builtin/packages/py-build/package.py
@@ -12,6 +12,8 @@ class PyBuild(PythonPackage):
     homepage = "https://github.com/pypa/build"
     pypi = "build/build-0.7.0.tar.gz"
 
+    version("1.0.3", sha256="538aab1b64f9828977f84bc63ae570b060a8ed1be419e7870b8b4fc5e6ea553b")
+    version("1.0.0", sha256="49a60f212df4d9925727c2118e1cbe3abf30b393eff7d0e7287d2170eb36844d")
     version("0.10.0", sha256="d5b71264afdb5951d6704482aac78de887c80691c52b88a9ad195983ca2c9269")
     version("0.9.0", sha256="1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c")
     version("0.8.0", sha256="887a6d471c901b1a6e6574ebaeeebb45e5269a79d095fe9a8f88d6614ed2e5f0")
@@ -19,17 +21,20 @@ class PyBuild(PythonPackage):
 
     variant("virtualenv", default=False, description="Install optional virtualenv dependency")
 
-    depends_on("python@3.6:", type=("build", "run"))
-    depends_on("python@3.7:", when="@0.10:", type=("build", "run"))
+    depends_on("py-flit-core@3.8:", when="@1:", type="build")
     depends_on("py-flit-core@3.4:", when="@0.10:", type="build")
-    depends_on("py-setuptools", when="@:0.9", type="build")
     depends_on("py-packaging@19:", type=("build", "run"))
     depends_on("py-pyproject-hooks", when="@0.10.0:", type=("build", "run"))
-    depends_on("py-pep517@0.9.1:", when="@:0.9", type=("build", "run"))
-    depends_on("py-tomli@1:", type=("build", "run"))
-    depends_on("py-tomli@1.1:", when="@0.10:", type=("build", "run"))
     depends_on("py-colorama", when="platform=windows", type=("build", "run"))
-    depends_on("py-importlib-metadata@0.22:", when="^python@:3.7", type=("build", "run"))
+    depends_on("py-importlib-metadata@4.6:", when="@1: ^python@:3.9", type=("build", "run"))
+    depends_on("py-importlib-metadata@0.22:", when="@0 ^python@:3.7", type=("build", "run"))
+    depends_on("py-tomli@1.1:", when="@1: ^python@:3.10", type=("build", "run"))
+    depends_on("py-tomli@1.1:", when="@0.10", type=("build", "run"))
+    depends_on("py-tomli@1:", when="@:0.9", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-setuptools", when="@:0.9", type="build")
+    depends_on("py-pep517@0.9.1:", when="@:0.9", type=("build", "run"))
     depends_on("py-virtualenv@20.0.35:", when="+virtualenv", type=("build", "run"))
 
     # https://github.com/pypa/build/issues/266
diff --git a/var/spack/repos/builtin/packages/py-bx-python/package.py b/var/spack/repos/builtin/packages/py-bx-python/package.py
index 17f642e0fbb26d..810f243fa133e6 100644
--- a/var/spack/repos/builtin/packages/py-bx-python/package.py
+++ b/var/spack/repos/builtin/packages/py-bx-python/package.py
@@ -16,10 +16,14 @@ class PyBxPython(PythonPackage):
     version("0.9.0", sha256="fe545c44d2ea74b239d41e9090618aaf6a859d1a1f64b4a21b133cb602dfdb49")
     version("0.8.8", sha256="ad0808ab19c007e8beebadc31827e0d7560ac0e935f1100fb8cc93607400bb47")
 
-    depends_on("python@3.7:", when="@0.8.13:", type=("build", "run"))
+    # See https://pypi.org/project/bx-python/(version)/#files for which Python versions
+    # work with which releases.
+    depends_on("python@3.7:3.11", when="@=0.9.0", type=("build", "run"))
+    depends_on("python@3.5:3.8", when="@=0.8.8", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("py-cython", type="build")
     depends_on("py-numpy", type=("build", "run"))
     depends_on("py-six", when="@:0.8.9", type=("build", "run"))
     # py-python-lzo is listed as a dependency in `tox.ini` rather than in `setup.cfg`
     depends_on("py-python-lzo@1.14:", type=("build", "run"))
+    depends_on("zlib", type="link")
diff --git a/var/spack/repos/builtin/packages/py-callmonitor/package.py b/var/spack/repos/builtin/packages/py-callmonitor/package.py
new file mode 100644
index 00000000000000..e8220a6395a7e1
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-callmonitor/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyCallmonitor(PythonPackage):
+    """A Simple Tool to Monitor and Log Function Calls"""
+
+    homepage = "https://github.com/JBlaschke/call-monitor"
+    pypi = "callmonitor/callmonitor-0.3.7.tar.gz"
+
+    maintainers("DaxLynch")
+
+    version("0.3.7", sha256="11bacfe5940c3f6aff223e8e761b033d540542b4d738f7fef38cd923b3be0cbc")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-numpy", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-cartopy/package.py b/var/spack/repos/builtin/packages/py-cartopy/package.py
index 9442c2f7448f3d..7091a3e965752a 100644
--- a/var/spack/repos/builtin/packages/py-cartopy/package.py
+++ b/var/spack/repos/builtin/packages/py-cartopy/package.py
@@ -15,6 +15,7 @@ class PyCartopy(PythonPackage):
     maintainers("adamjstewart")
     skip_modules = ["cartopy.tests"]
 
+    version("0.22.0", sha256="b300f90120931d43f11ef87c064ea1dacec1b59a4940aa76ebf82cf09548bb49")
     version("0.21.1", sha256="89d5649712c8582231c6e11825a04c85f6f0cee94dbb89e4db23eabca1cc250a")
     version("0.21.0", sha256="ce1d3a28a132e94c89ac33769a50f81f65634ab2bd40556317e15bd6cad1ce42")
     version("0.20.3", sha256="0d60fa2e2fbd77c4d1f6b1f9d3b588966147f07c1b179d2d34570ac1e1b49006")
@@ -36,9 +37,17 @@ class PyCartopy(PythonPackage):
     )
     variant("plotting", default=False, description="Add plotting functionality")
 
-    # pyproject.toml
+    # Based on wheel availability on PyPI
+    depends_on("python@3.9:3.11", when="@0.22:", type=("build", "link", "run"))
+    depends_on("python@3.8:3.11", when="@0.21", type=("build", "link", "run"))
+    depends_on("python@:3.11", when="@0.20", type=("build", "link", "run"))
+    depends_on("python@:3.10", when="@0.19", type=("build", "link", "run"))
+    depends_on("python@:3.9", when="@:0.18", type=("build", "link", "run"))
+
+    # Required dependencies
     depends_on("py-setuptools@40.6:", when="@0.19:", type="build")
     depends_on("py-setuptools@0.7.2:", type="build")
+    depends_on("py-cython@0.29.24:", when="@0.22:", type="build")
     depends_on("py-cython@0.29.13:", when="@0.20:", type="build")
     depends_on("py-cython@0.29.2:", when="@0.19:", type="build")
     depends_on("py-cython@0.28:", when="@0.18:", type="build")
@@ -46,54 +55,55 @@ class PyCartopy(PythonPackage):
     depends_on("py-cython", type="build")
     depends_on("py-setuptools-scm@7:", when="@0.20.3:", type="build")
     depends_on("py-setuptools-scm", when="@0.19:", type="build")
-    depends_on("py-setuptools-scm-git-archive", when="@0.19:0.20.2", type="build")
-
-    # setup.py
-    depends_on("python@3.8:", when="@0.21:", type=("build", "run"))
-    depends_on("python@:3.9", when="@:0.18", type=("build", "run"))
-    depends_on("geos@3.7.2:", when="@0.20:")
-    depends_on("geos@3.3.3:")
-    depends_on("proj@8:", when="@0.20")
-    depends_on("proj@4.9:7", when="@0.17:0.19")
-    depends_on("proj@4.9:5", when="@:0.16")
-
-    # requirements/default.txt
-    depends_on("py-numpy@1.18:", when="@0.20:", type=("build", "run"))
-    depends_on("py-numpy@1.13.3:", when="@0.19:", type=("build", "run"))
-    depends_on("py-numpy@1.10:", when="@0.17:", type=("build", "run"))
-    depends_on("py-numpy@1.6:", type=("build", "run"))
-    depends_on("py-matplotlib@3.1:", when="@0.21:", type=("build", "run"))
+    depends_on("py-numpy@1.21:", when="@0.22:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.18:", when="@0.20:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.13.3:", when="@0.19:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.10:", when="@0.17:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.6:", type=("build", "link", "run"))
+    depends_on("py-matplotlib@3.4:", when="@0.22:", type=("build", "run"))
+    depends_on("py-matplotlib@3.1:", when="@0.21", type=("build", "run"))
     # https://github.com/SciTools/cartopy/issues/2086
     depends_on("py-matplotlib@3.1:3.5", when="@0.20", type=("build", "run"))
-    depends_on("py-shapely@1.6.4:", when="@0.21.1:", type=("build", "run"))
+    depends_on("py-shapely@1.7:", when="@0.22:", type=("build", "run"))
+    depends_on("py-shapely@1.6.4:", when="@0.21.1:0.21", type=("build", "run"))
     depends_on("py-shapely@1.6.4:1", when="@0.20:0.21.0", type=("build", "run"))
     depends_on("py-shapely@1.5.6:1", when="@:0.19", type=("build", "run"))
+    depends_on("py-packaging@20:", when="@0.22:", type=("build", "run"))
     depends_on("py-pyshp@2.1:", when="@0.20:", type=("build", "run"))
     depends_on("py-pyshp@2:", when="@0.19:", type=("build", "run"))
     depends_on("py-pyshp@1.1.4:", type=("build", "run"))
+    depends_on("py-pyproj@3.1:", when="@0.22:", type=("build", "run"))
     depends_on("py-pyproj@3:", when="@0.20:", type=("build", "run"))
-    depends_on("py-six@1.3:", when="@:0.18", type=("build", "run"))
-
-    # requirements/epsg.txt
-    with when("+epsg"):
-        depends_on("py-pyepsg@0.2:", type="run")
-        depends_on("py-pyepsg@0.4:", when="@0.18:", type="run")
 
-    # requirements/ows.txt
     with when("+ows"):
+        depends_on("py-owslib@0.20:", when="@0.22:", type="run")
         depends_on("py-owslib@0.18:", when="@0.20:", type="run")
         depends_on("py-owslib@0.8.11:", type="run")
         depends_on("pil@6.1:", when="@0.20:", type="run")
         depends_on("pil@1.7.8:", type="run")
 
-    # requirements/plotting.txt
     with when("+plotting"):
-        depends_on("gdal@2.3.2:+python", when="@0.20:", type="run")
-        depends_on("gdal@1.10:+python", type="run")
         depends_on("pil@6.1:", when="@0.20:", type="run")
         depends_on("pil@1.7.8:", type="run")
         depends_on("py-scipy@1.3.1:", when="@0.20:", type="run")
         depends_on("py-scipy@0.10:", type="run")
+
+    # Historical dependencies
+    depends_on("py-setuptools-scm-git-archive", when="@0.19:0.20.2", type="build")
+    depends_on("py-six@1.3:", when="@:0.18", type=("build", "run"))
+    depends_on("geos@3.7.2:", when="@0.20:0.21")
+    depends_on("geos@3.3.3:", when="@:0.19")
+    depends_on("proj@8:", when="@0.20")
+    depends_on("proj@4.9:7", when="@0.17:0.19")
+    depends_on("proj@4.9:5", when="@:0.16")
+
+    with when("+epsg"):
+        depends_on("py-pyepsg@0.2:", type="run")
+        depends_on("py-pyepsg@0.4:", when="@0.18:", type="run")
+
+    with when("+plotting"):
+        depends_on("gdal@2.3.2:+python", when="@0.20:0.21", type="run")
+        depends_on("gdal@1.10:+python", when="@:0.19", type="run")
         depends_on("py-matplotlib@1.5.1:3.5", when="@0.17:0.19", type="run")
         depends_on("py-matplotlib@1.3:3.5", when="@0.16", type="run")
 
diff --git a/var/spack/repos/builtin/packages/py-certifi/package.py b/var/spack/repos/builtin/packages/py-certifi/package.py
index bb26b43fbc6b5e..bcf14be2f068f0 100644
--- a/var/spack/repos/builtin/packages/py-certifi/package.py
+++ b/var/spack/repos/builtin/packages/py-certifi/package.py
@@ -14,6 +14,7 @@ class PyCertifi(PythonPackage):
     homepage = "https://github.com/certifi/python-certifi"
     pypi = "certifi/certifi-2020.6.20.tar.gz"
 
+    version("2023.7.22", sha256="539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082")
     version("2023.5.7", sha256="0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7")
     version("2022.12.7", sha256="35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3")
     version("2022.9.14", sha256="36973885b9542e6bd01dea287b2b4b3b21236307c56324fcc3f1160f2d655ed5")
diff --git a/var/spack/repos/builtin/packages/py-cffi/package.py b/var/spack/repos/builtin/packages/py-cffi/package.py
index 3d982b6c5c7889..e43d40e2c36d85 100644
--- a/var/spack/repos/builtin/packages/py-cffi/package.py
+++ b/var/spack/repos/builtin/packages/py-cffi/package.py
@@ -24,6 +24,10 @@ class PyCffi(PythonPackage):
     version("1.10.0", sha256="b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5")
     version("1.1.2", sha256="390970b602708c91ddc73953bb6929e56291c18a4d80f360afa00fad8b6f3339")
 
+    # ./spack-src/cffi/ffiplatform.py has _hack_at_distutils which imports
+    # setuptools before distutils, but only on Windows. This could be made
+    # unconditional to support Python 3.12
+    depends_on("python@:3.11", type=("build", "run"))
     depends_on("pkgconfig", type="build")
     depends_on("py-setuptools", type="build")
     depends_on("py-pycparser", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-cfgv/package.py b/var/spack/repos/builtin/packages/py-cfgv/package.py
index df26dcae22e23e..9158f42762f5b6 100644
--- a/var/spack/repos/builtin/packages/py-cfgv/package.py
+++ b/var/spack/repos/builtin/packages/py-cfgv/package.py
@@ -12,10 +12,13 @@ class PyCfgv(PythonPackage):
     homepage = "https://github.com/asottile/cfgv/"
     pypi = "cfgv/cfgv-2.0.1.tar.gz"
 
+    version("3.4.0", sha256="e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560")
     version("3.3.1", sha256="f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736")
     version("2.0.1", sha256="edb387943b665bf9c434f717bf630fa78aecd53d5900d2e05da6ad6048553144")
 
+    depends_on("python@3.8:", when="@3.4:", type=("build", "run"))
     depends_on("python@3.6.1:", when="@3.1:", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.4:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
+
+    # Historical dependencies
     depends_on("py-six", when="@:2", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-chardet/package.py b/var/spack/repos/builtin/packages/py-chardet/package.py
index 246f00c207f57c..f7a3f5cc621ff0 100644
--- a/var/spack/repos/builtin/packages/py-chardet/package.py
+++ b/var/spack/repos/builtin/packages/py-chardet/package.py
@@ -12,6 +12,7 @@ class PyChardet(PythonPackage):
     homepage = "https://github.com/chardet/chardet"
     pypi = "chardet/chardet-3.0.4.tar.gz"
 
+    version("5.2.0", sha256="1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7")
     version("5.1.0", sha256="0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5")
     version("5.0.0", sha256="0368df2bfd78b5fc20572bb4e9bb7fb53e2c094f60ae9993339e8671d0afb8aa")
     version("4.0.0", sha256="0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa")
@@ -20,4 +21,6 @@ class PyChardet(PythonPackage):
     version("2.3.0", sha256="e53e38b3a4afe6d1132de62b7400a4ac363452dc5dfcf8d88e8e0cce663c68aa")
 
     depends_on("py-setuptools", type="build")
+
+    # Historical dependencies
     depends_on("py-pytest-runner", when="@3", type="build")
diff --git a/var/spack/repos/builtin/packages/py-charm4py/package.py b/var/spack/repos/builtin/packages/py-charm4py/package.py
index efbfc40a753d08..f84a0b06d69b47 100644
--- a/var/spack/repos/builtin/packages/py-charm4py/package.py
+++ b/var/spack/repos/builtin/packages/py-charm4py/package.py
@@ -34,10 +34,15 @@ class PyCharm4py(PythonPackage):
     # Builds its own charm++, so no charmpp dependency
     depends_on("python@2.7:2.8,3.4:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
+    # in newer pip versions --install-option does not exist
+    depends_on("py-pip@:23.0", type="build")
     depends_on("py-cython", type="build")
     depends_on("py-cffi@1.7:", type="build")
     depends_on("py-numpy@1.10.0:", type=("build", "run"))
     depends_on("py-greenlet", type=("build", "run"))
+
+    depends_on("autoconf", type="build")
+    depends_on("automake", type="build")
     depends_on("cuda")
     depends_on("mpi", when="+mpi")
 
diff --git a/var/spack/repos/builtin/packages/py-charset-normalizer/package.py b/var/spack/repos/builtin/packages/py-charset-normalizer/package.py
index eab217431524cd..706c49c9884606 100644
--- a/var/spack/repos/builtin/packages/py-charset-normalizer/package.py
+++ b/var/spack/repos/builtin/packages/py-charset-normalizer/package.py
@@ -13,6 +13,7 @@ class PyCharsetNormalizer(PythonPackage):
     homepage = "https://github.com/ousret/charset_normalizer"
     pypi = "charset-normalizer/charset-normalizer-2.0.7.tar.gz"
 
+    version("3.3.0", sha256="63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6")
     version("3.1.0", sha256="34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5")
     version("2.1.1", sha256="5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845")
     version("2.0.12", sha256="2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597")
diff --git a/var/spack/repos/builtin/packages/py-chex/package.py b/var/spack/repos/builtin/packages/py-chex/package.py
index 7523c868ef33b6..434a8c93578b3f 100644
--- a/var/spack/repos/builtin/packages/py-chex/package.py
+++ b/var/spack/repos/builtin/packages/py-chex/package.py
@@ -13,6 +13,7 @@ class PyChex(PythonPackage):
     homepage = "https://github.com/deepmind/chex"
     pypi = "chex/chex-0.1.0.tar.gz"
 
+    version("0.1.5", sha256="686858320f8f220c82a6c7eeb54dcdcaa4f3d7f66690dacd13a24baa1ee8299e")
     version("0.1.0", sha256="9e032058f5fed2fc1d5e9bf8e12ece5910cf6a478c12d402b6d30984695f2161")
 
     depends_on("python@3.7:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-clean-text/package.py b/var/spack/repos/builtin/packages/py-clean-text/package.py
index d9aee44fd903d1..b4aa902dd81fad 100644
--- a/var/spack/repos/builtin/packages/py-clean-text/package.py
+++ b/var/spack/repos/builtin/packages/py-clean-text/package.py
@@ -13,9 +13,12 @@ class PyCleanText(PythonPackage):
 
     pypi = "clean-text/clean-text-0.5.0.tar.gz"
 
+    version("0.6.0", sha256="8374b385fc2a26e06383f62aed076fa6be115e5832239e2a7fd8b344fa8d2ab2")
     version("0.5.0", sha256="e525951bef0c8b72e03c987fdac2c475b61d7debf7a8834366fd75716179b6e1")
 
     depends_on("python@3.6:", type=("build", "run"))
-    depends_on("py-setuptools", type="build")
+    depends_on("python@3.7:", when="@0.6:", type=("build", "run"))
+    depends_on("py-poetry@0.12:", type="build")
     depends_on("py-emoji", type=("build", "run"))
-    depends_on("py-ftfy@6.0:6.999", type=("build", "run"))
+    depends_on("py-emoji@1", when="@0.6:", type=("build", "run"))
+    depends_on("py-ftfy@6", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-click/package.py b/var/spack/repos/builtin/packages/py-click/package.py
index d5830049ea13fd..16585406035fb4 100644
--- a/var/spack/repos/builtin/packages/py-click/package.py
+++ b/var/spack/repos/builtin/packages/py-click/package.py
@@ -11,8 +11,9 @@ class PyClick(PythonPackage):
 
     homepage = "https://click.palletsprojects.com"
     pypi = "click/click-7.1.2.tar.gz"
-    git = "https://github.com/pallets/click/"
+    git = "https://github.com/pallets/click.git"
 
+    version("8.1.7", sha256="ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de")
     version("8.1.3", sha256="7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e")
     version("8.0.3", sha256="410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b")
     version("8.0.1", sha256="8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a")
@@ -24,9 +25,8 @@ class PyClick(PythonPackage):
     )
     version("6.6", sha256="cc6a19da8ebff6e7074f731447ef7e112bd23adf3de5c597cf9989f2fd8defe9")
 
+    # Needed to ensure that Spack can bootstrap black with Python 3.6
     depends_on("python@3.7:", when="@8.1:", type=("build", "run"))
-    depends_on("python@3.6:", when="@8:", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.5:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
 
     depends_on("py-importlib-metadata", when="@8: ^python@:3.7", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-cmaes/package.py b/var/spack/repos/builtin/packages/py-cmaes/package.py
new file mode 100644
index 00000000000000..26730ea43d0c66
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-cmaes/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyCmaes(PythonPackage):
+    """Lightweight Covariance Matrix Adaptation Evolution Strategy (CMA-ES) implementation."""
+
+    homepage = "https://github.com/CyberAgentAILab/cmaes"
+    pypi = "cmaes/cmaes-0.10.0.tar.gz"
+
+    maintainers("eugeneswalker")
+
+    version("0.10.0", sha256="48afc70df027114739872b50489ae6b32461c307b92d084a63c7090a9742faf9")
+
+    depends_on("py-setuptools@61:", type="build")
+
+    depends_on("py-numpy", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-cmocean/package.py b/var/spack/repos/builtin/packages/py-cmocean/package.py
index 032827edba473b..89e9dc22dded35 100644
--- a/var/spack/repos/builtin/packages/py-cmocean/package.py
+++ b/var/spack/repos/builtin/packages/py-cmocean/package.py
@@ -13,8 +13,11 @@ class PyCmocean(PythonPackage):
     homepage = "https://matplotlib.org/cmocean/"
     pypi = "cmocean/cmocean-2.0.tar.gz"
 
+    version("3.0.3", sha256="abaf99383c1a60f52970c86052ae6c14eafa84fc16984488040283c02db77c0b")
     version("2.0", sha256="13eea3c8994d8e303e32a2db0b3e686f6edfb41cb21e7b0e663c2b17eea9b03a")
 
+    depends_on("python@3.8:", when="@3:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("py-matplotlib", type=("build", "run"))
     depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-packaging", when="@3:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-colorama/package.py b/var/spack/repos/builtin/packages/py-colorama/package.py
index 47000049238fe8..7f8b9188ce2637 100644
--- a/var/spack/repos/builtin/packages/py-colorama/package.py
+++ b/var/spack/repos/builtin/packages/py-colorama/package.py
@@ -12,11 +12,14 @@ class PyColorama(PythonPackage):
     homepage = "https://github.com/tartley/colorama"
     pypi = "colorama/colorama-0.3.7.tar.gz"
 
+    version("0.4.6", sha256="08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44")
     version("0.4.5", sha256="e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4")
     version("0.4.4", sha256="5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b")
     version("0.4.1", sha256="05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d")
     version("0.3.7", sha256="e043c8d32527607223652021ff648fbb394d5e19cba9f1a698670b338c9d782b")
 
-    depends_on("python@2.7:2.8,3.5:", when="@0.4.2:", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.4:", type=("build", "run"))
-    depends_on("py-setuptools", type="build")
+    depends_on("python@2.7:2,3.7:", when="@0.4.6:", type=("build", "run"))
+    depends_on("python@2.7:2,3.5:", when="@0.4.2:", type=("build", "run"))
+    depends_on("python@2.7:2,3.4:", type=("build", "run"))
+    depends_on("py-setuptools", when="@:0.4.5", type="build")
+    depends_on("py-hatchling@0.25.1:", when="@0.4.6:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-comm/package.py b/var/spack/repos/builtin/packages/py-comm/package.py
index fd195b1f4f7644..5e82ade2c04fd2 100644
--- a/var/spack/repos/builtin/packages/py-comm/package.py
+++ b/var/spack/repos/builtin/packages/py-comm/package.py
@@ -12,7 +12,10 @@ class PyComm(PythonPackage):
     homepage = "https://github.com/ipython/comm"
     pypi = "comm/comm-0.1.3.tar.gz"
 
+    version("0.1.4", sha256="354e40a59c9dd6db50c5cc6b4acc887d82e9603787f83b68c01a80a923984d15")
     version("0.1.3", sha256="a61efa9daffcfbe66fd643ba966f846a624e4e6d6767eda9cf6e993aadaab93e")
 
     depends_on("py-hatchling@1.10:", type="build")
-    depends_on("py-traitlets@5.3:", type=("build", "run"))
+
+    depends_on("py-traitlets@4:", when="@0.1.4:", type=("build", "run"))
+    depends_on("py-traitlets@5.3:", when="@0.1.3", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-conda-souschef/package.py b/var/spack/repos/builtin/packages/py-conda-souschef/package.py
new file mode 100644
index 00000000000000..5c4ddc7d937e41
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-conda-souschef/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyCondaSouschef(PythonPackage):
+    """Project to handle conda recipes."""
+
+    homepage = "https://github.com/marcelotrevisani/souschef"
+    pypi = "conda-souschef/conda-souschef-2.2.3.tar.gz"
+
+    version("2.2.3", sha256="9bf3dba0676bc97616636b80ad4a75cd90582252d11c86ed9d3456afb939c0c3")
+
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-setuptools@30.3:", type="build")
+    depends_on("py-setuptools-scm", type="build")
+    depends_on("py-ruamel-yaml@0.15.3:", type=("build", "run"))
+    depends_on("py-ruamel-yaml-jinja2", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-confection/package.py b/var/spack/repos/builtin/packages/py-confection/package.py
new file mode 100644
index 00000000000000..c4e3813b8e143a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-confection/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyConfection(PythonPackage):
+    """The sweetest config system for Python"""
+
+    homepage = "https://github.com/explosion/confection"
+    pypi = "confection/confection-0.0.4.tar.gz"
+
+    version("0.0.4", sha256="b1ddf5885da635f0e260a40b339730806dfb1bd17d30e08764f35af841b04ecf")
+
+    depends_on("python@3.6:", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
+    depends_on("py-pydantic@1.7.4:1.7,1.9:1.10", type=("build", "run"))
+    depends_on("py-typing-extensions@3.7.4.1:4.4", type=("build", "run"), when="^python@3.7")
+    depends_on("py-srsly@2.4:2", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-consolekit/package.py b/var/spack/repos/builtin/packages/py-consolekit/package.py
new file mode 100644
index 00000000000000..6acef963218c21
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-consolekit/package.py
@@ -0,0 +1,25 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyConsolekit(PythonPackage):
+    """Additional utilities for click."""
+
+    homepage = "https://github.com/domdfcoding/consolekit"
+    pypi = "consolekit/consolekit-1.5.1.tar.gz"
+
+    version("1.5.1", sha256="55ea43e226863e1d618ec9b860c9842d84249d895c3376c03b158d8f3a335626")
+
+    depends_on("py-flit-core@3.2:3", type="build")
+    depends_on("py-click@7.1.2:", type=("build", "run"))
+    depends_on("py-colorama@0.4.3:", type=("build", "run"), when="^python@:3.9 platform=windows")
+    depends_on("py-deprecation-alias@0.1.1:", type=("build", "run"))
+    depends_on("py-domdf-python-tools@2.6:", type=("build", "run"))
+    depends_on("py-mistletoe@0.7.2:", type=("build", "run"))
+    depends_on("py-typing-extensions@3.10:", type=("build", "run"))
+    conflicts("^py-typing-extensions@3.10.0.1")
diff --git a/var/spack/repos/builtin/packages/py-corner/package.py b/var/spack/repos/builtin/packages/py-corner/package.py
new file mode 100644
index 00000000000000..81ae512123dd39
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-corner/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyCorner(PythonPackage):
+    """Make some beautiful corner plots."""
+
+    homepage = "https://corner.readthedocs.io"
+    pypi = "corner/corner-2.2.2.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("2.2.2", sha256="4bc79f3b6778c270103f0926e64ef2606c48c3b6f92daf5382fc4babf5d608d1")
+
+    depends_on("python@3.9:", type=("build", "run"))
+    depends_on("py-setuptools@62.0:", type="build")
+    depends_on("py-setuptools-scm", type="build")
+    depends_on("py-matplotlib@2.1:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-cryptography/package.py b/var/spack/repos/builtin/packages/py-cryptography/package.py
index cdbbe863f7d8e0..cf8e782998dfb6 100644
--- a/var/spack/repos/builtin/packages/py-cryptography/package.py
+++ b/var/spack/repos/builtin/packages/py-cryptography/package.py
@@ -13,6 +13,7 @@ class PyCryptography(PythonPackage):
     homepage = "https://github.com/pyca/cryptography"
     pypi = "cryptography/cryptography-1.8.1.tar.gz"
 
+    version("41.0.3", sha256="6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34")
     version("40.0.2", sha256="c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99")
     version("38.0.1", sha256="1db3d807a14931fa317f96435695d9ec386be7b84b618cc61cfa5d08b0ae33d7")
     version("37.0.4", sha256="63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82")
@@ -27,15 +28,23 @@ class PyCryptography(PythonPackage):
     version("1.8.1", sha256="323524312bb467565ebca7e50c8ae5e9674e544951d28a2904a50012a8828190")
 
     variant("idna", default=False, when="@2.5:3.0", description="Deprecated U-label support")
+    variant("rust_bootstrap", default=False, when="@3.4.3:", description="Use pre-compiled Rust")
 
+    # distutils required in version <= 40
+    depends_on("python@:3.11", when="@:40", type=("build", "run"))
+    depends_on("py-setuptools@61.0:", when="@41:", type="build")
     depends_on("py-setuptools@40.6:60.8,60.9.1:", when="@37:", type="build")
     depends_on("py-setuptools@40.6:", when="@2.7:36", type="build")
     depends_on("py-setuptools@18.5:", when="@2.2:2.6", type="build")
     depends_on("py-setuptools@11.3:", when="@:2.1", type="build")
-    depends_on("py-setuptools-rust@0.11.4:", when="@3.4:", type=("build", "run"))
-    depends_on("rust@1.48:", when="@38:", type="build")
-    depends_on("rust@1.41:", when="@3.4.5:", type="build")
-    depends_on("rust@1.45:", when="@3.4.3:3.4.4", type="build")
+    depends_on("py-setuptools-rust@0.11.4:", when="@3.4.2:", type="build")
+    depends_on("py-setuptools-rust@0.11.4:", when="@3.4:3.4.1", type=("build", "run"))
+    with when("~rust_bootstrap"):
+        depends_on("rust@1.56:", when="@41:", type="build")
+        depends_on("rust@1.48:", when="@38:", type="build")
+        depends_on("rust@1.41:", when="@3.4.5:", type="build")
+        depends_on("rust@1.45:", when="@3.4.3:3.4.4", type="build")
+    depends_on("rust-bootstrap", when="+rust_bootstrap", type="build")
     depends_on("pkgconfig", when="@40:", type="build")
 
     depends_on("py-cffi@1.12:", when="@3.3:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-css-parser/package.py b/var/spack/repos/builtin/packages/py-css-parser/package.py
new file mode 100644
index 00000000000000..7ed99c56ca6e20
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-css-parser/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyCssParser(PythonPackage):
+    """A CSS Cascading Style Sheets library for Python."""
+
+    homepage = "https://github.com/ebook-utils/css-parser"
+    pypi = "css-parser/css-parser-1.0.9.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("1.0.9", sha256="196db822cef22745af6a58d180cf8206949ced58b48f5f3ee98f1de1627495bb")
+
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-cssutils/package.py b/var/spack/repos/builtin/packages/py-cssutils/package.py
new file mode 100644
index 00000000000000..3a6772edc5f134
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-cssutils/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyCssutils(PythonPackage):
+    """A CSS Cascading Style Sheets library for Python."""
+
+    homepage = "https://github.com/jaraco/cssutils"
+    pypi = "cssutils/cssutils-2.7.1.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("2.7.1", sha256="340ecfd9835d21df8f98500f0dfcea0aee41cb4e19ecbc2cf94f0a6d36d7cb6c")
+
+    depends_on("py-setuptools@56:", type="build")
+    depends_on("py-setuptools-scm@3.4.1:+toml", type="build")
+    depends_on("py-importlib-metadata", type=("build", "run"), when="^python@:3.7")
diff --git a/var/spack/repos/builtin/packages/py-csvkit/package.py b/var/spack/repos/builtin/packages/py-csvkit/package.py
index f5b702c11042f3..90970797e060ee 100644
--- a/var/spack/repos/builtin/packages/py-csvkit/package.py
+++ b/var/spack/repos/builtin/packages/py-csvkit/package.py
@@ -12,21 +12,26 @@ class PyCsvkit(PythonPackage):
 
     homepage = "http://csvkit.rtfd.org/"
     pypi = "csvkit/csvkit-0.9.1.tar.gz"
+    git = "https://github.com/wireservice/csvkit.git"
 
+    version("1.1.1", sha256="beddb7b78f6b22adbed6ead5ad5de4bfb31dd2c55f3211b2a2b3b65529049223")
     version("1.0.4", sha256="1353a383531bee191820edfb88418c13dfe1cdfa9dd3dc46f431c05cd2a260a0")
     version("0.9.1", sha256="92f8b8647becb5cb1dccb3af92a13a4e85702d42ba465ce8447881fb38c9f93a")
 
     depends_on("py-setuptools", type=("build", "run"))
-    depends_on("py-six@1.6.1:", type=("build", "run"))
-    depends_on("py-python-dateutil@2.2", type=("build", "run"), when="@0.9.1")
-    depends_on("py-dbf@0.94.003", type=("build", "run"), when="@0.9.1")
-    depends_on("py-xlrd@0.7.1:", type=("build", "run"), when="@0.9.1")
-    depends_on("py-sqlalchemy@0.6.6:", type=("build", "run"), when="@0.9.1")
-    depends_on("py-openpyxl@2.2.0", type=("build", "run"), when="@0.9.1")
-    depends_on("py-agate@1.6.1:", type=("build", "run"), when="@1:")
-    depends_on("py-agate-excel@0.2.2:", type=("build", "run"), when="@1:")
-    depends_on("py-agate-dbf@0.2.0:", type=("build", "run"), when="@1:")
-    depends_on("py-agate-sql@0.5.3:", type=("build", "run"), when="@1:")
+    depends_on("py-agate@1.6.1:", when="@1:", type=("build", "run"))
+    depends_on("py-agate-excel@0.2.2:", when="@1:", type=("build", "run"))
+    depends_on("py-agate-dbf@0.2.2:", when="@1.0.7:", type=("build", "run"))
+    depends_on("py-agate-dbf@0.2.0:", when="@1:", type=("build", "run"))
+    depends_on("py-agate-sql@0.5.3:", when="@1:", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-six@1.6.1:", when="@:1.0", type=("build", "run"))
+    with when("@0.9.1"):
+        depends_on("py-xlrd@0.7.1:", type=("build", "run"))
+        depends_on("py-sqlalchemy@0.6.6:", type=("build", "run"))
+        depends_on("py-openpyxl@2.2.0", type=("build", "run"))
+        depends_on("py-python-dateutil@2.2", type=("build", "run"))
 
     @when("@0.9.1")
     def patch(self):
diff --git a/var/spack/repos/builtin/packages/py-cupy/package.py b/var/spack/repos/builtin/packages/py-cupy/package.py
index b9dee225b40b4f..feffd1dcf66d46 100644
--- a/var/spack/repos/builtin/packages/py-cupy/package.py
+++ b/var/spack/repos/builtin/packages/py-cupy/package.py
@@ -3,10 +3,12 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import os
+
 from spack.package import *
 
 
-class PyCupy(PythonPackage, CudaPackage):
+class PyCupy(PythonPackage, CudaPackage, ROCmPackage):
     """CuPy is an open-source array library accelerated with
     NVIDIA CUDA. CuPy provides GPU accelerated computing with
     Python. CuPy uses CUDA-related libraries including cuBLAS,
@@ -16,23 +18,86 @@ class PyCupy(PythonPackage, CudaPackage):
     homepage = "https://cupy.dev/"
     pypi = "cupy/cupy-8.0.0.tar.gz"
 
+    version("12.1.0", sha256="f6d31989cdb2d96581da12822e28b102f29e254427195c2017eac327869b7320")
+    version("12.0.0", sha256="61ddbbef73d50d606bd5087570645f3c91ec9176c2566784c1d486d6a3404545")
+    version("11.6.0", sha256="53dbb840072bb32d4bfbaa6bfa072365a30c98b1fcd1f43e48969071ad98f1a7")
+    version("11.5.0", sha256="4bc8565bded22cc89b210fd9fb48a5d5316f30701e12bb23852a60314e1f9f6e")
+    version("11.4.0", sha256="03d52b2626e02a3a2b46d714c1cd03e702c8fe33915fcca6ed8de5c539964f49")
+    version("11.3.0", sha256="d057cc2f73ecca06fae8b9c270d9e14116203abfd211a704810cc50a453b4c9e")
     version("11.2.0", sha256="c33361f117a347a63f6996ea97446d17f1c038f1a1f533e502464235076923e2")
 
-    depends_on("python@3.7:", type=("build", "run"))
+    variant("all", default=False, description="Enable optional py-scipy, optuna, and cython")
+
+    depends_on("python@3.7:", when="@:11", type=("build", "run"))
+    depends_on("python@3.8:", when="@12:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("py-cython@0.29.22:2", type="build")
     depends_on("py-fastrlock@0.5:", type=("build", "run"))
-    depends_on("py-numpy@1.20:1.25", type=("build", "run"))
-    depends_on("cuda")
-    depends_on("nccl")
-    depends_on("cudnn")
-    depends_on("cutensor")
+    depends_on("py-numpy@1.20:1.25", when="@:11", type=("build", "run"))
+    depends_on("py-numpy@1.20:1.26", when="@12:", type=("build", "run"))
+
+    depends_on("py-scipy@1.6:1.12", when="+all", type=("build", "run"))
+    depends_on("py-cython@0.29.22:2", when="+all", type=("build", "run"))
+    depends_on("py-optuna@2:", when="+all", type=("build", "run"))
+
+    # Based on https://github.com/cupy/cupy/releases
+    depends_on("cuda@:11.9", when="@:11 +cuda")
+    depends_on("cuda@:12.1", when="@12: +cuda")
+
+    for a in CudaPackage.cuda_arch_values:
+        depends_on("nccl +cuda cuda_arch={0}".format(a), when="+cuda cuda_arch={0}".format(a))
+
+    depends_on("cudnn", when="+cuda")
+    depends_on("cutensor", when="+cuda")
 
-    conflicts("~cuda")
+    for _arch in ROCmPackage.amdgpu_targets:
+        arch_str = "amdgpu_target={0}".format(_arch)
+        rocm_str = "+rocm {0}".format(arch_str)
+        depends_on("rocprim {0}".format(arch_str), when=rocm_str, type=("link"))
+        depends_on("rocsolver {0}".format(arch_str), when=rocm_str, type=("link"))
+        depends_on("rocthrust {0}".format(arch_str), when=rocm_str, type=("link"))
+        depends_on("rocrand {0}".format(arch_str), when=rocm_str, type=("link"))
+        depends_on("hipcub {0}".format(rocm_str), when=rocm_str, type=("link"))
+        depends_on("hipblas {0}".format(rocm_str), when=rocm_str, type=("link"))
+        depends_on("hiprand {0}".format(rocm_str), when=rocm_str, type=("link"))
+        depends_on("hipsparse {0}".format(rocm_str), when=rocm_str, type=("link"))
+        depends_on("hipfft {0}".format(rocm_str), when=rocm_str, type=("link"))
+
+    depends_on("rccl", when="+rocm", type=("link"))
+    depends_on("roctracer-dev", when="+rocm", type=("link"))
+    depends_on("rocprofiler-dev", when="+rocm", type=("link"))
+
+    conflicts("~cuda ~rocm")
+    conflicts("+cuda +rocm")
+    conflicts("+cuda cuda_arch=none")
 
     def setup_build_environment(self, env):
         env.set("CUPY_NUM_BUILD_JOBS", make_jobs)
-        if not self.spec.satisfies("cuda_arch=none"):
+        if self.spec.satisfies("+cuda"):
             cuda_arch = self.spec.variants["cuda_arch"].value
             arch_str = ";".join("arch=compute_{0},code=sm_{0}".format(i) for i in cuda_arch)
             env.set("CUPY_NVCC_GENERATE_CODE", arch_str)
+        elif self.spec.satisfies("+rocm"):
+            spec = self.spec
+
+            incs = {
+                "roctracer-dev": ["include/roctracer"],
+                "hiprand": ["include"],
+                "rocrand": ["include"],
+                "rocthrust": ["include"],
+                "rocprim": ["include"],
+                "hip": ["include", "include/hip"],
+            }
+
+            inc_dirs = []
+            for pkg, ds in incs.items():
+                for d in ds:
+                    p = os.path.join(spec[pkg].prefix, d)
+                    if os.path.exists(p):
+                        inc_dirs.append(p)
+
+            env.set("CUPY_INCLUDE_PATH", ":".join(inc_dirs))
+
+            env.set("HIPCC", self.spec["hip"].hipcc)
+            env.set("ROCM_HOME", self.spec["hipcub"].prefix)
+            env.set("CUPY_INSTALL_USE_HIP", 1)
diff --git a/var/spack/repos/builtin/packages/py-currentscape/package.py b/var/spack/repos/builtin/packages/py-currentscape/package.py
new file mode 100644
index 00000000000000..eb6d75be89c8ab
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-currentscape/package.py
@@ -0,0 +1,23 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyCurrentscape(PythonPackage):
+    """Module to easily plot the currents in electrical neuron models."""
+
+    homepage = "https://github.com/BlueBrain/Currentscape"
+    git = "https://github.com/BlueBrain/Currentscape.git"
+    pypi = "currentscape/currentscape-1.0.12.tar.gz"
+
+    version("1.0.12", sha256="d83c5a58074e4d612553472a487e5d1d2854dc4d5c161817c6bafdf4a5988011")
+
+    depends_on("py-setuptools", type=("build", "run"))
+    depends_on("py-setuptools-scm", type=("build",))
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-matplotlib", type=("build", "run"))
+    depends_on("py-palettable", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-cykhash/package.py b/var/spack/repos/builtin/packages/py-cykhash/package.py
new file mode 100644
index 00000000000000..85d67fb9799f9e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-cykhash/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyCykhash(PythonPackage):
+    """Cython wrapper for khash-sets/maps, efficient implementation of isin and unique."""
+
+    homepage = "https://github.com/realead/cykhash"
+    pypi = "cykhash/cykhash-2.0.1.tar.gz"
+
+    maintainers("snehring")
+
+    version("2.0.1", sha256="b4794bc9f549114d8cf1d856d9f64e08ff5f246bf043cf369fdb414e9ceb97f7")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-cython@0.28:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-cylc-flow/package.py b/var/spack/repos/builtin/packages/py-cylc-flow/package.py
new file mode 100644
index 00000000000000..bed0abd1b19788
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-cylc-flow/package.py
@@ -0,0 +1,36 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyCylcFlow(PythonPackage):
+    """A workflow engine for cycling systems."""
+
+    homepage = "https://cylc.org"
+    pypi = "cylc-flow/cylc-flow-8.1.4.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("8.2.0", sha256="cbe35e0d72d1ca36f28a4cebe9b9040a3445a74253bc94051a3c906cf179ded0")
+    version("8.1.4", sha256="d1835ac18f6f24f3115c56b2bc821185484e834a86b12fd0033ff7e4dc3c1f63")
+
+    depends_on("py-setuptools@49:66,68:", type=("build", "run"))
+    depends_on("py-aiofiles@0.7", type=("build", "run"), when="@:8.1")
+    depends_on("py-ansimarkup@1.0.0:", type=("build", "run"))
+    depends_on("py-async-timeout@3.0.0:", type=("build", "run"))
+    depends_on("py-colorama@0.4:1", type=("build", "run"))
+    depends_on("py-graphene@2.1:2", type=("build", "run"))
+    depends_on("py-jinja2@3.0", type=("build", "run"))
+    depends_on("py-metomi-isodatetime@3.0", type=("build", "run"))
+    depends_on("py-protobuf@4.21.2:4.21", type=("build", "run"))
+    depends_on("py-psutil@5.6.0:", type=("build", "run"))
+    depends_on("py-pyzmq@22:", type=("build", "run"), when="@8.2:")
+    depends_on("py-pyzmq@22", type=("build", "run"), when="@:8.1")
+    depends_on("py-importlib-metadata", type=("build", "run"), when="^python@:3.7")
+    depends_on("py-urwid@2", type=("build", "run"))
+    depends_on("py-rx", type=("build", "run"))
+    depends_on("py-promise", type=("build", "run"))
+    depends_on("py-tomli@2:", type=("build", "run"), when="^python@:3.10")
diff --git a/var/spack/repos/builtin/packages/py-cylc-rose/package.py b/var/spack/repos/builtin/packages/py-cylc-rose/package.py
new file mode 100644
index 00000000000000..37805c66d0fe8a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-cylc-rose/package.py
@@ -0,0 +1,23 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyCylcRose(PythonPackage):
+    """A Cylc plugin providing support for the Rose rose-suite.conf file."""
+
+    homepage = "https://cylc.github.io/cylc-doc/latest/html/plugins/cylc-rose.html"
+    pypi = "cylc-rose/cylc-rose-1.3.0.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("1.3.0", sha256="017072b69d7a50fa6d309a911d2428743b07c095f308529b36b1b787ebe7ab88")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-metomi-rose@2.1", type=("build", "run"))
+    depends_on("py-cylc-flow@8.2", type=("build", "run"))
+    depends_on("py-metomi-isodatetime", type=("build", "run"))
+    depends_on("py-jinja2", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-cylc-uiserver/package.py b/var/spack/repos/builtin/packages/py-cylc-uiserver/package.py
new file mode 100644
index 00000000000000..57345f8a11257f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-cylc-uiserver/package.py
@@ -0,0 +1,32 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyCylcUiserver(PythonPackage):
+    """Cylc UI Server."""
+
+    homepage = "https://github.com/cylc/cylc-uiserver/"
+    pypi = "cylc-uiserver/cylc-uiserver-1.3.0.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("1.3.0", sha256="f3526e470c7ac2b61bf69e9b8d17fc7a513392219d28baed9b1166dcc7033d7a")
+
+    depends_on("py-wheel", type="build")
+    depends_on("py-setuptools@40.9.0:", type="build")
+    depends_on("py-cylc-flow@8.2", type=("build", "run"))
+    depends_on("py-ansimarkup@1.0.0:", type=("build", "run"))
+    depends_on("py-graphene", type=("build", "run"))
+    depends_on("py-graphene-tornado@2.6", type=("build", "run"))
+    depends_on("py-graphql-ws@0.4.4", type=("build", "run"))
+    depends_on("py-jupyter-server@1.10.2:1", type=("build", "run"))
+    depends_on("py-requests", type=("build", "run"))
+    depends_on("py-tornado@6.1.0:", type=("build", "run"))
+    depends_on("py-traitlets@5.2.1:", type=("build", "run"))
+    depends_on("py-pyzmq", type=("build", "run"))
+    depends_on("py-graphql-core", type=("build", "run"))
+    depends_on("py-rx@:1", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-cython/5712.patch b/var/spack/repos/builtin/packages/py-cython/5712.patch
new file mode 100644
index 00000000000000..e820d5f3377a40
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-cython/5712.patch
@@ -0,0 +1,23 @@
+From d6b6df173dc43625c22f23503b22253db35848e2 Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels 
+Date: Tue, 19 Sep 2023 21:36:24 +0200
+Subject: [PATCH] cythonize 0.29.x: use force=None as default instead of
+ force=False
+
+---
+ Cython/Build/Dependencies.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Cython/Build/Dependencies.py b/Cython/Build/Dependencies.py
+index 28c48ed8c3b..92fa96aa013 100644
+--- a/Cython/Build/Dependencies.py
++++ b/Cython/Build/Dependencies.py
+@@ -878,7 +878,7 @@ class Extension_setuptools(Extension): pass
+ 
+ 
+ # This is the user-exposed entry point.
+-def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, force=False, language=None,
++def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, force=None, language=None,
+               exclude_failures=False, **options):
+     """
+     Compile a set of source modules into C/C++ files and return a list of distutils
diff --git a/var/spack/repos/builtin/packages/py-cython/package.py b/var/spack/repos/builtin/packages/py-cython/package.py
index 869a3b2d377847..5726db29972eef 100644
--- a/var/spack/repos/builtin/packages/py-cython/package.py
+++ b/var/spack/repos/builtin/packages/py-cython/package.py
@@ -13,13 +13,17 @@ class PyCython(PythonPackage):
 
     homepage = "https://github.com/cython/cython"
     pypi = "cython/Cython-0.29.21.tar.gz"
+    tags = ["build-tools"]
 
-    version("3.0.0a9", sha256="23931c45877432097cef9de2db2dc66322cbc4fc3ebbb42c476bb2c768cecff0")
+    version("3.0.4", sha256="2e379b491ee985d31e5faaf050f79f4a8f59f482835906efe4477b33b4fbe9ff")
+    version("3.0.0", sha256="350b18f9673e63101dbbfcf774ee2f57c20ac4636d255741d76ca79016b1bd82")
     version(
-        "0.29.35",
-        sha256="6e381fa0bf08b3c26ec2f616b19ae852c06f5750f4290118bf986b6f85c8c527",
-        preferred=True,
+        "3.0.0a9",
+        sha256="23931c45877432097cef9de2db2dc66322cbc4fc3ebbb42c476bb2c768cecff0",
+        deprecated=True,
     )
+    version("0.29.36", sha256="41c0cfd2d754e383c9eeb95effc9aa4ab847d0c9747077ddd7c0dcb68c3bc01f")
+    version("0.29.35", sha256="6e381fa0bf08b3c26ec2f616b19ae852c06f5750f4290118bf986b6f85c8c527")
     version("0.29.34", sha256="1909688f5d7b521a60c396d20bba9e47a1b2d2784bfb085401e1e1e7d29a29a8")
     version("0.29.33", sha256="5040764c4a4d2ce964a395da24f0d1ae58144995dab92c6b96f44c3f4d72286a")
     version("0.29.32", sha256="8733cf4758b79304f2a4e39ebfac5e92341bce47bcceb26c1254398b2f8c1af7")
@@ -44,8 +48,8 @@ class PyCython(PythonPackage):
     version("0.23.5", sha256="0ae5a5451a190e03ee36922c4189ca2c88d1df40a89b4f224bc842d388a0d1b6")
     version("0.23.4", sha256="fec42fecee35d6cc02887f1eef4e4952c97402ed2800bfe41bbd9ed1a0730d8e")
 
-    depends_on("python@2.7:2,3.4:", when="@3:", type=("build", "link", "run"))
-    depends_on("python@2.6:2,3.3:", type=("build", "link", "run"))
+    # https://github.com/cython/cython/issues/5751 (distutils not yet dropped)
+    depends_on("python@:3.11", type=("build", "link", "run"))
 
     # https://github.com/cython/cython/commit/1cd24026e9cf6d63d539b359f8ba5155fd48ae21
     # collections.Iterable was removed in Python 3.10
@@ -59,6 +63,7 @@ class PyCython(PythonPackage):
 
     # Backports CYTHON_FORCE_REGEN environment variable
     patch("5307.patch", when="@0.29:0.29.33")
+    patch("5712.patch", when="@0.29")
 
     @property
     def prefix(self):
diff --git a/var/spack/repos/builtin/packages/py-dadi/package.py b/var/spack/repos/builtin/packages/py-dadi/package.py
index 1230c13cb53b04..9944896e502659 100644
--- a/var/spack/repos/builtin/packages/py-dadi/package.py
+++ b/var/spack/repos/builtin/packages/py-dadi/package.py
@@ -16,7 +16,7 @@ class PyDadi(PythonPackage):
 
     maintainers("dorton21")
 
-    version("2020-12-02", commit="047bac0")
+    version("2020-12-02", commit="047bac0db5245009d9c724e91a851149c34c9de0")
     version("2.1.0", sha256="97a15aa7ef501850cad4cff66b11b66ecb65d5d68acbf2ff713585c81c3a1038")
 
     depends_on("py-setuptools", type=("build"))
diff --git a/var/spack/repos/builtin/packages/py-darshan/package.py b/var/spack/repos/builtin/packages/py-darshan/package.py
index 12ee277cdac106..5c7450e0d762d9 100644
--- a/var/spack/repos/builtin/packages/py-darshan/package.py
+++ b/var/spack/repos/builtin/packages/py-darshan/package.py
@@ -14,6 +14,7 @@ class PyDarshan(PythonPackage):
 
     maintainers("jeanbez", "shanedsnyder")
 
+    version("3.4.4.0", sha256="2d218a1b2a450934698a78148c6603e453c246ec852679432bf217981668e56b")
     version("3.4.3.0", sha256="e0708fc5445f2d491ebd381a253cd67534cef13b963f1d749dd605a10f5c0f8f")
     version("3.4.2.0", sha256="eb00eb758c96899c0d523b71eb00caa3b967509c27fd504c579ac8c9b521845c")
     version("3.4.1.0", sha256="41a033ebac6fcd0ca05b8ccf07e11191286dee923ec334b876a7ec8e8a6add84")
diff --git a/var/spack/repos/builtin/packages/py-dask/package.py b/var/spack/repos/builtin/packages/py-dask/package.py
index cff78bd133e030..df07a6e03e2c89 100644
--- a/var/spack/repos/builtin/packages/py-dask/package.py
+++ b/var/spack/repos/builtin/packages/py-dask/package.py
@@ -103,7 +103,7 @@ class PyDask(PythonPackage):
     depends_on("py-bokeh@2.4.2:2", type=("build", "run"), when="@2022.10.2:2023.3 +diagnostics")
     depends_on("py-bokeh@2.4.2:", type=("build", "run"), when="@2023.4.0: +diagnostics")
     depends_on("py-jinja2", type=("build", "run"), when="@2022.10.2: +diagnostics")
-    depends_on("py-jinja2@2.10.3", type=("build", "run"), when="@2023.4.0: +diagnostics")
+    depends_on("py-jinja2@2.10.3:", type=("build", "run"), when="@2023.4.0: +diagnostics")
 
     # Requirements for dask.delayed
     # The dependency on py-cloudpickle is non-optional starting version 2021.3.1
diff --git a/var/spack/repos/builtin/packages/py-dataclasses-json/package.py b/var/spack/repos/builtin/packages/py-dataclasses-json/package.py
new file mode 100644
index 00000000000000..465265f834bcd0
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-dataclasses-json/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyDataclassesJson(PythonPackage):
+    """Easily serialize dataclasses to and from JSON."""
+
+    homepage = "https://github.com/lidatong/dataclasses-json"
+    pypi = "dataclasses_json/dataclasses_json-0.5.12.tar.gz"
+
+    version("0.5.12", sha256="70e28da52e36f4be6b724e1f1e77fbcd19e0e0a6bf9a4c4c6e5abf713d4dab5a")
+
+    depends_on("python@3.7:3.11", type=("build", "run"))
+    depends_on("py-poetry-core@1.2:", type="build")
+    depends_on("py-typing-inspect@0.4:0", type=("build", "run"))
+    depends_on("py-marshmallow@3.18:3", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-dcm2bids/package.py b/var/spack/repos/builtin/packages/py-dcm2bids/package.py
index 2aa34b5eab1914..fe3deb15629ada 100644
--- a/var/spack/repos/builtin/packages/py-dcm2bids/package.py
+++ b/var/spack/repos/builtin/packages/py-dcm2bids/package.py
@@ -13,11 +13,15 @@ class PyDcm2bids(PythonPackage):
     homepage = "https://github.com/unfmontreal/Dcm2Bids"
     pypi = "dcm2bids/dcm2bids-2.1.9.tar.gz"
 
+    version("3.1.0", sha256="53a8a177d556df897e19d72bd517fdae0245927a8938bb9fbbd51f9f33f54f84")
     version("2.1.9", sha256="d962bd0a7f1ed200ecb699e8ddb29ff58f09ab2f850a7f37511b79c62189f715")
 
-    depends_on("python@3.7:", type=("build", "run"))
+    depends_on("python@3.8:", when="@3:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
-    depends_on("py-setuptools-scm", type="build")
 
-    depends_on("py-future@0.17.1:", type=("build", "run"))
+    depends_on("py-packaging@23.1:", when="@3:", type=("build", "run"))
     depends_on("dcm2niix", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-setuptools-scm", when="@2", type="build")
+    depends_on("py-future@0.17.1:", when="@2", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-deepspeed/package.py b/var/spack/repos/builtin/packages/py-deepspeed/package.py
new file mode 100644
index 00000000000000..390aa6e00489a5
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-deepspeed/package.py
@@ -0,0 +1,32 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyDeepspeed(PythonPackage):
+    """DeepSpeed library.
+
+    DeepSpeed enables world's most powerful language models like MT-530B and BLOOM. It is an
+    easy-to-use deep learning optimization software suite that powers unprecedented scale and
+    speed for both training and inference.
+    """
+
+    homepage = "http://deepspeed.ai/"
+    pypi = "deepspeed/deepspeed-0.10.0.tar.gz"
+
+    version("0.10.0", sha256="afb06a97fde2a33d0cbd60a8357a70087c037b9f647ca48377728330c35eff3e")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-hjson", type=("build", "run"))
+    depends_on("py-ninja", type=("build", "run"))
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-packaging@20:", type=("build", "run"))
+    depends_on("py-psutil", type=("build", "run"))
+    depends_on("py-py-cpuinfo", type=("build", "run"))
+    depends_on("py-pydantic@:1", type=("build", "run"))
+    # https://github.com/microsoft/DeepSpeed/issues/2830
+    depends_on("py-torch+distributed", type=("build", "run"))
+    depends_on("py-tqdm", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-deprecation-alias/package.py b/var/spack/repos/builtin/packages/py-deprecation-alias/package.py
new file mode 100644
index 00000000000000..b16d2ffb2baa4c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-deprecation-alias/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyDeprecationAlias(PythonPackage):
+    """Wrapper providing support for deprecated aliases."""
+
+    homepage = "https://github.com/domdfcoding/deprecation-alias"
+    pypi = "deprecation_alias/deprecation-alias-0.3.2.tar.gz"
+
+    version("0.3.2", sha256="1c9e1a5ddd0a276a1a18e7a4f9d56b53232217491c4549eaa45e51753013ce76")
+
+    depends_on("py-wheel@0.34.2:", type="build")
+    depends_on("py-setuptools@40.6:", type="build")
+    conflicts("^py-setuptools@61")
+    depends_on("py-deprecation@2.1:", type=("build", "run"))
+    depends_on("py-packaging@20.4:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-dgl/package.py b/var/spack/repos/builtin/packages/py-dgl/package.py
index 3ea7e748f95d32..4a1d4b73cd2c1b 100644
--- a/var/spack/repos/builtin/packages/py-dgl/package.py
+++ b/var/spack/repos/builtin/packages/py-dgl/package.py
@@ -22,9 +22,15 @@ class PyDgl(CMakePackage, PythonExtension, CudaPackage):
     maintainers("adamjstewart", "meyersbs")
 
     version("master", branch="master", submodules=True)
-    version("1.0.1", tag="1.0.1", submodules=True)
-    version("0.4.3", tag="0.4.3", submodules=True)
-    version("0.4.2", tag="0.4.2", submodules=True)
+    version(
+        "1.0.1", tag="1.0.1", commit="cc2e9933f309f585fae90965ab61ad11ac1eecd5", submodules=True
+    )
+    version(
+        "0.4.3", tag="0.4.3", commit="e1d90f9b5eeee7359a6b4f5edca7473a497984ba", submodules=True
+    )
+    version(
+        "0.4.2", tag="0.4.2", commit="55e056fbae8f25f3da4aab0a0d864d72c2a445ff", submodules=True
+    )
 
     variant("cuda", default=True, description="Build with CUDA")
     variant("openmp", default=True, description="Build with OpenMP")
diff --git a/var/spack/repos/builtin/packages/py-dict2css/package.py b/var/spack/repos/builtin/packages/py-dict2css/package.py
new file mode 100644
index 00000000000000..7e962e56d75e0a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-dict2css/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyDict2css(PythonPackage):
+    """A μ-library for constructing cascading style sheets from Python dictionaries."""
+
+    homepage = "https://github.com/sphinx-toolbox/dict2css"
+    pypi = "dict2css/dict2css-0.3.0.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("0.3.0", sha256="1e8b1bf580dca2083198f88a60ec88c878a8829d760dfe45483ef80fe2905117")
+
+    depends_on("py-whey", type="build")
+    depends_on("py-cssutils@2.2.0:", type=("build", "run"))
+    depends_on("py-domdf-python-tools@2.2.0:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-dipy/package.py b/var/spack/repos/builtin/packages/py-dipy/package.py
index 4979e83e001991..467cb201602605 100644
--- a/var/spack/repos/builtin/packages/py-dipy/package.py
+++ b/var/spack/repos/builtin/packages/py-dipy/package.py
@@ -19,14 +19,20 @@ class PyDipy(PythonPackage):
     homepage = "https://dipy.org/"
     pypi = "dipy/dipy-1.4.1.tar.gz"
 
+    version("1.7.0", sha256="59bb647128aae7793215c813bb8ea35dae260ac9f0d938c724064f0af5a05cc3")
     version("1.4.1", sha256="b4bf830feae751f3f985d54cb71031fc35cea612838320f1f74246692b8a3cc0")
 
     depends_on("python@3.6:", type=("build", "run"))
+    depends_on("python@:3.9", type=("build", "run"), when="@:1.4")
     depends_on("py-setuptools", type="build")
     depends_on("py-cython@0.29:", type=("build"))
+    depends_on("py-cython@0.29.24:", type=("build"), when="@1.7:")
     depends_on("py-numpy@1.12.0:", type=("build", "run"))
+    depends_on("py-numpy@1.14.5:", type=("build", "run"), when="@1.7:")
     depends_on("py-scipy@1.0:", type=("build", "run"))
+    depends_on("py-scipy@1.1:", type=("build", "run"), when="@1.7:")
     depends_on("py-nibabel@3.0.0:", type=("build", "run"))
     depends_on("py-h5py@2.5.0:", type=("build", "run"))
+    depends_on("py-h5py@2.8.0:", type=("build", "run"), when="@1.7:")
     depends_on("py-packaging@19.0:", type=("build", "run"))
     depends_on("py-tqdm@4.30.0:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-dist-meta/package.py b/var/spack/repos/builtin/packages/py-dist-meta/package.py
new file mode 100644
index 00000000000000..c8ceed07f7a8ad
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-dist-meta/package.py
@@ -0,0 +1,23 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyDistMeta(PythonPackage):
+    """Parse and create Python distribution metadata."""
+
+    homepage = "https://github.com/repo-helper/dist-meta"
+    pypi = "dist_meta/dist-meta-0.8.0.tar.gz"
+
+    version("0.8.0", sha256="541d51f75b7f580c80d8d7b23112d0b4bf3edbc9442e425a7c4fcd75f4138551")
+
+    depends_on("py-wheel@0.34.2:", type="build")
+    depends_on("py-setuptools@40.6:", type="build")
+    conflicts("^py-setuptools@61")
+    depends_on("py-domdf-python-tools@3.1:", type=("build", "run"))
+    depends_on("py-handy-archives@0.1:", type=("build", "run"))
+    depends_on("py-packaging@20.9:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-distlib/package.py b/var/spack/repos/builtin/packages/py-distlib/package.py
index f4356b532829c2..f0a4c8d6e2a59d 100644
--- a/var/spack/repos/builtin/packages/py-distlib/package.py
+++ b/var/spack/repos/builtin/packages/py-distlib/package.py
@@ -12,6 +12,7 @@ class PyDistlib(PythonPackage):
     homepage = "https://bitbucket.org/pypa/distlib"
     pypi = "distlib/distlib-0.3.6.tar.gz"
 
+    version("0.3.7", sha256="9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8")
     version("0.3.6", sha256="14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46")
     version("0.3.4", sha256="e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579")
     version("0.3.3", sha256="d982d0751ff6eaaab5e2ec8e691d949ee80eddf01a62eaa96ddb11531fe16b05")
diff --git a/var/spack/repos/builtin/packages/py-distributed/package.py b/var/spack/repos/builtin/packages/py-distributed/package.py
index 2fbeda844d2e97..ef4dbe7c15205d 100644
--- a/var/spack/repos/builtin/packages/py-distributed/package.py
+++ b/var/spack/repos/builtin/packages/py-distributed/package.py
@@ -48,7 +48,7 @@ class PyDistributed(PythonPackage):
     depends_on("py-click@8.0:", type=("build", "run"), when="@2023.4.1:")
     depends_on("py-cloudpickle@1.5.0:", type=("build", "run"))
     depends_on("py-jinja2", type=("build", "run"), when="@2022.2.1:")
-    depends_on("py-jinja2@2.10.3", type=("build", "run"), when="@2023.4.1:")
+    depends_on("py-jinja2@2.10.3:", type=("build", "run"), when="@2023.4.1:")
     depends_on("py-locket@1:", type=("build", "run"), when="@2022.2.1:")
     depends_on("py-msgpack@0.6.0:", type=("build", "run"))
     depends_on("py-msgpack@1.0.0:", type=("build", "run"), when="@2023.4.1:")
@@ -57,6 +57,7 @@ class PyDistributed(PythonPackage):
     depends_on("py-psutil@5.7.0:", type=("build", "run"), when="@2023.4.1:")
     depends_on("py-sortedcontainers@:1,2.0.2:", type=("build", "run"))
     depends_on("py-sortedcontainers@2.0.5:", type=("build", "run"), when="@2023.4.1:")
+    depends_on("py-tblib@1.6:", type=("build", "run"))
     depends_on("py-toolz@0.8.2:", type=("build", "run"))
     # Note that the setup.py is wrong for py-toolz, when="@2022.10.2".
     # See https://github.com/dask/distributed/pull/7309
diff --git a/var/spack/repos/builtin/packages/py-dlio-profiler-py/package.py b/var/spack/repos/builtin/packages/py-dlio-profiler-py/package.py
new file mode 100644
index 00000000000000..ee18886ed1b15f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-dlio-profiler-py/package.py
@@ -0,0 +1,30 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyDlioProfilerPy(PythonPackage):
+    """A low-level profiler for capture I/O calls from deep learning applications."""
+
+    homepage = "https://github.com/hariharan-devarajan/dlio-profiler.git"
+    git = "https://github.com/hariharan-devarajan/dlio-profiler.git"
+    maintainers("hariharan-devarajan")
+
+    version("develop", branch="dev")
+    version("master", branch="master")
+    version("0.0.1", tag="v0.0.1", commit="28affe716211315dd6936ddc8e25ce6c43cdf491")
+    depends_on("cpp-logger@0.0.1")
+    depends_on("brahma@0.0.1")
+    depends_on("gotcha@develop")
+    depends_on("py-setuptools@42:", type="build")
+    depends_on("py-pybind11", type=("build", "run"))
+    depends_on("py-ninja", type="build")
+    depends_on("py-cmake@3.12:", type="build")
+
+    def setup_build_environment(self, env):
+        env.set("DLIO_PROFILER_DIR", self.prefix)
+        env.set("DLIO_PYTHON_SITE", python_purelib)
+        env.set("DLIO_BUILD_DEPENDENCIES", "0")
diff --git a/var/spack/repos/builtin/packages/py-dm-tree/package.py b/var/spack/repos/builtin/packages/py-dm-tree/package.py
index 71a6ef4a0f8ff5..02002412e57a11 100644
--- a/var/spack/repos/builtin/packages/py-dm-tree/package.py
+++ b/var/spack/repos/builtin/packages/py-dm-tree/package.py
@@ -52,14 +52,7 @@ def patch(self):
             "'--jobs={0}',\n".format(make_jobs),
             # Enable verbose output for failures
             "'--verbose_failures',\n",
-            # Show (formatted) subcommands being executed
-            "'--subcommands=pretty_print',\n",
             "'--spawn_strategy=local',\n",
-            # Ask bazel to explain what it's up to
-            # Needs a filename as argument
-            "'--explain=explainlogfile.txt',\n",
-            # Increase verbosity of explanation,
-            "'--verbose_explanations',\n",
             # bazel uses system PYTHONPATH instead of spack paths
             "'--action_env', 'PYTHONPATH={0}',\n".format(env["PYTHONPATH"]),
         ]
diff --git a/var/spack/repos/builtin/packages/py-doit/package.py b/var/spack/repos/builtin/packages/py-doit/package.py
new file mode 100644
index 00000000000000..aad45a5c2fa6a8
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-doit/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyDoit(PythonPackage):
+    """doit - Automation Tool."""
+
+    homepage = "http://pydoit.org/"
+    pypi = "doit/doit-0.36.0.tar.gz"
+
+    version("0.36.0", sha256="71d07ccc9514cb22fe59d98999577665eaab57e16f644d04336ae0b4bae234bc")
+
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
+    depends_on("py-cloudpickle", type=("build", "run"))
+    depends_on("py-importlib-metadata@4.4:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-dom-toml/package.py b/var/spack/repos/builtin/packages/py-dom-toml/package.py
new file mode 100644
index 00000000000000..54c43de68c0b28
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-dom-toml/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyDomToml(PythonPackage):
+    """Dom's tools for Tom's Obvious, Minimal Language."""
+
+    homepage = "https://github.com/domdfcoding/dom_toml"
+    pypi = "dom_toml/dom_toml-0.6.1.tar.gz"
+
+    version("0.6.1", sha256="a0bfc204ae32c72ed36e526dce56108a3b20741ac3c055207206ce3b2f302868")
+
+    depends_on("py-flit-core@3.2:3", type="build")
+    depends_on("py-domdf-python-tools@2.8:", type=("build", "run"))
+    depends_on("py-toml@0.10.2:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-domdf-python-tools/package.py b/var/spack/repos/builtin/packages/py-domdf-python-tools/package.py
new file mode 100644
index 00000000000000..d8e5444300fd90
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-domdf-python-tools/package.py
@@ -0,0 +1,23 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyDomdfPythonTools(PythonPackage):
+    """Helpful functions for Python"""
+
+    homepage = "https://github.com/domdfcoding/domdf_python_tools"
+    pypi = "domdf_python_tools/domdf_python_tools-3.6.1.tar.gz"
+
+    version("3.6.1", sha256="acc04563d23bce4d437dd08af6b9bea788328c412772a044d8ca428a7ad861be")
+
+    depends_on("py-wheel@0.34.2:", type="build")
+    depends_on("py-setuptools@40.6:", type="build")
+    conflicts("^py-setuptools@61")
+    depends_on("py-importlib-metadata@3.6:", type=("build", "run"), when="^python@:3.8")
+    depends_on("py-natsort@7.0.1:", type=("build", "run"))
+    depends_on("py-typing-extensions@3.7.4.1:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-dunamai/package.py b/var/spack/repos/builtin/packages/py-dunamai/package.py
index a8b608a77c421f..ea9d2d89312d7d 100644
--- a/var/spack/repos/builtin/packages/py-dunamai/package.py
+++ b/var/spack/repos/builtin/packages/py-dunamai/package.py
@@ -12,6 +12,7 @@ class PyDunamai(PythonPackage):
     homepage = "https://github.com/mtkennerly/dunamai"
     pypi = "dunamai/dunamai-1.13.1.tar.gz"
 
+    version("1.18.0", sha256="5200598561ea5ba956a6174c36e402e92206c6a6aa4a93a6c5cb8003ee1e0997")
     version("1.17.0", sha256="459381b585a1e78e4070f0d38a6afb4d67de2ee95064bf6b0438ec620dde0820")
     version("1.13.1", sha256="49597bdf653bdacdeb51ec6e0f1d4d2327309376fc83e6f1d42af6e29600515f")
 
@@ -19,4 +20,4 @@ class PyDunamai(PythonPackage):
     depends_on("py-poetry-core@1:", type="build")
 
     depends_on("py-packaging@20.9:", type=("build", "run"))
-    depends_on("py-importlib-metadata@1.6:", when="^pyhton@:3.7", type=("build", "run"))
+    depends_on("py-importlib-metadata@1.6:", when="^python@:3.7", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-dynaconf/package.py b/var/spack/repos/builtin/packages/py-dynaconf/package.py
new file mode 100644
index 00000000000000..64c38a87683a8c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-dynaconf/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyDynaconf(PythonPackage):
+    """Dynaconf is a dynamic configuration management package for Python projects"""
+
+    homepage = "https://github.com/dynaconf/dynaconf"
+    pypi = "dynaconf/dynaconf-3.2.2.tar.gz"
+
+    version("3.2.2", sha256="2f98ec85a2b8edb767b3ed0f82c6d605d30af116ce4622932a719ba70ff152fc")
+
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-setuptools@38.6.0:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-editdistance/package.py b/var/spack/repos/builtin/packages/py-editdistance/package.py
index b0f7dfbfb437fb..3b1157937fa921 100644
--- a/var/spack/repos/builtin/packages/py-editdistance/package.py
+++ b/var/spack/repos/builtin/packages/py-editdistance/package.py
@@ -17,7 +17,7 @@ class PyEditdistance(PythonPackage):
 
     # PyPI tarball for version 0.6.2 missing bycython.pyx file
     # https://github.com/roy-ht/editdistance/issues/94#issuecomment-1426279375
-    version("0.6.2", tag="v0.6.2")
+    version("0.6.2", tag="v0.6.2", commit="3f5a5b0299f36662349df0917352a42c620e3dd4")
     version("0.4", sha256="c765db6f8817d38922e4a50be4b9ab338b2c539377b6fcf0bca11dea72eeb8c1")
 
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-efel/package.py b/var/spack/repos/builtin/packages/py-efel/package.py
new file mode 100644
index 00000000000000..a33749b9af75eb
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-efel/package.py
@@ -0,0 +1,24 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class PyEfel(PythonPackage):
+    """The Electrophys Feature Extract Library (eFEL) allows
+    neuroscientists to automatically extract features from time series data
+    recorded from neurons (both in vitro and in silico).
+    Examples are the action potential width and amplitude in
+    voltage traces recorded during whole-cell patch clamp experiments.
+    The user of the library provides a set of traces and selects the
+    features to be calculated. The library will then extract the requested
+    features and return the values to the user."""
+
+    homepage = "https://github.com/BlueBrain/eFEL"
+    pypi = "efel/efel-3.0.80.tar.gz"
+
+    version("5.2.0", sha256="ed2c5efe22a4c703a4d9e47775b939009e1456713ac896898ebabf177c60b1dc")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-numpy@1.6:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-einops/package.py b/var/spack/repos/builtin/packages/py-einops/package.py
index 9394cdfc4621a7..d5d8eedebe6cf4 100644
--- a/var/spack/repos/builtin/packages/py-einops/package.py
+++ b/var/spack/repos/builtin/packages/py-einops/package.py
@@ -14,10 +14,12 @@ class PyEinops(PythonPackage):
     homepage = "https://github.com/arogozhnikov/einops"
     pypi = "einops/einops-0.3.2.tar.gz"
 
+    version("0.7.0", sha256="b2b04ad6081a3b227080c9bf5e3ace7160357ff03043cd66cc5b2319eb7031d1")
     version("0.6.1", sha256="f95f8d00f4ded90dbc4b19b6f98b177332614b0357dde66997f3ae5d474dc8c8")
     version("0.6.0", sha256="6f6c78739316a2e3ccbce8052310497e69da092935e4173f2e76ec4e3a336a35")
     version("0.5.0", sha256="8b7a83cffc1ea88e306df099b7cbb9c3ba5003bd84d05ae44be5655864abb8d3")
     version("0.3.2", sha256="5200e413539f0377f4177ef00dc019968f4177c49b1db3e836c7883df2a5fe2e")
 
+    depends_on("python@3.8:", when="@0.7:", type=("build", "run"))
     depends_on("py-hatchling@1.10:", when="@0.5:", type="build")
     depends_on("py-setuptools", when="@:0.4", type="build")
diff --git a/var/spack/repos/builtin/packages/py-enum-tools/package.py b/var/spack/repos/builtin/packages/py-enum-tools/package.py
new file mode 100644
index 00000000000000..e8bef99f69b127
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-enum-tools/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyEnumTools(PythonPackage):
+    """Tools to expand Python's enum module."""
+
+    homepage = "https://github.com/domdfcoding/enum_tools"
+    pypi = "enum_tools/enum_tools-0.10.0.tar.gz"
+
+    version("0.10.0", sha256="777dc3cfb4314780bb9ca2460b518be58c3f29e13bd77b33badead4c2c136976")
+
+    depends_on("py-whey", type="build")
+    depends_on("py-pygments@2.6.1:", type=("build", "run"))
+    depends_on("py-typing-extensions@3.7.4.3:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-eprosima-fastdds/package.py b/var/spack/repos/builtin/packages/py-eprosima-fastdds/package.py
new file mode 100644
index 00000000000000..bf01f4edc45696
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-eprosima-fastdds/package.py
@@ -0,0 +1,29 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyEprosimaFastdds(CMakePackage, PythonExtension):
+    """eprosima Fast DDS (formerly Fast RTPS) is a C++ implementation of the DDS
+    (Data Distribution Service) standard of the OMG (Object Management Group).
+    eProsima Fast DDS implements the RTPS (Real Time Publish Subscribe) protocol,
+    which provides publisher-subscriber communications over unreliable transports
+    such as UDP, as defined and maintained by the Object Management Group (OMG) consortium.
+    This is the python interface for eprosima Fast DDS."""
+
+    homepage = "https://www.eprosima.com/"
+    url = "https://github.com/eProsima/Fast-DDS-python/archive/v1.2.2.tar.gz"
+
+    version("1.2.2", sha256="78c53739a66544b8c91d0016560c267e11bd7fdaf727b3bfbffd44ae65c93c62")
+
+    depends_on("cmake@3.15:", type="build")
+    extends("python")
+    depends_on("py-pytest", type="test")
+    depends_on("swig", type=("build"))
+    depends_on("eprosima-fastdds")
+    depends_on("openssl")
+
+    root_cmakelists_dir = "fastdds_python"
diff --git a/var/spack/repos/builtin/packages/py-exarl/package.py b/var/spack/repos/builtin/packages/py-exarl/package.py
index c3106e87786d17..0506a21778cbcf 100644
--- a/var/spack/repos/builtin/packages/py-exarl/package.py
+++ b/var/spack/repos/builtin/packages/py-exarl/package.py
@@ -19,7 +19,7 @@ class PyExarl(PythonPackage):
     version("master", branch="master")
     version("develop", branch="develop")
     version("update-spack", branch="update-spack")
-    version("0.1.0", tag="v0.1.0")
+    version("0.1.0", tag="v0.1.0", commit="5f5b99884a92f86ea9f637524eca6f4393b9635f")
 
     depends_on("python@3.6:", type=("build", "run"))
     depends_on("git-lfs", type=("build"))
diff --git a/var/spack/repos/builtin/packages/py-expecttest/package.py b/var/spack/repos/builtin/packages/py-expecttest/package.py
new file mode 100644
index 00000000000000..60857d5a9bdb8a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-expecttest/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyExpecttest(PythonPackage):
+    """This library implements expect tests (also known as "golden" tests)."""
+
+    homepage = "https://github.com/ezyang/expecttest"
+    pypi = "expecttest/expecttest-0.1.6.tar.gz"
+
+    version("0.1.6", sha256="fd49563b6703b9c060a0bc946dfafc62bad74898867432192927eb1e5f9d8952")
+
+    depends_on("python@:3", type=("build", "run"))
+    depends_on("py-poetry-core@1:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-f90nml/package.py b/var/spack/repos/builtin/packages/py-f90nml/package.py
index cfc06bc6e2267c..3e018f79396d49 100644
--- a/var/spack/repos/builtin/packages/py-f90nml/package.py
+++ b/var/spack/repos/builtin/packages/py-f90nml/package.py
@@ -14,7 +14,7 @@ class PyF90nml(PythonPackage):
     pypi = "f90nml/f90nml-1.4.3.tar.gz"
 
     version("1.4.3", sha256="e2f3cd23d821ebcaef66ce406485b35aa08aae0df92c4bece76e227e5bd146e1")
-    version('1.4.2', sha256='becacc8bed78efa3873438027f898fdd42f3b447c94cd29ae3033d6ff88ab9bb')
+    version("1.4.2", sha256="becacc8bed78efa3873438027f898fdd42f3b447c94cd29ae3033d6ff88ab9bb")
     version("1.4.1", sha256="9df312aa13b9c21936f059cab9ab40afebc280f1ab54e6854c3873d0b7b7865c")
     version("1.3.1", sha256="145c1f2c55bad628d225af22fc9bf06347eb0d33e7bae8a05869a68274b8fb2d")
 
diff --git a/var/spack/repos/builtin/packages/py-fairscale/package.py b/var/spack/repos/builtin/packages/py-fairscale/package.py
new file mode 100644
index 00000000000000..f740269beb921a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-fairscale/package.py
@@ -0,0 +1,37 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyFairscale(PythonPackage):
+    """FairScale is a PyTorch extension library for high performance and large
+    scale training. This library extends basic PyTorch capabilities while adding
+    new SOTA scaling techniques."""
+
+    homepage = "https://github.com/facebookresearch/fairscale"
+    pypi = "fairscale/fairscale-0.4.13.tar.gz"
+
+    version("0.4.13", sha256="1b797825c427f5dba92253fd0d8daa574e8bd651a2423497775fab1b30cfb768")
+    version("0.4.4", sha256="7719898743dc58c04a2294c896ee6308c92ccb3af9e10632b2a62f77cb689357")
+
+    variant("extra", default=False, description="support for cuda.list.gpu, scaler, weight")
+
+    # from setup.py
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("ninja", type="build")
+    # from pyproject.toml
+    depends_on("py-wheel@0.30.0:", type="build")
+    depends_on("py-setuptools@40.6.2:", type="build")
+    # from requirements.txt
+    depends_on("py-torch@1.8.0:", type=("build", "run"))
+    depends_on("py-numpy@1.22.0:", type=("build", "run"))
+    # added extra to support cuda.list.gpu, scaler, and weight (not in pip install)
+    # see requirements-dev.txt in github. This does not include the tools for testing
+    with when("+extra"):
+        depends_on("py-pynvml@8.0.4", type=("build", "run"))
+        depends_on("py-scikit-learn@1.1.3", type=("build", "run"))
+        depends_on("py-pygit2@1.11.1", type=("build", "run"))
+        depends_on("py-pgzip@0.3.1", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-fenics-basix/package.py b/var/spack/repos/builtin/packages/py-fenics-basix/package.py
index 9e6d5b44b7a925..6d3ffa6cae3754 100644
--- a/var/spack/repos/builtin/packages/py-fenics-basix/package.py
+++ b/var/spack/repos/builtin/packages/py-fenics-basix/package.py
@@ -15,22 +15,26 @@ class PyFenicsBasix(PythonPackage):
     maintainers("chrisrichardson", "mscroggs", "garth-wells", "jhale")
 
     version("main", branch="main")
+    version("0.7.0", sha256="9bee81b396ee452eec8d9735f278cb44cb6994c6bc30aec8ed9bb4b12d83fa7f")
     version("0.6.0", sha256="687ae53153c98facac4080dcdc7081701db1dcea8c5e7ae3feb72aec17f83304")
     version("0.5.1", sha256="69133476ac35f0bd0deccb480676030378c341d7dfb2adaca22cd16b7e1dc1cb")
     version("0.4.2", sha256="a54f5e442b7cbf3dbb6319c682f9161272557bd7f42e2b8b8ccef88bc1b7a22f")
 
     depends_on("fenics-basix@main", type=("build", "run"), when="@main")
+    depends_on("fenics-basix@0.7.0", type=("build", "run"), when="@0.7.0")
     depends_on("fenics-basix@0.6.0", type=("build", "run"), when="@0.6.0")
     depends_on("fenics-basix@0.5.1", type=("build", "run"), when="@0.5.1")
     depends_on("fenics-basix@0.4.2", type=("build", "run"), when="@0.4.2")
 
+    # See python/CMakeLists.txt
+    depends_on("cmake@3.16:", type="build")
+
+    # See python/pyproject.toml
+    depends_on("python@3.8:", when="@0.7.0:", type=("build", "run"))
     depends_on("py-setuptools@42:", type="build")
-    depends_on("py-setuptools@40:", type="build")
     depends_on("py-numpy@1.21:", type=("build", "run"))
-    depends_on("cmake@3.19:", type="build")
-    depends_on("python@3.7:", type=("build", "run"))
-    depends_on("py-pybind11@2.9.1:", type="build")
-    depends_on("py-pybind11@2.2.4:", type="build")
+    depends_on("py-pybind11@2.9.1:", when="@:0.7", type="build")
+    depends_on("py-nanobind@1.5.1:", when="@0.8:", type="build")
 
     depends_on("xtensor@0.23.10:", type="build", when="@:0.4")
 
diff --git a/var/spack/repos/builtin/packages/py-fenics-dolfinx/package.py b/var/spack/repos/builtin/packages/py-fenics-dolfinx/package.py
index 9106bd80a56df5..8c07879efd608c 100644
--- a/var/spack/repos/builtin/packages/py-fenics-dolfinx/package.py
+++ b/var/spack/repos/builtin/packages/py-fenics-dolfinx/package.py
@@ -24,8 +24,10 @@ class PyFenicsDolfinx(PythonPackage):
     depends_on("cmake@3.19:", type="build")
     depends_on("hdf5", type="build")
     depends_on("pkgconfig", type="build")
-    depends_on("python@3.8:", type=("build", "run"), when="@0.5:")
-    depends_on("python@3.7:", type=("build", "run"))
+
+    depends_on("python@3.8:", when="@0.6.1:", type=("build", "run"))
+    depends_on("python@3.8:3.10", when="@0.5:0.6.0", type=("build", "run"))
+    depends_on("python@3.7:3.10", when="@0.4", type=("build", "run"))
 
     depends_on("fenics-dolfinx@main", when="@main")
     depends_on("fenics-dolfinx@0.6.0", when="@0.6.0")
diff --git a/var/spack/repos/builtin/packages/py-fenics-ffcx/package.py b/var/spack/repos/builtin/packages/py-fenics-ffcx/package.py
index 45c11e7952f596..f2cec1e21e9b0d 100644
--- a/var/spack/repos/builtin/packages/py-fenics-ffcx/package.py
+++ b/var/spack/repos/builtin/packages/py-fenics-ffcx/package.py
@@ -15,24 +15,43 @@ class PyFenicsFfcx(PythonPackage):
     maintainers("chrisrichardson", "garth-wells", "jhale")
 
     version("main", branch="main")
+    version("0.7.0", sha256="7f3c3ca91d63ce7831d37799cc19d0551bdcd275bdfa4c099711679533dd1c71")
     version("0.6.0", sha256="076fad61d406afffd41019ae1abf6da3f76406c035c772abad2156127667980e")
     version(
         "0.5.0.post0", sha256="039908c9998b51ba53e5deb3a97016062c262f0a4285218644304f7d3cd35882"
     )
     version("0.4.2", sha256="3be6eef064d6ef907245db5b6cc15d4e603762e68b76e53e099935ca91ef1ee4")
 
-    depends_on("python@3.7:", type=("build", "run"))
-    depends_on("py-setuptools@58:", type=("build", "run"))
+    depends_on("python@3.8:", when="@0.7:", type=("build", "run"))
+    depends_on("py-setuptools@62:", when="@0.7:", type="build")
+    # Runtime dependency on pkg_resources from setuptools at 0.6.0
+    depends_on("py-setuptools@58:", when="@0.4.2:0.6", type=("build", "run"))
 
+    # CFFI is required at runtime for JIT support
     depends_on("py-cffi", type=("build", "run"))
+    # py-numpy>=1.21 required because FFCx uses NumPy typing (version
+    # requirement not properly set in the FFCx pyproject.toml file)
     depends_on("py-numpy@1.21:", type=("build", "run"))
 
     depends_on("py-fenics-ufl@main", type=("build", "run"), when="@main")
+    depends_on("py-fenics-ufl@2023.3.0:", type=("build", "run"), when="@0.8")
+    depends_on("py-fenics-ufl@2023.2.0", type=("build", "run"), when="@0.7")
     depends_on("py-fenics-ufl@2023.1", type=("build", "run"), when="@0.6")
     depends_on("py-fenics-ufl@2022.2.0", type=("build", "run"), when="@0.5.0:0.5")
     depends_on("py-fenics-ufl@2022.1.0", type=("build", "run"), when="@0.4.2")
 
     depends_on("py-fenics-basix@main", type=("build", "run"), when="@main")
+    depends_on("py-fenics-basix@0.7", type=("build", "run"), when="@0.7")
     depends_on("py-fenics-basix@0.6.0:0.6", type=("build", "run"), when="@0.6.0:0.6")
     depends_on("py-fenics-basix@0.5.1:0.5", type=("build", "run"), when="@0.5.0:0.5")
     depends_on("py-fenics-basix@0.4.2", type=("build", "run"), when="@0.4.2")
+
+    depends_on("py-pytest@6:", type="test")
+    depends_on("py-sympy", type="test")
+
+    @run_after("install")
+    @on_package_attributes(run_tests=True)
+    def check_build(self):
+        with working_dir("test"):
+            pytest = which("pytest")
+            pytest("--ignore=test_cmdline.py")
diff --git a/var/spack/repos/builtin/packages/py-fenics-fiat/package.py b/var/spack/repos/builtin/packages/py-fenics-fiat/package.py
index 16efcf4b7207ce..86c10ae64fcc74 100644
--- a/var/spack/repos/builtin/packages/py-fenics-fiat/package.py
+++ b/var/spack/repos/builtin/packages/py-fenics-fiat/package.py
@@ -29,7 +29,7 @@ class PyFenicsFiat(PythonPackage):
         "2017.1.0.post1", sha256="1784fe1cb9479ca7cd85f63b0afa6e07634feec8d8e82fa8be4c480649cb9621"
     )
     version("2017.1.0", sha256="d4288401ad16c4598720f9db0810a522f7f0eadad35d8211bac7120bce5fde94")
-    version("2016.2.0", tag="fiat-2016.2.0")
+    version("2016.2.0", tag="fiat-2016.2.0", commit="e021c825cab938ae8809b738410cda26e392db05")
 
     depends_on("python@3:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-fenics-ufl/package.py b/var/spack/repos/builtin/packages/py-fenics-ufl/package.py
index 81690fcd115790..890b1934649b07 100644
--- a/var/spack/repos/builtin/packages/py-fenics-ufl/package.py
+++ b/var/spack/repos/builtin/packages/py-fenics-ufl/package.py
@@ -19,6 +19,7 @@ class PyFenicsUfl(PythonPackage):
     maintainers("chrisrichardson", "garth-wells", "jhale")
 
     version("main", branch="main")
+    version("2023.2.0", sha256="d1d3209e8ebd4bd70513c26890f51823bac90edc956233c47bd8e686e064436e")
     version(
         "2023.1.1.post0", sha256="9e6e87f1447635029cec42604f62a76bba84899beb4b8822af10389d1f93a9b6"
     )
@@ -34,9 +35,19 @@ class PyFenicsUfl(PythonPackage):
     version(
         "2017.1.0.post1", sha256="82c8170f44c2392c7e60aa86495df22cc209af50735af8115dc35aeda4b0ca96"
     )
-    version("2016.2.0", tag="ufl-2016.2.0")
+    version("2016.2.0", tag="ufl-2016.2.0", commit="962d56f65821fb9c50ca4a5a858882c472243431")
 
-    depends_on("python@3.7:", type=("build", "run"))
-    depends_on("py-setuptools@58:", type=("build", "run"), when="@2022.1.0:")
-    depends_on("py-setuptools@40:", type=("build", "run"))
-    depends_on("py-numpy@1.21:", type=("build", "run"))
+    depends_on("python@3.8:", when="@2023.2.0:", type=("build", "run"))
+
+    depends_on("py-setuptools@62:", when="@2023.2.0:", type="build")
+    depends_on("py-setuptools@58:", when="@2022.1.0:2023.1.1.post0", type="build")
+    depends_on("py-setuptools@40:", when="@2016.2.0:2021.1.0", type="build")
+    depends_on("py-numpy", type=("build", "run"))
+
+    depends_on("py-pytest", type="test")
+
+    @run_after("install")
+    @on_package_attributes(run_tests=True)
+    def check_build(self):
+        with working_dir("test"):
+            Executable("py.test")()
diff --git a/var/spack/repos/builtin/packages/py-ffmpy/package.py b/var/spack/repos/builtin/packages/py-ffmpy/package.py
new file mode 100644
index 00000000000000..65ec6762b62fab
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-ffmpy/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyFfmpy(PythonPackage):
+    """A simple Python wrapper for ffmpeg"""
+
+    homepage = "https://github.com/Ch00k/ffmpy"
+    pypi = "ffmpy/ffmpy-0.3.0.tar.gz"
+
+    version("0.3.0", sha256="757591581eee25b4a50ac9ffb9b58035a2794533db47e0512f53fb2d7b6f9adc")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("ffmpeg", type="run")
diff --git a/var/spack/repos/builtin/packages/py-filelock/package.py b/var/spack/repos/builtin/packages/py-filelock/package.py
index 5feb61cb544cee..be882c5ebc84b0 100644
--- a/var/spack/repos/builtin/packages/py-filelock/package.py
+++ b/var/spack/repos/builtin/packages/py-filelock/package.py
@@ -16,6 +16,7 @@ class PyFilelock(PythonPackage):
     homepage = "https://github.com/tox-dev/py-filelock"
     pypi = "filelock/filelock-3.0.4.tar.gz"
 
+    version("3.12.4", sha256="2e6f249f1f3654291606e046b09f1fd5eac39b360664c27f5aad072012f8bcbd")
     version("3.12.0", sha256="fc03ae43288c013d2ea83c8597001b1129db351aad9c57fe2409327916b8e718")
     version("3.8.0", sha256="55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc")
     version("3.5.0", sha256="137b661e657f7850eec9def2a001efadba3414be523b87cd3f9a037372d80a15")
@@ -31,11 +32,13 @@ class PyFilelock(PythonPackage):
     version("2.0.9", sha256="0f91dce339c9f25d6f2e0733a17e4f9a47b139dffda52619a0e61e013e5c6782")
     version("2.0.8", sha256="7e48e4906de3c9a5d64d8f235eb3ae1050dfefa63fd65eaf318cc915c935212b")
 
-    depends_on("python@3.7:", when="@3.4.2:", type=("build", "run"))
+    depends_on("python@3.8:", when="@3.12.3:", type=("build", "run"))
 
     depends_on("py-hatch-vcs@0.3:", when="@3.8:", type="build")
+    depends_on("py-hatchling@1.18:", when="@3.12.3:", type="build")
     depends_on("py-hatchling@1.14:", when="@3.8:", type="build")
 
+    # Historical dependencies
     with when("@:3.8.0"):
         depends_on("py-setuptools@63.4:", when="@3.8:", type="build")
         depends_on("py-setuptools@41:", when="@3.1:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-fiona/package.py b/var/spack/repos/builtin/packages/py-fiona/package.py
index 37da11dd4cba8b..917a20ab09099d 100644
--- a/var/spack/repos/builtin/packages/py-fiona/package.py
+++ b/var/spack/repos/builtin/packages/py-fiona/package.py
@@ -10,12 +10,13 @@ class PyFiona(PythonPackage):
     """Fiona reads and writes spatial data files."""
 
     homepage = "https://github.com/Toblerity/Fiona"
-    pypi = "Fiona/Fiona-1.8.18.tar.gz"
+    pypi = "fiona/fiona-1.9.5.tar.gz"
     git = "https://github.com/Toblerity/Fiona.git"
 
     maintainers("adamjstewart")
 
     version("master", branch="master")
+    version("1.9.5", sha256="99e2604332caa7692855c2ae6ed91e1fffdf9b59449aa8032dd18e070e59a2f7")
     version("1.9.4", sha256="49f18cbcd3b1f97128c1bb038c3451b2e1be25baa52f02ce906c25cf75af95b6")
     version("1.9.3", sha256="60f3789ad9633c3a26acf7cbe39e82e3c7a12562c59af1d599fc3e4e8f7f8f25")
     version("1.9.2", sha256="f9263c5f97206bf2eb2c010d52e8ffc54e96886b0e698badde25ff109b32952a")
@@ -27,11 +28,13 @@ class PyFiona(PythonPackage):
     version("1.8.18", sha256="b732ece0ff8886a29c439723a3e1fc382718804bb057519d537a81308854967a")
 
     # pyproject.toml
-    depends_on("python@3.7:", when="@1.9:", type=("build", "link", "run"))
-    depends_on("python@2.6:", when="@1.8.22:1.8", type=("build", "link", "run"))
-    depends_on("python@2.6:3.10", when="@1.8.21", type=("build", "link", "run"))
-    depends_on("python@2.6:3.9", when="@:1.8.20", type=("build", "link", "run"))
-    depends_on("py-cython@0.29.29:0.29", when="@1.9:", type="build")
+    depends_on("python@:3.10", when="@1.8.21", type=("build", "link", "run"))
+    depends_on("python@:3.9", when="@:1.8.20", type=("build", "link", "run"))
+    depends_on("py-cython", type="build")
+    # Overly strict version requirements
+    # depends_on("py-cython@3.0.2:3", when="@1.9.5:", type="build")
+    # depends_on("py-cython@0.29.29:0.29", when="@1.9.0:1.9.4", type="build")
+    depends_on("py-setuptools@67.8:", when="@1.9.5:", type="build")
     depends_on("py-setuptools@61:", when="@1.9:", type="build")
     depends_on("py-attrs@19.2:", when="@1.9:", type=("build", "run"))
     depends_on("py-attrs@17:", type=("build", "run"))
@@ -43,12 +46,20 @@ class PyFiona(PythonPackage):
     depends_on("py-importlib-metadata", when="@1.9.2: ^python@:3.9", type=("build", "run"))
     depends_on("py-six", when="@1.9.4:", type=("build", "run"))
     depends_on("py-six@1.7:", when="@:1.8", type=("build", "run"))
+    depends_on("py-setuptools", when="@:1.9.1,1.9.5:", type="run")
 
     # setup.py or release notes
     depends_on("gdal@3.1:", when="@1.9:", type=("build", "link", "run"))
     depends_on("gdal@1.8:", type=("build", "link", "run"))
 
     # Historical dependencies
-    depends_on("py-setuptools", when="@:1.9.1", type=("build", "run"))
     depends_on("py-munch@2.3.2:", when="@1.9.0:1.9.3", type=("build", "run"))
     depends_on("py-munch", when="@:1.8", type=("build", "run"))
+
+    def url_for_version(self, version):
+        url = "https://files.pythonhosted.org/packages/source/{0}/{0}iona/{0}iona-{1}.tar.gz"
+        if version >= Version("1.9.5"):
+            letter = "f"
+        else:
+            letter = "F"
+        return url.format(letter, version)
diff --git a/var/spack/repos/builtin/packages/py-flake8/package.py b/var/spack/repos/builtin/packages/py-flake8/package.py
index 781b94d0f11584..7cc6124a5197b6 100644
--- a/var/spack/repos/builtin/packages/py-flake8/package.py
+++ b/var/spack/repos/builtin/packages/py-flake8/package.py
@@ -13,6 +13,7 @@ class PyFlake8(PythonPackage):
     homepage = "https://github.com/PyCQA/flake8"
     pypi = "flake8/flake8-4.0.1.tar.gz"
 
+    version("6.1.0", sha256="d5b3857f07c030bdb5bf41c7f53799571d75c4491748a3adcd47de929e34cd23")
     version("6.0.0", sha256="c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181")
     version("5.0.4", sha256="6fbe320aad8d6b95cec8b8e47bc933004678dc63095be98528b7bdd2a9f510db")
     version("5.0.2", sha256="9cc32bc0c5d16eacc014c7ec6f0e9565fd81df66c2092c3c9df06e3c1ac95e5d")
@@ -43,6 +44,11 @@ class PyFlake8(PythonPackage):
     # http://flake8.pycqa.org/en/latest/faq.html#why-does-flake8-use-ranges-for-its-dependencies
     # http://flake8.pycqa.org/en/latest/internal/releases.html#releasing-flake8
 
+    # Flake8 6.1.X
+    depends_on("py-mccabe@0.7", when="@6.1", type=("build", "run"))
+    depends_on("py-pycodestyle@2.11", when="@6.1", type=("build", "run"))
+    depends_on("py-pyflakes@3.1", when="@6.1", type=("build", "run"))
+
     # Flake8 6.0.X
     depends_on("py-mccabe@0.7", when="@6.0", type=("build", "run"))
     depends_on("py-pycodestyle@2.10", when="@6.0", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-flatbuffers/package.py b/var/spack/repos/builtin/packages/py-flatbuffers/package.py
index b03bd2a94dd69b..d87b1cbdc2d3db 100644
--- a/var/spack/repos/builtin/packages/py-flatbuffers/package.py
+++ b/var/spack/repos/builtin/packages/py-flatbuffers/package.py
@@ -18,6 +18,7 @@ class PyFlatbuffers(PythonPackage):
 
     maintainers("gperrotta")
 
+    version("23.5.26", sha256="9ea1144cac05ce5d86e2859f431c6cd5e66cd9c78c558317c7955fb8d4c78d89")
     version("2.0.7", sha256="0ae7d69c5b82bf41962ca5fde9cc43033bc9501311d975fd5a25e8a7d29c1245")
     version("2.0", sha256="12158ab0272375eab8db2d663ae97370c33f152b27801fa6024e1d6105fd4dd2")
     version("1.12", sha256="63bb9a722d5e373701913e226135b28a6f6ac200d5cc7b4d919fa38d73b44610")
diff --git a/var/spack/repos/builtin/packages/py-flit-core/package.py b/var/spack/repos/builtin/packages/py-flit-core/package.py
index 8bd8835a3a4a21..a05cd05fc102d7 100644
--- a/var/spack/repos/builtin/packages/py-flit-core/package.py
+++ b/var/spack/repos/builtin/packages/py-flit-core/package.py
@@ -9,10 +9,12 @@
 class PyFlitCore(PythonPackage):
     """Distribution-building parts of Flit."""
 
-    homepage = "https://github.com/takluyver/flit"
-    pypi = "flit-core/flit_core-3.3.0.tar.gz"
+    homepage = "https://github.com/pypa/flit"
+    pypi = "flit-core/flit_core-3.9.0.tar.gz"
     maintainers("takluyver")
 
+    version("3.9.0", sha256="72ad266176c4a3fcfab5f2930d76896059851240570ce9a98733b658cb786eba")
+    version("3.8.0", sha256="b305b30c99526df5e63d6022dd2310a0a941a187bd3884f4c8ef0418df6c39f3")
     version("3.7.1", sha256="14955af340c43035dbfa96b5ee47407e377ee337f69e70f73064940d27d0a44f")
     version("3.6.0", sha256="5892962ab8b8ea945835b3a288fe9dd69316f1903d5288c3f5cafdcdd04756ad")
     version("3.5.1", sha256="3083720351a6cb00e0634a1ec0e26eae7b273174c3c6c03d5b597a14203b282e")
diff --git a/var/spack/repos/builtin/packages/py-flit/package.py b/var/spack/repos/builtin/packages/py-flit/package.py
index 762ca569e45262..859594c01699f2 100644
--- a/var/spack/repos/builtin/packages/py-flit/package.py
+++ b/var/spack/repos/builtin/packages/py-flit/package.py
@@ -10,19 +10,23 @@ class PyFlit(PythonPackage):
     """Flit is a simple way to put Python packages and modules on PyPI."""
 
     homepage = "https://github.com/pypa/flit"
-    pypi = "flit/flit-3.3.0.tar.gz"
+    pypi = "flit/flit-3.9.0.tar.gz"
     maintainers("takluyver")
 
+    version("3.9.0", sha256="d75edf5eb324da20d53570a6a6f87f51e606eee8384925cd66a90611140844c7")
+    version("3.8.0", sha256="d0f2a8f4bd45dc794befbf5839ecc0fd3830d65a57bd52b5997542fac5d5e937")
     version("3.7.1", sha256="3c9bd9c140515bfe62dd938c6610d10d6efb9e35cc647fc614fe5fb3a5036682")
     version("3.6.0", sha256="b1464e006df4df4c8eeb37671c0e0ce66e1d04e4a36d91b702f180a25fde3c11")
     version("3.3.0", sha256="65fbe22aaa7f880b776b20814bd80b0afbf91d1f95b17235b608aa256325ce57")
 
     depends_on("python@3.6:", type=("build", "run"))
-    depends_on("py-flit-core@3.7.1:3", when="@3.7.1:3", type=("build", "run"))
+    depends_on("py-flit-core@3.9.0:3", when="@3.9.0:3", type=("build", "run"))
+    depends_on("py-flit-core@3.8.0:3", when="@3.8.0:3.8", type=("build", "run"))
+    depends_on("py-flit-core@3.7.1:3", when="@3.7.1:3.7", type=("build", "run"))
     depends_on("py-flit-core@3.6.0:3", when="@3.6.0:3.6", type=("build", "run"))
     depends_on("py-flit-core@3.3.0:3", when="@3.3.0:3.3", type=("build", "run"))
     depends_on("py-requests", type=("build", "run"))
     depends_on("py-docutils", type=("build", "run"))
-    depends_on("py-tomli", when="@3.4:", type=("build", "run"))
+    depends_on("py-tomli", when="@3.4:3.7", type=("build", "run"))
     depends_on("py-tomli-w", when="@3.4:", type=("build", "run"))
     depends_on("py-toml", when="@3.3.0:3.3", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-flye/package.py b/var/spack/repos/builtin/packages/py-flye/package.py
index c3f57fc841e956..2e19485f86bbf7 100644
--- a/var/spack/repos/builtin/packages/py-flye/package.py
+++ b/var/spack/repos/builtin/packages/py-flye/package.py
@@ -25,7 +25,7 @@ class PyFlye(PythonPackage):
     depends_on("python@2.7:2.8,3.5:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("gmake", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     msg = "C++ compiler with C++11 support required"
     conflicts("%gcc@:4.7", msg=msg)
diff --git a/var/spack/repos/builtin/packages/py-fn-py/package.py b/var/spack/repos/builtin/packages/py-fn-py/package.py
index 543bbe5a366d3f..253ebf7ead826f 100644
--- a/var/spack/repos/builtin/packages/py-fn-py/package.py
+++ b/var/spack/repos/builtin/packages/py-fn-py/package.py
@@ -13,6 +13,10 @@ class PyFnPy(PythonPackage):
     homepage = "https://github.com/fnpy/fn.py"
     url = "https://github.com/fnpy/fn.py/archive/v0.5.2.tar.gz"
 
+    version("0.6.0", sha256="85d4d4ae6ce3c13e9dbe45df2895188f742ebddbf100a95d360d73a8965ee0c8")
     version("0.5.2", sha256="fda2253d792867a79514496932630622df9340f214a2f4b2d597b60a8cc3d96b")
 
     depends_on("py-setuptools", type="build")
+
+    # Uses inspect.getargspec
+    depends_on("python@:3.10", when="@:0.5", type=("run", "build"))
diff --git a/var/spack/repos/builtin/packages/py-fraction/package.py b/var/spack/repos/builtin/packages/py-fraction/package.py
new file mode 100644
index 00000000000000..919d14cadf0c19
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-fraction/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyFraction(PythonPackage):
+    """
+    Fraction carries out all the fraction operations including addition, subtraction, multiplicati
+    on, division, reciprocation.
+    """
+
+    homepage = "https://github.com/bradley101/fraction"
+    pypi = "Fraction/Fraction-2.2.0.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("2.2.0", sha256="2c1179f20c8b749622935fe04db1c7f2987f011f2376bdad84c2a39c8e3d0fdb")
+
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-furo/package.py b/var/spack/repos/builtin/packages/py-furo/package.py
new file mode 100644
index 00000000000000..a4ca3e1ce4d2d7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-furo/package.py
@@ -0,0 +1,23 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyFuro(PythonPackage):
+    """A clean customisable Sphinx documentation theme.."""
+
+    homepage = "https://github.com/pradyunsg/furo"
+    pypi = "furo/furo-2023.5.20.tar.gz"
+
+    version("2023.9.10", sha256="5707530a476d2a63b8cad83b4f961f3739a69f4b058bcf38a03a39fa537195b2")
+    version("2023.5.20", sha256="40e09fa17c6f4b22419d122e933089226dcdb59747b5b6c79363089827dea16f")
+
+    depends_on("py-sphinx-theme-builder@0.2.0a10:", type="build")
+
+    depends_on("py-beautifulsoup4", type=("build", "run"))
+    depends_on("py-sphinx@6:7", type=("build", "run"))
+    depends_on("py-sphinx-basic-ng", type=("build", "run"))
+    depends_on("py-pygments@2.7:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-generateds/package.py b/var/spack/repos/builtin/packages/py-generateds/package.py
new file mode 100644
index 00000000000000..bcf82787cf4dd3
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-generateds/package.py
@@ -0,0 +1,27 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyGenerateds(PythonPackage):
+    """Generate Python data structures and XML parser from Xschema."""
+
+    homepage = "http://www.davekuhlman.org/generateDS.html"
+    pypi = "generateDS/generateDS-2.41.4.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("2.43.2", sha256="e86f033f4d93414dd5b04cab9544a68b8f46d559073d85cd0990266b7b9ec09e")
+    version("2.43.1", sha256="2d3d71b42a09ba153bc51d2204324d04e384d0f15e41bdba881ee2daff9bbd68")
+    version("2.42.2", sha256="1d322aa7e074c262062b068660dd0c53bbdb0bb2b30152bb9e0074bd29fd365a")
+    version("2.42.1", sha256="87e4654449d34150802ca0cfb2330761382510d1385880f4d607cd34466abc2d")
+    version("2.41.5", sha256="8800c09454bb22f8f80f2ee138072d4e58bd5b6c14dbdf0a2a7ca13f06ba72e4")
+    version("2.41.4", sha256="804592eef573fa514741528a0bf9998f0c57ee29960c87f54608011f1fc722ea")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-six", type=("build", "run"))
+    depends_on("py-lxml", type=("build", "run"))
+    depends_on("py-requests@2.21:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-gensim/package.py b/var/spack/repos/builtin/packages/py-gensim/package.py
index f6d702e4a539a6..9ab2c5317fc1a9 100644
--- a/var/spack/repos/builtin/packages/py-gensim/package.py
+++ b/var/spack/repos/builtin/packages/py-gensim/package.py
@@ -16,18 +16,27 @@ class PyGensim(PythonPackage):
 
     maintainers("adamjstewart")
 
+    version("4.3.1", sha256="8b5f11c0e6a5308086b48e8f6841223a4fa1a37d513684612b7ee854b533015f")
     version("3.8.3", sha256="786adb0571f75114e9c5f7a31dd2e6eb39a9791f22c8757621545e2ded3ea367")
     version("3.8.1", sha256="33277fc0a8d7b0c7ce70fcc74bb82ad39f944c009b334856c6e86bf552b1dfdc")
     version("3.8.0", sha256="ec5de7ff2bfa8692fa96a846bb5aae52f267fc322fbbe303c1f042d258af5766")
 
     depends_on("python@2.7:2.8,3.5:", type=("build", "run"))
+    depends_on("python@3.8:", type=("build", "run"), when="@4.3.1:")
     depends_on("py-setuptools", type="build")
 
+    depends_on("py-cython", type=("build", "run"), when="@4.3.1:")
+
     depends_on("py-numpy@1.11.3:", type=("build", "run"))
+    depends_on("py-numpy@1.18.5:", type=("build", "run"), when="@4.3.1:")
 
     depends_on("py-scipy@0.18.1:", type=("build", "run"))
+    depends_on("py-scipy@1.7.0:", type=("build", "run"), when="@4.3.1:")
 
-    depends_on("py-six@1.5.0:", type=("build", "run"))
+    depends_on("py-six@1.5.0:", type=("build", "run"), when="@:3.8.3")
 
     depends_on("py-smart-open@1.7.0:", when="@3.8.0", type=("build", "run"))
     depends_on("py-smart-open@1.8.1:", when="@3.8.1:", type=("build", "run"))
+
+    def setup_build_environment(self, env):
+        env.set("GENSIM_CYTHON_REQUIRES", "Cython=={0}".format(self.spec["py-cython"].version))
diff --git a/var/spack/repos/builtin/packages/py-gevent/package.py b/var/spack/repos/builtin/packages/py-gevent/package.py
index c5b59423fba0a5..c41b815ef24320 100644
--- a/var/spack/repos/builtin/packages/py-gevent/package.py
+++ b/var/spack/repos/builtin/packages/py-gevent/package.py
@@ -10,32 +10,34 @@ class PyGevent(PythonPackage):
     """gevent is a coroutine-based Python networking library."""
 
     homepage = "https://www.gevent.org"
-    pypi = "gevent/gevent-1.3a2.tar.gz"
+    pypi = "gevent/gevent-23.7.0.tar.gz"
+    git = "https://github.com/gevent/gevent.git"
 
+    version("23.7.0", sha256="d0d3630674c1b344b256a298ab1ff43220f840b12af768131b5d74e485924237")
     version("21.12.0", sha256="f48b64578c367b91fa793bf8eaaaf4995cb93c8bc45860e473bf868070ad094e")
     version("21.8.0", sha256="43e93e1a4738c922a2416baf33f0afb0a20b22d3dba886720bc037cd02a98575")
     version("1.5.0", sha256="b2814258e3b3fb32786bb73af271ad31f51e1ac01f33b37426b66cb8491b4c29")
-    version("1.3a2", sha256="f7ab82697111ea233c7beeadf5240f669dfad9c4bbc89a3ec80a49e2c48a65bd")
 
-    depends_on("python@2.7:2,3.6:", when="@21.8:", type=("build", "run"))
-    depends_on("python@2.7:2,3.5:", when="@1.5:", type=("build", "run"))
-    depends_on("python@2.7:2,3.4:", type=("build", "run"))
+    depends_on("python@3.8:", when="@23.7.0:", type=("build", "run"))
+    depends_on("python@:3.10", when="@:21.12", type=("build", "run"))
+
     depends_on("py-setuptools@40.8:", when="@20.5.1:", type=("build", "run"))
     depends_on("py-setuptools@40.8:", when="@1.5:", type="build")
     depends_on("py-setuptools@24.2:", when="@:1.4", type="build")
-    depends_on("py-cython@3.0.0a9:", when="@20.5.1:", type="build")
+    depends_on("py-cython@3:", when="@20.5.1:", type="build")
     depends_on("py-cython@0.29.14:", when="@1.5:", type="build")
-    depends_on("py-cython@0.27:", when="@:1.4", type="build")
-    depends_on("py-cython@0.27:", when="@:1.4", type="build")
-    depends_on("py-cffi@1.12.3:", when="@1.5:", type=("build", "run"))  # from pyproject.toml
-    depends_on("py-cffi@1.4:", when="@:1.4", type=("build", "run"))
-    depends_on("py-greenlet@1.1:1", when="@21.8:", type=("build", "run"))
-    depends_on("py-greenlet@0.4.17:1", when="@20.12:", type=("build", "run"))
-    depends_on("py-greenlet@0.4.14:", when="@1.5:", type=("build", "run"))
-    depends_on("py-greenlet@0.4.13:", when="@:1.4", type=("build", "run"))
+    depends_on("py-cffi@1.12.3:", type=("build", "run"))
+    depends_on("py-greenlet@3:", when="@23.7: ^python@3.12:", type=("build", "run"))
+    depends_on("py-greenlet@2:", when="@22.10.2: ^python@:3.11", type=("build", "run"))
+    depends_on("py-greenlet@1.1:1", when="@21.8:21.12.0", type=("build", "run"))
+    depends_on("py-greenlet@0.4.17:1", when="@20.12:21.1.2", type=("build", "run"))
+    depends_on("py-greenlet@0.4.14:", type=("build", "run"))
     depends_on("py-zope-event", when="@20.5.1:", type=("build", "run"))
     depends_on("py-zope-interface", when="@20.5.1:", type=("build", "run"))
 
+    # https://github.com/gevent/gevent/issues/1599
+    conflicts("^py-cython@3:", when="@:20.5.0")
+
     # Deprecated compiler options. upstream PR: https://github.com/gevent/gevent/pull/1896
     patch("icc.patch", when="%intel")
 
diff --git a/var/spack/repos/builtin/packages/py-gluoncv/no-unicode-readme.patch b/var/spack/repos/builtin/packages/py-gluoncv/no-unicode-readme.patch
deleted file mode 100644
index 43d0a6c5e87a32..00000000000000
--- a/var/spack/repos/builtin/packages/py-gluoncv/no-unicode-readme.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- a/README.md	2020-04-03 11:56:41.820271316 -0400
-+++ b/README.md	2020-04-03 11:56:55.901335611 -0400
-@@ -72,7 +72,7 @@
- 
- There are multiple versions of MXNet pre-built package available. Please refer to [mxnet packages](https://gluon-crash-course.mxnet.io/mxnet_packages.html) if you need more details about MXNet versions.
- 
--# Docs 📖
-+# Docs
- GluonCV documentation is available at [our website](https://gluon-cv.mxnet.io/index.html).
- 
- # Examples
diff --git a/var/spack/repos/builtin/packages/py-gluoncv/package.py b/var/spack/repos/builtin/packages/py-gluoncv/package.py
index d5bd60eb599e48..3099006c4025c7 100644
--- a/var/spack/repos/builtin/packages/py-gluoncv/package.py
+++ b/var/spack/repos/builtin/packages/py-gluoncv/package.py
@@ -14,11 +14,16 @@ class PyGluoncv(PythonPackage):
     vision."""
 
     homepage = "https://gluon-cv.mxnet.io/"
-    url = "https://github.com/dmlc/gluon-cv/archive/v0.6.0.tar.gz"
+    pypi = "gluoncv/gluoncv-0.6.0.tar.gz"
+    git = "https://github.com/dmlc/gluon-cv.git"
 
-    version("0.6.0", sha256="5ac89d73f34d02b2e60595a5cc35f46d0a69376567fae3a9518005dd89161305")
+    version(
+        "0.10.5.post0", sha256="4598b9612e8b459a5a14ebeffedefcdae4a5700302a91f9b99fc82e9b08928a5"
+    )
+    version("0.6.0", sha256="313848b939c30e9e4c0040078421c02e32a350b8ebf2a966313fd893d7b3bdf6")
 
     depends_on("py-setuptools", type="build")
+
     depends_on("py-numpy", type=("build", "run"))
     depends_on("py-tqdm", type=("build", "run"))
     depends_on("py-requests", type=("build", "run"))
@@ -26,9 +31,7 @@ class PyGluoncv(PythonPackage):
     depends_on("py-portalocker", type=("build", "run"))
     depends_on("pil", type=("build", "run"))
     depends_on("py-scipy", type=("build", "run"))
-    depends_on("py-cython", type="build")
-
-    patch("no-unicode-readme.patch")
-
-    def install_options(self, spec, prefix):
-        return ["--with-cython"]
+    depends_on("py-yacs", when="@0.8:", type=("build", "run"))
+    depends_on("py-pandas", when="@0.9:", type=("build", "run"))
+    depends_on("py-pyyaml", when="@0.9:", type=("build", "run"))
+    depends_on("py-autocfg", when="@0.9:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-gpaw/package.py b/var/spack/repos/builtin/packages/py-gpaw/package.py
index 0f5072e927c534..f6759fb279ea53 100644
--- a/var/spack/repos/builtin/packages/py-gpaw/package.py
+++ b/var/spack/repos/builtin/packages/py-gpaw/package.py
@@ -35,7 +35,7 @@ class PyGpaw(PythonPackage):
     depends_on("py-ase@3.19.0:", type=("build", "run"), when="@20.1.0")
     depends_on("py-ase@3.20.1:", type=("build", "run"), when="@20.10.0")
     depends_on("py-ase@3.21.0:", type=("build", "run"), when="@21.1.0")
-    depends_on("py-numpy +blas +lapack", type=("build", "run"))
+    depends_on("py-numpy", type=("build", "run"))
     depends_on("py-scipy", type=("build", "run"))
     depends_on("libxc@3:4.3.4")
     depends_on("blas")
diff --git a/var/spack/repos/builtin/packages/py-gpy/package.py b/var/spack/repos/builtin/packages/py-gpy/package.py
index ad21efa1eb76fa..82a85e94ec4687 100644
--- a/var/spack/repos/builtin/packages/py-gpy/package.py
+++ b/var/spack/repos/builtin/packages/py-gpy/package.py
@@ -13,13 +13,23 @@ class PyGpy(PythonPackage):
     pypi = "gpy/GPy-1.9.9.tar.gz"
     maintainers("liuyangzhuan")
 
+    version("1.10.0", sha256="a2b793ef8d0ac71739e7ba1c203bc8a5afa191058b42caa617e0e29aa52aa6fb")
     version("1.9.9", sha256="04faf0c24eacc4dea60727c50a48a07ddf9b5751a3b73c382105e2a31657c7ed")
     version("0.8.8", sha256="e135d928cf170e2ec7fb058a035b5a7e334dc6b84d0bfb981556782528341988")
 
     depends_on("py-setuptools", type="build")
     depends_on("py-numpy@1.7:", type=("build", "run"))
     depends_on("py-scipy@0.16:", type=("build", "run"))
+    depends_on("py-scipy@1.3:", when="@1.10.0:", type=("build", "run"))
     depends_on("py-six", type=("build", "run"))
     depends_on("py-paramz@0.9.0:", type=("build", "run"))
+    # cython is install_requires, but not used at runtime, so stick to build type
     depends_on("py-cython@0.29:", type="build")
-    depends_on("python@:3.8", type=("build", "run"))
+
+    @run_before("install")
+    def touch_sources(self):
+        # This packages uses deprecated build_ext, for which we cannot
+        # automatically force recythonization.
+        # See also https://github.com/SheffieldML/GPy/pull/1020
+        for src in find(".", "*.pyx"):
+            touch(src)
diff --git a/var/spack/repos/builtin/packages/py-gradio-client/package.py b/var/spack/repos/builtin/packages/py-gradio-client/package.py
new file mode 100644
index 00000000000000..bcedd71047c8ec
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-gradio-client/package.py
@@ -0,0 +1,27 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyGradioClient(PythonPackage):
+    """Python library for easily interacting with trained machine learning models"""
+
+    homepage = "https://github.com/gradio-app/gradio"
+    pypi = "gradio_client/gradio_client-0.2.9.tar.gz"
+
+    version("0.2.9", sha256="d4071709ab45a3dbacdbd0797fde5d66d87a98424559e060d576fbe9b0171f4d")
+
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-hatchling", type="build")
+    depends_on("py-hatch-requirements-txt", type="build")
+    depends_on("py-hatch-fancy-pypi-readme@22.5:", type="build")
+    depends_on("py-requests", type=("build", "run"))
+    depends_on("py-websockets", type=("build", "run"))
+    depends_on("py-packaging", type=("build", "run"))
+    depends_on("py-fsspec", type=("build", "run"))
+    depends_on("py-huggingface-hub@0.13:", type=("build", "run"))
+    depends_on("py-typing-extensions", type=("build", "run"))
+    depends_on("py-httpx", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-gradio/package.py b/var/spack/repos/builtin/packages/py-gradio/package.py
new file mode 100644
index 00000000000000..43f9b74c4ea8bb
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-gradio/package.py
@@ -0,0 +1,46 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyGradio(PythonPackage):
+    """Python library for easily interacting with trained machine learning models"""
+
+    homepage = "https://github.com/gradio-app/gradio"
+    pypi = "gradio/gradio-3.36.1.tar.gz"
+
+    version("3.36.1", sha256="1d821cee15da066c24c197248ba9aaed5f5e59505d17754561c2f96f90e73a89")
+
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-hatchling", type="build")
+    depends_on("py-hatch-requirements-txt", type="build")
+    depends_on("py-hatch-fancy-pypi-readme@22.5.0:", type="build")
+    depends_on("py-aiofiles", type=("build", "run"))
+    depends_on("py-aiohttp", type=("build", "run"))
+    depends_on("py-altair@4.2.0:", type=("build", "run"))
+    depends_on("py-fastapi", type=("build", "run"))
+    depends_on("py-ffmpy", type=("build", "run"))
+    depends_on("py-gradio-client@0.2.7:", type=("build", "run"))
+    depends_on("py-httpx", type=("build", "run"))
+    depends_on("py-huggingface-hub@0.14.0:", type=("build", "run"))
+    depends_on("py-jinja2", type=("build", "run"))
+    depends_on("py-markdown-it-py@2.0.0:+linkify", type=("build", "run"))
+    depends_on("py-pygments@2.12.0:", type=("build", "run"))
+    depends_on("py-mdit-py-plugins@:0.3.3", type=("build", "run"))
+    depends_on("py-markupsafe", type=("build", "run"))
+    depends_on("py-matplotlib", type=("build", "run"))
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-orjson", type=("build", "run"))
+    depends_on("py-pandas", type=("build", "run"))
+    depends_on("pil", type=("build", "run"))
+    depends_on("py-pydantic", type=("build", "run"))
+    depends_on("py-python-multipart", type=("build", "run"))
+    depends_on("py-pydub", type=("build", "run"))
+    depends_on("py-pyyaml", type=("build", "run"))
+    depends_on("py-requests", type=("build", "run"))
+    depends_on("py-semantic-version", type=("build", "run"))
+    depends_on("py-uvicorn@0.14.0:", type=("build", "run"))
+    depends_on("py-websockets@10.0:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-graphene-tornado/package.py b/var/spack/repos/builtin/packages/py-graphene-tornado/package.py
new file mode 100644
index 00000000000000..ba44045b4c6d79
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-graphene-tornado/package.py
@@ -0,0 +1,25 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyGrapheneTornado(PythonPackage):
+    """Graphene Tornado integration."""
+
+    homepage = "https://github.com/graphql-python/graphene-tornado"
+    pypi = "graphene-tornado/graphene-tornado-2.6.1.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("2.6.1", sha256="953bf812267177224ce1ac2a93c669069d85a8fa187a9fac681b76b63dffebc2")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-six@1.10.0:", type=("build", "run"))
+    depends_on("py-graphene@2.1:2", type=("build", "run"))
+    depends_on("py-jinja2@2.10.1:", type=("build", "run"))
+    depends_on("py-tornado@5.1.0:", type=("build", "run"))
+    depends_on("py-werkzeug@0.12.2", type=("build", "run"))
+    depends_on("py-pytest", type=("build"))
diff --git a/var/spack/repos/builtin/packages/py-graphene/package.py b/var/spack/repos/builtin/packages/py-graphene/package.py
new file mode 100644
index 00000000000000..407156127f32ec
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-graphene/package.py
@@ -0,0 +1,23 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyGraphene(PythonPackage):
+    """GraphQL Framework for Python."""
+
+    homepage = "https://github.com/graphql-python/graphene"
+    pypi = "graphene/graphene-3.3.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("2.1.9", sha256="b9f2850e064eebfee9a3ef4a1f8aa0742848d97652173ab44c82cc8a62b9ed93")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-graphql-core@2.1:2", type=("build", "run"))
+    depends_on("py-graphql-relay@2", type=("build", "run"))
+    depends_on("py-aniso8601@3:7", type=("build", "run"))
+    depends_on("py-six@1.10.0:1", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-graphql-core/package.py b/var/spack/repos/builtin/packages/py-graphql-core/package.py
index a9cb29523751c8..492444f5ccc429 100644
--- a/var/spack/repos/builtin/packages/py-graphql-core/package.py
+++ b/var/spack/repos/builtin/packages/py-graphql-core/package.py
@@ -20,4 +20,8 @@ class PyGraphqlCore(PythonPackage):
     version("2.3.2", sha256="aac46a9ac524c9855910c14c48fc5d60474def7f99fd10245e76608eba7af746")
 
     depends_on("python@3.6:3", type=("build", "run"))
-    depends_on("py-setuptools", type="build")
+    depends_on("py-poetry@1", when="@3:", type="build")
+    depends_on("py-setuptools", when="@2", type="build")
+    depends_on("py-six@1.10.0:", type=("build", "run"), when="@2.3.2")
+    depends_on("py-promise@2.3:2", type=("build", "run"), when="@2.3.2")
+    depends_on("py-rx@1.6:1", type=("build", "run"), when="@2.3.2")
diff --git a/var/spack/repos/builtin/packages/py-graphql-relay/package.py b/var/spack/repos/builtin/packages/py-graphql-relay/package.py
new file mode 100644
index 00000000000000..67aaa8d25a2c6a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-graphql-relay/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyGraphqlRelay(PythonPackage):
+    """Relay library for graphql-core."""
+
+    homepage = "https://github.com/graphql-python/graphql-relay-py"
+    pypi = "graphql-relay/graphql-relay-2.0.1.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("2.0.1", sha256="870b6b5304123a38a0b215a79eace021acce5a466bf40cd39fa18cb8528afabb")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-graphql-core@2.2:2", type=("build", "run"), when="@2")
+    depends_on("py-six@1.12:", type=("build", "run"), when="@2")
+    depends_on("py-promise@2.2:2", type=("build", "run"), when="@2")
diff --git a/var/spack/repos/builtin/packages/py-graphql-ws/package.py b/var/spack/repos/builtin/packages/py-graphql-ws/package.py
new file mode 100644
index 00000000000000..c89e5d8fd9d800
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-graphql-ws/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyGraphqlWs(PythonPackage):
+    """Websocket backend for GraphQL subscriptions."""
+
+    homepage = "https://github.com/graphql-python/graphql-ws"
+    pypi = "graphql-ws/graphql-ws-0.4.4.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("0.4.4", sha256="2ad38db70f37964f54d7eb3e2ede86dbe3f2a1ed7ea0a9f9a3b8b17162a22852")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-graphql-core@2", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-grayskull/package.py b/var/spack/repos/builtin/packages/py-grayskull/package.py
new file mode 100644
index 00000000000000..f1375aa4567062
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-grayskull/package.py
@@ -0,0 +1,35 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyGrayskull(PythonPackage):
+    """Project to generate recipes for conda packages."""
+
+    homepage = "https://github.com/conda/grayskull"
+    pypi = "grayskull/grayskull-2.5.0.tar.gz"
+
+    version("2.5.0", sha256="b021138655be550fd1b93b8db08b9c66169fac9cba6bcdad1411263e12fc703f")
+
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-setuptools@61:", type="build")
+    depends_on("py-setuptools-scm@6.2:+toml", type="build")
+    depends_on("py-beautifulsoup4", type=("build", "run"))
+    depends_on("py-colorama", type=("build", "run"))
+    depends_on("py-conda-souschef@2.2.3:", type=("build", "run"))
+    depends_on("py-packaging@21.3:", type=("build", "run"))
+    depends_on("py-pip", type=("build", "run"))
+    depends_on("py-pkginfo", type=("build", "run"))
+    depends_on("py-progressbar2@3.53:", type=("build", "run"))
+    depends_on("py-rapidfuzz@3:", type=("build", "run"))
+    depends_on("py-requests", type=("build", "run"))
+    depends_on("py-ruamel-yaml@0.16.10:", type=("build", "run"))
+    depends_on("py-ruamel-yaml-jinja2", type=("build", "run"))
+    depends_on("py-setuptools@30.3:", type=("build", "run"))
+    depends_on("py-semver@3.0", type=("build", "run"))
+    depends_on("py-stdlib-list", type=("build", "run"))
+    depends_on("py-tomli", type=("build", "run"))
+    depends_on("py-tomli-w", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-greenlet/package.py b/var/spack/repos/builtin/packages/py-greenlet/package.py
index 01dae7356a2fe1..5a379ff7d427bc 100644
--- a/var/spack/repos/builtin/packages/py-greenlet/package.py
+++ b/var/spack/repos/builtin/packages/py-greenlet/package.py
@@ -12,7 +12,12 @@ class PyGreenlet(PythonPackage):
     homepage = "https://github.com/python-greenlet/greenlet"
     pypi = "greenlet/greenlet-0.4.17.tar.gz"
 
-    version("2.0.2", sha256="e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0")
+    version("3.0.0a1", sha256="1bd4ea36f0aeb14ca335e0c9594a5aaefa1ac4e2db7d86ba38f0be96166b3102")
+    version(
+        "2.0.2",
+        sha256="e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0",
+        preferred=True,
+    )
     version("1.1.3", sha256="bcb6c6dd1d6be6d38d6db283747d07fda089ff8c559a835236560a4410340455")
     version("1.1.2", sha256="e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a")
     version("1.1.0", sha256="c87df8ae3f01ffb4483c796fe1b15232ce2b219f0b18126948616224d3f658ee")
diff --git a/var/spack/repos/builtin/packages/py-grpcio-tools/package.py b/var/spack/repos/builtin/packages/py-grpcio-tools/package.py
index c4bebb05840539..e2cc8d79db6289 100644
--- a/var/spack/repos/builtin/packages/py-grpcio-tools/package.py
+++ b/var/spack/repos/builtin/packages/py-grpcio-tools/package.py
@@ -25,7 +25,7 @@ class PyGrpcioTools(PythonPackage):
     depends_on("py-grpcio@1.39.0:", when="@1.39.0:1.41", type=("build", "run"))
     depends_on("py-cython@0.23:", type="build")
     depends_on("openssl")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("c-ares")
     depends_on("re2+shared")
 
@@ -56,7 +56,7 @@ def patch(self):
         )
         filter_file(
             r"(\s+ZLIB_INCLUDE = ).*",
-            r"\1('{0}',)".format(self.spec["zlib"].prefix.include),
+            r"\1('{0}',)".format(self.spec["zlib-api"].prefix.include),
             "setup.py",
         )
         filter_file(
diff --git a/var/spack/repos/builtin/packages/py-grpcio/package.py b/var/spack/repos/builtin/packages/py-grpcio/package.py
index b0d6658fcff7f3..8a185d480265f0 100644
--- a/var/spack/repos/builtin/packages/py-grpcio/package.py
+++ b/var/spack/repos/builtin/packages/py-grpcio/package.py
@@ -37,9 +37,9 @@ class PyGrpcio(PythonPackage):
 
     depends_on("py-setuptools", type="build")
     depends_on("py-six@1.5.2:", when="@:1.48", type=("build", "run"))
-    depends_on("py-cython@0.23:", type="build")
+    depends_on("py-cython@0.23:2", type="build")
     depends_on("openssl")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("c-ares")
     depends_on("re2+shared")
 
@@ -69,7 +69,7 @@ def patch(self):
         )
         filter_file(
             r"(\s+ZLIB_INCLUDE = ).*",
-            r"\1('{0}',)".format(self.spec["zlib"].prefix.include),
+            r"\1('{0}',)".format(self.spec["zlib-api"].prefix.include),
             "setup.py",
         )
         filter_file(
diff --git a/var/spack/repos/builtin/packages/py-gtdbtk/package.py b/var/spack/repos/builtin/packages/py-gtdbtk/package.py
index 6e5328a34838cc..75b5171ffa95c0 100644
--- a/var/spack/repos/builtin/packages/py-gtdbtk/package.py
+++ b/var/spack/repos/builtin/packages/py-gtdbtk/package.py
@@ -15,15 +15,15 @@ class PyGtdbtk(PythonPackage):
     homepage = "https://github.com/Ecogenomics/GTDBTk"
     pypi = "gtdbtk/gtdbtk-2.1.0.tar.gz"
 
+    version("2.3.2", sha256="80efd31e10007d835f56a3d6fdf039a59db3b6ba4be26b234692da5e688aa99f")
     version("2.3.0", sha256="4f237a03657be4540ac653c276fe31c002b6923af0411316719a9541d6e97d4b")
     version("2.1.0", sha256="980885141f13502afdf05e720871427e3de4fe27f4f3f97e74af6fed87eb50a7")
 
-    depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-setuptools", type=("build"))
     depends_on("py-dendropy@4.1.0:", type=("build", "run"))
     depends_on("py-numpy@1.9.0:", type=("build", "run"))
     depends_on("py-tqdm@4.35.0:", type=("build", "run"))
-    depends_on("py-pydantic", type=("build", "run"), when="@2.3.0:")
+    depends_on("py-pydantic@1.9.2:1", type=("build", "run"), when="@2.3.0:")
     depends_on("prodigal@2.6.2:", type=("build", "run"))
     depends_on("hmmer@3.1b2:", type=("build", "run"))
     depends_on("pplacer@1.1:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-handy-archives/package.py b/var/spack/repos/builtin/packages/py-handy-archives/package.py
new file mode 100644
index 00000000000000..f9a3dba92ae2d5
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-handy-archives/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyHandyArchives(PythonPackage):
+    """Some handy archive helpers for Python."""
+
+    homepage = "https://github.com/domdfcoding/handy-archives"
+    pypi = "handy_archives/handy_archives-0.2.0.tar.gz"
+
+    version("0.2.0", sha256="fba21101fd9e29d5e3b72823261aaae06b9350686f0d2067786d64dce73eb3f6")
+
+    depends_on("py-flit-core@3.2:3", type="build")
diff --git a/var/spack/repos/builtin/packages/py-hatch-requirements-txt/package.py b/var/spack/repos/builtin/packages/py-hatch-requirements-txt/package.py
new file mode 100644
index 00000000000000..6f2489e2a9e9da
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-hatch-requirements-txt/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyHatchRequirementsTxt(PythonPackage):
+    """Hatchling plugin to read project dependencies from requirements.txt"""
+
+    homepage = "https://github.com/repo-helper/hatch-requirements-txt"
+    pypi = "hatch_requirements_txt/hatch_requirements_txt-0.4.0.tar.gz"
+
+    version("0.4.0", sha256="800509946e85d9e56d73242fab223ec36db50372e870a04e2dd1fd9bad98455d")
+
+    depends_on("python@3.6.1:", type=("build", "run"))
+    depends_on("py-hatchling@0.21:", type=("build", "run"))
+    depends_on("py-packaging@21.3:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-hatchling/package.py b/var/spack/repos/builtin/packages/py-hatchling/package.py
index 5ff82874128be4..29fcce5df9aa2d 100644
--- a/var/spack/repos/builtin/packages/py-hatchling/package.py
+++ b/var/spack/repos/builtin/packages/py-hatchling/package.py
@@ -13,6 +13,7 @@ class PyHatchling(PythonPackage):
     pypi = "hatchling/hatchling-1.4.1.tar.gz"
     git = "https://github.com/pypa/hatch"
 
+    version("1.18.0", sha256="50e99c3110ce0afc3f7bdbadff1c71c17758e476731c27607940cfa6686489ca")
     version("1.17.0", sha256="b1244db3f45b4ef5a00106a46612da107cdfaf85f1580b8e1c059fefc98b0930")
     version("1.14.0", sha256="462ea91df03ff5d52813b5613fec1313a1a2059d2e37343e572b3f979867c5da")
     version("1.13.0", sha256="f8d275a2cc720735286b7c2e2bc35da05761e6d3695c2fa416550395f10c53c7")
@@ -20,6 +21,7 @@ class PyHatchling(PythonPackage):
     version("1.8.1", sha256="448b04b23faed669b2b565b998ac955af4feea66c5deed3a1212ac9399d2e1cd")
     version("1.4.1", sha256="13461b42876ade4f75ee5d2a2c656b288ca0aab7f048ef66657ef166996b2118")
 
+    depends_on("python@3.8:", when="@1.18:", type=("build", "run"))
     depends_on("py-editables@0.3:", type=("build", "run"))
     depends_on("py-importlib-metadata", when="^python@:3.7", type=("build", "run"))
     depends_on("py-packaging@21.3:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-hepunits/package.py b/var/spack/repos/builtin/packages/py-hepunits/package.py
index 8e771898c3073c..d096f1193adbae 100644
--- a/var/spack/repos/builtin/packages/py-hepunits/package.py
+++ b/var/spack/repos/builtin/packages/py-hepunits/package.py
@@ -18,6 +18,7 @@ class PyHepunits(PythonPackage):
     maintainers("vvolkl")
 
     version("master", branch="master")
+    version("2.3.2", sha256="8a3366fa5d72c16af1166ed579cdaa81edd2676acb8f6a1fe7da290cefca3b08")
     version("2.3.1", sha256="b1174bba4d575b9939c01f341e24d9bdbe0e0cd4cc4ce2e7d77692da19145cfb")
     version("2.3.0", sha256="33b9ae9a8b7b3af355141a74901cb5aa557dce2e4c9992a0a30ef0443a1b2206")
     version("2.2.1", sha256="6097e69547a483bdc0cfe4d106e447b5eba87c5501060d312cd9d61aa9e22414")
diff --git a/var/spack/repos/builtin/packages/py-hist/package.py b/var/spack/repos/builtin/packages/py-hist/package.py
index d70dfdc7deec52..3e606e79dbbaf1 100644
--- a/var/spack/repos/builtin/packages/py-hist/package.py
+++ b/var/spack/repos/builtin/packages/py-hist/package.py
@@ -15,7 +15,7 @@ class PyHist(PythonPackage):
     version("2.6.1", sha256="ee9034795fd2feefed923461aaccaf76f87c1f8d5414b1e704faa293ceb4fc27")
     version("2.5.2", sha256="0bafb8b956cc041f1b26e8f5663fb8d3b8f7673f56336facb84d8cfdc30ae2cf")
 
-    variant("plot", default="False", description="Add support for drawing histograms")
+    variant("plot", default=False, description="Add support for drawing histograms")
 
     depends_on("python@3.7:", type=("build", "run"))
     depends_on("py-setuptools@45:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-hmmlearn/package.py b/var/spack/repos/builtin/packages/py-hmmlearn/package.py
new file mode 100644
index 00000000000000..16bfb20ccf3f66
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-hmmlearn/package.py
@@ -0,0 +1,28 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyHmmlearn(PythonPackage):
+    """hmmlearn is a set of algorithms for unsupervised learning and
+    inference of Hidden Markov Models."""
+
+    homepage = "https://github.com/hmmlearn/hmmlearn"
+    pypi = "hmmlearn/hmmlearn-0.3.0.tar.gz"
+
+    maintainers("snehring")
+
+    version("0.3.0", sha256="d13a91ea3695df881465e3d36132d7eef4e84d483f4ba538a4b46e24b5ea100f")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-setuptools-scm@3.3:", type="build")
+    depends_on("py-pybind11@2.6:", type="build")
+
+    depends_on("py-numpy@1.10:", type=("build", "run"))
+    depends_on("py-scikit-learn@0.16:", type=("build", "run"))
+    depends_on("py-scipy@0.19:", type=("build", "run"))
+
+    conflicts("py-scikit-learn@=0.22.0", msg="Not compatible with scikit-learn@0.22.0")
diff --git a/var/spack/repos/builtin/packages/py-horovod/package.py b/var/spack/repos/builtin/packages/py-horovod/package.py
index 45134b07d7bf91..5e221c0296824f 100644
--- a/var/spack/repos/builtin/packages/py-horovod/package.py
+++ b/var/spack/repos/builtin/packages/py-horovod/package.py
@@ -17,41 +17,111 @@ class PyHorovod(PythonPackage, CudaPackage):
     maintainers("adamjstewart", "aweits", "tgaddair", "thomas-bouvier")
 
     version("master", branch="master", submodules=True)
-    version("0.28.1", tag="v0.28.1", submodules=True)
-    version("0.28.0", tag="v0.28.0", submodules=True)
-    version("0.27.0", tag="v0.27.0", submodules=True)
-    version("0.26.1", tag="v0.26.1", submodules=True)
-    version("0.26.0", tag="v0.26.0", submodules=True)
-    version("0.25.0", tag="v0.25.0", submodules=True)
-    version("0.24.3", tag="v0.24.3", submodules=True)
-    version("0.24.2", tag="v0.24.2", submodules=True)
-    version("0.24.1", tag="v0.24.1", submodules=True)
-    version("0.24.0", tag="v0.24.0", submodules=True)
-    version("0.23.0", tag="v0.23.0", submodules=True)
-    version("0.22.1", tag="v0.22.1", submodules=True)
-    version("0.22.0", tag="v0.22.0", submodules=True)
-    version("0.21.3", tag="v0.21.3", submodules=True)
-    version("0.21.2", tag="v0.21.2", submodules=True)
-    version("0.21.1", tag="v0.21.1", submodules=True)
-    version("0.21.0", tag="v0.21.0", submodules=True)
-    version("0.20.3", tag="v0.20.3", submodules=True)
-    version("0.20.2", tag="v0.20.2", submodules=True)
-    version("0.20.1", tag="v0.20.1", submodules=True)
-    version("0.20.0", tag="v0.20.0", submodules=True)
-    version("0.19.5", tag="v0.19.5", submodules=True)
-    version("0.19.4", tag="v0.19.4", submodules=True)
-    version("0.19.3", tag="v0.19.3", submodules=True)
-    version("0.19.2", tag="v0.19.2", submodules=True)
-    version("0.19.1", tag="v0.19.1", submodules=True)
-    version("0.19.0", tag="v0.19.0", submodules=True)
-    version("0.18.2", tag="v0.18.2", submodules=True)
-    version("0.18.1", tag="v0.18.1", submodules=True)
-    version("0.18.0", tag="v0.18.0", submodules=True)
-    version("0.17.1", tag="v0.17.1", submodules=True)
-    version("0.17.0", tag="v0.17.0", submodules=True)
-    version("0.16.4", tag="v0.16.4", submodules=True)
-    version("0.16.3", tag="v0.16.3", submodules=True)
-    version("0.16.2", tag="v0.16.2", submodules=True)
+    version(
+        "0.28.1", tag="v0.28.1", commit="1d217b59949986d025f6db93c49943fb6b6cc78f", submodules=True
+    )
+    version(
+        "0.28.0", tag="v0.28.0", commit="587d72004736209a93ebda8cec0acdb7870db583", submodules=True
+    )
+    version(
+        "0.27.0", tag="v0.27.0", commit="bfaca90d5cf66780a97d8799d4e1573855b64560", submodules=True
+    )
+    version(
+        "0.26.1", tag="v0.26.1", commit="34604870eabd9dc670c222deb1da9acc6b9d7c03", submodules=True
+    )
+    version(
+        "0.26.0", tag="v0.26.0", commit="c638dcec972750d4a75b229bc208cff9dc76b00a", submodules=True
+    )
+    version(
+        "0.25.0", tag="v0.25.0", commit="48e0affcba962831668cd1222866af2d632920c2", submodules=True
+    )
+    version(
+        "0.24.3", tag="v0.24.3", commit="a2d9e280c1210a8e364a7dc83ca6c2182fefa99d", submodules=True
+    )
+    version(
+        "0.24.2", tag="v0.24.2", commit="b4c191c8d05086842517b3836285a85c6f96ab22", submodules=True
+    )
+    version(
+        "0.24.1", tag="v0.24.1", commit="ebd135098571722469bb6290a6d098a9e1c96574", submodules=True
+    )
+    version(
+        "0.24.0", tag="v0.24.0", commit="b089df66a29d3ba6672073eef3d42714d9d3626b", submodules=True
+    )
+    version(
+        "0.23.0", tag="v0.23.0", commit="66ad6d5a3586decdac356e8ec95c204990bbc3d6", submodules=True
+    )
+    version(
+        "0.22.1", tag="v0.22.1", commit="93a2f2583ed63391a904aaeb03b602729be90f15", submodules=True
+    )
+    version(
+        "0.22.0", tag="v0.22.0", commit="3ff94801fbb4dbf6bc47c23888c93cad4887435f", submodules=True
+    )
+    version(
+        "0.21.3", tag="v0.21.3", commit="6916985c9df111f36864724e2611827f64de8e11", submodules=True
+    )
+    version(
+        "0.21.2", tag="v0.21.2", commit="c64b1d60c6bad7834f3315f12707f8ebf11c9c3d", submodules=True
+    )
+    version(
+        "0.21.1", tag="v0.21.1", commit="a9dea74abc1f0b8e81cd2b6dd9fe81e2c4244e39", submodules=True
+    )
+    version(
+        "0.21.0", tag="v0.21.0", commit="7d71874258fc8625ad8952defad0ea5b24531248", submodules=True
+    )
+    version(
+        "0.20.3", tag="v0.20.3", commit="b3c4d81327590c9064d544622b6250d9a19ce2c2", submodules=True
+    )
+    version(
+        "0.20.2", tag="v0.20.2", commit="cef4393eb980d4137bb91256da4dd847b7f44d1c", submodules=True
+    )
+    version(
+        "0.20.1", tag="v0.20.1", commit="4099c2b7f34f709f0db1c09f06b2594d7b4b9615", submodules=True
+    )
+    version(
+        "0.20.0", tag="v0.20.0", commit="396c1319876039ad8f5a56c007a020605ccb8277", submodules=True
+    )
+    version(
+        "0.19.5", tag="v0.19.5", commit="b52e4b3e6ce5b1b494b77052878a0aad05c2e3ce", submodules=True
+    )
+    version(
+        "0.19.4", tag="v0.19.4", commit="31f1f700b8fa6d3b6df284e291e302593fbb4fa3", submodules=True
+    )
+    version(
+        "0.19.3", tag="v0.19.3", commit="ad63bbe9da8b41d0940260a2dd6935fa0486505f", submodules=True
+    )
+    version(
+        "0.19.2", tag="v0.19.2", commit="f8fb21e0ceebbdc6ccc069c43239731223d2961d", submodules=True
+    )
+    version(
+        "0.19.1", tag="v0.19.1", commit="9ad69e78e83c34568743e8e97b1504c6c7af34c3", submodules=True
+    )
+    version(
+        "0.19.0", tag="v0.19.0", commit="1a805d9b20224069b294f361e47f5d9b55f426ff", submodules=True
+    )
+    version(
+        "0.18.2", tag="v0.18.2", commit="bb2134b427e0e0c5a83624d02fafa4f14de623d9", submodules=True
+    )
+    version(
+        "0.18.1", tag="v0.18.1", commit="0008191b3e61b5dfccddabe0129bbed7cd544c56", submodules=True
+    )
+    version(
+        "0.18.0", tag="v0.18.0", commit="a639de51e9a38d5c1f99f458c045aeaebe70351e", submodules=True
+    )
+    version(
+        "0.17.1", tag="v0.17.1", commit="399e70adc0f74184b5848d9a46b9b6ad67b5fe6d", submodules=True
+    )
+    version(
+        "0.17.0", tag="v0.17.0", commit="2fed0410774b480ad19057320be9027be06b309e", submodules=True
+    )
+    version(
+        "0.16.4", tag="v0.16.4", commit="2aac48c95c035bee7d68f9aff30e59319f46c21e", submodules=True
+    )
+    version(
+        "0.16.3", tag="v0.16.3", commit="30a2148784478415dc31d65a6aa08d237f364b42", submodules=True
+    )
+    version(
+        "0.16.2", tag="v0.16.2", commit="217774652eeccfcd60aa6e268dfd6b766d71b768", submodules=True
+    )
 
     # https://github.com/horovod/horovod/blob/master/docs/install.rst
     variant(
@@ -155,6 +225,8 @@ class PyHorovod(PythonPackage, CudaPackage):
     conflicts(
         "controllers=gloo", when="@:0.20.0 platform=darwin", msg="Gloo cannot be compiled on MacOS"
     )
+    # https://github.com/horovod/horovod/issues/3996
+    conflicts("^py-torch@2.1:")
 
     # https://github.com/horovod/horovod/pull/1835
     patch("fma.patch", when="@0.19.0:0.19.1")
diff --git a/var/spack/repos/builtin/packages/py-igraph/package.py b/var/spack/repos/builtin/packages/py-igraph/package.py
new file mode 100644
index 00000000000000..c995834236c493
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-igraph/package.py
@@ -0,0 +1,28 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyIgraph(PythonPackage):
+    """Python interface to the igraph high performance graph library,
+    primarily aimed at complex network research and analysis."""
+
+    homepage = "https://github.com/igraph/python-igraph"
+    pypi = "igraph/igraph-0.10.6.tar.gz"
+
+    version("0.10.6", sha256="76f7aad294514412f835366a7d9a9c1e7a34c3e6ef0a6c3a1a835234323228e8")
+
+    variant("matplotlib", default=False, description="Enable plotting with Matplotlib")
+
+    depends_on("cmake", type="build")
+    depends_on("igraph+shared@0.10.6", when="@0.10.6")
+    depends_on("pkgconfig", type="build")
+    depends_on("py-setuptools", type="build")
+    depends_on("py-texttable@1.6.2:", type=("build", "run"))
+    depends_on("py-matplotlib@3.5", type="run", when="+matplotlib")
+
+    def setup_build_environment(self, env):
+        env.set("IGRAPH_USE_PKG_CONFIG", "1")
diff --git a/var/spack/repos/builtin/packages/py-imagecodecs/package.py b/var/spack/repos/builtin/packages/py-imagecodecs/package.py
index 1a84287556b107..03195578b6087f 100644
--- a/var/spack/repos/builtin/packages/py-imagecodecs/package.py
+++ b/var/spack/repos/builtin/packages/py-imagecodecs/package.py
@@ -42,7 +42,8 @@ class PyImagecodecs(PythonPackage):
     depends_on("libwebp@1.2.2:")
     depends_on("openjpeg@2.4.0:")
     depends_on("snappy@1.1.9:")
-    depends_on("zlib@1.2.11:")
+    depends_on("zlib-api")
+    depends_on("zlib@1.2.11:", when="^zlib")
     depends_on("zopfli@1.0.3: +shared")
     depends_on("zstd@1.5.2:")
 
diff --git a/var/spack/repos/builtin/packages/py-importlib-metadata/package.py b/var/spack/repos/builtin/packages/py-importlib-metadata/package.py
index 319f35b6c7b633..0abad1fe97ef49 100644
--- a/var/spack/repos/builtin/packages/py-importlib-metadata/package.py
+++ b/var/spack/repos/builtin/packages/py-importlib-metadata/package.py
@@ -16,9 +16,12 @@ class PyImportlibMetadata(PythonPackage):
     version("6.6.0", sha256="92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705")
     version("5.1.0", sha256="d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b")
     version("4.12.0", sha256="637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670")
+    version("4.11.4", sha256="5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700")
     version("4.11.1", sha256="175f4ee440a0317f6e8d81b7f8d4869f93316170a65ad2b007d2929186c8052c")
+    version("4.8.3", sha256="766abffff765960fcc18003801f7044eb6755ffae4521c8e8ce8e83b9c9b0668")
     version("4.8.2", sha256="75bdec14c397f528724c1bfd9709d660b33a4d2e77387a3358f20b848bb5e5fb")
     version("4.8.1", sha256="f284b3e11256ad1e5d03ab86bb2ccd6f5339688ff17a4d797a0fe7df326f23b1")
+    version("4.6.4", sha256="7b30a78db2922d78a6f47fb30683156a14f3c6aa5cc23f77cc8967e9ab2d002f")
     version("4.6.1", sha256="079ada16b7fc30dfbb5d13399a5113110dab1aa7c2bc62f66af75f0b717c8cac")
     version("3.10.1", sha256="c9356b657de65c53744046fa8f7358afe0714a1af7d570c00c3835c2d724a7c1")
     version("3.10.0", sha256="c9db46394197244adf2f0b08ec5bc3cf16757e9590b02af1fca085c16c0d600a")
@@ -29,6 +32,9 @@ class PyImportlibMetadata(PythonPackage):
     version("0.19", sha256="23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8")
     version("0.18", sha256="cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db")
 
+    depends_on("python@3.8:", when="@6.8.0:", type=("build", "run"))
+    # lowerbound needed as spack itself supports python 3.6 (can be dropped in spack 0.21)
+    depends_on("python@3.7:", when="@4.9.0:", type=("build", "run"))
     depends_on("py-setuptools@56:", when="@4.6.4:", type="build")
     depends_on("py-setuptools", type="build")
     depends_on("py-setuptools-scm@3.4.1:+toml", when="@3:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-installer/package.py b/var/spack/repos/builtin/packages/py-installer/package.py
index 30a4a62072ab03..1bedecf074160a 100644
--- a/var/spack/repos/builtin/packages/py-installer/package.py
+++ b/var/spack/repos/builtin/packages/py-installer/package.py
@@ -40,6 +40,5 @@ def install(self, spec, prefix):
 
     def setup_dependent_package(self, module, dependent_spec):
         installer = dependent_spec["python"].command
-        installer.add_default_arg("-m")
-        installer.add_default_arg("installer")
+        installer.add_default_arg("-m", "installer")
         setattr(module, "installer", installer)
diff --git a/var/spack/repos/builtin/packages/py-ipycanvas/package.py b/var/spack/repos/builtin/packages/py-ipycanvas/package.py
index 82f832ac8043e3..d61e222fb47a0d 100644
--- a/var/spack/repos/builtin/packages/py-ipycanvas/package.py
+++ b/var/spack/repos/builtin/packages/py-ipycanvas/package.py
@@ -17,9 +17,7 @@ class PyIpycanvas(PythonPackage):
 
     depends_on("python@3.5:", type=("build", "run"))
     depends_on("py-setuptools@40.8:", type="build")
-    # TODO: replace this after concretizer learns how to concretize separate build deps
-    depends_on("py-jupyter-packaging7", type="build")
-    # depends_on('py-jupyter-packaging@0.7.0:0.7', type='build')
+    depends_on("py-jupyter-packaging@0.7", type="build")
     depends_on("py-jupyterlab@3.0:3", type="build")
     depends_on("py-ipywidgets@7.6:", type=("build", "run"))
     depends_on("pil@6:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-ipyevents/package.py b/var/spack/repos/builtin/packages/py-ipyevents/package.py
index 8f9d717d75e571..042cff2119353b 100644
--- a/var/spack/repos/builtin/packages/py-ipyevents/package.py
+++ b/var/spack/repos/builtin/packages/py-ipyevents/package.py
@@ -16,8 +16,6 @@ class PyIpyevents(PythonPackage):
 
     depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-setuptools@40.8:", type="build")
-    # TODO: replace this after concretizer learns how to concretize separate build deps
-    depends_on("py-jupyter-packaging7", type="build")
-    # depends_on('py-jupyter-packaging@0.7.0:0.7', type='build')
+    depends_on("py-jupyter-packaging@0.7", type="build")
     depends_on("py-jupyterlab@3.0:3", type="build")
     depends_on("py-ipywidgets@7.6:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-ipympl/package.py b/var/spack/repos/builtin/packages/py-ipympl/package.py
index 158345349f4518..56eefb8429f3e6 100644
--- a/var/spack/repos/builtin/packages/py-ipympl/package.py
+++ b/var/spack/repos/builtin/packages/py-ipympl/package.py
@@ -23,8 +23,6 @@ class PyIpympl(PythonPackage):
     depends_on("py-traitlets@:5", type=("build", "run"))
     depends_on("py-ipywidgets@7.6:7", type=("build", "run"))
     depends_on("py-matplotlib@2:3", type=("build", "run"))
-    # TODO: replace this after concretizer learns how to concretize separate build deps
-    depends_on("py-jupyter-packaging7", type="build")
-    # depends_on('py-jupyter-packaging@0.7', type='build')
+    depends_on("py-jupyter-packaging@0.7", type="build")
     depends_on("py-jupyterlab@3", type="build")
     depends_on("yarn", type="build")
diff --git a/var/spack/repos/builtin/packages/py-isort/package.py b/var/spack/repos/builtin/packages/py-isort/package.py
index 9f81cd8a8b4b59..aca4dd29047753 100644
--- a/var/spack/repos/builtin/packages/py-isort/package.py
+++ b/var/spack/repos/builtin/packages/py-isort/package.py
@@ -12,6 +12,7 @@ class PyIsort(PythonPackage):
     homepage = "https://github.com/timothycrosley/isort"
     pypi = "isort/isort-4.2.15.tar.gz"
 
+    version("5.12.0", sha256="8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504")
     version("5.10.1", sha256="e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951")
     version("5.9.3", sha256="9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899")
     version("5.9.1", sha256="83510593e07e433b77bd5bff0f6f607dbafa06d1a89022616f02d8b699cfcd56")
@@ -20,8 +21,12 @@ class PyIsort(PythonPackage):
 
     variant("colors", default=False, description="Install colorama for --color support")
 
-    depends_on("python@3.6.1:3", type=("build", "run"), when="@5:")
-    depends_on("python@2.7:2.8,3.4:", type=("build", "run"), when="@4.3:")
-    depends_on("python@2.6:2.8,3.3:", type=("build", "run"))
-    depends_on("py-poetry-core@1:", type="build")
-    depends_on("py-colorama@0.4.3:0.4", type=("build", "run"), when="+colors")
+    depends_on("python@3.8:", when="@5.12:", type=("build", "run"))
+    depends_on("python@3.6.1:3", when="@5:5.10", type=("build", "run"))
+    depends_on("py-setuptools", when="@:4", type=("build", "run"))
+    depends_on("py-poetry-core@1:", when="@5:", type="build")
+    depends_on("py-colorama@0.4.3:", when="+colors @5.12:", type=("build", "run"))
+    depends_on("py-colorama@0.4.3:0.4", when="+colors @:5.11", type=("build", "run"))
+
+    # https://github.com/PyCQA/isort/issues/2077
+    conflicts("^py-poetry-core@1.5:", when="@:5.11.4")
diff --git a/var/spack/repos/builtin/packages/py-jax/package.py b/var/spack/repos/builtin/packages/py-jax/package.py
index 279e4a00f7e309..a2a3900c5d4ea7 100644
--- a/var/spack/repos/builtin/packages/py-jax/package.py
+++ b/var/spack/repos/builtin/packages/py-jax/package.py
@@ -34,11 +34,7 @@ class PyJax(PythonPackage):
     depends_on("py-scipy@1.2.1:", type=("build", "run"))
 
     # See _minimum_jaxlib_version in jax/version.py
-    jax_to_jaxlib = {
-        "0.4.3": "0.4.2",
-        "0.3.23": "0.3.15",
-        "0.2.25": "0.1.69",
-    }
+    jax_to_jaxlib = {"0.4.3": "0.4.2", "0.3.23": "0.3.15", "0.2.25": "0.1.69"}
 
     for jax, jaxlib in jax_to_jaxlib.items():
         depends_on(f"py-jaxlib@{jaxlib}:", when=f"@{jax}", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-jaxlib/package.py b/var/spack/repos/builtin/packages/py-jaxlib/package.py
index d8def42008afce..b4c7daba27c817 100644
--- a/var/spack/repos/builtin/packages/py-jaxlib/package.py
+++ b/var/spack/repos/builtin/packages/py-jaxlib/package.py
@@ -47,6 +47,13 @@ class PyJaxlib(PythonPackage, CudaPackage):
     depends_on("py-absl-py", when="@:0.3", type=("build", "run"))
     depends_on("py-flatbuffers@1.12:2", when="@0.1", type=("build", "run"))
 
+    conflicts(
+        "cuda_arch=none",
+        when="+cuda",
+        msg="Must specify CUDA compute capabilities of your GPU, see "
+        "https://developer.nvidia.com/cuda-gpus",
+    )
+
     def patch(self):
         self.tmp_path = tempfile.mkdtemp(prefix="spack")
         self.buildtmp = tempfile.mkdtemp(prefix="spack")
diff --git a/var/spack/repos/builtin/packages/py-jinja2/package.py b/var/spack/repos/builtin/packages/py-jinja2/package.py
index a94f81dce0cac7..a4c92d359415cf 100644
--- a/var/spack/repos/builtin/packages/py-jinja2/package.py
+++ b/var/spack/repos/builtin/packages/py-jinja2/package.py
@@ -39,3 +39,6 @@ class PyJinja2(PythonPackage):
     depends_on("py-markupsafe@0.23:", type=("build", "run"))
     depends_on("py-babel@2.7:", when="@3:+i18n", type=("build", "run"))
     depends_on("py-babel@0.8:", when="+i18n", type=("build", "run"))
+
+    # https://github.com/pallets/jinja/issues/1585
+    conflicts("^py-markupsafe@2.1:", when="@:2")
diff --git a/var/spack/repos/builtin/packages/py-jsonargparse/package.py b/var/spack/repos/builtin/packages/py-jsonargparse/package.py
index 2f4584e48b2352..a84a23c7fc6926 100644
--- a/var/spack/repos/builtin/packages/py-jsonargparse/package.py
+++ b/var/spack/repos/builtin/packages/py-jsonargparse/package.py
@@ -15,13 +15,17 @@ class PyJsonargparse(PythonPackage):
     homepage = "https://github.com/omni-us/jsonargparse"
     pypi = "jsonargparse/jsonargparse-4.19.0.tar.gz"
 
+    version("4.25.0", sha256="4eaadae69c387a3d83a76b1eaf20ca98d5274d8637f180dca0754ce5405adb6b")
     version("4.19.0", sha256="63aa3c7bbdb219d0f254a5ae86f3d54384ebc1ffa905e776cc19283bc843787b")
 
     variant("signatures", default=False, description="Enable signature features")
 
+    depends_on("py-setuptools@65.6.3:", when="@4.25:", type="build")
     depends_on("py-setuptools", type="build")
+    depends_on("py-wheel@0.38.4:", when="@4.25:", type="build")
     depends_on("py-pyyaml@3.13:", type=("build", "run"))
 
     with when("+signatures"):
+        depends_on("py-typing-extensions@3.10:", when="@4.25: ^python@:3.9", type=("build", "run"))
         depends_on("py-docstring-parser@0.15:", type=("build", "run"))
         depends_on("py-typeshed-client@2.1:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-jupyter-packaging/package.py b/var/spack/repos/builtin/packages/py-jupyter-packaging/package.py
index 472d7e6bc93afc..cf333579fd2031 100644
--- a/var/spack/repos/builtin/packages/py-jupyter-packaging/package.py
+++ b/var/spack/repos/builtin/packages/py-jupyter-packaging/package.py
@@ -12,6 +12,8 @@ class PyJupyterPackaging(PythonPackage):
     homepage = "https://github.com/jupyter/jupyter-packaging"
     pypi = "jupyter_packaging/jupyter_packaging-0.10.4.tar.gz"
 
+    tags = ["build-tools"]
+
     version("0.12.0", sha256="b27455d60adc93a7baa2e0b8f386be81b932bb4e3c0116046df9ed230cd3faac")
     version("0.11.1", sha256="6f5c7eeea98f7f3c8fb41d565a94bf59791768a93f93148b3c2dfb7ebade8eec")
     version("0.10.6", sha256="a8a2c90bf2e0cae83be63ccb0b7035032a1589f268cc08b1d479e37ce50fc940")
diff --git a/var/spack/repos/builtin/packages/py-jupyter-packaging11/package.py b/var/spack/repos/builtin/packages/py-jupyter-packaging11/package.py
index c74a7a5231d0a0..b15cfe8752e480 100644
--- a/var/spack/repos/builtin/packages/py-jupyter-packaging11/package.py
+++ b/var/spack/repos/builtin/packages/py-jupyter-packaging11/package.py
@@ -16,9 +16,21 @@ class PyJupyterPackaging11(PythonPackage):
     homepage = "https://github.com/jupyter/jupyter-packaging"
     pypi = "jupyter_packaging/jupyter_packaging-0.11.1.tar.gz"
 
-    version("0.12.3", sha256="9d9b2b63b97ffd67a8bc5391c32a421bc415b264a32c99e4d8d8dd31daae9cf4")
-    version("0.12.0", sha256="b27455d60adc93a7baa2e0b8f386be81b932bb4e3c0116046df9ed230cd3faac")
-    version("0.11.1", sha256="6f5c7eeea98f7f3c8fb41d565a94bf59791768a93f93148b3c2dfb7ebade8eec")
+    version(
+        "0.12.3",
+        sha256="9d9b2b63b97ffd67a8bc5391c32a421bc415b264a32c99e4d8d8dd31daae9cf4",
+        deprecated=True,
+    )
+    version(
+        "0.12.0",
+        sha256="b27455d60adc93a7baa2e0b8f386be81b932bb4e3c0116046df9ed230cd3faac",
+        deprecated=True,
+    )
+    version(
+        "0.11.1",
+        sha256="6f5c7eeea98f7f3c8fb41d565a94bf59791768a93f93148b3c2dfb7ebade8eec",
+        deprecated=True,
+    )
 
     depends_on("python@3.7:", type=("build", "run"))
     depends_on("py-packaging", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-jupyter-packaging7/package.py b/var/spack/repos/builtin/packages/py-jupyter-packaging7/package.py
index be160db5350ee1..8f0da9b9999cb4 100644
--- a/var/spack/repos/builtin/packages/py-jupyter-packaging7/package.py
+++ b/var/spack/repos/builtin/packages/py-jupyter-packaging7/package.py
@@ -16,7 +16,11 @@ class PyJupyterPackaging7(PythonPackage):
     homepage = "https://github.com/jupyter/jupyter-packaging"
     pypi = "jupyter_packaging/jupyter-packaging-0.7.12.tar.gz"
 
-    version("0.7.12", sha256="b140325771881a7df7b7f2d14997b619063fe75ae756b9025852e4346000bbb8")
+    version(
+        "0.7.12",
+        sha256="b140325771881a7df7b7f2d14997b619063fe75ae756b9025852e4346000bbb8",
+        deprecated=True,
+    )
 
     depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-packaging", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-jupyter-server-mathjax/package.py b/var/spack/repos/builtin/packages/py-jupyter-server-mathjax/package.py
index ee2234d14747f5..c86e2f18147c0c 100644
--- a/var/spack/repos/builtin/packages/py-jupyter-server-mathjax/package.py
+++ b/var/spack/repos/builtin/packages/py-jupyter-server-mathjax/package.py
@@ -18,6 +18,6 @@ class PyJupyterServerMathjax(PythonPackage):
     depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("py-jupyter-packaging", type="build")
-    depends_on("py-jupyter-packaging11@:1", when="@0.2.6:", type="build")
+    depends_on("py-jupyter-packaging@0.10:1", when="@0.2.6:", type="build")
     depends_on("py-jupyter-server@1.1:1", when="@0.2.3", type=("build", "run"))
     depends_on("py-jupyter-server@1.1:", when="@0.2.6:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-jupyter-server-proxy/package.py b/var/spack/repos/builtin/packages/py-jupyter-server-proxy/package.py
index a44d656f773a63..4e111148069580 100644
--- a/var/spack/repos/builtin/packages/py-jupyter-server-proxy/package.py
+++ b/var/spack/repos/builtin/packages/py-jupyter-server-proxy/package.py
@@ -19,7 +19,7 @@ class PyJupyterServerProxy(PythonPackage):
 
     version("3.2.2", sha256="54690ea9467035d187c930c599e76065017baf16e118e6eebae0d3a008c4d946")
 
-    depends_on("py-jupyter-packaging7@0.7.9:0.7", type="build")
+    depends_on("py-jupyter-packaging@0.7.9:0.7", type="build")
     depends_on("py-jupyterlab@3.0:3", type="build")
     depends_on("py-setuptools@40.8.0:", type="build")
 
diff --git a/var/spack/repos/builtin/packages/py-jupyter-server/package.py b/var/spack/repos/builtin/packages/py-jupyter-server/package.py
index 5f371a5403f437..4f461dca212cf3 100644
--- a/var/spack/repos/builtin/packages/py-jupyter-server/package.py
+++ b/var/spack/repos/builtin/packages/py-jupyter-server/package.py
@@ -32,9 +32,7 @@ class PyJupyterServer(PythonPackage):
     depends_on("py-hatch-jupyter-builder@0.8.1:", when="@2:", type="build")
 
     with when("@:1"):
-        # TODO: replace this after concretizer learns how to concretize separate build deps
-        depends_on("py-jupyter-packaging11", when="@1.6.2:", type="build")
-        # depends_on('py-jupyter-packaging@0.9:0', when='@1.6.2:', type='build')
+        depends_on("py-jupyter-packaging@0.9:0", when="@1.6.2:", type="build")
         depends_on("py-pre-commit", when="@1.16:", type="build")
         depends_on("py-setuptools", type="build")
 
diff --git a/var/spack/repos/builtin/packages/py-jupyterlab-server/package.py b/var/spack/repos/builtin/packages/py-jupyterlab-server/package.py
index 35e5be5e8e198b..d6791671f645c4 100644
--- a/var/spack/repos/builtin/packages/py-jupyterlab-server/package.py
+++ b/var/spack/repos/builtin/packages/py-jupyterlab-server/package.py
@@ -24,9 +24,8 @@ class PyJupyterlabServer(PythonPackage):
 
     with when("@:2.14"):
         depends_on("py-setuptools", type="build")
-        # TODO: replace this after concretizer learns how to concretize separate build deps
-        depends_on("py-jupyter-packaging11", type="build")
-        # depends_on('py-jupyter-packaging@0.9:0', type='build')
+        depends_on("py-jupyter-packaging@0.10:1", when="@2.10.3", type="build")
+        depends_on("py-jupyter-packaging@0.9:0", when="@:2.6", type="build")
 
     depends_on("py-babel@2.10:", when="@2.16.4:", type=("build", "run"))
     depends_on("py-babel", when="@2.5.1:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-jupyterlab/package.py b/var/spack/repos/builtin/packages/py-jupyterlab/package.py
index 8865713bf5d270..ccacf77c8e8d62 100644
--- a/var/spack/repos/builtin/packages/py-jupyterlab/package.py
+++ b/var/spack/repos/builtin/packages/py-jupyterlab/package.py
@@ -24,7 +24,11 @@ class PyJupyterlab(PythonPackage):
     version("3.1.14", sha256="13174cb6076dd5da6f1b85725ccfcc9518d8f98e86b8b644fc89b1dfaeda63a9")
     version("3.0.18", sha256="0e4bb4b89014607a16658b54f13df2f0af14f3c286109a0e14d5a46cbbe28caf")
     version("3.0.16", sha256="7ad4fbe1f6d38255869410fd151a8b15692a663ca97c0a8146b3f5c40e275c23")
-    version("3.0.14", sha256="713a84991dfcca8c0bc260911f1bd54ac25a386a86285713b9555a60f795059b")
+    version(
+        "3.0.14",
+        sha256="713a84991dfcca8c0bc260911f1bd54ac25a386a86285713b9555a60f795059b",
+        deprecated=True,
+    )
     version("2.2.7", sha256="a72ffd0d919cba03a5ef8422bc92c3332a957ff97b0490494209c83ad93826da")
     version("2.1.0", sha256="8c239aababf5baa0b3d36e375fddeb9fd96f3a9a24a8cda098d6a414f5bbdc81")
 
@@ -50,12 +54,9 @@ class PyJupyterlab(PythonPackage):
 
     with when("@:3"):
         depends_on("py-setuptools", when="@:3", type=("build", "run"))
-        # TODO: replace this after concretizer learns how to concretize separate build deps
-        depends_on("py-jupyter-packaging11", when="@3.0.15:3", type="build")
-        depends_on("py-jupyter-packaging7", when="@3.0.0:3.0.14", type="build")
-        # depends_on('py-jupyter-packaging@0.9:0', when='@3.0.15:', type='build')
-        # depends_on('py-jupyter-packaging@0.7.3:0.7', when='@3.0.0:3.0.14',
-        #            type=('build', 'run'))
+        depends_on("py-jupyter-packaging@0.9:1", when="@3.4.8", type="build")
+        depends_on("py-jupyter-packaging@0.9:0", when="@3.0.15:3.4.2", type="build")
+        depends_on("py-jupyter-packaging@0.7.3:0.7", when="@3.0.0:3.0.14", type=("build", "run"))
         depends_on("py-pre-commit", when="@3.4:3.4.3", type="build")
 
         depends_on("py-ipython", when="@3", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-jupytext/package.py b/var/spack/repos/builtin/packages/py-jupytext/package.py
index f9bf4efa8d1219..6491fd575611dc 100644
--- a/var/spack/repos/builtin/packages/py-jupytext/package.py
+++ b/var/spack/repos/builtin/packages/py-jupytext/package.py
@@ -31,6 +31,4 @@ class PyJupytext(PythonPackage):
     # todo: in order to use jupytext as a jupyterlab extension,
     # some additional dependencies need to be added (and checked):
     depends_on("py-jupyterlab@3", type=("build", "run"))
-    # TODO: replace this after concretizer learns how to concretize separate build deps
-    depends_on("py-jupyter-packaging7", type="build")
-    # depends_on('py-jupyter-packaging@0.7.9:0.7', type='build')```
+    depends_on("py-jupyter-packaging@0.7.9:0.7", type="build")
diff --git a/var/spack/repos/builtin/packages/py-keras/package.py b/var/spack/repos/builtin/packages/py-keras/package.py
index ba5ca20a2bf9ce..c6f65dc957cb07 100644
--- a/var/spack/repos/builtin/packages/py-keras/package.py
+++ b/var/spack/repos/builtin/packages/py-keras/package.py
@@ -21,6 +21,8 @@ class PyKeras(PythonPackage):
     git = "https://github.com/keras-team/keras.git"
     url = "https://github.com/keras-team/keras/archive/refs/tags/v2.7.0.tar.gz"
 
+    version("2.14.0", sha256="a845d446b6ae626f61dde5ab2fa952530b6c17b4f9ed03e9362bd20172d00cca")
+    version("2.13.1", sha256="b3591493cce75a69adef7b192cec6be222e76e2386d132cd4e34aa190b0ecbd5")
     version("2.12.0", sha256="6336cebb6b2b0a91f7efd3ff3a9db3a94f2abccf07a40323138afb80826aec62")
     version("2.11.0", sha256="e7a7c4199ac76ea750d145c1d84ae1b932e68b9bca34e83596bd66b2fc2ad79e")
     version("2.10.0", sha256="b1d8d9358700f4a585455854a142d88cc987419c1638ef935b440842d593ad04")
@@ -60,7 +62,7 @@ class PyKeras(PythonPackage):
     depends_on("py-pydot", type=("build", "run"))
     depends_on("py-scipy", type=("build", "run"))
     depends_on("py-six", type=("build", "run"))
-    for minor_ver in range(6, 13):
+    for minor_ver in range(6, 15):
         depends_on(
             "py-tensorflow@2.{}".format(minor_ver),
             type=("build", "run"),
@@ -114,14 +116,7 @@ def install(self, spec, prefix):
             "--jobs={0}".format(make_jobs),
             # Enable verbose output for failures
             "--verbose_failures",
-            # Show (formatted) subcommands being executed
-            "--subcommands=pretty_print",
             "--spawn_strategy=local",
-            # Ask bazel to explain what it's up to
-            # Needs a filename as argument
-            "--explain=explainlogfile.txt",
-            # Increase verbosity of explanation,
-            "--verbose_explanations",
             # bazel uses system PYTHONPATH instead of spack paths
             "--action_env",
             "PYTHONPATH={0}".format(env["PYTHONPATH"]),
diff --git a/var/spack/repos/builtin/packages/py-kiwisolver/package.py b/var/spack/repos/builtin/packages/py-kiwisolver/package.py
index 4f71b99ec68e69..9c83a2b25894a6 100644
--- a/var/spack/repos/builtin/packages/py-kiwisolver/package.py
+++ b/var/spack/repos/builtin/packages/py-kiwisolver/package.py
@@ -12,6 +12,7 @@ class PyKiwisolver(PythonPackage):
     homepage = "https://github.com/nucleic/kiwi"
     pypi = "kiwisolver/kiwisolver-1.1.0.tar.gz"
 
+    version("1.4.5", sha256="e57e563a57fb22a142da34f38acc2fc1a5c864bc29ca1517a88abc963e60d6ec")
     version("1.4.4", sha256="d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955")
     version("1.3.2", sha256="fc4453705b81d03568d5b808ad8f09c77c47534f6ac2e72e733f9ca4714aa75c")
     version("1.3.1", sha256="950a199911a8d94683a6b10321f9345d5a3a8433ec58b217ace979e18f16e248")
diff --git a/var/spack/repos/builtin/packages/py-kombu/package.py b/var/spack/repos/builtin/packages/py-kombu/package.py
index 9c732796cf30a6..6f13c380ffb840 100644
--- a/var/spack/repos/builtin/packages/py-kombu/package.py
+++ b/var/spack/repos/builtin/packages/py-kombu/package.py
@@ -24,6 +24,9 @@ class PyKombu(PythonPackage):
     variant("redis", default=False, description="Use redis transport")
 
     depends_on("py-setuptools", type="build")
+    # "pytz>dev" in tests_require: setuptools parser changed in v60 and errors.
+    depends_on("py-setuptools@:59", when="@4.6:5.2", type="build")
+
     depends_on("py-amqp@2.5.2:2.5", when="@:4.6.6", type=("build", "run"))
     depends_on("py-amqp@2.6.0:2.6", when="@4.6.7:4", type=("build", "run"))
     depends_on("py-amqp@5.0.0:5", when="@5.0.0:5.0.2", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-kornia/package.py b/var/spack/repos/builtin/packages/py-kornia/package.py
index 98b5a9cf0316f8..2ba1a1a4aed52d 100644
--- a/var/spack/repos/builtin/packages/py-kornia/package.py
+++ b/var/spack/repos/builtin/packages/py-kornia/package.py
@@ -12,6 +12,7 @@ class PyKornia(PythonPackage):
     homepage = "https://www.kornia.org/"
     pypi = "kornia/kornia-0.5.10.tar.gz"
 
+    version("0.7.0", sha256="72cba6a0965a15caf10a664647654412effb7c0b9afcf40e458bc005f976ffac")
     version("0.6.12", sha256="e30bd3d830226f7a159dff1f7757c6200e8f27d1333f06e9d2f98bdb33ce18d3")
     version("0.6.11", sha256="ba77198f2c2e4044c87e4503ff277aadbfc0db20495da5a5289663f380b4be32")
     version("0.6.10", sha256="00b4e8990b5ccdc589e3ecc2613446623b0ffb2fe1db5598aac55513b4016903")
@@ -27,6 +28,7 @@ class PyKornia(PythonPackage):
     version("0.5.10", sha256="428b4b934a2ba7360cc6cba051ed8fd96c2d0f66611fdca0834e82845f14f65d")
 
     # pyproject.toml
+    depends_on("python@3.8:", when="@0.7:", type=("build", "run"))
     depends_on("py-setuptools@61.2:", when="@0.6.11:", type="build")
     depends_on("py-setuptools", type="build")
 
diff --git a/var/spack/repos/builtin/packages/py-langsmith/package.py b/var/spack/repos/builtin/packages/py-langsmith/package.py
new file mode 100644
index 00000000000000..e5c8363487b6e3
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-langsmith/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyLangsmith(PythonPackage):
+    """Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."""
+
+    pypi = "langsmith/langsmith-0.0.10.tar.gz"
+
+    version("0.0.11", sha256="7c1be28257d6c7279c85f81e6d8359d1006af3b1238fc198d13ca75c8fe421c8")
+    version("0.0.10", sha256="11e5db0d8e29ee5583cabd872eeece8ce50738737b1f52f316ac984f4a1a58c5")
+    version("0.0.7", sha256="2f18e51cfd4e42f2b3cf00fa87e9d03012eb7269cdafd8e7c0cf7aa828dcc03e")
+
+    depends_on("python@3.8.1:3", type=("build", "run"))
+    depends_on("py-poetry-core", type="build")
+    depends_on("py-pydantic@1", type=("build", "run"))
+    depends_on("py-requests@2", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-ldap3/package.py b/var/spack/repos/builtin/packages/py-ldap3/package.py
new file mode 100644
index 00000000000000..646870b2971ec0
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-ldap3/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyLdap3(PythonPackage):
+    """A strictly RFC 4510 conforming LDAP V3 pure Python client library."""
+
+    homepage = "https://github.com/cannatag/ldap3"
+    pypi = "ldap3/ldap3-2.9.1.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("2.9.1", sha256="f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-pyasn1@0.4.6:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-lhsmdu/package.py b/var/spack/repos/builtin/packages/py-lhsmdu/package.py
index 3d3e33dc5a6f70..866c70e04d6f81 100644
--- a/var/spack/repos/builtin/packages/py-lhsmdu/package.py
+++ b/var/spack/repos/builtin/packages/py-lhsmdu/package.py
@@ -21,5 +21,5 @@ class PyLhsmdu(PythonPackage):
     version("0.1", sha256="ef462054b354cd20b10c6d80876c8fdb552a8d2e23eaf74179dc91956d68d32a")
 
     depends_on("py-setuptools", type="build")
-    depends_on("py-numpy", type="build")
-    depends_on("py-scipy", type="build")
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-scipy", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-libensemble/package.py b/var/spack/repos/builtin/packages/py-libensemble/package.py
index 0f7cbf07bf0e39..d47a3d68fb932e 100644
--- a/var/spack/repos/builtin/packages/py-libensemble/package.py
+++ b/var/spack/repos/builtin/packages/py-libensemble/package.py
@@ -12,13 +12,16 @@ class PyLibensemble(PythonPackage):
     """Library for managing ensemble-like collections of computations."""
 
     homepage = "https://libensemble.readthedocs.io"
-    pypi = "libensemble/libensemble-0.10.0.tar.gz"
+    pypi = "libensemble/libensemble-1.0.0.tar.gz"
     git = "https://github.com/Libensemble/libensemble.git"
     maintainers("shuds13", "jlnav")
 
     tags = ["e4s"]
 
     version("develop", branch="develop")
+    version("1.0.0", sha256="b164e044f16f15b68fd565684ad8ce876c93aaeb84e5078f4ea2a29684b110ca")
+    version("0.10.2", sha256="ef8dfe5d233dcae2636a3d6aa38f3c2ad0f42c65bd38f664e99b3e63b9f86622")
+    version("0.10.1", sha256="56ae42ec9a28d3df8f46bdf7d016db9526200e9df2a28d849902e3c44fe5c1ba")
     version("0.10.0", sha256="f800f38d02def526f1d2a325710d01fdd3637cd1e33a9a083a3cf4a7f419a726")
     version("0.9.3", sha256="00e5a65d6891feee6a686c048d8de72097b8bff164431f163be96ec130a9c390")
     version("0.9.2", sha256="e46598e5696f770cbff4cb90507b52867faad5654f1b80de35405a95228c909f")
@@ -38,7 +41,9 @@ class PyLibensemble(PythonPackage):
     version("0.2.0", sha256="ecac7275d4d0f4a5e497e5c9ef2cd998da82b2c020a0fb87546eeea262f495ff")
     version("0.1.0", sha256="0b27c59ae80f7af8b1bee92fcf2eb6c9a8fd3494bf2eb6b3ea17a7c03d3726bb")
 
-    variant("mpi", default=True, description="Install with MPI")
+    variant("mpi", default=True, description="Install with MPI")  # Optional communications method
+
+    # The following variants are for optional built-in generators
     variant("scipy", default=False, description="Install with scipy")
     variant("petsc4py", default=False, description="Install with petsc4py")
     variant("nlopt", default=False, description="Install with nlopt")
@@ -46,23 +51,28 @@ class PyLibensemble(PythonPackage):
     variant("deap", default=False, description="Install with DEAP")
     variant("tasmanian", default=False, description="Install with tasmanian")
 
+    depends_on("py-numpy@1.21:", when="@1:", type=("build", "run"))
     depends_on("py-numpy", type=("build", "run"))
-    depends_on("py-psutil", type=("build", "run"), when="@0.7.1:")
-    depends_on("py-setuptools", type=("build", "run"))
-    depends_on("py-pydantic", type=("build", "run"), when="@0.10:")
-    depends_on("py-tomli", type=("build", "run"), when="@0.10:")
-    depends_on("py-pyyaml", type=("build", "run"), when="@0.10:")
+    depends_on("py-psutil@5.9.4:", when="@1:", type=("build", "run"))
+    depends_on("py-psutil", when="@0.7.1:", type=("build", "run"))
+    depends_on("py-setuptools", when="@0.10.2:", type="build")
+    depends_on("py-setuptools", when="@:0.10.1", type=("build", "run"))
+    depends_on("py-pydantic@:1", when="@0.10:", type=("build", "run"))
+    depends_on("py-tomli@1.2.1:", when="@1:", type=("build", "run"))
+    depends_on("py-tomli", when="@0.10:", type=("build", "run"))
+    depends_on("py-pyyaml@6.0:", when="@1:", type=("build", "run"))
+    depends_on("py-pyyaml", when="@0.10:", type=("build", "run"))
     depends_on("mpi", when="@:0.4.1")
     depends_on("mpi", when="+mpi")
-    depends_on("py-mpi4py@2.0:", type=("build", "run"), when="@:0.4.1")
-    depends_on("py-mpi4py@2.0:", type=("build", "run"), when="+mpi")
-    depends_on("py-scipy", type=("build", "run"), when="+scipy")
-    depends_on("py-petsc4py", type=("build", "run"), when="+petsc4py")
-    depends_on("py-petsc4py@main", type=("build", "run"), when="@develop+petsc4py")
-    depends_on("nlopt", type=("build", "run"), when="+nlopt")
-    depends_on("py-mpmath", type=("build", "run"), when="+mpmath")
-    depends_on("py-deap", type=("build", "run"), when="+deap")
-    depends_on("tasmanian+python", type=("build", "run"), when="+tasmanian")
+    depends_on("py-mpi4py@2.0:", when="@:0.4.1", type=("build", "run"))
+    depends_on("py-mpi4py@2.0:", when="+mpi", type=("build", "run"))
+    depends_on("py-scipy", when="+scipy", type=("build", "run"))
+    depends_on("py-petsc4py", when="+petsc4py", type=("build", "run"))
+    depends_on("py-petsc4py@main", when="@develop+petsc4py", type=("build", "run"))
+    depends_on("nlopt", when="+nlopt", type=("build", "run"))
+    depends_on("py-mpmath", when="+mpmath", type=("build", "run"))
+    depends_on("py-deap", when="+deap", type=("build", "run"))
+    depends_on("tasmanian+python", when="+tasmanian", type=("build", "run"))
     conflicts("~mpi", when="@:0.4.1")
 
     @run_after("install")
diff --git a/var/spack/repos/builtin/packages/py-lightly/package.py b/var/spack/repos/builtin/packages/py-lightly/package.py
index 2d7990c7869b46..260679e84696ea 100644
--- a/var/spack/repos/builtin/packages/py-lightly/package.py
+++ b/var/spack/repos/builtin/packages/py-lightly/package.py
@@ -15,6 +15,13 @@ class PyLightly(PythonPackage):
 
     maintainers("adamjstewart")
 
+    version("1.4.18", sha256="41794f6815db178b031236793b379e5573e074fdf730506872b73766396a6bdf")
+    version("1.4.17", sha256="1533ddf28c8a08b3eafd404964d03f9a62fe76405fcf8dc7206ca4093725285e")
+    version("1.4.16", sha256="9bd2af53e144e4f9823409cd33b39651f579ed671ff242a1445640c9df504d92")
+    version("1.4.15", sha256="9b02f523b35621a98c3e87efdda735897b726b39cf4a8bf106769f54b3df154e")
+    version("1.4.14", sha256="edbad4c95866fea6951a4fc5e851518f3afb2ff19381648accb4e1a005366720")
+    version("1.4.13", sha256="d3819d75b85534f98603743486b263eb1dc62cd5b20de30d5bad07227f1e0d98")
+    version("1.4.12", sha256="66aea90f096c474b26515bafbef728632516d6f8f13ab5caba38b09903d1b86e")
     version("1.4.11", sha256="3c767a15368c4294a14067a93a4b07e91bd45295313f1e132f28d00270bb69d0")
     version("1.4.10", sha256="77381c913d06f3c02ad7b36a0434bfd7e27a738648f555a51c9912d80dd11125")
     version("1.4.9", sha256="33c7988b41447c9beeb1781805ad24c8a61aa174e1a37b0b95d8e91e4a7c7f96")
@@ -43,6 +50,7 @@ class PyLightly(PythonPackage):
 
     # requirements/openapi.txt
     depends_on("py-python-dateutil@2.5.3:", when="@1.4.8:", type=("build", "run"))
+    depends_on("py-setuptools@21:", when="@1.4.15:", type=("build", "run"))
     depends_on("py-urllib3@1.25.3:", when="@1.4.8:", type=("build", "run"))
     depends_on("py-pydantic@1.10.5:1", when="@1.4.8:", type=("build", "run"))
     depends_on("py-aenum@3.1.11:", when="@1.4.8:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-lightning-cloud/package.py b/var/spack/repos/builtin/packages/py-lightning-cloud/package.py
index e0f0afca089710..80417b11c17cc2 100644
--- a/var/spack/repos/builtin/packages/py-lightning-cloud/package.py
+++ b/var/spack/repos/builtin/packages/py-lightning-cloud/package.py
@@ -12,10 +12,14 @@ class PyLightningCloud(PythonPackage):
     homepage = "https://lightning.ai/"
     pypi = "lightning_cloud/lightning_cloud-0.5.31.tar.gz"
 
+    version("0.5.38", sha256="86fd5144b721bb289e9fd863604a67d55515688ccbadf0d858adfca1cefaf78f")
+    version("0.5.37", sha256="1e0d577e1696e3aef974c589c4bd1ed22c70a332750863a393ec3949166137e0")
     version("0.5.36", sha256="990558d93a1b67d8bcbf8a87feb2ac455e13ec5223916ad2d5707d96df9558c9")
     version("0.5.31", sha256="a5a138f4abbeffe66ee476fb9a8d621befac0434ffeeeec1cc00ccd3d72ffc09")
 
     depends_on("py-setuptools", type="build")
+
+    # requirements.txt
     depends_on("py-click", type=("build", "run"))
     depends_on("py-fastapi", when="@0.5.36:", type=("build", "run"))
     depends_on("py-fastapi+all", when="@:0.5.31", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-lightning/package.py b/var/spack/repos/builtin/packages/py-lightning/package.py
index 2c86aee35dfe72..8bec9806ee3478 100644
--- a/var/spack/repos/builtin/packages/py-lightning/package.py
+++ b/var/spack/repos/builtin/packages/py-lightning/package.py
@@ -7,15 +7,20 @@
 
 
 class PyLightning(PythonPackage):
-    """Use Lightning Apps to build everything from production-ready,
-    multi-cloud ML systems to simple research demos.
-    """
+    """The Deep Learning framework to train, deploy, and ship AI products Lightning fast."""
 
     homepage = "https://github.com/Lightning-AI/lightning"
     pypi = "lightning/lightning-2.0.0.tar.gz"
+    skip_modules = ["lightning.app", "lightning.data", "lightning.store"]
 
     maintainers("adamjstewart")
 
+    version("2.1.0", sha256="1f78f5995ae7dcffa1edf34320db136902b73a0d1b304404c48ec8be165b3a93")
+    version("2.0.9", sha256="2395ece6e29e12064718ff16b8edec5685df7f7095d4fee78edb0a654f5cd7eb")
+    version("2.0.8", sha256="db914e211b5c3b079a821be6e4344e72d0a729163676a65c4e00aae98390ae7b")
+    version("2.0.7", sha256="f05acd4ba846505d40125b4f9f0bda0804b2b0356e2ad2fd4e4bf7d1c61c8cc6")
+    version("2.0.6", sha256="bff959f65eed2f626dd65e7b2cfd0d3ddcd0c4ca19ffc8f5f49a4ba4494ca528")
+    version("2.0.5", sha256="77df233129b29c11df7b5e071e24e29420d5efbdbbac9cb6fb4602b7b5afce8a")
     version("2.0.4", sha256="f5f5ed75a657caa8931051590ed000d46bf1b8311ae89bb17a961c3f299dbf33")
     version("2.0.3", sha256="5a70f05e40f1d7882f81eace0d4a86fe2604b423f8df42beaabd187bfdb420cf")
     version("2.0.2", sha256="fa32d671850a5be2d961c6705c927f6f48d1cf9696f61f7d865244142e684430")
@@ -28,45 +33,67 @@ class PyLightning(PythonPackage):
     depends_on("py-setuptools", type="build")
 
     # src/lightning.egg-info/requires.txt
-    depends_on("py-jinja2@:4", type=("build", "run"))
     depends_on("py-pyyaml@5.4:7", type=("build", "run"))
-    depends_on("py-arrow@1.2:2", type=("build", "run"))
-    depends_on("py-beautifulsoup4@4.8:5", type=("build", "run"))
-    depends_on("py-click@:9", type=("build", "run"))
-    depends_on("py-croniter@1.3", type=("build", "run"))
-    depends_on("py-dateutils@:1", type=("build", "run"))
-    depends_on("py-deepdiff@5.7:7", type=("build", "run"))
-    depends_on("py-fastapi@0.92:1", when="@2.0.4:", type=("build", "run"))
-    depends_on("py-fastapi@0.69:0.88", when="@2.0.3", type=("build", "run"))
-    depends_on("py-fastapi@:0.88", when="@:2.0.2", type=("build", "run"))
-    depends_on("py-fsspec@2022.5:2023+http", type=("build", "run"))
-    depends_on("py-inquirer@2.10:4", type=("build", "run"))
-    depends_on("py-lightning-cloud@0.5.34:", when="@2.0.3:", type=("build", "run"))
-    depends_on("py-lightning-cloud@0.5.31:", when="@2:", type=("build", "run"))
-    depends_on("py-lightning-cloud@0.5.27:", when="@:1", type=("build", "run"))
-    depends_on("py-lightning-utilities@0.7:1", when="@2:", type=("build", "run"))
+    depends_on("py-fsspec@2021.6.1:2024+http", when="@2.1:", type=("build", "run"))
+    depends_on("py-fsspec@2022.5:2024+http", when="@2.0.5:2.0", type=("build", "run"))
+    depends_on("py-fsspec@2022.5:2023+http", when="@:2.0.4", type=("build", "run"))
+    depends_on("py-lightning-utilities@0.8:1", when="@2.1:", type=("build", "run"))
+    depends_on("py-lightning-utilities@0.7:1", when="@2.0", type=("build", "run"))
     depends_on("py-lightning-utilities@0.6.0.post0:1", when="@:1", type=("build", "run"))
     depends_on("py-numpy@1.17.2:2", type=("build", "run"))
-    depends_on("py-packaging@17.1:24", type=("build", "run"))
-    depends_on("py-psutil@:6", type=("build", "run"))
-    depends_on("py-pydantic@1.7.4:3", when="@2.0.3:", type=("build", "run"))
-    depends_on("py-pydantic@:2", when="@:2.0.2", type=("build", "run"))
-    depends_on("py-python-multipart@0.0.5:1", type=("build", "run"))
-    depends_on("py-requests@:3", type=("build", "run"))
-    depends_on("py-rich@12.3:14", when="@2:", type=("build", "run"))
-    depends_on("py-rich@:14", when="@:1", type=("build", "run"))
-    depends_on("py-starlette", when="@2.0.3:", type=("build", "run"))
-    depends_on("py-starlette@:1", when="@:2.0.2", type=("build", "run"))
-    depends_on("py-starsessions@1.2.1:1", type=("build", "run"))
-    depends_on("py-torch@1.11:3", when="@2:", type=("build", "run"))
+    depends_on("py-packaging@20:24", when="@2.1:", type=("build", "run"))
+    depends_on("py-packaging@17.1:24", when="@:2.0", type=("build", "run"))
+    depends_on("py-torch@1.12:3", when="@2.1:", type=("build", "run"))
+    depends_on("py-torch@1.11:3", when="@2.0", type=("build", "run"))
     depends_on("py-torch@1.10:3", when="@:1", type=("build", "run"))
-    depends_on("py-torchmetrics@0.7:1", type=("build", "run"))
+    depends_on("py-torchmetrics@0.7:2", when="@2.0.9:", type=("build", "run"))
+    depends_on("py-torchmetrics@0.7:1", when="@:2.0.8", type=("build", "run"))
     depends_on("py-tqdm@4.57:5", type=("build", "run"))
-    depends_on("py-traitlets@5.3:6", type=("build", "run"))
     depends_on("py-typing-extensions@4:5", type=("build", "run"))
-    depends_on("py-urllib3@:3", when="@2.0.4:", type=("build", "run"))
-    depends_on("py-urllib3@:2", when="@:2.0.3", type=("build", "run"))
-    depends_on("py-uvicorn@:1", type=("build", "run"))
-    depends_on("py-websocket-client@:2", type=("build", "run"))
-    depends_on("py-websockets@:11", type=("build", "run"))
-    depends_on("py-pytorch-lightning", when="@2:", type=("build", "run"))
+
+    # Only an alias, not actually used by the library
+    # depends_on("py-pytorch-lightning", when="@2:", type=("build", "run"))
+
+    # Historical requirements
+    with when("@:2.0"):
+        depends_on("py-jinja2@:4", type=("build", "run"))
+        depends_on("py-arrow@1.2:2", type=("build", "run"))
+        depends_on("py-backoff@2.2.1:3", when="@2.0.5:", type=("build", "run"))
+        depends_on("py-beautifulsoup4@4.8:5", type=("build", "run"))
+        depends_on("py-click@:9", type=("build", "run"))
+        depends_on("py-croniter@1.3:1.4", when="@2.0.5:", type=("build", "run"))
+        depends_on("py-croniter@1.3", when="@:2.0.4", type=("build", "run"))
+        depends_on("py-dateutils@:1", type=("build", "run"))
+        depends_on("py-deepdiff@5.7:7", type=("build", "run"))
+        depends_on("py-fastapi@0.92:1", when="@2.0.4:", type=("build", "run"))
+        depends_on("py-fastapi@0.69:0.88", when="@2.0.3", type=("build", "run"))
+        depends_on("py-fastapi@:0.88", when="@:2.0.2", type=("build", "run"))
+        depends_on("py-inquirer@2.10:4", type=("build", "run"))
+        depends_on("py-lightning-cloud@0.5.38:", when="@2.0.9:", type=("build", "run"))
+        depends_on("py-lightning-cloud@0.5.37:", when="@2.0.5:", type=("build", "run"))
+        depends_on("py-lightning-cloud@0.5.34:", when="@2.0.3:", type=("build", "run"))
+        depends_on("py-lightning-cloud@0.5.31:", when="@2:", type=("build", "run"))
+        depends_on("py-lightning-cloud@0.5.27:", when="@:1", type=("build", "run"))
+        depends_on("py-psutil@:6", type=("build", "run"))
+        depends_on("py-pydantic@1.7.4:2.1", when="@2.0.7:", type=("build", "run"))
+        depends_on("py-pydantic@1.7.4:2.0", when="@2.0.6", type=("build", "run"))
+        depends_on("py-pydantic@1.7.4:1", when="@2.0.5", type=("build", "run"))
+        depends_on("py-pydantic@1.7.4:3", when="@2.0.3:2.0.4", type=("build", "run"))
+        depends_on("py-pydantic@:2", when="@:2.0.2", type=("build", "run"))
+        depends_on("py-python-multipart@0.0.5:1", type=("build", "run"))
+        depends_on("py-requests@:3", type=("build", "run"))
+        depends_on("py-rich@12.3:14", when="@2:", type=("build", "run"))
+        depends_on("py-rich@:14", when="@:1", type=("build", "run"))
+        depends_on("py-starlette", when="@2.0.3:", type=("build", "run"))
+        depends_on("py-starlette@:1", when="@:2.0.2", type=("build", "run"))
+        depends_on("py-starsessions@1.2.1:1", type=("build", "run"))
+        depends_on("py-traitlets@5.3:6", type=("build", "run"))
+        depends_on("py-urllib3@:3", when="@2.0.4:", type=("build", "run"))
+        depends_on("py-urllib3@:2", when="@:2.0.3", type=("build", "run"))
+        depends_on("py-uvicorn@:1", type=("build", "run"))
+        depends_on("py-websocket-client@:2", type=("build", "run"))
+        depends_on("py-websockets@:12", when="@2.0.5:", type=("build", "run"))
+        depends_on("py-websockets@:11", when="@:2.0.4", type=("build", "run"))
+
+    # https://github.com/Lightning-AI/lightning/issues/18858
+    conflicts("^py-torch~distributed", when="@2.1.0")
diff --git a/var/spack/repos/builtin/packages/py-linkify-it-py/package.py b/var/spack/repos/builtin/packages/py-linkify-it-py/package.py
new file mode 100644
index 00000000000000..e6671f3e8e3694
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-linkify-it-py/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyLinkifyItPy(PythonPackage):
+    """Links recognition library with FULL unicode support."""
+
+    homepage = "https://github.com/tsutsu3/linkify-it-py"
+    pypi = "linkify-it-py/linkify-it-py-2.0.2.tar.gz"
+
+    version("2.0.2", sha256="19f3060727842c254c808e99d465c80c49d2c7306788140987a1a7a29b0d6ad2")
+    version("1.0.3", sha256="2b3f168d5ce75e3a425e34b341a6b73e116b5d9ed8dbbbf5dc7456843b7ce2ee")
+
+    depends_on("python@3.6:", when="@1.0.3", type=("build", "run"))
+    depends_on("python@3.7:", when="@2.0.2:", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
+    depends_on("py-uc-micro-py", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-llnl-sina/package.py b/var/spack/repos/builtin/packages/py-llnl-sina/package.py
index 3bec899480770c..8084a43dd4262a 100644
--- a/var/spack/repos/builtin/packages/py-llnl-sina/package.py
+++ b/var/spack/repos/builtin/packages/py-llnl-sina/package.py
@@ -25,8 +25,8 @@ class PyLlnlSina(PythonPackage):
 
     # notify when the package is updated.
     maintainers("HaluskaR", "estebanpauli", "murray55", "doutriaux1")
-    version("1.11.0", tag="v1.11.0")
-    version("1.10.0", tag="v1.10.0")
+    version("1.11.0", tag="v1.11.0", commit="f3e9bb3a122cfae2a9fd82c3c5613cff939d3aa1")
+    version("1.10.0", tag="v1.10.0", commit="9c3c0acca5f0d4ac02470571688f00ab0bd61a30")
 
     # let's remove dependency on orjson
     patch("no_orjson.patch")
diff --git a/var/spack/repos/builtin/packages/py-llvmlite/package.py b/var/spack/repos/builtin/packages/py-llvmlite/package.py
index b4a5ad65736e4b..a6f7cd8a4db2bd 100644
--- a/var/spack/repos/builtin/packages/py-llvmlite/package.py
+++ b/var/spack/repos/builtin/packages/py-llvmlite/package.py
@@ -26,19 +26,18 @@ class PyLlvmlite(PythonPackage):
     version("0.27.1", sha256="48a1c3ae69fd8920cba153bfed8a46ac46474bc706a2100226df4abffe0000ab")
     version("0.26.0", sha256="13e84fe6ebb0667233074b429fd44955f309dead3161ec89d9169145dbad2ebf")
     version("0.25.0", sha256="fd64def9a51dd7dc61913a7a08eeba5b9785522740bec5a7c5995b2a90525025")
-    version("0.23.0", sha256="bc8b1b46274d05b578fe9e980a6d98fa71c8727f6f9ed31d4d8468dce7aa5762")
 
     depends_on("py-setuptools", type="build")
-    depends_on("python@3.8:3.11", type=("build", "run"), when="@0.40.0:")
-    depends_on("python@3.7:3.10", type=("build", "run"), when="@0.38.0:0.39")
-    depends_on("python@3.7:3.9", type=("build", "run"), when="@0.37")
-    depends_on("python@3.6:", type=("build", "run"), when="@0.33:")
-    depends_on("python@2.6:2.8,3.4:", type=("build", "run"))
+    depends_on("python@3.8:3.11", when="@0.40:", type=("build", "run"))
+    depends_on("python@:3.10", when="@0.38:0.39", type=("build", "run"))
+    depends_on("python@:3.9", when="@0.36:0.37", type=("build", "run"))
+    depends_on("python@:3.8", when="@0.31:0.35", type=("build", "run"))
+    depends_on("python@:3.7", when="@:0.30", type=("build", "run"))
 
-    # llvmlite compatibility information taken from https://github.com/numba/llvmlite#compatibility
-    depends_on("llvm@14:~flang", when="@0.41:")
-    depends_on("llvm@11:14~flang", when="@0.40")
-    depends_on("llvm@11~flang", when="@0.37.0:0.39")
+    # https://github.com/numba/llvmlite#compatibility
+    depends_on("llvm@14", when="@0.41:")
+    depends_on("llvm@11:14", when="@0.40")
+    depends_on("llvm@11", when="@0.37:0.39")
     for t in [
         "arm:",
         "ppc:",
@@ -50,12 +49,12 @@ class PyLlvmlite(PythonPackage):
         "x86:",
         "x86_64:",
     ]:
-        depends_on("llvm@10.0.0:10.0~flang", when="@0.34.0:0.36 target={0}".format(t))
-    depends_on("llvm@9.0.0:9.0~flang", when="@0.34.0:0.36 target=aarch64:")
-    depends_on("llvm@9.0.0:9.0~flang", when="@0.33.0:0.33")
-    depends_on("llvm@7.0.0:8.0~flang", when="@0.29.0:0.32")
-    depends_on("llvm@7.0.0:7.0~flang", when="@0.27.0:0.28")
-    depends_on("llvm@6.0.0:6.0~flang", when="@0.23.0:0.26")
+        depends_on("llvm@10.0", when=f"@0.34:0.36 target={t}")
+    depends_on("llvm@9.0", when="@0.34:0.36 target=aarch64:")
+    depends_on("llvm@9.0", when="@0.33")
+    depends_on("llvm@7.0:7.1,8.0", when="@0.29:0.32")
+    depends_on("llvm@7.0", when="@0.27:0.28")
+    depends_on("llvm@6.0", when="@0.23:0.26")
     depends_on("binutils", type="build")
 
     def setup_build_environment(self, env):
diff --git a/var/spack/repos/builtin/packages/py-lvis/package.py b/var/spack/repos/builtin/packages/py-lvis/package.py
new file mode 100644
index 00000000000000..4824f487d5fe52
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-lvis/package.py
@@ -0,0 +1,36 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+
+
+class PyLvis(PythonPackage):
+    """Python API for LVIS dataset."""
+
+    pypi = "lvis/lvis-0.5.3.tar.gz"
+
+    version("0.5.3", sha256="55aeeb84174abea2ed0d6985a8e93aa9bdbb60c61c6db130c8269a275ef61a6e")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-cycler@0.10:", type=("build", "run"))
+    depends_on("py-cython@0.29.12:", type=("build", "run"))
+    depends_on("py-kiwisolver@1.1:", type=("build", "run"))
+    depends_on("py-matplotlib@3.1.1:", type=("build", "run"))
+    depends_on("py-numpy@1.18.2:", type=("build", "run"))
+    depends_on("opencv@4.1.0.25:+python3", type=("build", "run"))
+    depends_on("py-pyparsing@2.4.0:", type=("build", "run"))
+    depends_on("py-python-dateutil@2.8:", type=("build", "run"))
+    depends_on("py-six@1.12:", type=("build", "run"))
+
+    # imported at lvis/lvis.py:15
+    depends_on("py-pycocotools", type=("build", "run"))
+
+    def patch(self):
+        os.rename(
+            join_path(self.stage.source_path, "lvis.egg-info", "requires.txt"),
+            join_path(self.stage.source_path, "requirements.txt"),
+        )
diff --git a/var/spack/repos/builtin/packages/py-macs3/package.py b/var/spack/repos/builtin/packages/py-macs3/package.py
new file mode 100644
index 00000000000000..be94b9c290cd7c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-macs3/package.py
@@ -0,0 +1,26 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyMacs3(PythonPackage):
+    """MACS: Model-based Analysis for ChIP-Seq"""
+
+    homepage = "https://github.com/macs3-project/MACS/"
+    pypi = "MACS3/MACS3-3.0.0b3.tar.gz"
+
+    maintainers("snehring")
+
+    version("3.0.0b3", sha256="caa794d4cfcd7368447eae15878505315dac44c21546e8fecebb3561e9cee362")
+
+    depends_on("python@3.9:", type=("build", "run"))
+
+    depends_on("py-setuptools@60.0:", type="build")
+    depends_on("py-cython@0.29:0", type=("build", "run"))
+
+    depends_on("py-numpy@1.19:", type=("build", "run"))
+    depends_on("py-cykhash@2", type=("build", "run"))
+    depends_on("py-hmmlearn@0.3:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-maestrowf/package.py b/var/spack/repos/builtin/packages/py-maestrowf/package.py
index 587d2e77b4faee..1cbcbe7c5acc79 100644
--- a/var/spack/repos/builtin/packages/py-maestrowf/package.py
+++ b/var/spack/repos/builtin/packages/py-maestrowf/package.py
@@ -21,18 +21,9 @@ class PyMaestrowf(PythonPackage):
     version("develop", branch="develop")
     version("master", branch="master")
 
-    # Development versions
-    version("1.1.9dev1", tag="1.1.9dev1")
-
-    # Pre-release candidates
-    version("1.1.7dev0", sha256="bcef838f13da396dd33cc7f503655de7a8f16ee5fe7b1e2a553044334a03f1f0")
-
     # pypi releases
-    version(
-        "1.1.8",
-        sha256="fa8f8eb8dd3adfb9646d7b0dfd498a00423d2131adbc8dbc8016c4159b2ec1d5",
-        preferred=True,
-    )
+    version("1.1.9", sha256="92d964fee7c944a2b08bb6148cbab66d0b8a02893d281d395971368cc2ecf957")
+    version("1.1.8", sha256="fa8f8eb8dd3adfb9646d7b0dfd498a00423d2131adbc8dbc8016c4159b2ec1d5")
     version("1.1.7", sha256="ff1b6696f30254b105fcadd297ad437c0c666ebc70124b231a713b89f47f4e94")
     version("1.1.6", sha256="9812e67d9bd83c452cc99d82fbceb3017b5e36dafdf52eda939748bad4a88756")
     version("1.1.4", sha256="6603b93494e8e9d939a4ab40ecdfe7923a85960a8a8bddea4734e230d8144016")
@@ -42,15 +33,27 @@ class PyMaestrowf(PythonPackage):
     version("1.1.0", sha256="1bfec546831f2ef577d7823bb50dcd12622644dad0d3d761998eafd0905b6977")
     version("1.0.1", sha256="dd42ffeac1f0492a576c630b37e5d3593273e59664407f2ebf78d49322d37146")
 
+    version(
+        "1.1.9dev1",
+        tag="1.1.9dev1",
+        commit="097e6b842fd4a9540432a9dec1e41f0d1a2c2c2f",
+        deprecated=True,
+    )
+    version(
+        "1.1.7dev0",
+        sha256="bcef838f13da396dd33cc7f503655de7a8f16ee5fe7b1e2a553044334a03f1f0",
+        deprecated=True,
+    )
+
     depends_on("python@2.7:2.8,3.5:", type=("build", "run"))
     depends_on("python@3.6:", type=("build", "run"), when="@1.1.9:")
     depends_on("py-setuptools", type=("build"), when="@:1.1.8")
-    depends_on("py-poetry@0.12:", type=("build"), when="@1.1.9:")
-    depends_on("py-cryptography@:2", type=("build"))
-    depends_on("py-pyyaml@4.2:", type=("build", "run"))
-    depends_on("py-six", type=("build", "run"))
-    depends_on("py-tabulate", type=("build", "run"), when="@1.1.0:")
-    depends_on("py-filelock", type=("build", "run"), when="@1.1.0:")
+    depends_on("py-poetry-core@1.0.8:", type=("build"), when="@1.1.9:")
     depends_on("py-coloredlogs", type=("build", "run"), when="@1.1.7:")
     depends_on("py-dill", type=("build", "run"), when="@1.1.7:")
+    depends_on("py-filelock", type=("build", "run"), when="@1.1.0:")
     depends_on("py-jsonschema@3.2.0:", type=("build", "run"), when="@1.1.7:")
+    depends_on("py-pyyaml@4.2:", type=("build", "run"))
+    depends_on("py-rich", type=("build", "run"), when="@1.1.9:")
+    depends_on("py-six", type=("build", "run"))
+    depends_on("py-tabulate", type=("build", "run"), when="@1.1.0:")
diff --git a/var/spack/repos/builtin/packages/py-markdown-it-py/package.py b/var/spack/repos/builtin/packages/py-markdown-it-py/package.py
index 11e04270073ad0..3ff44a6088f38a 100644
--- a/var/spack/repos/builtin/packages/py-markdown-it-py/package.py
+++ b/var/spack/repos/builtin/packages/py-markdown-it-py/package.py
@@ -16,14 +16,20 @@ class PyMarkdownItPy(PythonPackage):
     pypi = "markdown-it-py/markdown-it-py-1.1.0.tar.gz"
 
     version("3.0.0", sha256="e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb")
+    version("2.2.0", sha256="7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1")
     version("1.1.0", sha256="36be6bb3ad987bfdb839f5ba78ddf094552ca38ccbd784ae4f74a4e1419fc6e3")
 
+    variant("linkify", default=False, description="Linkify support")
+
     depends_on("python@3.8:", when="@2.1:", type=("build", "run"))
     depends_on("python@3.6:3", when="@:2.0", type=("build", "run"))
     depends_on("py-flit-core@3.4:3", when="@2.1:", type="build")
 
     depends_on("py-mdurl@0.1:0", when="@2:", type=("build", "run"))
 
+    depends_on("py-linkify-it-py@1.0", when="@1.1.0+linkify", type=("build", "run"))
+    depends_on("py-linkify-it-py@1:2", when="@2.2:+linkify", type=("build", "run"))
+
     # Historical dependencies
     depends_on("py-setuptools", when="@:2.0", type="build")
     depends_on("py-attrs@19:21", when="@:2.0", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-marshmallow/package.py b/var/spack/repos/builtin/packages/py-marshmallow/package.py
index 53e9ef2d50b0ca..f8de1f05346078 100644
--- a/var/spack/repos/builtin/packages/py-marshmallow/package.py
+++ b/var/spack/repos/builtin/packages/py-marshmallow/package.py
@@ -15,8 +15,10 @@ class PyMarshmallow(PythonPackage):
 
     maintainers("haralmha")
 
+    version("3.19.0", sha256="90032c0fd650ce94b6ec6dc8dfeb0e3ff50c144586462c389b81a07205bedb78")
     version("3.15.0", sha256="2aaaab4f01ef4f5a011a21319af9fce17ab13bf28a026d1252adab0e035648d5")
 
     depends_on("python@3.7:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("py-packaging", type=("build", "run"))
+    depends_on("py-packaging@17:", when="@3.19.0:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-matplotlib/package.py b/var/spack/repos/builtin/packages/py-matplotlib/package.py
index 0c6f99dd85ceb8..18d8d98f0ac880 100644
--- a/var/spack/repos/builtin/packages/py-matplotlib/package.py
+++ b/var/spack/repos/builtin/packages/py-matplotlib/package.py
@@ -17,26 +17,16 @@ class PyMatplotlib(PythonPackage):
     pypi = "matplotlib/matplotlib-3.3.2.tar.gz"
 
     maintainers("adamjstewart")
-    import_modules = [
-        "mpl_toolkits.axes_grid1",
-        "mpl_toolkits.axes_grid",
-        "mpl_toolkits.mplot3d",
-        "mpl_toolkits.axisartist",
-        "matplotlib",
-        "matplotlib.compat",
-        "matplotlib.tri",
-        "matplotlib.axes",
-        "matplotlib.sphinxext",
-        "matplotlib.cbook",
-        "matplotlib.backends",
-        "matplotlib.backends.qt_editor",
-        "matplotlib.style",
-        "matplotlib.projections",
-        "matplotlib.testing",
-        "matplotlib.testing.jpl_units",
-        "pylab",
+    skip_modules = [
+        "matplotlib.tests",
+        "mpl_toolkits.axes_grid1.tests",
+        "mpl_toolkits.axisartist.tests",
+        "mpl_toolkits.mplot3d.tests",
     ]
 
+    version("3.8.1", sha256="044df81c1f6f3a8e52d70c4cfcb44e77ea9632a10929932870dfaa90de94365d")
+    version("3.8.0", sha256="df8505e1c19d5c2c26aff3497a7cbd3ccfc2e97043d1e4db3e76afa399164b69")
+    version("3.7.3", sha256="f09b3dd6bdeb588de91f853bbb2d6f0ff8ab693485b0c49035eaa510cb4f142e")
     version("3.7.2", sha256="a8cdb91dddb04436bd2f098b8fdf4b81352e68cf4d2c6756fcc414791076569b")
     version("3.7.1", sha256="7b73305f25eab4541bd7ee0b96d87e53ae9c9f1823be5659b806cd85786fe882")
     version("3.7.0", sha256="8f6efd313430d7ef70a38a3276281cb2e8646b3a22b3b21eb227da20e15e6813")
@@ -141,28 +131,31 @@ class PyMatplotlib(PythonPackage):
     # https://matplotlib.org/stable/devel/dependencies.html
     # Runtime dependencies
     # Mandatory dependencies
+    depends_on("python@3.9:", when="@3.8:", type=("build", "link", "run"))
     depends_on("python@3.8:", when="@3.6:", type=("build", "link", "run"))
+    depends_on("python", type=("build", "link", "run"))
     depends_on("py-contourpy@1.0.1:", when="@3.6:", type=("build", "run"))
     depends_on("py-cycler@0.10:", type=("build", "run"))
     depends_on("py-fonttools@4.22:", when="@3.5:", type=("build", "run"))
-    depends_on("py-kiwisolver@1.0.1:", type=("build", "run"), when="@2.2.0:")
-    depends_on("py-numpy@1.20:", when="@3.7:", type=("build", "run"))
-    depends_on("py-numpy@1.19:", when="@3.6:", type=("build", "run"))
-    depends_on("py-numpy@1.17:", when="@3.5:", type=("build", "run"))
-    depends_on("py-numpy@1.16:", when="@3.4:", type=("build", "run"))
-    depends_on("py-numpy@1.15:", when="@3.3:", type=("build", "run"))
+    depends_on("py-kiwisolver@1.3.1:", when="@3.8.1:", type=("build", "run"))
+    depends_on("py-kiwisolver@1.0.1:", when="@2.2:", type=("build", "run"))
+    depends_on("py-numpy@1.21:1", when="@3.8:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.20:", when="@3.7:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.19:", when="@3.6:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.17:", when="@3.5:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.16:", when="@3.4:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.15:", when="@3.3:", type=("build", "link", "run"))
     depends_on("py-numpy@1.11:", type=("build", "run"))
     depends_on("py-packaging@20:", when="@3.6:", type=("build", "run"))
     depends_on("py-packaging", when="@3.5:", type=("build", "run"))
+    depends_on("pil@8:", when="@3.8.1:", type=("build", "run"))
     depends_on("pil@6.2:", when="@3.3:", type=("build", "run"))
-    depends_on("py-pyparsing@2.3.1:3.0", when="@3.7.2:", type=("build", "run"))
+    depends_on("py-pyparsing@2.3.1:3.0", when="@3.7.2", type=("build", "run"))
     depends_on("py-pyparsing@2.3.1:", when="@3.7:", type=("build", "run"))
     depends_on("py-pyparsing@2.2.1:", when="@3.4:", type=("build", "run"))
     depends_on("py-pyparsing@2.0.3,2.0.5:2.1.1,2.1.3:2.1.5,2.1.7:", type=("build", "run"))
     depends_on("py-python-dateutil@2.7:", when="@3.4:", type=("build", "run"))
     depends_on("py-python-dateutil@2.1:", type=("build", "run"))
-    depends_on("py-setuptools@42:", when="@3.7.2:", type=("build", "run"))
-    depends_on("py-setuptools", type=("build", "run"))
     depends_on("py-importlib-resources@3.2:", when="@3.7: ^python@:3.9", type=("build", "run"))
 
     # Historical dependencies
@@ -177,13 +170,16 @@ class PyMatplotlib(PythonPackage):
         depends_on("tk@8.4:8.5,8.6.2:", when="backend=" + backend, type="run")
         depends_on("python+tkinter", when="backend=" + backend, type="run")
     # Qt
+    # matplotlib/backends/qt_compat.py
     for backend in ["qt4agg", "qt4cairo"]:
         depends_on("py-pyqt4@4.6:", when="backend=" + backend, type="run")
+        depends_on("qt+gui", when="backend=" + backend, type="run")
     for backend in ["qt5agg", "qt5cairo"]:
         depends_on("py-pyqt5", when="backend=" + backend, type="run")
-    # https://github.com/spack/spack/pull/32696
-    # for backend in ["qtagg", "qtcairo"]:
-    #     depends_on("py-pyqt6@6.1:", when="backend=" + backend, type="run")
+        depends_on("qt+gui", when="backend=" + backend, type="run")
+    for backend in ["qtagg", "qtcairo"]:
+        depends_on("py-pyqt6@6.1:", when="backend=" + backend, type="run")
+        depends_on("qt-base+gui+widgets", when="backend=" + backend, type="run")
     # GTK
     for backend in ["gtk", "gtkagg", "gtkcairo", "gtk3agg", "gtk3cairo", "gtk4agg", "gtk4cairo"]:
         depends_on("py-pygobject", when="backend=" + backend, type="run")
@@ -228,7 +224,12 @@ class PyMatplotlib(PythonPackage):
     # Dependencies for building matplotlib
     # Setup dependencies
     depends_on("py-certifi@2020.6.20:", when="@3.3.1:", type="build")
+    depends_on("py-numpy@1.25:", when="@3.8:", type="build")
     depends_on("py-pybind11@2.6:", when="@3.7:", type="build")
+    depends_on("py-setuptools@64:", when="@3.8.1:", type="build")
+    depends_on("py-setuptools@42:", when="@3.8:", type="build")
+    depends_on("py-setuptools@42:", when="@3.7.2:3.7", type=("build", "run"))
+    depends_on("py-setuptools", when="@:3.7.1", type=("build", "run"))
     depends_on("py-setuptools-scm@7:", when="@3.6:", type="build")
     depends_on("py-setuptools-scm@4:6", when="@3.5", type="build")
     depends_on("py-setuptools-scm-git-archive", when="@3.5", type="build")
@@ -258,6 +259,12 @@ def config_file(self):
     def archive_files(self):
         return [os.path.join(self.build_directory, self.config_file)]
 
+    def flag_handler(self, name, flags):
+        if name == "cxxflags":
+            if self.spec.satisfies("%oneapi"):
+                flags.append("-Wno-error=register")
+        return (flags, None, None)
+
     def setup_build_environment(self, env):
         include = []
         library = []
diff --git a/var/spack/repos/builtin/packages/py-maturin/package.py b/var/spack/repos/builtin/packages/py-maturin/package.py
index 926511ce750b40..487e808e6aaf38 100644
--- a/var/spack/repos/builtin/packages/py-maturin/package.py
+++ b/var/spack/repos/builtin/packages/py-maturin/package.py
@@ -15,6 +15,7 @@ class PyMaturin(PythonPackage):
     pypi = "maturin/maturin-0.13.7.tar.gz"
 
     version("1.1.0", sha256="4650aeaa8debd004b55aae7afb75248cbd4d61cd7da2dcf4ead8b22b58cecae0")
+    version("0.14.17", sha256="fb4e3311e8ce707843235fbe8748a05a3ae166c3efd6d2aa335b53dfc2bd3b88")
     version("0.13.7", sha256="c0a77aa0c57f945649ca711c806203a1b6888ad49c2b8b85196ffdcf0421db77")
 
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-mda-xdrlib/package.py b/var/spack/repos/builtin/packages/py-mda-xdrlib/package.py
new file mode 100644
index 00000000000000..e33c8354863ee1
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-mda-xdrlib/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyMdaXdrlib(PythonPackage):
+    """A stand-alone XDRLIB module extracted from CPython 3.10.8"""
+
+    homepage = "https://github.com/MDAnalysis/mda-xdrlib"
+    pypi = "mda_xdrlib/mda_xdrlib-0.2.0.tar.gz"
+
+    maintainers("RMeli")
+
+    version("0.2.0", sha256="f26f7158a83c32b96d15b530fce2cbc1190c4b7024e41faa4ab3e3db74e272af")
+
+    depends_on("py-setuptools@61.2:", type="build")
+    depends_on("py-tomli", when="^python@:3.10", type="build")
diff --git a/var/spack/repos/builtin/packages/py-mdanalysis/package.py b/var/spack/repos/builtin/packages/py-mdanalysis/package.py
index d26c95c568a258..f961975e3c5372 100644
--- a/var/spack/repos/builtin/packages/py-mdanalysis/package.py
+++ b/var/spack/repos/builtin/packages/py-mdanalysis/package.py
@@ -18,6 +18,8 @@ class PyMdanalysis(PythonPackage):
 
     maintainers("RMeli")
 
+    version("2.6.1", sha256="9cc69b94bddd026f26ffcaf5bdbed6d568c1c10e19a341d84f8d37a2a70222f2")
+    version("2.6.0", sha256="210b198a115165004c36fbbbe5eb83a13323f52b10ccaef30dd40bfe25ba3e61")
     version("2.5.0", sha256="06ce4efab6ca1dbd2ee2959fc668049e1d574a8fe94ab948a4608244da1d016b")
     version("2.4.3", sha256="c4fbdc414e4fdda69052fff2a6e412180fe6fa90a42c24793beee04123648c92")
     version("2.4.2", sha256="ae2ee5627391e73f74eaa3c547af3ec6ab8b040d27dedffe3a7ece8e0cd27636")
@@ -37,6 +39,7 @@ class PyMdanalysis(PythonPackage):
     depends_on("py-cython@0.28:", type="build")
 
     # MDAnalysis required dependencies (install_requires)
+    depends_on("py-numpy@1.22.3:1", when="@2.6.0:", type=("build", "run"))
     depends_on("py-numpy@1.21.0:", when="@2.5.0:", type=("build", "run"))
     depends_on("py-numpy@1.20.0:", type=("build", "run"))
 
@@ -53,14 +56,23 @@ class PyMdanalysis(PythonPackage):
     depends_on("py-threadpoolctl", type=("build", "run"))
     depends_on("py-packaging", type=("build", "run"))
     depends_on("py-fasteners", type=("build", "run"))
-    depends_on("py-gsd@1.9.3:", type=("build", "run"))
+    depends_on("py-gsd@1.9.3:", when="@:2.5.0", type=("build", "run"))
 
     # extra_format (extras_require)
     depends_on("py-netcdf4@1.0:", when="+extra_formats", type=("build", "run"))
     depends_on("py-h5py@2.10:", when="+extra_formats", type=("build", "run"))
+    depends_on("py-pytng@0.2.3:", when="+extra_formats", type=("build", "run"))
     depends_on("py-chemfiles@0.10:", when="+extra_formats", type=("build", "run"))
     depends_on("py-pyedr@0.7.0:", when="+extra_formats", type=("build", "run"))
-    depends_on("py-pytng@0.2.3:", when="+extra_formats", type=("build", "run"))
+    # py-gsd is now an optional dependency and requires >3.0.0
+    # gsd>=2.9.0 requires setuptools>=64 and can't be concretised with py-numpy
+    #    depends_on("py-gsd@3.0.1:", when="+extra_formats @2.6.0:", type=("build", "run"))
+    depends_on(
+        "rdkit@2020.03.1: +python ~coordgen ~maeparser ~yaehmop ~descriptors3d",
+        when="+extra_formats @2.6.0:",
+        type=("build", "run"),
+    )
+    depends_on("py-parmed", when="+extra_formats @2.6.0:", type=("build", "run"))
 
     # analysis (extras_require)
     depends_on("py-seaborn", when="+analysis", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-mdanalysistests/package.py b/var/spack/repos/builtin/packages/py-mdanalysistests/package.py
index b6934d9a50dfdc..df2633ef74c8b3 100644
--- a/var/spack/repos/builtin/packages/py-mdanalysistests/package.py
+++ b/var/spack/repos/builtin/packages/py-mdanalysistests/package.py
@@ -14,16 +14,20 @@ class PyMdanalysistests(PythonPackage):
 
     maintainers("RMeli")
 
+    version("2.6.1", sha256="043f7451f4d9c42ea9e6609a81a6002948e2c74fd268282e0831416789b22e5e")
+    version("2.6.0", sha256="16fdd10e5240b606e8f9210b7cbd9e4be110e6b8d79bb6e72ce6250c4731a817")
     version("2.5.0", sha256="a15b53b7f8bed67900a2bf542bbb3cab81dc71674fa6cddb3248dd11880e4c9d")
     version("2.4.3", sha256="6fbdeccdbfb249f76520ee3605d007cd70292187e3754d0184c71e5afe133abb")
     version("2.4.2", sha256="6e8fb210a4268691c77717ea5157e82d85874a4f7ee0f8f177718451a44ee793")
 
     # Version need to match MDAnalysis'
+    depends_on("py-mdanalysis@2.6.1", when="@2.6.1", type=("build", "run"))
+    depends_on("py-mdanalysis@2.6.0", when="@2.6.0", type=("build", "run"))
     depends_on("py-mdanalysis@2.5.0", when="@2.5.0", type=("build", "run"))
     depends_on("py-mdanalysis@2.4.3", when="@2.4.3", type=("build", "run"))
     depends_on("py-mdanalysis@2.4.2", when="@2.4.2", type=("build", "run"))
 
-    depends_on("python@3.9:", when="@2.5.0", type=("build", "run"))
+    depends_on("python@3.9:", when="@2.5.0:", type=("build", "run"))
     depends_on("python@3.8:", type=("build", "run"))
 
     depends_on("py-pytest@3.3.0:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-melissa-core/package.py b/var/spack/repos/builtin/packages/py-melissa-core/package.py
index fb2eab70ff9f14..b9f4d00babe41b 100644
--- a/var/spack/repos/builtin/packages/py-melissa-core/package.py
+++ b/var/spack/repos/builtin/packages/py-melissa-core/package.py
@@ -17,7 +17,9 @@ class PyMelissaCore(PythonPackage):
     git = "https://gitlab.inria.fr/melissa/melissa.git"
     maintainers("robcaulk", "mschouler", "raffino")
 
-    version("develop", branch="develop")
+    version("develop", branch="develop", preferred=True)
+    version("joss", tag="JOSS_v2", commit="20bbe68c1a7b73aa2ea3ad35681c332c7a5fc516")
+    version("sc23", tag="SC23", commit="8bb5b6817d4abe4eaa5893552d711150e53535f3")
 
     # define variants for the deep learning server (torch, tf)
     variant(
@@ -31,17 +33,24 @@ class PyMelissaCore(PythonPackage):
     depends_on("py-setuptools@46.4:", type=("build"))
     # requirements.txt (SA)
     depends_on("py-pyzmq@22.3.0:", type="run")
-    depends_on("py-mpi4py@3.1.3:", type="run")
+    depends_on("py-mpi4py@3.1.3:", when="@develop,joss", type="run")
+    depends_on("py-mpi4py@3.1.3", when="@sc23", type="run")
     depends_on("py-numpy@1.21:", type="run")
     depends_on("py-jsonschema@4.5:", type="run")
-    depends_on("py-python-rapidjson@1.8:", type="run")
+    depends_on("py-python-rapidjson@1.8:", when="@develop,joss", type="run")
+    depends_on("py-python-rapidjson@1.9:", when="@sc23", type="run")
     depends_on("py-scipy@1.10.0:", type="run")
-    depends_on("py-cloudpickle@2.2.0:", type="run")
+    depends_on("py-plotext@5.2.8:", type="run")
+    depends_on("py-cloudpickle@2.2.0:", type="run", when="@develop,joss")
+    depends_on("py-iterative-stats@0.1.0", type="run", when="@develop")
+    depends_on("py-iterative-stats@0.0.4", type="run", when="@joss")
     # requirements_deep_learning.txt (DL with torch)
     depends_on("py-tensorboard@2.10.0:", type="run", when="+torch")
     depends_on("py-matplotlib", type="run", when="+torch")
     depends_on("py-torch@1.12.1:", type="run", when="+torch")
+    depends_on("py-python-hostlist", type="run", when="@sc23+torch")
     # requirements_deep_learning.txt  (DL with tensorflow)
     depends_on("py-tensorboard@2.10.0:", type="run", when="+tf")
     depends_on("py-matplotlib", type="run", when="+tf")
     depends_on("py-tensorflow@2.8.0:", type="run", when="+tf")
+    conflicts("@sc23", when="+tf", msg="tensorflow is only supported with newer versions")
diff --git a/var/spack/repos/builtin/packages/py-merlin/package.py b/var/spack/repos/builtin/packages/py-merlin/package.py
index 64ef08ad9118b2..35296b5c394d5e 100644
--- a/var/spack/repos/builtin/packages/py-merlin/package.py
+++ b/var/spack/repos/builtin/packages/py-merlin/package.py
@@ -14,6 +14,7 @@ class PyMerlin(PythonPackage):
     git = "https://github.com/LLNL/merlin.git"
     tags = ["radiuss"]
 
+    version("1.10.3", sha256="6edaf17b502db090cef0bc53ae0118c55f77d7a16f43c7a235e0dd1770decadb")
     version("1.7.5", sha256="1994c1770ec7fc9da216f9d0ca8214684dcc0daa5fd55337b96e308b2e68daaa")
     version("1.7.4", sha256="9a6f8a84a1b52d0bfb0dc7bdbd7e15677e4a1041bd25a49a2d965efe503a6c20")
     version("1.4.1", sha256="9d515cfdbcde2443892afd92b78dbc5bf2aed2060ed3a336e683188e015bca7c")
@@ -23,14 +24,23 @@ class PyMerlin(PythonPackage):
     depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-setuptools", type=("build", "run"))
     depends_on("py-cached-property", type=("build", "run"))
-    depends_on("py-celery@5.0.0+redis+sqlalchemy", when="@1.7.5:", type=("build", "run"))
-    depends_on("py-celery@4.3.0:4+redis+sqlalchemy", when="@:1.7.4", type=("build", "run"))
-    depends_on("py-coloredlogs@10.0:", type=("build", "run"))
+    depends_on("py-celery@5.0.3:+redis+sqlalchemy", when="@1.7.6:", type=("build", "run"))
+    depends_on("py-celery@5.0.0:+redis+sqlalchemy", when="@1.7.5", type=("build", "run"))
+    depends_on("py-celery@4.4.5:+redis+sqlalchemy", when="@1.6.2:1.7.4", type=("build", "run"))
+    depends_on("py-celery@4.3.0:+redis+sqlalchemy", when="@:1.5.2", type=("build", "run"))
+    depends_on("py-celery@4.3.0:+redis", when="@1.4.1:1.5.1", type=("build", "run"))
+    # The upperbound on py-celery is not in requirements.txt because there was no 5.x release
+    # then. See commit 61b4fc708a3d6fd22709b35836065c778bf6304e for why it's needed.
+    depends_on("py-celery@:4", when="@:1.7.4", type=("build", "run"))
+    depends_on("py-coloredlogs", type=("build", "run"))
     depends_on("py-cryptography", type=("build", "run"))
-    depends_on("py-maestrowf@1.1.7dev0", when="@1.2.0:", type=("build", "run"))
+    depends_on("py-importlib-metadata@:4", when="@1.10: ^python@3.7", type=("build", "run"))
+    depends_on("py-maestrowf@1.1.9:", when="@1.9.0:", type=("build", "run"))
+    depends_on("py-maestrowf@1.1.7dev0", when="@1.2.0:1.8", type=("build", "run"))
     depends_on("py-maestrowf@1.1.6:", when="@:1.1", type=("build", "run"))
     depends_on("py-numpy", type=("build", "run"))
     depends_on("py-parse", type=("build", "run"))
     depends_on("py-psutil@5.1.0:", type=("build", "run"))
     depends_on("py-pyyaml@5.1.2:", type=("build", "run"))
     depends_on("py-tabulate", type=("build", "run"))
+    depends_on("py-redis@4.3.4:", when="@1.9.0:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-metomi-isodatetime/package.py b/var/spack/repos/builtin/packages/py-metomi-isodatetime/package.py
new file mode 100644
index 00000000000000..feaa89161827dd
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-metomi-isodatetime/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyMetomiIsodatetime(PythonPackage):
+    """Python ISO 8601 date time parser and data model/manipulation utilities."""
+
+    homepage = "https://github.com/metomi/isodatetime"
+    pypi = "metomi-isodatetime/metomi-isodatetime-1!3.0.0.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("3.0.0", sha256="2141e8aaa526ea7f7f1cb883e6c8ed83ffdab73269658d84d0624f63a6e1357e")
+
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-metomi-rose/package.py b/var/spack/repos/builtin/packages/py-metomi-rose/package.py
new file mode 100644
index 00000000000000..161edeb582ce8e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-metomi-rose/package.py
@@ -0,0 +1,27 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyMetomiRose(PythonPackage):
+    """Rose, a framework for meteorological suites."""
+
+    homepage = "https://metomi.github.io/rose/doc/html/index.html"
+    pypi = "metomi-rose/metomi-rose-2.1.0.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("2.1.0", sha256="1b60135a434fe4325d364a57e8f5e81e90f39b373b9d68733458c1adc2513c05")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-aiofiles", type=("build", "run"))
+    depends_on("py-jinja2@2.10.1:", type=("build", "run"))
+    depends_on("py-keyring@23", type=("build", "run"))
+    depends_on("py-ldap3", type=("build", "run"))
+    depends_on("py-metomi-isodatetime@3", type=("build", "run"))
+    depends_on("py-psutil@5.6.0:", type=("build", "run"))
+    depends_on("py-requests", type=("build", "run"))
+    depends_on("py-sqlalchemy@1", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-minkowskiengine/package.py b/var/spack/repos/builtin/packages/py-minkowskiengine/package.py
index 719ea80ba6625e..03b5609140e921 100644
--- a/var/spack/repos/builtin/packages/py-minkowskiengine/package.py
+++ b/var/spack/repos/builtin/packages/py-minkowskiengine/package.py
@@ -19,6 +19,11 @@ class PyMinkowskiengine(PythonPackage, CudaPackage):
 
     depends_on("py-setuptools", type="build")
     depends_on("py-pybind11", type="link")
+    # in newer pip versions --install-option does not exist
+    depends_on("py-pip@:23.0", type="build")
+    # According to the documentation other BLAS should be possible too, but
+    # they didn't work, see https://github.com/spack/spack/pull/38742
+    depends_on("openblas")
 
     depends_on("py-numpy", type=("build", "run"))
     depends_on("py-torch", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-mistletoe/package.py b/var/spack/repos/builtin/packages/py-mistletoe/package.py
new file mode 100644
index 00000000000000..74e0d8928e1374
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-mistletoe/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyMistletoe(PythonPackage):
+    """A fast, extensible Markdown parser in pure Python."""
+
+    homepage = "https://github.com/miyuchina/mistletoe"
+    pypi = "mistletoe/mistletoe-1.2.1.tar.gz"
+
+    version("1.2.1", sha256="7d0c1ab3747047d169f9fc4b925d1cba3f5c13eaf0b90c365b72e47e59d00a02")
+
+    depends_on("python@3.5:3", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-ml-dtypes/package.py b/var/spack/repos/builtin/packages/py-ml-dtypes/package.py
new file mode 100644
index 00000000000000..192069e21973d1
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-ml-dtypes/package.py
@@ -0,0 +1,25 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyMlDtypes(PythonPackage):
+    """A stand-alone implementation of several NumPy dtype extensions
+    used in machine learning libraries."""
+
+    homepage = "https://github.com/jax-ml/ml_dtypes"
+    pypi = "ml_dtypes/ml_dtypes-0.3.1.tar.gz"
+    git = "https://github.com/jax-ml/ml_dtypes.git"
+    submodules = True
+
+    version("0.3.1", tag="v0.3.1", commit="bbeedd470ecac727c42e97648c0f27bfc312af30")
+    version("0.2.0", tag="v0.2.0", commit="5b9fc9ad978757654843f4a8d899715dbea30e88")
+
+    depends_on("python@3.9:", when="@0.3:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.21:", type=("build", "link", "run"))
+    # Build dependencies are overconstrained, older versions work just fine
+    depends_on("py-pybind11", type=("build", "link"))
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-moarchiving/package.py b/var/spack/repos/builtin/packages/py-moarchiving/package.py
new file mode 100644
index 00000000000000..a43a9c9efad260
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-moarchiving/package.py
@@ -0,0 +1,25 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyMoarchiving(PythonPackage):
+    """
+    Biobjective Archive class with hypervolume indicator and uncrowded
+    hypervolume improvement computation.
+    """
+
+    homepage = "https://github.com/CMA-ES/moarchiving"
+    pypi = "moarchiving/moarchiving-0.6.0.tar.gz"
+
+    maintainers("LydDeb")
+
+    version("0.6.0", sha256="705ded992d399bc1ac703e68391bded6f64e1bde81b2bb25061eaa6208b5b29a")
+
+    variant("arbitrary_precision", default=False, description="Build with Fraction support")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-fraction", when="+arbitrary_precision", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-model-index/package.py b/var/spack/repos/builtin/packages/py-model-index/package.py
new file mode 100644
index 00000000000000..111d44af30ea41
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-model-index/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyModelIndex(PythonPackage):
+    """Create a source of truth for ML model results and browse it on Papers with Code"""
+
+    homepage = "https://github.com/paperswithcode/model-index"
+    git = "https://github.com/paperswithcode/model-index.git"
+
+    version("0.1.11", commit="a39af5f8aaa2a90b8fc7180744a855282360067a")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-pyyaml", type=("build", "run"))
+    depends_on("py-markdown", type=("build", "run"))
+    depends_on("py-ordered-set", type=("build", "run"))
+    depends_on("py-click", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-modred/package.py b/var/spack/repos/builtin/packages/py-modred/package.py
index 27a45fc013e62c..565da9c41e86b8 100644
--- a/var/spack/repos/builtin/packages/py-modred/package.py
+++ b/var/spack/repos/builtin/packages/py-modred/package.py
@@ -14,11 +14,11 @@ class PyModred(PythonPackage):
     homepage = "https://github.com/belson17/modred"
     git = "https://github.com/belson17/modred.git"
 
-    version("2.0.4", tag="v2.0.4")
-    version("2.0.3", tag="v2.0.3")
-    version("2.0.2", tag="v2.0.2")
-    version("2.0.1", tag="v2.0.1")
-    version("2.0.0", tag="v2.0.0")
+    version("2.0.4", tag="v2.0.4", commit="b793efd353434799ec8c4c350757037f87dcf99a")
+    version("2.0.3", tag="v2.0.3", commit="70f61fddf4192a33952f5c98103d2b90955c4e79")
+    version("2.0.2", tag="v2.0.2", commit="674d6962f87c93697e4cbb4efd0785cd3398c4b1")
+    version("2.0.1", tag="v2.0.1", commit="a557e72d6416b4beefd006ab3e90df7ae1ec6ddd")
+    version("2.0.0", tag="v2.0.0", commit="ebf219ccedb3d753fad825d9b20ac9b14b84f404")
 
     depends_on("py-setuptools", type="build")
     depends_on("py-numpy", type="run")
diff --git a/var/spack/repos/builtin/packages/py-morph-tool/package.py b/var/spack/repos/builtin/packages/py-morph-tool/package.py
new file mode 100644
index 00000000000000..7927b468c07edf
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-morph-tool/package.py
@@ -0,0 +1,39 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyMorphTool(PythonPackage):
+    """Python morphology manipulation toolkit"""
+
+    homepage = "https://github.com/BlueBrain/morph-tool"
+    git = "https://github.com/BlueBrain/morph-tool.git"
+    pypi = "morph-tool/morph-tool-2.9.1.tar.gz"
+
+    version("master", branch="master")
+    version("2.9.1", sha256="305e9456c8047726588b23dfa070eb95ccbe5573e9fea3e0a83dc93eacdf61dc")
+    version("2.9.0", sha256="c60d4010e17ddcc3f53c864c374fffee05713c8f8fd2ba4eed7706041ce1fa47")
+
+    variant("nrn", default=False, description="Enable additional neuron support")
+    variant("plot", default=False, description="Enable additional plotly support")
+    variant("parallel", default=False, description="Enable additional parallel support")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-setuptools-scm", type="build")
+
+    depends_on("py-click@6.7:", type=("build", "run"))
+    depends_on("py-deprecation@2.1.0:", type=("build", "run"))
+    depends_on("py-more-itertools@8.6.0:", type=("build", "run"))
+    depends_on("py-morphio@3", type=("build", "run"))
+    depends_on("py-neurom@3", type=("build", "run"))
+    depends_on("py-numpy@1.14:", type=("build", "run"))
+    depends_on("py-pandas@1.0.3:", type=("build", "run"))
+    depends_on("py-xmltodict@0.12.0:", type=("build", "run"))
+
+    depends_on("py-plotly@4.1.0:", type=("build", "run"), when="+plot")
+    depends_on("py-dask+bag@2.19.0:", type=("build", "run"), when="+parallel")
+    depends_on("neuron+python@7.8:", type=("build", "run"), when="+nrn")
+    depends_on("py-bluepyopt@1.9.37:", type=("build", "run"), when="+nrn")
diff --git a/var/spack/repos/builtin/packages/py-morphio/package.py b/var/spack/repos/builtin/packages/py-morphio/package.py
new file mode 100644
index 00000000000000..a5a9fee7deaf3c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-morphio/package.py
@@ -0,0 +1,30 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import sys
+
+from spack.package import *
+
+
+class PyMorphio(PythonPackage):
+    """Python library for reading / writing morphology files"""
+
+    homepage = "https://github.com/BlueBrain/MorphIO"
+    git = "https://github.com/BlueBrain/MorphIO.git"
+    pypi = "morphio/MorphIO-3.3.2.tar.gz"
+
+    version("master", branch="master", submodules=True)
+
+    version("3.3.6", sha256="0f2e55470d92a3d89f2141ae905ee104fd16257b93dafb90682d90171de2f4e6")
+
+    depends_on("py-setuptools@24.2:", type="build")
+    depends_on("py-setuptools-scm", type="build")
+
+    depends_on("ninja", type="build")
+    depends_on("cmake@3.2:", type="build")
+    depends_on("py-numpy@1.14.1:", type=("build", "run"))
+    depends_on("py-h5py@3", when="platform=windows", type=("build", "run"))
+    if sys.platform != "win32":
+        depends_on("hdf5")
diff --git a/var/spack/repos/builtin/packages/py-mpi4py/package.py b/var/spack/repos/builtin/packages/py-mpi4py/package.py
index 96133bff2cebed..402429b6216ed0 100644
--- a/var/spack/repos/builtin/packages/py-mpi4py/package.py
+++ b/var/spack/repos/builtin/packages/py-mpi4py/package.py
@@ -28,17 +28,15 @@ class PyMpi4py(PythonPackage):
     version("2.0.0", sha256="6543a05851a7aa1e6d165e673d422ba24e45c41e4221f0993fe1e5924a00cb81")
     version("1.3.1", sha256="e7bd2044aaac5a6ea87a87b2ecc73b310bb6efe5026031e33067ea3c2efc3507")
 
-    depends_on("python@2.6:2.7,3.2:")
-    depends_on("python@2.7:2.8,3.5:", when="@3.1:")
     depends_on("py-setuptools@40.9:", type="build")
-    # in newer pip versions --install-option does not exist
-    depends_on("py-pip@:23.0", type="build")
     depends_on("mpi")
     depends_on("py-cython@0.27.0:", type="build")
 
-    @when("@3.1:")
-    def install_options(self, spec, prefix):
-        return ["--mpicc=%s -shared" % spec["mpi"].mpicc]
+    # https://github.com/mpi4py/mpi4py/pull/311
+    conflicts("^py-cython@3:")
+
+    def setup_build_environment(self, env):
+        env.set("MPICC", f"{self.spec['mpi'].mpicc} -shared")
 
     @run_before("install")
     def cythonize(self):
diff --git a/var/spack/repos/builtin/packages/py-multiqc/package.py b/var/spack/repos/builtin/packages/py-multiqc/package.py
index 71efcb3c98cb72..c4f79799862fdd 100644
--- a/var/spack/repos/builtin/packages/py-multiqc/package.py
+++ b/var/spack/repos/builtin/packages/py-multiqc/package.py
@@ -14,6 +14,7 @@ class PyMultiqc(PythonPackage):
     homepage = "https://multiqc.info"
     pypi = "multiqc/multiqc-1.0.tar.gz"
 
+    version("1.15", sha256="ce5359a12226cf4ce372c6fdad142cfe2ae7501ffa97ac7aab544ced4db5ea3c")
     version("1.14", sha256="dcbba405f0c9521ed2bbd7e8f7a9200643047311c9619878b81d167300149362")
     version("1.13", sha256="0564fb0f894e6ca0822a0f860941b3ed2c33dce407395ac0c2103775d45cbfa0")
     version("1.7", sha256="02e6a7fac7cd9ed036dcc6c92b8f8bcacbd28983ba6be53afb35e08868bd2d68")
@@ -24,7 +25,7 @@ class PyMultiqc(PythonPackage):
     depends_on("python@2.7:", when="@:1.7", type=("build", "run"))
     depends_on("python@3:", when="@1.9:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
-    depends_on("py-matplotlib@2.1.1:", type=("build", "run"), when="@:1.13")
+    depends_on("py-matplotlib@2.1.1:", type=("build", "run"), when="@1.13:")
     depends_on("py-matplotlib@2.1.1:2", type=("build", "run"), when="@1.7")
     depends_on("py-matplotlib@:2.1.0", type=("build", "run"), when="@1.5")
     depends_on("py-matplotlib", type=("build", "run"), when="@:1.3")
diff --git a/var/spack/repos/builtin/packages/py-nanobind/package.py b/var/spack/repos/builtin/packages/py-nanobind/package.py
index de2a1c7cc7aaf3..5c39cf271cfc14 100644
--- a/var/spack/repos/builtin/packages/py-nanobind/package.py
+++ b/var/spack/repos/builtin/packages/py-nanobind/package.py
@@ -20,15 +20,39 @@ class PyNanobind(PythonPackage):
     url = "https://github.com/wjakob/nanobind/archive/refs/tags/v1.2.0.tar.gz"
     git = "https://github.com/wjakob/nanobind.git"
 
-    maintainers("ma595")
+    maintainers("chrisrichardson", "garth-wells", "ma595")
 
     version("master", branch="master", submodules=True)
-    version("1.4.0", tag="v1.4.0", submodules=True)
-    version("1.2.0", tag="v1.2.0", submodules=True)
+    version(
+        "1.7.0", tag="v1.7.0", commit="555ec7595c89c60ce7cf53e803bc226dc4899abb", submodules=True
+    )
+    version(
+        "1.6.2", tag="v1.6.2", commit="cc5ac7e61def198db2a8b65c6d630343987a9f1d", submodules=True
+    )
+    version(
+        "1.5.2", tag="v1.5.2", commit="b0e24d5b0ab0d518317d6b263a257ae72d4d29a2", submodules=True
+    )
+    version(
+        "1.5.1", tag="v1.5.1", commit="ec6168d06dbf2ab94c31858223bd1d7617222706", submodules=True
+    )
+    version(
+        "1.5.0", tag="v1.5.0", commit="e85a51049db500383808aaa4a77306ff37d96131", submodules=True
+    )
+    version(
+        "1.4.0", tag="v1.4.0", commit="05cba0ef85ba2bb68aa115af4b74c30aa2aa7bec", submodules=True
+    )
+    version(
+        "1.2.0", tag="v1.2.0", commit="ec9350b805d2fe568f65746fd69225eedc5e37ae", submodules=True
+    )
 
     depends_on("python@3.8:", type=("build", "run"))
     depends_on("py-setuptools@42:", type="build")
     depends_on("py-scikit-build", type="build")
 
-    depends_on("py-cmake@3.17:", type="build")
-    depends_on("py-ninja", type="build")
+    depends_on("cmake@3.17:", type="build")
+    depends_on("ninja", type="build")
+
+    @property
+    def cmake_prefix_paths(self):
+        paths = [join_path(self.prefix, self.spec["python"].package.platlib, "nanobind", "cmake")]
+        return paths
diff --git a/var/spack/repos/builtin/packages/py-nbclassic/package.py b/var/spack/repos/builtin/packages/py-nbclassic/package.py
index 0f9bf98d9c9465..e46a6cd01e48e4 100644
--- a/var/spack/repos/builtin/packages/py-nbclassic/package.py
+++ b/var/spack/repos/builtin/packages/py-nbclassic/package.py
@@ -18,9 +18,7 @@ class PyNbclassic(PythonPackage):
     version("0.3.1", sha256="f920f8d09849bea7950e1017ff3bd101763a8d68f565a51ce053572e65aa7947")
 
     depends_on("py-setuptools", type="build")
-    # TODO: replace this after concretizer learns how to concretize separate build deps
-    depends_on("py-jupyter-packaging11", when="@0.3.3:", type="build")
-    # depends_on('py-jupyter-packaging@0.9:1', when='@0.3.3:', type='build')
+    depends_on("py-jupyter-packaging@0.9:0", when="@0.3.3:", type="build")
     depends_on("py-babel", when="@0.4:", type="build")
 
     depends_on("py-jinja2", when="@0.4:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-nbmake/package.py b/var/spack/repos/builtin/packages/py-nbmake/package.py
index 8754947b56bcae..7fc05fe369e363 100644
--- a/var/spack/repos/builtin/packages/py-nbmake/package.py
+++ b/var/spack/repos/builtin/packages/py-nbmake/package.py
@@ -12,6 +12,7 @@ class PyNbmake(PythonPackage):
     homepage = "https://github.com/treebeardtech/nbmake"
     pypi = "nbmake/nbmake-0.5.tar.gz"
 
+    version("1.4.3", sha256="9afc46ba05cc22f5a78047a758dca32386c95eaaa41501b25ce108cf733d9622")
     version("1.4.1", sha256="7f602ba5195e80e4f2527944bb06d3b4df0d1520e73ba66126b51132b1f646ea")
     version("1.4", sha256="2d3b97b83a8a378d5d828ad7b5412e509b82ed883662af16533236c909cfa20a")
     version("1.3.5", sha256="95d4716928171120fae562e69440989a636e2af8616c829573e9574f5bea30db")
@@ -33,7 +34,6 @@ class PyNbmake(PythonPackage):
 
     depends_on("py-setuptools", type="build")
     depends_on("py-poetry-core@1:", type="build")
-    depends_on("py-pydantic@1.7.2:1", type=("build", "run"))
     depends_on("py-pytest@6.1.0:", when="@1.2.1:", type=("build", "run"))
     depends_on("py-pytest@6.1.0:6", when="@0.10:1.2.0", type=("build", "run"))
     depends_on("py-pytest@6.1.2:6", when="@:0.9", type=("build", "run"))
@@ -47,3 +47,6 @@ class PyNbmake(PythonPackage):
     depends_on("py-pygments@2.7.3:2", type=("build", "run"))
     depends_on("py-ipykernel@5.4.0:", when="@0.7:", type=("build", "run"))
     depends_on("py-ipykernel@5.4.0:5", when="@0.5", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-pydantic@1.7.2:1", when="@:1.4.1", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-ndindex/package.py b/var/spack/repos/builtin/packages/py-ndindex/package.py
new file mode 100644
index 00000000000000..b8d579c08a33fc
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-ndindex/package.py
@@ -0,0 +1,17 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyNdindex(PythonPackage):
+    """A Python library for manipulating indices of ndarrays."""
+
+    homepage = "https://quansight-labs.github.io/ndindex/"
+    pypi = "ndindex/ndindex-1.7.tar.gz"
+
+    version("1.7", sha256="bf9bd0b76eeada1c8275e04091f8291869ed2b373b7af48e56faf7579fd2efd2")
+
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-networkit/package.py b/var/spack/repos/builtin/packages/py-networkit/package.py
index b3a337455f8758..6279cb0a017868 100644
--- a/var/spack/repos/builtin/packages/py-networkit/package.py
+++ b/var/spack/repos/builtin/packages/py-networkit/package.py
@@ -44,6 +44,8 @@ class PyNetworkit(PythonPackage):
     depends_on("py-scipy", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("python@3:", type=("build", "run"))
+    # in newer pip versions --install-option does not exist
+    depends_on("py-pip@:23.0", type="build")
 
     def install_options(self, spec, prefix):
         # Enable ext. core-library + parallel build
diff --git a/var/spack/repos/builtin/packages/py-neurom/package.py b/var/spack/repos/builtin/packages/py-neurom/package.py
new file mode 100644
index 00000000000000..19bad5fc7b94c7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-neurom/package.py
@@ -0,0 +1,35 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyNeurom(PythonPackage):
+    """Python library neuron morphology analysis"""
+
+    homepage = "https://github.com/BlueBrain/NeuroM"
+    git = "https://github.com/BlueBrain/NeuroM.git"
+    pypi = "neurom/neurom-2.2.1.tar.gz"
+
+    version("master", branch="master")
+    version("3.2.4", sha256="a584e0979b54deee906dd716ea90de20773e20b527d83960d0fe655b0905eb4a")
+
+    variant("plotly", default=False, description="Enable plotly support")
+
+    depends_on("py-setuptools@42:", type=("build", "run"))
+    depends_on("py-setuptools-scm", type="build")
+    depends_on("python@3.8:", type=("build", "run"))
+
+    depends_on("py-click@7.0:", type=("build", "run"))
+    depends_on("py-matplotlib@3.2.1:", type=("build", "run"))
+    depends_on("py-morphio@3.3.6:", type=("build", "run"))
+    depends_on("py-numpy@1.8.0:", type=("build", "run"))
+    depends_on("py-pandas@1.0.5:", type=("build", "run"))
+    depends_on("py-pyyaml@3.10:", type=("build", "run"))
+    depends_on("py-scipy@1.2.0:", type=("build", "run"))
+    depends_on("py-tqdm@4.8.4:", type=("build", "run"))
+
+    depends_on("py-plotly@3.6.0:", type=("build", "run"), when="+plotly")
+    depends_on("py-psutil@5.5.1:", type=("build", "run"), when="+plotly")
diff --git a/var/spack/repos/builtin/packages/py-nltk/resourcegen.py b/var/spack/repos/builtin/packages/py-nltk/resourcegen.py
index 77cf25cc266c7e..1a6e747b49f29f 100644
--- a/var/spack/repos/builtin/packages/py-nltk/resourcegen.py
+++ b/var/spack/repos/builtin/packages/py-nltk/resourcegen.py
@@ -8,7 +8,7 @@
 import xml.etree.ElementTree
 from typing import Optional
 
-url = None  # type: Optional[str]
+url: Optional[str] = None
 url = "https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml"
 if url is not None:
     document = urllib.request.urlopen(url).read()
diff --git a/var/spack/repos/builtin/packages/py-notebook/package.py b/var/spack/repos/builtin/packages/py-notebook/package.py
index 6c3a5a6983eb8d..382c0aa91d2e18 100644
--- a/var/spack/repos/builtin/packages/py-notebook/package.py
+++ b/var/spack/repos/builtin/packages/py-notebook/package.py
@@ -40,9 +40,7 @@ class PyNotebook(PythonPackage):
 
     depends_on("python@3.7:", type=("build", "run"), when="@6.4:")
     depends_on("python@3.6:", type=("build", "run"), when="@6.3:")
-    # TODO: replace this after concretizer learns how to concretize separate build deps
-    depends_on("py-jupyter-packaging11", when="@6.4.1:", type="build")
-    # depends_on('py-jupyter-packaging@0.9:0', when='@6.4.1:', type='build')
+    depends_on("py-jupyter-packaging@0.9:0", when="@6.4.1:", type="build")
 
     depends_on("py-setuptools", when="@5:", type="build")
     depends_on("py-jinja2", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-numcodecs/package.py b/var/spack/repos/builtin/packages/py-numcodecs/package.py
index 5a9d4b5d832808..6d466c19175c48 100644
--- a/var/spack/repos/builtin/packages/py-numcodecs/package.py
+++ b/var/spack/repos/builtin/packages/py-numcodecs/package.py
@@ -21,17 +21,42 @@ class PyNumcodecs(PythonPackage):
     import_modules = ["numcodecs"]
 
     version("master", branch="master", submodules=True)
+    version("0.11.0", sha256="6c058b321de84a1729299b0eae4d652b2e48ea1ca7f9df0da65cb13470e635eb")
     version("0.7.3", sha256="022b12ad83eb623ec53f154859d49f6ec43b15c36052fa864eaf2d9ee786dd85")
     version("0.6.4", sha256="ef4843d5db4d074e607e9b85156835c10d006afc10e175bda62ff5412fca6e4d")
 
     variant("msgpack", default=False, description="Codec to encode data as msgpacked bytes.")
 
-    depends_on("python@3.6:3", when="@0.7:", type=("build", "link", "run"))
-    depends_on("python@2.7:2.8,3.5:", when="@:0.6", type=("build", "link", "run"))
+    depends_on("python@3.8:", when="@0.11:", type=("build", "link", "run"))
+    depends_on("python@3.6:3", when="@0.7:0.10", type=("build", "link", "run"))
+    depends_on("py-setuptools@64:", when="@0.11:", type="build")
     depends_on("py-setuptools@18.1:", type="build")
-    depends_on("py-setuptools-scm@1.5.5:", type="build")
+    depends_on("py-setuptools-scm@6.2: +toml", when="@0.11:", type="build")
+    depends_on("py-setuptools-scm@1.5.5: +toml", type="build")
     depends_on("py-cython", type="build")
     depends_on("py-numpy@1.7:", type=("build", "run"))
+    depends_on("py-py-cpuinfo", when="@0.11:", type="build")
+    depends_on("py-entrypoints", when="@0.10.1:0.11", type=("build", "run"))
     depends_on("py-msgpack", type=("build", "run"), when="+msgpack")
 
     patch("apple-clang-12.patch", when="%apple-clang@12:")
+
+    # TODO: this package should really depend on blosc, zstd, lz4, zlib, but right now it vendors
+    # those libraries without any way to use the system versions.
+    # https://github.com/zarr-developers/numcodecs/issues/464
+
+    def setup_build_environment(self, env):
+        # This package likes to compile natively by checking cpu features and then setting flags
+        # -msse2 and -mavx2, which we want to avoid in Spack. This could go away if the package
+        # supports external libraries.
+        if self.spec.satisfies("target=x86_64:"):
+            if "avx2" not in self.spec.target.features:
+                env.set("DISABLE_NUMCODECS_AVX2", "1")
+            if "sse2" not in self.spec.target.features:
+                env.set("DISABLE_NUMCODECS_SSE2", "1")
+
+    def flag_handler(self, name, flags):
+        if name == "cflags":
+            if self.spec.satisfies("%oneapi"):
+                flags.append("-Wno-error=implicit-function-declaration")
+        return (flags, None, None)
diff --git a/var/spack/repos/builtin/packages/py-numpy/package.py b/var/spack/repos/builtin/packages/py-numpy/package.py
index b11e504e20529c..24bc86134b0dc9 100644
--- a/var/spack/repos/builtin/packages/py-numpy/package.py
+++ b/var/spack/repos/builtin/packages/py-numpy/package.py
@@ -5,16 +5,13 @@
 
 import platform
 import subprocess
+from typing import Tuple
 
 from spack.package import *
 
 
 class PyNumpy(PythonPackage):
-    """NumPy is the fundamental package for scientific computing with Python.
-    It contains among other things: a powerful N-dimensional array object,
-    sophisticated (broadcasting) functions, tools for integrating C/C++ and
-    Fortran code, and useful linear algebra, Fourier transform, and random
-    number capabilities"""
+    """Fundamental package for array computing in Python."""
 
     homepage = "https://numpy.org/"
     pypi = "numpy/numpy-1.23.0.tar.gz"
@@ -23,6 +20,10 @@ class PyNumpy(PythonPackage):
     maintainers("adamjstewart", "rgommers")
 
     version("main", branch="main")
+    version("1.26.1", sha256="c8c6c72d4a9f831f328efb1312642a1cafafaa88981d9ab76368d50d07d93cbe")
+    version("1.26.0", sha256="f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf")
+    version("1.25.2", sha256="fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760")
+    version("1.25.1", sha256="9a3a9f3a61480cc086117b426a8bd86869c213fc4072e606f01c4e4b66eb92bf")
     version("1.25.0", sha256="f1accae9a28dc3cda46a91de86acf69de0d1b5f4edd44a9b0c3ceb8036dfff19")
     version("1.24.4", sha256="80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463")
     version("1.24.3", sha256="ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155")
@@ -85,11 +86,8 @@ class PyNumpy(PythonPackage):
     version("1.14.6", sha256="1250edf6f6c43e1d7823f0967416bc18258bb271dc536298eb0ea00a9e45b80a")
     version("1.14.5", sha256="a4a433b3a264dbc9aa9c7c241e87c0358a503ea6394f8737df1683c7c9a102ac")
 
-    variant("blas", default=True, description="Build with BLAS support")
-    variant("lapack", default=True, description="Build with LAPACK support")
-
-    # Based on wheel availability on PyPI
-    depends_on("python@3.9:3.11", when="@1.25:", type=("build", "link", "run"))
+    depends_on("python@3.9:3.12", when="@1.26:", type=("build", "link", "run"))
+    depends_on("python@3.9:3.11", when="@1.25", type=("build", "link", "run"))
     depends_on("python@3.8:3.11", when="@1.23.2:1.24", type=("build", "link", "run"))
     depends_on("python@3.8:3.10", when="@1.22:1.23.1", type=("build", "link", "run"))
     depends_on("python@:3.10", when="@1.21.2:1.21", type=("build", "link", "run"))
@@ -97,19 +95,30 @@ class PyNumpy(PythonPackage):
     depends_on("python@:3.8", when="@1.17.3:1.19.2", type=("build", "link", "run"))
     depends_on("python@:3.7", when="@1.14.5:1.17.2", type=("build", "link", "run"))
 
+    depends_on("py-cython@0.29.34:3", when="@1.26:", type="build")
+    depends_on("py-cython@0.29.34:2", when="@1.25", type="build")
+    depends_on("py-cython@0.29.30:2", when="@1.22.4:1.24", type="build")
+    depends_on("py-cython@0.29.24:2", when="@1.21.2:1.22.3", type="build")
+    depends_on("py-cython@0.29.21:2", when="@1.19.1:1.21.1", type="build")
+    depends_on("py-cython@0.29.14:2", when="@1.18.1:1.19.0", type="build")
+    depends_on("py-cython@0.29.13:2", when="@1.18.0", type="build")
+    depends_on("py-pyproject-metadata@0.7.1:", when="@1.26:", type="build")
+    depends_on("py-tomli@1:", when="@1.26: ^python@:3.10", type="build")
+    depends_on("py-setuptools@60:", when="@1.26: ^python@3.12:", type="build")
     # https://github.com/spack/spack/pull/32078
-    depends_on("py-setuptools@:63", type=("build", "run"))
+    depends_on("py-setuptools@:63", when="@:1.25", type=("build", "run"))
     depends_on("py-setuptools@:59", when="@:1.22.1", type=("build", "run"))
-    # Check pyproject.toml for updates to the required cython version
-    depends_on("py-cython@0.29.34:2", when="@1.25:", type="build")
-    depends_on("py-cython@0.29.13:2", when="@1.18.0:", type="build")
-    depends_on("py-cython@0.29.14:2", when="@1.18.1:", type="build")
-    depends_on("py-cython@0.29.21:2", when="@1.19.1:", type="build")
-    depends_on("py-cython@0.29.24:2", when="@1.21.2:", type="build")
-    depends_on("py-cython@0.29.30:2", when="@1.22.4:", type="build")
-    depends_on("blas", when="+blas")
-    depends_on("lapack", when="+lapack")
+    depends_on("py-colorama", when="@1.26: platform=windows", type="build")
+
+    # Required to use --config-settings
+    depends_on("py-pip@23.1:", when="@1.26:", type="build")
+    # meson is vendored, ninja and pkgconfig are not
+    depends_on("ninja@1.8.2:", when="@1.26:", type="build")
+    depends_on("pkgconfig", when="@1.26:", type="build")
+    depends_on("blas")
+    depends_on("lapack")
 
+    # test_requirements.txt
     depends_on("py-nose@1.0.0:", when="@:1.14", type="test")
     depends_on("py-pytest", when="@1.15:", type="test")
     depends_on("py-hypothesis", when="@1.19:", type="test")
@@ -143,13 +152,21 @@ class PyNumpy(PythonPackage):
         when="@1.22.0:1.22.3",
     )
 
-    # version 1.21.0 runs into an infinit loop during printing
+    # meson.build
+    # https://docs.scipy.org/doc/scipy/dev/toolchain.html#compilers
+    conflicts("%gcc@:8.3", when="@1.26:", msg="NumPy requires GCC >= 8.4")
+    conflicts("%gcc@:4.7", msg="NumPy requires GCC >= 4.8")
+    conflicts(
+        "%msvc@:19.19",
+        when="@1.26:",
+        msg="NumPy requires at least vc142 (default with Visual Studio 2019) "
+        "when building with MSVC",
+    )
+
+    # version 1.21.0 runs into an infinite loop during printing
     # (e.g. print(numpy.ones(1000)) when compiled with gcc 11
     conflicts("%gcc@11:", when="@1.21.0")
 
-    # GCC 4.8 is the minimum version that works
-    conflicts("%gcc@:4.7", msg="GCC 4.8+ required")
-
     # NVHPC support added in https://github.com/numpy/numpy/pull/17344
     conflicts("%nvhpc", when="@:1.19")
 
@@ -157,6 +174,10 @@ class PyNumpy(PythonPackage):
     conflicts("%intel", when="@1.23.0:1.23.3")
     conflicts("%oneapi", when="@1.23.0:1.23.3")
 
+    @property
+    def archive_files(self):
+        return [join_path(self.stage.source_path, "build", "meson-logs", "meson-log.txt")]
+
     def url_for_version(self, version):
         url = "https://files.pythonhosted.org/packages/source/n/numpy/numpy-{}.{}"
         if version >= Version("1.23"):
@@ -166,19 +187,9 @@ def url_for_version(self, version):
         return url.format(version, ext)
 
     def flag_handler(self, name, flags):
-        # -std=c99 at least required, old versions of GCC default to -std=c90
         if self.spec.satisfies("%gcc@:5.1") and name == "cflags":
             flags.append(self.compiler.c99_flag)
-        # Check gcc version in use by intel compiler
-        # This will essentially check the system gcc compiler unless a gcc
-        # module is already loaded.
         if self.spec.satisfies("%intel") and name == "cflags":
-            # Note that the compiler environment variables and modules
-            # aren"t loaded for the flag_handler phase ...
-            # See https://github.com/spack/spack/issues/2056
-            #
-            # Newer/other flavors of Cray systems using the Intel compilers directly
-            # (icc, ...), therefore use this workaround only if "cc" is used.
             if self.compiler.cc == "cc":
                 gcc_version = Version(
                     spack.compiler.get_compiler_version_output("gcc", "-dumpversion")
@@ -217,16 +228,68 @@ def flag_handler(self, name, flags):
 
         return (flags, None, None)
 
-    @run_before("install")
-    def set_blas_lapack(self):
-        # https://numpy.org/devdocs/user/building.html
-        # https://github.com/numpy/numpy/blob/master/site.cfg.example
+    def blas_lapack_pkg_config(self) -> Tuple[str, str]:
+        """Convert library names to pkg-config names.
+
+        Returns:
+            The names of the blas and lapack libs that pkg-config should search for.
+        """
+        spec = self.spec
+        blas = spec["blas"].libs.names[0]
+        lapack = spec["lapack"].libs.names[0]
+
+        if spec["blas"].name in ["intel-mkl", "intel-parallel-studio", "intel-oneapi-mkl"]:
+            blas = "mkl-dynamic-lp64-seq"
+
+        if spec["lapack"].name in ["intel-mkl", "intel-parallel-studio", "intel-oneapi-mkl"]:
+            lapack = "mkl-dynamic-lp64-seq"
+
+        if spec["blas"].name in ["blis", "amdblis"]:
+            blas = "blis"
+
+        if spec["blas"].name == "cray-libsci":
+            blas = "libsci"
 
-        # Skip if no BLAS/LAPACK requested
+        if spec["lapack"].name == "cray-libsci":
+            lapack = "libsci"
+
+        if "armpl" in blas:
+            if "_mp" in blas:
+                blas = "armpl-dynamic-lp64-omp"
+            else:
+                blas = "armpl-dynamic-lp64-seq"
+
+        if "armpl" in lapack:
+            if "_mp" in lapack:
+                lapack = "armpl-dynamic-lp64-omp"
+            else:
+                lapack = "armpl-dynamic-lp64-seq"
+
+        return blas, lapack
+
+    @when("@1.26:")
+    def config_settings(self, spec, prefix):
+        blas, lapack = self.blas_lapack_pkg_config()
+        return {
+            "builddir": "build",
+            "compile-args": f"-j{make_jobs}",
+            "setup-args": {
+                # https://scipy.github.io/devdocs/building/blas_lapack.html
+                "-Dblas": blas,
+                "-Dlapack": lapack,
+                # https://numpy.org/doc/stable/reference/simd/build-options.html
+                # TODO: get this working in CI
+                # "-Dcpu-baseline": "native",
+                # "-Dcpu-dispatch": "none",
+            },
+        }
+
+    def blas_lapack_site_cfg(self) -> None:
+        """Write a site.cfg file to configure BLAS/LAPACK."""
         spec = self.spec
-        if "+blas" not in spec and "+lapack" not in spec:
-            return
 
+        # https://numpy.org/doc/1.25/user/building.html
+        # https://github.com/numpy/numpy/blob/v1.25.2/site.cfg.example
         def write_library_dirs(f, dirs):
             f.write("library_dirs = {0}\n".format(dirs))
             if not (
@@ -235,17 +298,11 @@ def write_library_dirs(f, dirs):
             ):
                 f.write("rpath = {0}\n".format(dirs))
 
-        blas_libs = LibraryList([])
-        blas_headers = HeaderList([])
-        if "+blas" in spec:
-            blas_libs = spec["blas"].libs
-            blas_headers = spec["blas"].headers
+        blas_libs = spec["blas"].libs
+        blas_headers = spec["blas"].headers
 
-        lapack_libs = LibraryList([])
-        lapack_headers = HeaderList([])
-        if "+lapack" in spec:
-            lapack_libs = spec["lapack"].libs
-            lapack_headers = spec["lapack"].headers
+        lapack_libs = spec["lapack"].libs
+        lapack_headers = spec["lapack"].headers
 
         lapackblas_libs = lapack_libs + blas_libs
         lapackblas_headers = lapack_headers + blas_headers
@@ -358,15 +415,25 @@ def write_library_dirs(f, dirs):
                     write_library_dirs(f, lapack_lib_dirs)
                     f.write("include_dirs = {0}\n".format(lapack_header_dirs))
 
+    @when("@:1.25")
+    @run_before("install")
+    def set_blas_lapack(self):
+        self.blas_lapack_site_cfg()
+
+    @when("@1.26:")
+    def setup_build_environment(self, env):
+        # https://github.com/scipy/scipy/issues/19357
+        if self.spec.satisfies("%apple-clang@15:"):
+            env.append_flags("LDFLAGS", "-Wl,-ld_classic")
+
+    @when("@:1.25")
     def setup_build_environment(self, env):
         # Tell numpy which BLAS/LAPACK libraries we want to use.
-        # https://github.com/numpy/numpy/pull/13132
-        # https://numpy.org/devdocs/user/building.html#accelerated-blas-lapack-libraries
         spec = self.spec
-        # https://numpy.org/devdocs/user/building.html#blas
-        if "blas" not in spec:
-            blas = ""
-        elif (
+        # https://github.com/numpy/numpy/pull/13132
+        # https://numpy.org/doc/1.25/user/building.html#accelerated-blas-lapack-libraries
+        # https://numpy.org/doc/1.25/user/building.html#blas
+        if (
             spec["blas"].name == "intel-mkl"
             or spec["blas"].name == "intel-parallel-studio"
             or spec["blas"].name == "intel-oneapi-mkl"
@@ -385,10 +452,8 @@ def setup_build_environment(self, env):
 
         env.set("NPY_BLAS_ORDER", blas)
 
-        # https://numpy.org/devdocs/user/building.html#lapack
-        if "lapack" not in spec:
-            lapack = ""
-        elif (
+        # https://numpy.org/doc/1.25/user/building.html#lapack
+        if (
             spec["lapack"].name == "intel-mkl"
             or spec["lapack"].name == "intel-parallel-studio"
             or spec["lapack"].name == "intel-oneapi-mkl"
diff --git a/var/spack/repos/builtin/packages/py-omegaconf/package.py b/var/spack/repos/builtin/packages/py-omegaconf/package.py
index bfb16b0dd42814..4fc4afb417680a 100644
--- a/var/spack/repos/builtin/packages/py-omegaconf/package.py
+++ b/var/spack/repos/builtin/packages/py-omegaconf/package.py
@@ -18,11 +18,12 @@ class PyOmegaconf(PythonPackage):
     maintainers("calebrob6")
 
     version("2.3.0", sha256="d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7")
+    version("2.2.2", sha256="10a89b5cb81887d68137b69a7c5c046a060e2239af4e37f20c3935ad2e5fd865")
     version("2.1.0", sha256="a08aec03a63c66449b550b85d70238f4dee9c6c4a0541d6a98845dcfeb12439d")
 
     depends_on("py-setuptools", type="build")
     depends_on("py-pytest-runner", when="@2.1", type="build")
-    depends_on("py-antlr4-python3-runtime@4.9", when="@2.3:", type=("build", "run"))
+    depends_on("py-antlr4-python3-runtime@4.9", when="@2.2.2:", type=("build", "run"))
     depends_on("py-antlr4-python3-runtime@4.8", when="@2.1", type=("build", "run"))
-    depends_on("py-pyyaml@5.1.0:", type=("build", "run"))
+    depends_on("py-pyyaml@5.1:", type=("build", "run"))
     depends_on("java", type="build")
diff --git a/var/spack/repos/builtin/packages/py-onnxruntime/package.py b/var/spack/repos/builtin/packages/py-onnxruntime/package.py
index 68929f6d3c4a0c..205785a4db66f9 100644
--- a/var/spack/repos/builtin/packages/py-onnxruntime/package.py
+++ b/var/spack/repos/builtin/packages/py-onnxruntime/package.py
@@ -19,8 +19,12 @@ class PyOnnxruntime(CMakePackage, PythonExtension):
     homepage = "https://github.com/microsoft/onnxruntime"
     git = "https://github.com/microsoft/onnxruntime.git"
 
-    version("1.10.0", tag="v1.10.0", submodules=True)
-    version("1.7.2", tag="v1.7.2", submodules=True)
+    version(
+        "1.10.0", tag="v1.10.0", commit="0d9030e79888d1d5828730b254fedc53c7b640c1", submodules=True
+    )
+    version(
+        "1.7.2", tag="v1.7.2", commit="5bc92dff16b0ddd5063b717fb8522ca2ad023cb0", submodules=True
+    )
 
     variant("cuda", default=False, description="Build with CUDA support")
 
@@ -39,7 +43,7 @@ class PyOnnxruntime(CMakePackage, PythonExtension):
     depends_on("py-wheel", type="build")
     depends_on("py-onnx", type=("build", "run"))
     depends_on("py-flatbuffers", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libpng")
     depends_on("py-pybind11", type="build")
     depends_on("cuda", when="+cuda")
diff --git a/var/spack/repos/builtin/packages/py-openai/package.py b/var/spack/repos/builtin/packages/py-openai/package.py
new file mode 100644
index 00000000000000..62efe6ad1cc4a3
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-openai/package.py
@@ -0,0 +1,58 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyOpenai(PythonPackage):
+    """The OpenAI Python library provides convenient access to the OpenAI API
+    from applications written in the Python language. It includes a pre-defined
+    set of classes for API resources that initialize themselves dynamically from
+    API responses which makes it compatible with a wide range of versions of the
+    OpenAI API."""
+
+    homepage = "https://github.com/openai/openai-python"
+    pypi = "openai/openai-0.27.8.tar.gz"
+
+    version("0.27.8", sha256="2483095c7db1eee274cebac79e315a986c4e55207bb4fa7b82d185b3a2ed9536")
+
+    variant("datalib", default=False, description="facilities for data loading")
+    variant(
+        "wandb",
+        default=False,
+        description="keeps track of hyperparameters, system metrics, and predictions",
+    )
+    variant("embeddings", default=False, description="represents a text string vector")
+
+    depends_on("python@3.7.1:", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
+    depends_on("py-requests@2.20:", type=("build", "run"))
+    depends_on("py-tqdm", type=("build", "run"))
+    depends_on("py-typing-extensions", when="^python@3.7", type=("build", "run"))
+    depends_on("py-aiohttp", type=("build", "run"))
+
+    with when("+datalib"):
+        depends_on("py-numpy", type=("build", "run"))
+        depends_on("py-pandas@1.2.3:", type=("build", "run"))
+        depends_on("py-pandas-stubs@1.1.0.11:", type=("build", "run"))
+        depends_on("py-openpyxl@3.0.7:", type=("build", "run"))
+
+    with when("+wandb"):
+        depends_on("py-wandb", type=("build", "run"))
+        depends_on("py-numpy", type=("build", "run"))
+        depends_on("py-pandas@1.2.3:", type=("build", "run"))
+        depends_on("py-pandas-stubs@1.1.0.11:", type=("build", "run"))
+        depends_on("py-openpyxl@3.0.7:", type=("build", "run"))
+
+    with when("+embeddings"):
+        depends_on("py-scikit-learn@1.0.2:", type=("build", "run"))
+        depends_on("py-tenacity@8.0.1:", type=("build", "run"))
+        depends_on("py-matplotlib", type=("build", "run"))
+        depends_on("py-plotly", type=("build", "run"))
+        depends_on("py-numpy", type=("build", "run"))
+        depends_on("py-scipy", type=("build", "run"))
+        depends_on("py-pandas@1.2.3:", type=("build", "run"))
+        depends_on("py-pandas-stubs@1.1.0.11:", type=("build", "run"))
+        depends_on("py-openpyxl@3.0.7:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-openapi-schema-pydantic/package.py b/var/spack/repos/builtin/packages/py-openapi-schema-pydantic/package.py
new file mode 100644
index 00000000000000..26d014da638371
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-openapi-schema-pydantic/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyOpenapiSchemaPydantic(PythonPackage):
+    """OpenAPI (v3) specification schema as pydantic class"""
+
+    homepage = "https://github.com/kuimono/openapi-schema-pydantic"
+    pypi = "openapi-schema-pydantic/openapi-schema-pydantic-1.2.4.tar.gz"
+
+    version("1.2.4", sha256="3e22cf58b74a69f752cc7e5f1537f6e44164282db2700cbbcd3bb99ddd065196")
+
+    depends_on("python@3.6.1:", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
+    depends_on("py-pydantic@1.8.2:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-opendatalab/package.py b/var/spack/repos/builtin/packages/py-opendatalab/package.py
new file mode 100644
index 00000000000000..886b3c5eac31ad
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-opendatalab/package.py
@@ -0,0 +1,26 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyOpendatalab(PythonPackage):
+    """OpenDataLab Python SDK"""
+
+    homepage = "https://github.com/opendatalab/opendatalab-python-sdk"
+    pypi = "opendatalab/opendatalab-0.0.9.tar.gz"
+
+    version("0.0.9", sha256="4648b66d5be096ba38fa087b6c7906c24218d02a49906c8b41c069b9a8747530")
+
+    depends_on("python@3.7:", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
+    depends_on("py-pycryptodome", type=("build", "run"))
+    depends_on("py-click@7:", type=("build", "run"))
+    depends_on("py-requests@2.4.2:", type=("build", "run"))
+    depends_on("py-tqdm", type=("build", "run"))
+    depends_on("py-colorama", type=("build", "run"))
+    depends_on("py-rich", type=("build", "run"))
+    # depends_on("py-pywin32", when="platform=windows", type=("build", "run"))
+    conflicts("platform=windows", msg="Requires py-pywin32 to be packaged")
diff --git a/var/spack/repos/builtin/packages/py-openmim/package.py b/var/spack/repos/builtin/packages/py-openmim/package.py
new file mode 100644
index 00000000000000..e9765051fbf72a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-openmim/package.py
@@ -0,0 +1,26 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyOpenmim(PythonPackage):
+    """MIM Installs OpenMMLab packages"""
+
+    homepage = "https://github.com/open-mmlab/mim"
+    pypi = "openmim/openmim-0.3.9.tar.gz"
+
+    version("0.3.9", sha256="b3977b92232b4b8c4d987cbc73e4515826d5543ccd3a66d49fcfc602cc5b3352")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-click", type=("build", "run"))
+    depends_on("py-colorama", type=("build", "run"))
+    depends_on("py-model-index", type=("build", "run"))
+    depends_on("py-opendatalab", type=("build", "run"))
+    depends_on("py-pandas", type=("build", "run"))
+    depends_on("py-pip@19.3:", type=("build", "run"))
+    depends_on("py-requests", type=("build", "run"))
+    depends_on("py-rich", type=("build", "run"))
+    depends_on("py-tabulate", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-openpmd-viewer/package.py b/var/spack/repos/builtin/packages/py-openpmd-viewer/package.py
index 481af0cebbc39e..543509ebb10d4c 100644
--- a/var/spack/repos/builtin/packages/py-openpmd-viewer/package.py
+++ b/var/spack/repos/builtin/packages/py-openpmd-viewer/package.py
@@ -19,7 +19,13 @@ class PyOpenpmdViewer(PythonPackage):
     version("1.3.0", sha256="236c065a37881fcb7603efde0bf2d61acc355a8acc595bebc3d6b9d03251b081")
     version("1.2.0", sha256="a27f8ac522c4c76fd774095e156a8b280c9211128f50aa07f16ac70d8222384d")
 
-    variant("backend", default="h5py,openpmd-api", multi=True, values=("h5py", "openpmd-api"))
+    variant(
+        "backend",
+        default="h5py,openpmd-api",
+        description="Visualization backend",
+        multi=True,
+        values=("h5py", "openpmd-api"),
+    )
     variant("jupyter", default=False, description="Enable Jupyter Widget GUI")
     variant("numba", default=False, description="Enable accelerated depositions for histograms")
     variant("plot", default=True, description="Enable plotting support")
diff --git a/var/spack/repos/builtin/packages/py-openpyxl/package.py b/var/spack/repos/builtin/packages/py-openpyxl/package.py
index 2e2fbb9105c1b3..246164f68837e7 100644
--- a/var/spack/repos/builtin/packages/py-openpyxl/package.py
+++ b/var/spack/repos/builtin/packages/py-openpyxl/package.py
@@ -10,21 +10,15 @@ class PyOpenpyxl(PythonPackage):
     """A Python library to read/write Excel 2010 xlsx/xlsm files"""
 
     homepage = "https://openpyxl.readthedocs.org/"
-    pypi = "openpyxl/openpyxl-3.0.3.tar.gz"
+    pypi = "openpyxl/openpyxl-3.1.2.tar.gz"
 
+    version("3.1.2", sha256="a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184")
     version("3.0.7", sha256="6456a3b472e1ef0facb1129f3c6ef00713cebf62e736cd7a75bcc3247432f251")
     version("3.0.3", sha256="547a9fc6aafcf44abe358b89ed4438d077e9d92e4f182c87e2dc294186dc4b64")
     version("2.4.5", sha256="78c331e819fb0a63a1339d452ba0b575d1a31f09fdcce793a31bec7e9ef4ef21")
     version("2.2.0", sha256="c34e3f7e3106dbe6d792f35d9a2f01c08fdd21a6fe582a2f540e39a70e7443c4")
     version("1.8.6", sha256="aa11a4acd2765392808bca2041f6f9ba17565c72dccc3f5d876bf78effa06126")
 
-    depends_on("python@3.6:", when="@3.0:", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.5:", when="@2.6:", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.4:", when="@2.5:", type=("build", "run"))
-    depends_on("python@2.6:2.8,3.3:", when="@2.1:", type=("build", "run"))
-    depends_on("python@2.6:", type=("build", "run"))
-
     depends_on("py-setuptools", type="build")
-
-    depends_on("py-jdcal", when="@2.2:", type=("build", "run"))
     depends_on("py-et-xmlfile", when="@2.4:", type=("build", "run"))
+    depends_on("py-jdcal", when="@2.2.0:3.0.6", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-opentuner/package.py b/var/spack/repos/builtin/packages/py-opentuner/package.py
index 9f1248c7e67c41..2623be661c8be1 100644
--- a/var/spack/repos/builtin/packages/py-opentuner/package.py
+++ b/var/spack/repos/builtin/packages/py-opentuner/package.py
@@ -14,8 +14,8 @@ class PyOpentuner(PythonPackage):
 
     maintainers("matthiasdiener")
 
-    version("0.8.7", commit="070c5ce")
-    version("0.8.2", commit="8e720a2")
+    version("0.8.7", commit="070c5cef6d933eb760a2f9cd5cd08c95f27aee75")
+    version("0.8.2", commit="8e720a2094e7964d7a1225e58aca40b0e78bff7d")
 
     depends_on("python@3:", type=("build", "run"), when="@0.8.1:")
 
diff --git a/var/spack/repos/builtin/packages/py-optuna/package.py b/var/spack/repos/builtin/packages/py-optuna/package.py
new file mode 100644
index 00000000000000..de74a405ece1cd
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-optuna/package.py
@@ -0,0 +1,33 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyOptuna(PythonPackage):
+    """Optuna is an automatic hyperparameter optimization software framework,
+    particularly designed for machine learning. It features an imperative,
+    define-by-run style user API. Thanks to our define-by-run API, the code
+    written with Optuna enjoys high modularity, and the user of Optuna can
+    dynamically construct the search spaces for the hyperparameters."""
+
+    homepage = "https://optuna.org/"
+    pypi = "optuna/optuna-3.2.0.tar.gz"
+
+    maintainers("elliottslaughter", "eugeneswalker")
+
+    version("3.2.0", sha256="683d8693643a761a41d251a6b8e13263b24acacf9fc46a9233d5f6aa3ce5c683")
+
+    depends_on("py-setuptools@61.1:", type="build")
+
+    depends_on("py-alembic@1.5:", type=("build", "run"))
+    depends_on("py-cmaes@0.9.1:", type=("build", "run"))
+    depends_on("py-colorlog", type=("build", "run"))
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-packaging@20:", type=("build", "run"))
+    depends_on("py-pyyaml", type=("build", "run"))
+    depends_on("py-sqlalchemy@1.3:", type=("build", "run"))
+    depends_on("py-tqdm", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pandas-stubs/package.py b/var/spack/repos/builtin/packages/py-pandas-stubs/package.py
new file mode 100644
index 00000000000000..8a11e327041dd4
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-pandas-stubs/package.py
@@ -0,0 +1,26 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPandasStubs(PythonPackage):
+    """These are public type stubs for pandas, following the convention of
+    providing stubs in a separate package, as specified in PEP 561. The stubs
+    cover the most typical use cases of pandas. In general, these stubs are
+    narrower than what is possibly allowed by pandas, but follow a convention of
+    suggesting best recommended practices for using pandas."""
+
+    homepage = "https://pandas.pydata.org/"
+    pypi = "pandas_stubs/pandas_stubs-2.0.2.230605.tar.gz"
+
+    version(
+        "2.0.2.230605", sha256="624c7bb06d38145a44b61be459ccd19b038e0bf20364a025ecaab78fea65e858"
+    )
+
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-poetry-core@1:", type="build")
+    depends_on("py-numpy@1.24.3:", type=("build", "run"))
+    depends_on("py-types-pytz@2022.1.1:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pandas/package.py b/var/spack/repos/builtin/packages/py-pandas/package.py
index 224af947aa177c..3dea26ff2c4111 100644
--- a/var/spack/repos/builtin/packages/py-pandas/package.py
+++ b/var/spack/repos/builtin/packages/py-pandas/package.py
@@ -17,8 +17,9 @@ class PyPandas(PythonPackage):
 
     maintainers("adamjstewart")
 
-    variant("excel", when="@1.4:", default=False, description="Build with support for Excel")
-
+    version("2.1.2", sha256="52897edc2774d2779fbeb6880d2cfb305daa0b1a29c16b91f531a18918a6e0f3")
+    version("2.1.1", sha256="fecb198dc389429be557cde50a2d46da8434a17fe37d7d41ff102e3987fd947b")
+    version("2.1.0", sha256="62c24c7fc59e42b775ce0679cfa7b14a5f9bfb7643cfbe708c960699e05fb918")
     version("2.0.3", sha256="c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c")
     version("2.0.2", sha256="dd5476b6c3fe410ee95926873f377b856dbc4e81a9c605a0dc05aaccc6a7c6c6")
     version("2.0.1", sha256="19b8e5270da32b41ebf12f0e7165efa7024492e9513fb46fb631c5022ae5709d")
@@ -63,68 +64,71 @@ class PyPandas(PythonPackage):
     version("0.24.2", sha256="4f919f409c433577a501e023943e582c57355d50a724c589e78bc1d551a535a2")
     version("0.24.1", sha256="435821cb2501eabbcee7e83614bd710940dc0cf28b5afbc4bdb816c31cec71af")
     version("0.23.4", sha256="5b24ca47acf69222e82530e89111dd9d14f9b970ab2cd3a1c2c78f0c4fbba4f4")
-    version("0.21.1", sha256="c5f5cba88bf0659554c41c909e1f78139f6fce8fa9315a29a23692b38ff9788a")
-    version("0.20.0", sha256="54f7a2bb2a7832c0446ad51d779806f07ec4ea2bb7c9aea4b83669fa97e778c4")
-    version("0.19.2", sha256="6f0f4f598c2b16746803c8bafef7c721c57e4844da752d36240c0acf97658014")
-    version("0.19.0", sha256="4697606cdf023c6b7fcb74e48aaf25cf282a1a00e339d2d274cf1b663748805b")
-    version("0.18.0", sha256="c975710ce8154b50f39a46aa3ea88d95b680191d1d9d4b5dd91eae7215e01814")
-    version("0.16.1", sha256="570d243f8cb068bf780461b9225d2e7bef7c90aa10d43cf908fe541fc92df8b6")
-    version("0.16.0", sha256="4013de6f8796ca9d2871218861823bd9878a8dfacd26e08ccf9afdd01bbad9f1")
+
+    variant("excel", when="@1.4:", default=False, description="Build with support for Excel")
 
     # Required dependencies
     # https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#python-version-support
-    depends_on("python@3.8:", type=("build", "run"), when="@1.4:")
-    depends_on("python@3.7.1:", type=("build", "run"), when="@1.2:")
-    depends_on("python@3.6.1:", type=("build", "run"), when="@1:")
-    depends_on("python@3.5.3:", type=("build", "run"), when="@0.25:")
+    depends_on("python@3.9:3.12", when="@2.1.1:", type=("build", "run"))
+    depends_on("python@3.9:3.11", when="@2.1.0", type=("build", "run"))
+    depends_on("python@3.8:3.11", when="@1.5:2.0", type=("build", "run"))
+    depends_on("python@3.8:3.10", when="@1.4", type=("build", "run"))
+    depends_on("python@:3.10", when="@1.3.3:1.3", type=("build", "run"))
+    depends_on("python@:3.9", when="@1.1.3:1.3.2", type=("build", "run"))
+    depends_on("python@:3.8", when="@0.25.2:1.1.2", type=("build", "run"))
+    depends_on("python@:3.7", when="@:0.25.1", type=("build", "run"))
 
     # pyproject.toml
-    depends_on("py-setuptools@61:", type="build", when="@2:")
-    depends_on("py-setuptools@51:", type="build", when="@1.3.2:")
-    depends_on("py-setuptools@38.6:", type="build", when="@1.3:")
-    depends_on("py-setuptools@24.2:", type="build")
-    depends_on("py-cython@0.29.33:2", type="build", when="@2:")
-    depends_on("py-cython@0.29.32:2", type="build", when="@1.4.4:")
-    depends_on("py-cython@0.29.30:2", type="build", when="@1.4.3:")
-    depends_on("py-cython@0.29.24:2", type="build", when="@1.3.4:")
-    depends_on("py-cython@0.29.21:2", type="build", when="@1.1.3:")
-    depends_on("py-cython@0.29.16:2", type="build", when="@1.1:")
-    depends_on("py-cython@0.29.13:2", type="build", when="@1:")
-    depends_on("py-versioneer+toml", type="build", when="@2:")
+    depends_on("py-meson-python@0.13.1", when="@2.1:", type="build")
+    depends_on("meson@1.2.1", when="@2.1.1:", type="build")
+    depends_on("meson@1.0.1", when="@2.1.0", type="build")
+    depends_on("py-cython@0.29.33:2", when="@2:", type="build")
+    depends_on("py-cython@0.29.32:2", when="@1.4.4:", type="build")
+    depends_on("py-cython@0.29.30:2", when="@1.4.3:", type="build")
+    depends_on("py-cython@0.29.24:2", when="@1.3.4:", type="build")
+    depends_on("py-cython@0.29.21:2", when="@1.1.3:", type="build")
+    depends_on("py-cython@0.29.16:2", when="@1.1:", type="build")
+    depends_on("py-cython@0.29.13:2", when="@1:", type="build")
+    depends_on("py-versioneer+toml", when="@2:", type="build")
 
     # https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#dependencies
-    depends_on("py-numpy@1.20.3:", type=("build", "run"), when="@1.5:")
-    depends_on("py-numpy@1.18.5:", type=("build", "run"), when="@1.4:")
-    depends_on("py-numpy@1.17.3:", type=("build", "run"), when="@1.3:")
-    depends_on("py-numpy@1.16.5:", type=("build", "run"), when="@1.2:")
-    depends_on("py-numpy@1.15.4:", type=("build", "run"), when="@1.1:")
-    depends_on("py-numpy@1.13.3:", type=("build", "run"), when="@0.25:")
+    depends_on("py-numpy@1.22.4:2", when="@2.1.2:", type=("build", "run"))
+    depends_on("py-numpy@1.22.4:", when="@2.1:", type=("build", "run"))
+    depends_on("py-numpy@1.20.3:", when="@1.5:", type=("build", "run"))
+    depends_on("py-numpy@1.18.5:", when="@1.4:", type=("build", "run"))
+    depends_on("py-numpy@1.17.3:", when="@1.3:", type=("build", "run"))
+    depends_on("py-numpy@1.16.5:", when="@1.2:", type=("build", "run"))
+    depends_on("py-numpy@1.15.4:", when="@1.1:", type=("build", "run"))
+    depends_on("py-numpy@1.13.3:", when="@0.25:", type=("build", "run"))
     depends_on("py-numpy", type=("build", "run"))
     # 'NUMPY_IMPORT_ARRAY_RETVAL' was removed in numpy@1.19
-    depends_on("py-numpy@:1.18", type=("build", "run"), when="@:0.25")
-    depends_on("py-python-dateutil@2.8.2:", type=("build", "run"), when="@2:")
-    depends_on("py-python-dateutil@2.8.1:", type=("build", "run"), when="@1.4:")
-    depends_on("py-python-dateutil@2.7.3:", type=("build", "run"), when="@1.1:")
-    depends_on("py-python-dateutil@2.6.1:", type=("build", "run"), when="@0.25:")
+    depends_on("py-numpy@:1.18", when="@:0.25", type=("build", "run"))
+    depends_on("py-python-dateutil@2.8.2:", when="@2:", type=("build", "run"))
+    depends_on("py-python-dateutil@2.8.1:", when="@1.4:", type=("build", "run"))
+    depends_on("py-python-dateutil@2.7.3:", when="@1.1:", type=("build", "run"))
+    depends_on("py-python-dateutil@2.6.1:", when="@0.25:", type=("build", "run"))
     depends_on("py-python-dateutil", type=("build", "run"))
-    depends_on("py-pytz@2020.1:", type=("build", "run"), when="@1.4:")
-    depends_on("py-pytz@2017.3:", type=("build", "run"), when="@1.2:")
+    depends_on("py-pytz@2020.1:", when="@1.4:", type=("build", "run"))
+    depends_on("py-pytz@2017.3:", when="@1.2:", type=("build", "run"))
     depends_on("py-pytz@2017.2:", type=("build", "run"))
-    depends_on("py-tzdata@2022.1:", type=("build", "run"), when="@2:")
+    depends_on("py-tzdata@2022.1:", when="@2:", type=("build", "run"))
 
     # Recommended dependencies
-    # https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#recommended-dependencies
-    depends_on("py-numexpr@2.7.3:", type=("build", "run"), when="@1.5:")
-    depends_on("py-numexpr@2.7.1:", type=("build", "run"), when="@1.4:")
-    depends_on("py-numexpr@2.7.0:", type=("build", "run"), when="@1.3:")
-    depends_on("py-numexpr@2.6.8:", type=("build", "run"), when="@1.2:")
-    depends_on("py-numexpr@2.6.2:", type=("build", "run"), when="@0.25:")
+    # https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#performance-dependencies-recommended
+    depends_on("py-numexpr@2.8.0:", when="@2.1:", type=("build", "run"))
+    depends_on("py-numexpr@2.7.3:", when="@1.5:", type=("build", "run"))
+    depends_on("py-numexpr@2.7.1:", when="@1.4:", type=("build", "run"))
+    depends_on("py-numexpr@2.7.0:", when="@1.3:", type=("build", "run"))
+    depends_on("py-numexpr@2.6.8:", when="@1.2:", type=("build", "run"))
+    depends_on("py-numexpr@2.6.2:", when="@0.25:", type=("build", "run"))
     depends_on("py-numexpr", type=("build", "run"))
-    depends_on("py-bottleneck@1.3.2:", type=("build", "run"), when="@1.5:")
-    depends_on("py-bottleneck@1.3.1:", type=("build", "run"), when="@1.4:")
-    depends_on("py-bottleneck@1.2.1:", type=("build", "run"), when="@0.25:")
+    depends_on("py-bottleneck@1.3.4:", when="@2.1:", type=("build", "run"))
+    depends_on("py-bottleneck@1.3.2:", when="@1.5:", type=("build", "run"))
+    depends_on("py-bottleneck@1.3.1:", when="@1.4:", type=("build", "run"))
+    depends_on("py-bottleneck@1.2.1:", when="@0.25:", type=("build", "run"))
     depends_on("py-bottleneck", type=("build", "run"))
-    depends_on("py-numba@0.53.1:", type=("build", "run"), when="@2:")
+    depends_on("py-numba@0.55.2:", when="@2.1:", type=("build", "run"))
+    depends_on("py-numba@0.53.1:", when="@2.0:", type=("build", "run"))
 
     # Optional dependencies
     # https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#optional-dependencies
@@ -150,3 +154,9 @@ class PyPandas(PythonPackage):
     depends_on("py-setuptools@24.2:", when="@:1.2", type="build")
 
     skip_modules = ["pandas.tests", "pandas.plotting._matplotlib", "pandas.core._numba.kernels"]
+
+    def flag_handler(self, name, flags):
+        if name == "cflags":
+            if self.spec.satisfies("@0.24.2 %oneapi"):
+                flags.append("-Wno-error=implicit-function-declaration")
+        return (flags, None, None)
diff --git a/var/spack/repos/builtin/packages/py-panedr/package.py b/var/spack/repos/builtin/packages/py-panedr/package.py
index ceebb267c37ad4..f1304ea809db56 100644
--- a/var/spack/repos/builtin/packages/py-panedr/package.py
+++ b/var/spack/repos/builtin/packages/py-panedr/package.py
@@ -15,9 +15,11 @@ class PyPanedr(PythonPackage):
 
     maintainers("RMeli")
 
+    version("0.7.2", sha256="2181009a962dc35745348d1146608c64fb1ba3ef5ae74043416fef739d1409e4")
     version("0.7.1", sha256="64c74863f72d51729ac5cb1e2dbb18747f7137588990c308ef8ca120fbf2ddd4")
 
     # PyEDR is released together with PanEDR, therefore versions should match
+    depends_on("py-pyedr@0.7.2", type=("build", "run"), when="@0.7.2")
     depends_on("py-pyedr@0.7.1", type=("build", "run"), when="@0.7.1")
 
     depends_on("py-pandas", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-paralleltask/package.py b/var/spack/repos/builtin/packages/py-paralleltask/package.py
new file mode 100644
index 00000000000000..53ce9066e92436
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-paralleltask/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyParalleltask(PythonPackage):
+    """Paralleltask is a simple and lightweight parallel task engine. It can launch a given
+    number of tasks from a batch of independent tasks, and keep this number of running tasks
+    until all tasks are completed."""
+
+    homepage = "https://github.com/moold/ParallelTask"
+    pypi = "Paralleltask/Paralleltask-0.2.2.tar.gz"
+
+    version("0.2.2", sha256="f00945e2bd5b6aff9cdc48fbd92aa7b48d23bb530d7f6643ac966fea11a7a9d5")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-psutil", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-parsl/package.py b/var/spack/repos/builtin/packages/py-parsl/package.py
index 5bc2c5ad9fc6fe..3b7b4771a21f33 100644
--- a/var/spack/repos/builtin/packages/py-parsl/package.py
+++ b/var/spack/repos/builtin/packages/py-parsl/package.py
@@ -16,32 +16,43 @@ class PyParsl(PythonPackage):
 
     maintainers("hategan")
 
+    version(
+        "2023.08.21", sha256="d7d6145ad5ab63baf9c9f9441a0a6ea5be6f896ef8094d47bf64d949a56b1782"
+    )
     version("1.2.0", sha256="342c74ee39fa210d74b8adfb455f0a9c20d9f059ec5bd9d60c5bdc9929abcdcc")
     version("1.1.0", sha256="6a623d3550329f028775950d23a2cafcb0f82b199f15940180410604aa5d102c")
 
     variant("monitoring", default=False, description="enable live monitoring")
     # See https://parsl.readthedocs.io/en/stable/userguide/monitoring.html
 
-    depends_on("python@3.6:", type=("build", "run"))
+    depends_on("python@3.8:", type=("build", "run"), when="@2023.08.21:")
+    depends_on("python@3.6:", type=("build", "run"), when="@:1.2")
     depends_on("py-setuptools", type="build")
     depends_on("py-pyzmq@17.1.2:", type=("build", "run"))
-    depends_on("py-typeguard@2.10:", type=("build", "run"))
-    depends_on("py-typing-extensions", type=("build", "run"))
+    depends_on("py-typeguard@2.10:2", type=("build", "run"), when="@2023.08.21:")
+    depends_on("py-typeguard@2.10:", type=("build", "run"), when="@:1.2")
+    depends_on("py-typing-extensions@4.6:4", type=("build", "run"), when="@2023.08.21:")
+    depends_on("py-typing-extensions", type=("build", "run"), when="@:1.2")
+    depends_on("py-six", type=("build", "run"), when="@2023.08.21:")
     depends_on("py-globus-sdk", type=("build", "run"))
     depends_on("py-dill", type=("build", "run"))
     depends_on("py-tblib", type=("build", "run"))
     depends_on("py-requests", type=("build", "run"))
     depends_on("py-paramiko", type=("build", "run"))
     depends_on("py-psutil@5.5.1:", type=("build", "run"))
+    depends_on("py-setproctitle", type=("build", "run"), when="@2023.08.21:")
 
     with when("+monitoring"):
-        depends_on("py-sqlalchemy@1.3", type=("build", "run"))
-        conflicts("^py-sqlalchemy@1.3.4")
-        depends_on("py-sqlalchemy-utils", type=("build", "run"))
+        depends_on("py-sqlalchemy@1.4:1", type=("build", "run"), when="@2023.08.21:")
+        depends_on("py-sqlalchemy@1.3", type=("build", "run"), when="@:1.2")
+        conflicts("^py-sqlalchemy@1.3.4", when="@:1.2")
+        depends_on("py-sqlalchemy-utils", type=("build", "run"), when="@:1.2")
         depends_on("py-pydot", type=("build", "run"))
-        depends_on("py-networkx", type=("build", "run"))
+        depends_on("py-networkx@2.5.0:2.5", type=("build", "run"), when="@2023.08.21:")
+        depends_on("py-networkx", type=("build", "run"), when="@:1.2")
         depends_on("py-flask@1.0.2:", type=("build", "run"))
         depends_on("py-flask-sqlalchemy", type=("build", "run"))
-        depends_on("py-pandas", type=("build", "run"))
+        depends_on("py-pandas@:1", type=("build", "run"), when="@2023.08.21:")
+        depends_on("py-pandas", type=("build", "run"), when="@:1.2")
         depends_on("py-plotly", type=("build", "run"))
         depends_on("py-python-daemon", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-particle/package.py b/var/spack/repos/builtin/packages/py-particle/package.py
index b3b819c0882e27..843e4c0e02e994 100644
--- a/var/spack/repos/builtin/packages/py-particle/package.py
+++ b/var/spack/repos/builtin/packages/py-particle/package.py
@@ -20,6 +20,9 @@ class PyParticle(PythonPackage):
     tags = ["hep"]
 
     version("master", branch="master")
+    version("0.23.0", sha256="d810f8fc27deb8e7fd64174017d9607d50522249c0973a0008e580f93db11750")
+    version("0.22.1", sha256="dcb45025cf7cff901e2c94922d150e1103245c46f2671eae4193c5fa767cc56c")
+    version("0.22.0", sha256="567bb3017cb7526f9c9ef4399e9ba5acbdb5b9ce93eb18e4da6479d3181c93a5")
     version("0.21.2", sha256="0434d39aab4fc72bce452a11f822736f95937c5f4116b0e831254ebcef6cfcdb")
     version("0.21.1", sha256="330f48550a17654f01c94e8e2365ad9a8dcf26e573e8acf31fb23ea4f624b2c4")
     version("0.21.0", sha256="483748834e7e81f2cd690fa0962b537f767fe8af4705c8bada68c8bbdac8a17d")
@@ -42,6 +45,7 @@ class PyParticle(PythonPackage):
     depends_on("py-hatch-vcs", when="@0.21:", type="build")
     depends_on("py-importlib-resources@2:", when="@0.16: ^python@:3.8", type=("build", "run"))
     depends_on("py-typing-extensions", when="@0.16: ^python@:3.7", type=("build", "run"))
+    depends_on("py-deprecated", when="@0.22.0:", type=("build", "run"))
 
     depends_on("py-attrs@19.2.0:", type=("build", "run"))
     depends_on("py-hepunits@1.2.0:", when="@:0.12", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pathy/package.py b/var/spack/repos/builtin/packages/py-pathy/package.py
new file mode 100644
index 00000000000000..93c7b8be5a52bc
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-pathy/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPathy(PythonPackage):
+    """pathlib.Path subclasses for local and cloud bucket storage"""
+
+    homepage = "https://github.com/justindujardin/pathy"
+    pypi = "pathy/pathy-0.10.1.tar.gz"
+
+    version("0.10.1", sha256="4cd6e71b4cd5ff875cfbb949ad9fa5519d8d1dbe69d5fc1d1b23aa3cb049618b")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-smart-open@5.2.1:6", type=("build", "run"))
+    depends_on("py-typer@0.3:0", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pennylane-lightning-kokkos/package.py b/var/spack/repos/builtin/packages/py-pennylane-lightning-kokkos/package.py
index e910e087e1ac89..8871ac3027338a 100644
--- a/var/spack/repos/builtin/packages/py-pennylane-lightning-kokkos/package.py
+++ b/var/spack/repos/builtin/packages/py-pennylane-lightning-kokkos/package.py
@@ -13,15 +13,20 @@ class PyPennylaneLightningKokkos(CMakePackage, PythonExtension, CudaPackage, ROC
 
     homepage = "https://docs.pennylane.ai/projects/lightning-kokkos"
     git = "https://github.com/PennyLaneAI/pennylane-lightning-kokkos.git"
-    url = "https://github.com/PennyLaneAI/pennylane-lightning-kokkos/archive/refs/tags/v0.30.0.tar.gz"
+    url = "https://github.com/PennyLaneAI/pennylane-lightning-kokkos/archive/refs/tags/v0.32.0.tar.gz"
 
     maintainers("AmintorDusko", "vincentmr")
 
     version("main", branch="main")
 
+    version("0.32.0", sha256="06f19dfb1073387ef9ee30c38ea44884844a771373256b694a0e1ceb87195bb2")
+    version("0.31.0", sha256="fe10322fee0fa7df45cd3a81d6c229a79c7dfa7f20ff7d67c65c9a28f494dc89")
     version("0.30.0", sha256="7c8f0e0431f8052993cd8033a316f53590c7bf5419445d0725e214b93cbc661b")
     version("0.29.1", sha256="f51ba7718defc7bb5064f690f381e04b2ec58cb09f22a171ae5f410860716e30")
 
+    depends_on("kokkos@:3.7.2", when="@:0.30", type=("run", "build"))
+    depends_on("kokkos@4:", when="@0.31:", type=("run", "build"))
+
     # kokkos backends
     backends = {
         "cuda": [False, "Whether to build CUDA backend"],
@@ -36,21 +41,16 @@ class PyPennylaneLightningKokkos(CMakePackage, PythonExtension, CudaPackage, ROC
     for backend in backends:
         deflt_bool, descr = backends[backend]
         variant(backend.lower(), default=deflt_bool, description=descr)
-        depends_on(
-            f"kokkos@3.7+{backend.lower()}", when=f"+{backend.lower()}", type=("run", "build")
-        )
-
+        depends_on(f"kokkos+{backend.lower()}", when=f"+{backend.lower()}", type=("run", "build"))
     # CUDA
     for val in CudaPackage.cuda_arch_values:
-        depends_on("kokkos@:3.7.01 cuda_arch={0}".format(val), when="cuda_arch={0}".format(val))
+        depends_on("kokkos cuda_arch={0}".format(val), when="cuda_arch={0}".format(val))
     # Use +wrapper when not %clang %cce
-    depends_on("kokkos@:3.7.01+wrapper", when="%gcc+cuda")
+    depends_on("kokkos+wrapper", when="%gcc+cuda")
 
     # ROCm
     for val in ROCmPackage.amdgpu_targets:
-        depends_on(
-            "kokkos@:3.7.01 amdgpu_target={0}".format(val), when="amdgpu_target={0}".format(val)
-        )
+        depends_on("kokkos amdgpu_target={0}".format(val), when="amdgpu_target={0}".format(val))
 
     conflicts(
         "+cuda",
@@ -72,9 +72,12 @@ class PyPennylaneLightningKokkos(CMakePackage, PythonExtension, CudaPackage, ROC
     depends_on("py-pybind11", type="link")
     depends_on("py-pip", type="build")
     depends_on("py-wheel", type="build")
-    depends_on("py-pennylane@0.28:", type=("build", "run"))
-    depends_on("py-pennylane-lightning@0.30:~kokkos", type=("build", "run"), when="@0.30.0:")
-    depends_on("py-pennylane-lightning@0.28:0.29~kokkos", type=("build", "run"), when="@0.29.1")
+    depends_on("py-pennylane@0.28:", type=("build", "run"), when="@:0.30")
+    depends_on("py-pennylane@0.30:", type=("build", "run"), when="@0.31:")
+    # This requirement differs from `pennylane>=0.30` in `setup.py`,
+    # but the introduction of `StatePrep` demands `pennylane>=0.32`
+    depends_on("py-pennylane@0.32:", type=("build", "run"), when="@0.32:")
+    depends_on("py-pennylane-lightning~kokkos", type=("build", "run"), when="@:0.31")
 
     # variant defined dependencies
     depends_on("llvm-openmp", when="+openmp %apple-clang")
diff --git a/var/spack/repos/builtin/packages/py-pennylane-lightning/package.py b/var/spack/repos/builtin/packages/py-pennylane-lightning/package.py
index e9edf6e1102526..5e931b9ac57628 100644
--- a/var/spack/repos/builtin/packages/py-pennylane-lightning/package.py
+++ b/var/spack/repos/builtin/packages/py-pennylane-lightning/package.py
@@ -12,11 +12,13 @@ class PyPennylaneLightning(CMakePackage, PythonExtension):
 
     homepage = "https://docs.pennylane.ai/projects/lightning"
     git = "https://github.com/PennyLaneAI/pennylane-lightning.git"
-    url = "https://github.com/PennyLaneAI/pennylane-lightning/archive/refs/tags/v0.30.0.tar.gz"
+    url = "https://github.com/PennyLaneAI/pennylane-lightning/archive/refs/tags/v0.32.0.tar.gz"
 
     maintainers("mlxd", "AmintorDusko")
 
     version("master", branch="master")
+    version("0.32.0", sha256="124edae1828c7e72e7b3bfbb0e75e98a07a490d7f1eab19eebb3311bfa8a23d4")
+    version("0.31.0", sha256="b177243625b6fdac0699d163bbc330c92ca87fb9f427643785069273d2a255f6")
     version("0.30.0", sha256="0f4032409d20d00991b5d14fe0b2b928baca4a13c5a1b16eab91f61f9273e58d")
     version("0.29.0", sha256="da9912f0286d1a54051cc19cf8bdbdcd732795636274c95f376db72a88e52d85")
 
@@ -26,7 +28,7 @@ class PyPennylaneLightning(CMakePackage, PythonExtension):
         default=True,
         description="Build with AVX2/AVX512 gate automatic dispatching support",
     )
-    variant("kokkos", default=True, description="Build with Kokkos support")
+    variant("kokkos", default=True, description="Build with Kokkos support", when="@:0.31")
     variant("openmp", default=True, description="Build with OpenMP support")
 
     variant("native", default=False, description="Build natively for given hardware")
@@ -43,11 +45,12 @@ class PyPennylaneLightning(CMakePackage, PythonExtension):
 
     # variant defined dependencies
     depends_on("blas", when="+blas")
-    depends_on("kokkos@:3.7.01", when="+kokkos")
-    depends_on("kokkos-kernels@:3.7.01", when="+kokkos")
+    depends_on("kokkos@:4.0.01", when="@:0.31+kokkos")
+    depends_on("kokkos-kernels@:4.0.01", when="@:0.31+kokkos")
     depends_on("llvm-openmp", when="+openmp %apple-clang")
 
-    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("python@3.8:", type=("build", "run"), when="@:0.31")
+    depends_on("python@3.9:", type=("build", "run"), when="@0.32:")
     depends_on("py-setuptools", type="build")
     depends_on("py-numpy", type=("build", "run"))
     depends_on("py-pybind11", type="link")
@@ -79,7 +82,7 @@ def cmake_args(self):
                 f"-DKokkos_Core_DIR={self.spec['kokkos'].home}",
                 f"-DKokkos_Kernels_DIR={self.spec['kokkos-kernels'].home}",
             ]
-        else:
+        elif self.spec.version < Version("0.32"):
             args += ["-DENABLE_KOKKOS=OFF"]
 
         return args
diff --git a/var/spack/repos/builtin/packages/py-pennylane/package.py b/var/spack/repos/builtin/packages/py-pennylane/package.py
index f9b31ad4a8e5f4..e3bc536ae98a75 100644
--- a/var/spack/repos/builtin/packages/py-pennylane/package.py
+++ b/var/spack/repos/builtin/packages/py-pennylane/package.py
@@ -12,34 +12,40 @@ class PyPennylane(PythonPackage):
 
     homepage = "https://docs.pennylane.ai/"
     git = "https://github.com/PennyLaneAI/pennylane.git"
-    url = "https://github.com/PennyLaneAI/pennylane/archive/refs/tags/v0.30.0.tar.gz"
+    url = "https://github.com/PennyLaneAI/pennylane/archive/refs/tags/v0.32.0.tar.gz"
 
     maintainers("mlxd", "AmintorDusko", "marcodelapierre")
 
     version("master", branch="master")
+    version("0.32.0", sha256="8a2206268d7cae0a59f9067b6075175eec93f4843519b371f02716c49a22e750")
+    version("0.31.0", sha256="f3b68700825c120e44434ed2b2ab71d0be9d3111f3043077ec0598661ec33477")
     version("0.30.0", sha256="7fe4821fbc733e3e40d7011e054bd2e31edde3151fd9539025c827a5a3579d6b")
     version("0.29.1", sha256="6ecfb305a3898347df8c539a89a67e748766941d159dbef9e34864872f13c45c")
 
-    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("python@3.8:", type=("build", "run"), when="@:0.31.0")
+    depends_on("python@3.9:", type=("build", "run"), when="@0.32.0:")
     depends_on("py-pip", type=("build", "run"))  # Runtime req for pennylane.about()
     depends_on("py-setuptools", type="build")
 
     depends_on("py-numpy@:1.23", type=("build", "run"))
     depends_on("py-scipy", type=("build", "run"))
+    depends_on("py-scipy@:1.10", type=("build", "run"), when="@:0.31.0")
     depends_on("py-networkx", type=("build", "run"))
     depends_on("py-rustworkx", type=("build", "run"), when="@0.30.0:")
     depends_on("py-retworkx", type=("build", "run"), when="@0.28.0:0.29.1")
-    depends_on("py-autograd", type=("build", "run"))
+    depends_on("py-autograd@:1.5", type=("build", "run"))
     depends_on("py-toml", type=("build", "run"))
     depends_on("py-appdirs", type=("build", "run"))
     depends_on("py-semantic-version@2.7:", type=("build", "run"))
     depends_on("py-autoray@0.3.1:", type=("build", "run"))
     depends_on("py-cachetools", type=("build", "run"))
-    depends_on("py-pennylane-lightning@0.30.0:", type=("build", "run"), when="@0.30.0:")
+    for v in range(30, 33):
+        depends_on(f"py-pennylane-lightning@0.{v}.0:", type=("build", "run"), when=f"@0.{v}.0:")
     depends_on(
         "py-pennylane-lightning@0.28.0:0.29.0", type=("build", "run"), when="@0.28.0:0.29.1"
     )
     depends_on("py-requests", type=("build", "run"))
+    depends_on("py-typing-extensions", type=("build", "run"), when="@0.32.0:")
 
     # Test deps
     depends_on("py-pytest", type="test")
diff --git a/var/spack/repos/builtin/packages/py-petsc4py/package.py b/var/spack/repos/builtin/packages/py-petsc4py/package.py
index 08491dd38482e1..06c551b9f4a1a1 100644
--- a/var/spack/repos/builtin/packages/py-petsc4py/package.py
+++ b/var/spack/repos/builtin/packages/py-petsc4py/package.py
@@ -10,12 +10,21 @@ class PyPetsc4py(PythonPackage):
     """This package provides Python bindings for the PETSc package."""
 
     homepage = "https://gitlab.com/petsc/petsc4py"
-    url = "https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc4py-3.15.0.tar.gz"
+    url = (
+        "https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc4py-3.20.0.tar.gz"
+    )
     git = "https://gitlab.com/petsc/petsc.git"
 
     maintainers("balay")
 
     version("main", branch="main")
+    version("3.20.1", sha256="dcc9092040d13130496f1961b79c36468f383b6ede398080e004f1966c06ad38")
+    version("3.20.0", sha256="c2461eef3977ae5c214ad252520adbb92ec3a31d00e79391dd92535077bbf03e")
+    version("3.19.6", sha256="bd7891b651eb83504c744e70706818cf63ecbabee3206c1fed7c3013873802b9")
+    version("3.19.5", sha256="e059fdb8b23936c3182c9226924029dbdc8f1f72a623be0fe8c2caf8646c7a45")
+    version("3.19.4", sha256="5621ddee63d0c631d2e8fed2d5d9763b183ad164c227dde8d3abcdb6c35c5ffb")
+    version("3.19.3", sha256="dcbadebf0c4fe78b4dc13b8cd910577b9cacf65636ea980523e61d95c6959e5b")
+    version("3.19.2", sha256="5f207eb95f87ddafa32229681a95af61912871cd7fbd38780bc63019dad3e7b8")
     version("3.19.1", sha256="d04def9995ed6395e125c605da169438d77d410d5019dc57be42e428ade30190")
     version("3.19.0", sha256="d1660092c9be9547e9a17d3d5bb139eaad737c3e1c4ef2ee41c71c8dc3bb9955")
     version("3.18.6", sha256="e4976e42895955cbb2c56f1b0f791c838338348a10664b8dcfc3fe56198bf943")
@@ -64,16 +73,10 @@ class PyPetsc4py(PythonPackage):
     depends_on("petsc+mpi", when="+mpi")
     depends_on("petsc~mpi", when="~mpi")
     depends_on("petsc@main", when="@main")
-    depends_on("petsc@3.19.0:3.19", when="@3.19.0:3.19")
-    depends_on("petsc@3.18.0:3.18", when="@3.18.0:3.18")
-    depends_on("petsc@3.17.0:3.17", when="@3.17.0:3.17")
-    depends_on("petsc@3.16.0:3.16", when="@3.16.0:3.16")
-    depends_on("petsc@3.15.0:3.15", when="@3.15.0:3.15")
+    for ver in ["3.20", "3.19", "3.18", "3.17", "3.16", "3.15", "3.13", "3.12", "3.11"]:
+        depends_on(f"petsc@{ver}", when=f"@{ver}")
     depends_on("petsc@3.14.2:3.14", when="@3.14.1:3.14")
     depends_on("petsc@3.14.0:3.14.1", when="@3.14.0")
-    depends_on("petsc@3.13.0:3.13", when="@3.13.0:3.13")
-    depends_on("petsc@3.12.0:3.12", when="@3.12.0:3.12")
-    depends_on("petsc@3.11.0:3.11", when="@3.11.0:3.11")
 
     @property
     def build_directory(self):
diff --git a/var/spack/repos/builtin/packages/py-pgzip/package.py b/var/spack/repos/builtin/packages/py-pgzip/package.py
new file mode 100644
index 00000000000000..62452ba417a5cb
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-pgzip/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPgzip(PythonPackage):
+    """A multi-threading implementation of Python gzip module"""
+
+    homepage = "https://github.com/pgzip/pgzip"
+    pypi = "pgzip/pgzip-0.3.4.tar.gz"
+
+    version("0.3.4", sha256="ef56449039bc6e88558e46fe6bb11e3faaeef445d3985a9fb286795ff842c480")
+    version("0.3.1", sha256="a9c2df369311473ec3c239f26bf01638bdc6b6094d89ff4c81c6ef5c84eb24b7")
+
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-picmistandard/package.py b/var/spack/repos/builtin/packages/py-picmistandard/package.py
index bb4eacd4cb3f66..bbc6b56212075f 100644
--- a/var/spack/repos/builtin/packages/py-picmistandard/package.py
+++ b/var/spack/repos/builtin/packages/py-picmistandard/package.py
@@ -11,27 +11,21 @@ class PyPicmistandard(PythonPackage):
 
     homepage = "https://picmi-standard.github.io"
     git = "https://github.com/picmi-standard/picmi.git"
-    pypi = "picmistandard/picmistandard-0.0.18.tar.gz"
+    pypi = "picmistandard/picmistandard-0.26.0.tar.gz"
 
     maintainers("ax3l", "dpgrote", "RemiLehe")
 
-    version("develop", branch="master")
-    version("0.25.0", sha256="0a78b3b17054d0861626287ace1fb9321469a9c95795b92981103b27d7797f67")
+    version("master", branch="master")
+    version("0.26.0", sha256="b22689f576d064bf0cd8f435621e912359fc2ee9347350eab845d2d36ebb62eb")
+    version("0.25.0", sha256="3fe6a524822d382e52bfc9d3378249546075d28620969954c5ffb43e7968fb02")
     version("0.24.0", sha256="55a82adcc14b41eb612caf0d9e47b0e2a56ffc196a58b41fa0cc395c6924be9a")
-    version("0.23.2", sha256="e6b4c46b23520d0a97b904df90d53ff6a3209b2b6b2fa741f684c594429a591c")
-    version("0.23.1", sha256="90ad1d3d2759d910cfdb88484516b7d0434ec98e881af46961b7e2faa534434d")
+    version("0.23.2", sha256="2853fcfaf2f226a88bb6063ae564832b7e69965294fd652cd2ac04756fa4599a")
+    version("0.23.1", sha256="c7375010b7a3431b519bc0accf097f2aafdb520e2a0126f42895cb96dcc7dcf1")
     version("0.0.22", sha256="e234a431274254b22cd70be64d6555b383d98426b2763ea0c174cf77bf4d0890")
     version("0.0.21", sha256="930056a23ed92dac7930198f115b6248606b57403bffebce3d84579657c8d10b")
     version("0.0.20", sha256="9c1822eaa2e4dd543b5afcfa97940516267dda3890695a6cf9c29565a41e2905")
     version("0.0.19", sha256="4b7ba1330964fbfd515e8ea2219966957c1386e0896b92d36bd9e134afb02f5a")
     version("0.0.18", sha256="68c208c0c54b4786e133bb13eef0dd4824998da4906285987ddee84e6d195e71")
-    # 0.15 - 0.17 have broken install logic: missing requirements.txt on pypi
-    version(
-        "0.0.16",
-        sha256="b7eefdae1c43119984226b2df358c86fdeef7495084e47b3575e3d07e790ba30",
-        url="https://github.com/picmi-standard/picmi/archive/refs/tags/0.0.14.tar.gz",
-    )
-    version("0.0.14", sha256="8f83b25b281fc0309a0c4f75c7605afd5fa0ef4df3b3ac115069478c119bc8c3")
 
     depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-numpy@1.15:1", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pika/package.py b/var/spack/repos/builtin/packages/py-pika/package.py
index 135568dd833078..0d16ae4071ebbe 100644
--- a/var/spack/repos/builtin/packages/py-pika/package.py
+++ b/var/spack/repos/builtin/packages/py-pika/package.py
@@ -11,13 +11,24 @@ class PyPika(PythonPackage):
 
     homepage = "https://pika.readthedocs.io/"
     git = "https://github.com/pika/pika.git"
-    pypi = "pika/pika-1.2.0.tar.gz"
+    pypi = "pika/pika-1.3.2.tar.gz"
 
+    version("1.3.2", sha256="b2a327ddddf8570b4965b3576ac77091b850262d34ce8c1d8cb4e4146aa4145f")
+    version("1.3.1", sha256="beb19ff6dd1547f99a29acc2c6987ebb2ba7c44bf44a3f8e305877c5ef7d2fdc")
+    version("1.3.0", sha256="15357ddc47a5c28f0b07d80e93d504cbbf7a1ad5e1cd129ecd27afe76472c529")
+    version("1.2.1", sha256="e5fbf3a0a3599f4e114f6e4a7af096f9413a8f24f975c2657ba2fac3c931434f")
     version("1.2.0", sha256="f023d6ac581086b124190cb3dc81dd581a149d216fa4540ac34f9be1e3970b89")
     version("1.1.0", sha256="9fa76ba4b65034b878b2b8de90ff8660a59d925b087c5bb88f8fdbb4b64a1dbf")
     version("1.0.0", sha256="fba41293b35c845bd96cfdd29981f0eeff91f705ac0c3ba361a771c4bfbc3485")
     version("0.13.1", sha256="b0640085f1d6398fd47bb16a17713053e26578192821ea5d928772b8e6a28789")
     version("0.13.0", sha256="5338d829d1edb3e5bcf1523b4a9e32c56dea5a8bda7018825849e35325580484")
 
-    depends_on("python@2.7,3.4:", type=("build", "run"))
+    variant("gevent", default=False, description="Build with gevent support")
+    variant("tornado", default=False, description="Build with tornado support")
+    variant("twisted", default=False, description="Build with twisted support")
+
     depends_on("py-setuptools", type="build")
+    depends_on("py-setuptools@61.2:", when="@1.3.0:", type="build")
+    depends_on("py-gevent", when="+gevent", type=("build", "run"))
+    depends_on("py-tornado", when="+tornado", type=("build", "run"))
+    depends_on("py-twisted", when="+twisted", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pillow/package.py b/var/spack/repos/builtin/packages/py-pillow/package.py
index 21152d37efd8c1..5a6007e9e3ed11 100644
--- a/var/spack/repos/builtin/packages/py-pillow/package.py
+++ b/var/spack/repos/builtin/packages/py-pillow/package.py
@@ -54,7 +54,7 @@ class PyPillowBase(PythonPackage):
 
     # Optional dependencies
     # https://pillow.readthedocs.io/en/latest/installation.html#external-libraries
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("jpeg", when="+jpeg")
     depends_on("libtiff", when="+tiff")
     depends_on("freetype", when="+freetype")
diff --git a/var/spack/repos/builtin/packages/py-pint/package.py b/var/spack/repos/builtin/packages/py-pint/package.py
index 85bb7a0f054acd..83cb92af0d6d67 100644
--- a/var/spack/repos/builtin/packages/py-pint/package.py
+++ b/var/spack/repos/builtin/packages/py-pint/package.py
@@ -18,6 +18,8 @@ class PyPint(PythonPackage):
     # any import tests for this package.
     import_modules = []  # type: List[str]
 
+    version("0.22", sha256="2d139f6abbcf3016cad7d3cec05707fe908ac4f99cf59aedfd6ee667b7a64433")
+    version("0.21.1", sha256="5d5b6b518d0c5a7ab03a776175db500f1ed1523ee75fb7fafe38af8149431c8d")
     version("0.20.1", sha256="387cf04078dc7dfe4a708033baad54ab61d82ab06c4ee3d4922b1e45d5626067")
     version("0.18", sha256="8c4bce884c269051feb7abc69dbfd18403c0c764abc83da132e8a7222f8ba801")
     version("0.17", sha256="f4d0caa713239e6847a7c6eefe2427358566451fe56497d533f21fb590a3f313")
@@ -27,11 +29,14 @@ class PyPint(PythonPackage):
     version("0.9", sha256="32d8a9a9d63f4f81194c0014b3b742679dce81a26d45127d9810a68a561fe4e2")
     version("0.8.1", sha256="afcf31443a478c32bbac4b00337ee9026a13d0e2ac83d30c79151462513bb0d4")
 
-    depends_on("python@3.8:", type=("build", "run"), when="@0.19:")
-    depends_on("py-setuptools@41:", when="@0.16:", type="build")
+    depends_on("python@3.9:", when="@0.22:", type=("build", "run"))
+    depends_on("python@3.8:", when="@0.19:0.21", type=("build", "run"))
+    depends_on("py-typing-extensions", when="@0.22:", type=("build", "run"))
+    depends_on("py-setuptools@61:", when="@0.21:", type="build")
+    depends_on("py-setuptools@41:", when="@0.16:0.20", type="build")
     depends_on("py-setuptools@41:", when="@0.11:0.15", type=("build", "run"))
     depends_on("py-setuptools", when="@:0.10", type=("build", "run"))
     depends_on("py-setuptools-scm@3.4.3:+toml", when="@0.11:", type="build")
     depends_on("py-setuptools-scm", when="@0.10", type="build")
-    depends_on("py-packaging", type=("build", "run"), when="@0.13:18")
-    depends_on("py-importlib-metadata", type=("build", "run"), when="@0.13:18 ^python@:3.7")
+    depends_on("py-packaging", when="@0.13:18", type=("build", "run"))
+    depends_on("py-importlib-metadata", when="@0.13:18 ^python@:3.7", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pip/package.py b/var/spack/repos/builtin/packages/py-pip/package.py
index 149f042756675c..d92a53671ed8b8 100644
--- a/var/spack/repos/builtin/packages/py-pip/package.py
+++ b/var/spack/repos/builtin/packages/py-pip/package.py
@@ -15,6 +15,8 @@ class PyPip(Package, PythonExtension):
     url = "https://files.pythonhosted.org/packages/py3/p/pip/pip-20.2-py3-none-any.whl"
     list_url = "https://pypi.org/simple/pip/"
 
+    tags = ["build-tools"]
+
     maintainers("adamjstewart", "pradyunsg")
 
     version(
@@ -86,6 +88,9 @@ class PyPip(Package, PythonExtension):
     extends("python")
     depends_on("python@3.7:", when="@22:", type=("build", "run"))
 
+    # Uses collections.MutableMapping
+    depends_on("python@:3.9", when="@:19.1", type=("build", "run"))
+
     def url_for_version(self, version):
         url = "https://files.pythonhosted.org/packages/{0}/p/pip/pip-{1}-{0}-none-any.whl"
         if version >= Version("21"):
@@ -105,6 +110,5 @@ def install(self, spec, prefix):
 
     def setup_dependent_package(self, module, dependent_spec):
         pip = dependent_spec["python"].command
-        pip.add_default_arg("-m")
-        pip.add_default_arg("pip")
+        pip.add_default_arg("-m", "pip")
         setattr(module, "pip", pip)
diff --git a/var/spack/repos/builtin/packages/py-pipdeptree/package.py b/var/spack/repos/builtin/packages/py-pipdeptree/package.py
new file mode 100644
index 00000000000000..92600763ede13c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-pipdeptree/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyPipdeptree(PythonPackage):
+    """Command line utility to show dependency tree of packages."""
+
+    homepage = "https://github.com/tox-dev/pipdeptree"
+    pypi = "pipdeptree/pipdeptree-2.13.0.tar.gz"
+
+    version("2.13.0", sha256="ff71a48abd0b1ab810c23734b47de6ebd93270857d6665e21ed5ef6136fcba6e")
+
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-hatch-vcs@0.3:", type="build")
+    depends_on("py-hatchling@1.18:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-platformdirs/package.py b/var/spack/repos/builtin/packages/py-platformdirs/package.py
index c655d702da2c3b..82929c5c8e8da8 100644
--- a/var/spack/repos/builtin/packages/py-platformdirs/package.py
+++ b/var/spack/repos/builtin/packages/py-platformdirs/package.py
@@ -14,6 +14,7 @@ class PyPlatformdirs(PythonPackage):
     homepage = "https://github.com/platformdirs/platformdirs"
     pypi = "platformdirs/platformdirs-2.4.0.tar.gz"
 
+    version("3.10.0", sha256="b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d")
     version("3.5.3", sha256="e48fabd87db8f3a7df7150a4a5ea22c546ee8bc39bc2473244730d4b56d2cc4e")
     version("3.5.0", sha256="7954a68d0ba23558d753f73437c55f89027cf8f5108c19844d4b82e5af396335")
     version("3.1.1", sha256="024996549ee88ec1a9aa99ff7f8fc819bb59e2c3477b410d90a16d32d6e707aa")
@@ -32,11 +33,13 @@ class PyPlatformdirs(PythonPackage):
     depends_on("python@3.7:", when="@2.4.1:", type=("build", "run"))
     depends_on("py-hatch-vcs@0.3:", when="@3:", type="build")
     depends_on("py-hatch-vcs", when="@2.5.2:", type="build")
+    depends_on("py-hatchling@1.17.1:", when="@3.10:", type="build")
     depends_on("py-hatchling@1.17:", when="@3.5.2:", type="build")
     depends_on("py-hatchling@1.14:", when="@3.3:", type="build")
     depends_on("py-hatchling@1.12.2:", when="@3:", type="build")
     depends_on("py-hatchling@0.22.0:", when="@2.5.2:", type="build")
 
+    depends_on("py-typing-extensions@4.7.1:", when="@3.10: ^python@:3.7", type=("build", "run"))
     depends_on("py-typing-extensions@4.6.3:", when="@3.5.2: ^python@:3.7", type=("build", "run"))
     depends_on("py-typing-extensions@4.5:", when="@3.2: ^python@:3.7", type=("build", "run"))
     depends_on("py-typing-extensions@4.4:", when="@3: ^python@:3.7", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-plotext/package.py b/var/spack/repos/builtin/packages/py-plotext/package.py
new file mode 100644
index 00000000000000..0d38a81d898fa7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-plotext/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPlotext(PythonPackage):
+    """Plotext plots directly on terminal."""
+
+    pypi = "plotext/plotext-5.2.8.tar.gz"
+    git = "https://github.com/piccolomo/plotext.git"
+
+    version("master", branch="master")
+    version("5.2.8", sha256="319a287baabeb8576a711995f973a2eba631c887aa6b0f33ab016f12c50ffebe")
+
+    # build dependencies
+    depends_on("python@3.5.0:", type=("build", "run"))
+    depends_on("py-setuptools", type=("build"))
diff --git a/var/spack/repos/builtin/packages/py-poetry-core/package.py b/var/spack/repos/builtin/packages/py-poetry-core/package.py
index c7b46373ff65e1..b49a291be9e5d3 100644
--- a/var/spack/repos/builtin/packages/py-poetry-core/package.py
+++ b/var/spack/repos/builtin/packages/py-poetry-core/package.py
@@ -10,17 +10,26 @@ class PyPoetryCore(PythonPackage):
     """Poetry PEP 517 Build Backend."""
 
     homepage = "https://github.com/python-poetry/poetry-core"
-    pypi = "poetry-core/poetry-core-1.0.7.tar.gz"
+    pypi = "poetry-core/poetry_core-1.6.1.tar.gz"
 
+    version("1.6.1", sha256="0f9b0de39665f36d6594657e7d57b6f463cc10f30c28e6d1c3b9ff54c26c9ac3")
     version("1.2.0", sha256="ceccec95487e46c63a41761fbac5211b809bca22658e25a049f4c7da96269f71")
     version("1.1.0", sha256="d145ae121cf79118a8901b60f2c951c4edcc16f55eb8aaefc156aa33aa921f07")
     version("1.0.8", sha256="951fc7c1f8d710a94cb49019ee3742125039fc659675912ea614ac2aa405b118")
     version("1.0.7", sha256="98c11c755a16ef6c5673c22ca94a3802a7df4746a0853a70b6fae8b9f5cac206")
 
-    depends_on("python@2.7,3.5:3", type=("build", "run"))
     depends_on("python@3.7:3", when="@1.1.0:", type=("build", "run"))
-    depends_on("py-importlib-metadata@1.7:1", when="@:1.0 ^python@:3.7", type=("build", "run"))
+    depends_on("python@:3", type=("build", "run"))
     depends_on("py-importlib-metadata@1.7:", when="@1.1: ^python@:3.7", type=("build", "run"))
+    depends_on("py-importlib-metadata@1.7:1", when="@:1.0 ^python@:3.7", type=("build", "run"))
+
+    def url_for_version(self, version):
+        url = "https://files.pythonhosted.org/packages/source/p/poetry-core/{0}-{1}.tar.gz"
+        if version >= Version("1.4"):
+            letter = "poetry_core"
+        else:
+            letter = "poetry-core"
+        return url.format(letter, version)
 
     # https://github.com/python-poetry/poetry/issues/5547
     def setup_build_environment(self, env):
diff --git a/var/spack/repos/builtin/packages/py-preshed/package.py b/var/spack/repos/builtin/packages/py-preshed/package.py
index 62bb8c4a37e57b..e6c8a175485b97 100644
--- a/var/spack/repos/builtin/packages/py-preshed/package.py
+++ b/var/spack/repos/builtin/packages/py-preshed/package.py
@@ -13,8 +13,10 @@ class PyPreshed(PythonPackage):
     homepage = "https://github.com/explosion/preshed"
     pypi = "preshed/preshed-3.0.2.tar.gz"
 
+    version("3.0.8", sha256="6c74c70078809bfddda17be96483c41d06d717934b07cab7921011d81758b357")
     version("3.0.2", sha256="61d73468c97c1d6d5a048de0b01d5a6fd052123358aca4823cdb277e436436cb")
 
     depends_on("py-setuptools", type="build")
+    depends_on("py-cython@0.28:", when="@3.0.8:", type="build")
     depends_on("py-cymem@2.0.2:2.0", type=("build", "run"))
     depends_on("py-murmurhash@0.28:1.0", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-protobuf/package.py b/var/spack/repos/builtin/packages/py-protobuf/package.py
index 9f688515cc86a7..41ee1b3f41cf7d 100644
--- a/var/spack/repos/builtin/packages/py-protobuf/package.py
+++ b/var/spack/repos/builtin/packages/py-protobuf/package.py
@@ -17,10 +17,8 @@ class PyProtobuf(PythonPackage):
     homepage = "https://developers.google.com/protocol-buffers/"
     pypi = "protobuf/protobuf-3.11.0.tar.gz"
 
-    variant("cpp", default=True, description="Enable the cpp implementation")
-
-    # Newer versions seem to require bazel to build?
-    # https://github.com/protocolbuffers/protobuf/tree/main/python
+    version("4.24.3", sha256="12e9ad2ec079b833176d2921be2cb24281fa591f0b119b208b788adc48c2561d")
+    version("4.23.3", sha256="7a92beb30600332a52cdadbedb40d33fd7c8a0d7f549c440347bc606fb3fe34b")
     version("4.21.9", sha256="61f21493d96d2a77f9ca84fefa105872550ab5ef71d21c458eb80edcf4885a99")
     version("4.21.7", sha256="71d9dba03ed3432c878a801e2ea51e034b0ea01cf3a4344fb60166cb5f6c8757")
     version("4.21.5", sha256="eb1106e87e095628e96884a877a51cdb90087106ee693925ec0a300468a9be3a")
@@ -61,18 +59,22 @@ class PyProtobuf(PythonPackage):
     version("3.3.0", sha256="1cbcee2c45773f57cb6de7ee0eceb97f92b9b69c0178305509b162c0160c1f04")
     version("3.0.0", sha256="ecc40bc30f1183b418fe0ec0c90bc3b53fa1707c4205ee278c6b90479e5b6ff5")
 
+    variant("cpp", default=False, when="@:4.21", description="Enable the cpp implementation")
+
+    depends_on("python", type=("build", "link", "run"))
     depends_on("py-setuptools", type=("build", "run"))
     # in newer pip versions --install-option does not exist
-    depends_on("py-pip@:23.0", when="@:4.21", type=("build", "run"))
+    depends_on("py-pip@:23.0", when="+cpp", type=("build", "run"))
     depends_on("py-six@1.9:", when="@3.0:3.17", type=("build", "run"))
 
     # Setup dependencies for protobuf to use the same minor version as py-protobuf
     # Handle mapping the 4.x release to the protobuf 3.x releases
-    for ver in list(range(21, 22)):
-        depends_on("protobuf@3." + str(ver), when="+cpp @4." + str(ver))
+    depends_on("protobuf@3.21", when="+cpp @4.21")
     # Handle the 3.x series releases
     for ver in list(range(0, 21)):
-        depends_on("protobuf@3." + str(ver), when="+cpp @3." + str(ver))
+        depends_on(f"protobuf@3.{ver}", when=f"@3.{ver}+cpp")
+
+    conflicts("+cpp", when="^python@3.11:")
 
     @property
     def build_directory(self):
diff --git a/var/spack/repos/builtin/packages/py-py-cpuinfo/package.py b/var/spack/repos/builtin/packages/py-py-cpuinfo/package.py
index 87d4e23e06ad8e..b98a15f82872f8 100644
--- a/var/spack/repos/builtin/packages/py-py-cpuinfo/package.py
+++ b/var/spack/repos/builtin/packages/py-py-cpuinfo/package.py
@@ -12,6 +12,7 @@ class PyPyCpuinfo(PythonPackage):
     homepage = "https://github.com/workhorsy/py-cpuinfo"
     pypi = "py-cpuinfo/py-cpuinfo-0.2.3.tar.gz"
 
+    version("9.0.0", sha256="3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690")
     version("8.0.0", sha256="5f269be0e08e33fd959de96b34cd4aeeeacac014dd8305f70eb28d06de2345c5")
     version("6.0.0", sha256="7ffb31dea845b9f359b99bd5f7eea72dc70f852e0e34547d261a630f2b8c9c61")
     version("0.2.3", sha256="f6a016fdbc4e7fadf2d519090fcb4fa9d0831bad4e85245d938e5c2fe7623ca6")
diff --git a/var/spack/repos/builtin/packages/py-pybids/package.py b/var/spack/repos/builtin/packages/py-pybids/package.py
index 7155b92c5248e4..8a77813ad6a05d 100644
--- a/var/spack/repos/builtin/packages/py-pybids/package.py
+++ b/var/spack/repos/builtin/packages/py-pybids/package.py
@@ -12,6 +12,7 @@ class PyPybids(PythonPackage):
     homepage = "https://github.com/bids-standard/pybids"
     pypi = "pybids/pybids-0.13.1.tar.gz"
 
+    version("0.16.3", sha256="10e279350c8d14ca602c0d4469a5e4bf7ff393e8643c831a546ae735b6b82cc3")
     version("0.16.1", sha256="1a6ab06d375f3b783e738826e6d220b2f4145419b4b02f4edbcc8cb7c9b2208a")
     version("0.15.3", sha256="4d99c979bc4bc209cff70a02d1da309c9bf8c6b0338e2a0b66ebea77c7f3c461")
     version("0.15.1", sha256="0253507a04dbfea43eb1f75a1f71aab04be21076bfe96c004888000b802e38f2")
@@ -22,7 +23,6 @@ class PyPybids(PythonPackage):
     version("0.8.0", sha256="fe60fa7d1e171e75a38a04220ed992f1b062531a7452fcb7ce5ba81bb6abfdbc")
 
     depends_on("python@3.8:", when="@0.16:", type=("build", "run"))
-    depends_on("python@3.7:", when="@0.15:", type=("build", "run"))
     depends_on("py-setuptools", when="@0.15.6:", type="build")
     depends_on("py-setuptools@30.3:60,61.0.1:", when="@:0.15.5", type="build")
     depends_on("py-versioneer+toml", when="@0.15.6:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-pybind11/package.py b/var/spack/repos/builtin/packages/py-pybind11/package.py
index 1dc422793c0346..63625651bcd964 100644
--- a/var/spack/repos/builtin/packages/py-pybind11/package.py
+++ b/var/spack/repos/builtin/packages/py-pybind11/package.py
@@ -27,6 +27,8 @@ class PyPybind11(CMakePackage, PythonExtension):
     maintainers("ax3l")
 
     version("master", branch="master")
+    version("2.11.1", sha256="d475978da0cdc2d43b73f30910786759d593a9d8ee05b1b6846d1eb16c6d2e0c")
+    version("2.11.0", sha256="7af30a84c6810e721829c4646e31927af9d8861e085aa5dd37c3c8b8169fcda1")
     version("2.10.4", sha256="832e2f309c57da9c1e6d4542dedd34b24e4192ecb4d62f6f4866a737454c9970")
     version("2.10.1", sha256="111014b516b625083bef701df7880f78c2243835abdb263065b6b59b960b6bad")
     version("2.10.0", sha256="eacf582fa8f696227988d08cfc46121770823839fe9e301a20fbce67e7cd70ec")
@@ -52,9 +54,6 @@ class PyPybind11(CMakePackage, PythonExtension):
 
     depends_on("py-setuptools@42:", type="build")
     depends_on("py-pytest", type="test")
-    depends_on("python@2.7:2.8,3.5:", type=("build", "run"))
-    depends_on("python@3.6:", when="@2.10.0:", type=("build", "run"))
-
     depends_on("py-pip", type="build")
     depends_on("py-wheel", type="build")
     extends("python")
@@ -64,10 +63,12 @@ class PyPybind11(CMakePackage, PythonExtension):
         depends_on("cmake@3.13:", type="build")
         depends_on("cmake@3.18:", type="build", when="@2.6.0:")
 
-    # compiler support
-    conflicts("%gcc@:4.7")
+    # https://github.com/pybind/pybind11/#supported-compilers
     conflicts("%clang@:3.2")
-    conflicts("%intel@:16")
+    conflicts("%apple-clang@:4")
+    conflicts("%gcc@:4.7")
+    conflicts("%msvc@:16")
+    conflicts("%intel@:17")
 
     # https://github.com/pybind/pybind11/pull/1995
     @when("@:2.4")
diff --git a/var/spack/repos/builtin/packages/py-pybrain/package.py b/var/spack/repos/builtin/packages/py-pybrain/package.py
index 0b9e4d86043016..758737efaf0a59 100644
--- a/var/spack/repos/builtin/packages/py-pybrain/package.py
+++ b/var/spack/repos/builtin/packages/py-pybrain/package.py
@@ -14,7 +14,7 @@ class PyPybrain(PythonPackage):
     url = "https://github.com/pybrain/pybrain/archive/refs/tags/0.3.3.tar.gz"
     git = "https://github.com/pybrain/pybrain.git"
 
-    version("0.3.3.post", commit="dcdf32b")
+    version("0.3.3.post", commit="dcdf32ba1805490cefbc0bdeb227260d304fdb42")
 
     depends_on("py-setuptools", type="build")
     depends_on("py-scipy", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pycocotools/package.py b/var/spack/repos/builtin/packages/py-pycocotools/package.py
index 60f577113211cf..8c892b58f5c5a2 100644
--- a/var/spack/repos/builtin/packages/py-pycocotools/package.py
+++ b/var/spack/repos/builtin/packages/py-pycocotools/package.py
@@ -22,3 +22,5 @@ class PyPycocotools(PythonPackage):
     depends_on("py-setuptools@18.0:", when="@:2.0.3", type=("build", "run"))
     depends_on("py-matplotlib@2.1.0:", type=("build", "run"))
     depends_on("py-numpy", type=("build", "link", "run"))
+
+    conflicts("^py-cython@3:", when="@:2.0.4")
diff --git a/var/spack/repos/builtin/packages/py-pycodestyle/package.py b/var/spack/repos/builtin/packages/py-pycodestyle/package.py
index 683d0728cdb517..326d79a58cc8e8 100644
--- a/var/spack/repos/builtin/packages/py-pycodestyle/package.py
+++ b/var/spack/repos/builtin/packages/py-pycodestyle/package.py
@@ -13,6 +13,7 @@ class PyPycodestyle(PythonPackage):
     homepage = "https://github.com/PyCQA/pycodestyle"
     pypi = "pycodestyle/pycodestyle-2.8.0.tar.gz"
 
+    version("2.11.0", sha256="259bcc17857d8a8b3b4a2327324b79e5f020a13c16074670f9c8c8f872ea76d0")
     version("2.10.0", sha256="347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053")
     version("2.9.1", sha256="2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785")
     version("2.9.0", sha256="beaba44501f89d785be791c9462553f06958a221d166c64e1f107320f839acc2")
diff --git a/var/spack/repos/builtin/packages/py-pydata-sphinx-theme/package.py b/var/spack/repos/builtin/packages/py-pydata-sphinx-theme/package.py
new file mode 100644
index 00000000000000..d1dfd45dab65b7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-pydata-sphinx-theme/package.py
@@ -0,0 +1,28 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPydataSphinxTheme(PythonPackage):
+    """A clean, three-column, Bootstrap-based Sphinx theme by and for the PyData community."""
+
+    homepage = "https://pydata-sphinx-theme.readthedocs.io/en/stable"
+    pypi = "pydata_sphinx_theme/pydata_sphinx_theme-0.14.1.tar.gz"
+
+    version("0.14.1", sha256="d8d4ac81252c16a002e835d21f0fea6d04cf3608e95045c816e8cc823e79b053")
+
+    depends_on("python@3.8:", type=("build", "run"))
+
+    depends_on("py-sphinx-theme-builder", type="build")
+
+    depends_on("py-sphinx@5:", type=("build", "run"))
+    depends_on("py-beautifulsoup4", type=("build", "run"))
+    depends_on("py-docutils@:0.16,0.17.1:", type=("build", "run"))
+    depends_on("py-packaging", type=("build", "run"))
+    depends_on("py-babel", type=("build", "run"))
+    depends_on("py-pygments@2.7:", type=("build", "run"))
+    depends_on("py-accessible-pygments", type=("build", "run"))
+    depends_on("py-typing-extensions", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pydevtool/package.py b/var/spack/repos/builtin/packages/py-pydevtool/package.py
new file mode 100644
index 00000000000000..ec9b69445fd66e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-pydevtool/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPydevtool(PythonPackage):
+    """CLI dev tools powered by pydoit."""
+
+    homepage = "https://github.com/pydoit/pydevtool"
+    pypi = "pydevtool/pydevtool-0.3.0.tar.gz"
+
+    version("0.3.0", sha256="25e3ba4f3d33ccac33ee2b9775995848d49e9b318b7a146477fb5d52f786fc8a")
+
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
+    depends_on("py-doit@0.36:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pydicom/package.py b/var/spack/repos/builtin/packages/py-pydicom/package.py
index 1001e3ddd47ec8..e2d1fadcecb614 100644
--- a/var/spack/repos/builtin/packages/py-pydicom/package.py
+++ b/var/spack/repos/builtin/packages/py-pydicom/package.py
@@ -15,15 +15,15 @@ class PyPydicom(PythonPackage):
     homepage = "https://github.com/pydicom/pydicom"
     pypi = "pydicom/pydicom-2.1.2.tar.gz"
 
+    version("2.4.3", sha256="51906e0b9fb6e184a0f56298cb43ed716b7cf7edc00f6b71d5c769bc1f982402")
     version("2.4.1", sha256="6cb210dbe5586841036e8eeb2d4feb4df22a48f39161ba7ee0bf3c89faaba946")
     version("2.3.0", sha256="dbfa081c9ad9ac8ff8a8efbd71784104db9eecf02fd775f7d7773f2183f89386")
     version("2.1.2", sha256="65f36820c5fec24b4e7ca45b7dae93e054ed269d55f92681863d39d30459e2fd")
 
     variant("numpy", default=False, description="Use NumPy for Pixel data")
 
-    depends_on("python@3.7:", when="@2.4:", type=("build", "run"))
-    depends_on("python@3.6.1:", type=("build", "run"))
-    depends_on("py-flit-core@3.2:3", when="@2.4:", type=("build", "run"))
+    depends_on("py-flit-core@3.2:3", when="@2.4:", type="build")
+
     depends_on("py-numpy", when="+numpy", type="run")
 
     # Historical dependencies
diff --git a/var/spack/repos/builtin/packages/py-pydub/package.py b/var/spack/repos/builtin/packages/py-pydub/package.py
new file mode 100644
index 00000000000000..0facdffc40b387
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-pydub/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPydub(PythonPackage):
+    """Manipulate audio with an simple and easy high level interface"""
+
+    homepage = "http://pydub.com/"
+    pypi = "pydub/pydub-0.25.1.tar.gz"
+
+    version("0.25.1", sha256="980a33ce9949cab2a569606b65674d748ecbca4f0796887fd6f46173a7b0d30f")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("ffmpeg", type="run")
diff --git a/var/spack/repos/builtin/packages/py-pyedr/package.py b/var/spack/repos/builtin/packages/py-pyedr/package.py
index c7efe5f859edf1..880ffe339af712 100644
--- a/var/spack/repos/builtin/packages/py-pyedr/package.py
+++ b/var/spack/repos/builtin/packages/py-pyedr/package.py
@@ -15,13 +15,13 @@ class PyPyedr(PythonPackage):
 
     maintainers("RMeli")
 
+    version("0.7.2", sha256="8a02b7d94f97f99083c489568f7816ee8ed37e2efca0c1ba3a2e4b83e932d5b9")
     version("0.7.1", sha256="ad7ccdeb739399acd11a25f2d2413ebb46a54223059a2b902ac604d29fabd767")
 
-    depends_on("python@3.6:", type=("build", "run"))
-
     # Minimal NumPy version only specified in requirements.txt
     depends_on("py-numpy@1.19.0:", type=("build", "run"))
     depends_on("py-pbr", type=("build", "run"))
     depends_on("py-tqdm", type=("build", "run"))
+    depends_on("py-mda-xdrlib", when="@0.7.2:", type=("build", "run"))
 
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-pyflakes/package.py b/var/spack/repos/builtin/packages/py-pyflakes/package.py
index 5efd78508cc032..2619199ad20d00 100644
--- a/var/spack/repos/builtin/packages/py-pyflakes/package.py
+++ b/var/spack/repos/builtin/packages/py-pyflakes/package.py
@@ -12,6 +12,7 @@ class PyPyflakes(PythonPackage):
     homepage = "https://github.com/PyCQA/pyflakes"
     pypi = "pyflakes/pyflakes-2.4.0.tar.gz"
 
+    version("3.1.0", sha256="a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc")
     version("3.0.1", sha256="ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd")
     version("2.5.0", sha256="491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3")
     version("2.4.0", sha256="05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c")
diff --git a/var/spack/repos/builtin/packages/py-pyfr/package.py b/var/spack/repos/builtin/packages/py-pyfr/package.py
index 7cbfe6ab71f67d..9f81ef7597fa51 100644
--- a/var/spack/repos/builtin/packages/py-pyfr/package.py
+++ b/var/spack/repos/builtin/packages/py-pyfr/package.py
@@ -41,7 +41,7 @@ class PyPyfr(PythonPackage, CudaPackage, ROCmPackage):
     depends_on("py-h5py@2.10:", type=("build", "run"))
     depends_on("py-mako@1.0.0:", type=("build", "run"))
     depends_on("py-mpi4py@3.1.0:", type=("build", "run"))
-    depends_on("py-numpy@1.20:+blas", type=("build", "run"))
+    depends_on("py-numpy@1.20:", type=("build", "run"))
     depends_on("py-platformdirs@2.2.0:", type=("build", "run"))
     depends_on("py-pytools@2016.2.1:", type=("build", "run"))
 
diff --git a/var/spack/repos/builtin/packages/py-pygit2/package.py b/var/spack/repos/builtin/packages/py-pygit2/package.py
index 325356a9b43e5a..78d9664c942284 100644
--- a/var/spack/repos/builtin/packages/py-pygit2/package.py
+++ b/var/spack/repos/builtin/packages/py-pygit2/package.py
@@ -12,30 +12,36 @@ class PyPygit2(PythonPackage):
     """
 
     homepage = "https://www.pygit2.org/"
-    pypi = "pygit2/pygit2-0.24.1.tar.gz"
+    pypi = "pygit2/pygit2-1.12.2.tar.gz"
 
+    version("1.12.1", sha256="56e85d0e66de957d599d1efb2409d39afeefd8f01009bfda0796b42a4b678358")
+    version("1.11.1", sha256="793f583fd33620f0ac38376db0f57768ef2922b89b459e75b1ac440377eb64ec")
     version("1.6.0", sha256="7aacea4e57011777f4774421228e5d0ddb9a6ddb87ac4b542346d17ab12a4d62")
     version("1.4.0", sha256="cbeb38ab1df9b5d8896548a11e63aae8a064763ab5f1eabe4475e6b8a78ee1c8")
     version("1.3.0", sha256="0be93f6a8d7cbf0cc79ae2f0afb1993fc055fc0018c27e2bd01ba143e51d4452")
     version("0.28.2", sha256="4d8c3fbbf2e5793a9984681a94e6ac2f1bc91a92cbac762dbdfbea296b917f86")
-    version("0.24.1", sha256="4d1d0196b38d6012faf0a7c45e235c208315672b6035da504566c605ba494064")
 
     depends_on("py-setuptools", type="build")
-    # Version must match with libgit2
-    # See: https://www.pygit2.org/install.html
-    depends_on("libgit2@1.1.0:1.1", when="@1.4:")
-    depends_on("libgit2@1.0.0:1.0", when="@1.2:1.3")
-    depends_on("libgit2@0:1.0", when="@1.1.0:1.1")
-    depends_on("libgit2@0.28.0:0.28", when="@0.28:1.0")
-    depends_on("libgit2@0.24:0.27", when="@0.24:0.27")
-    depends_on("python@3.6:", when="@1.4.0:")
-    depends_on("python@3.6:3.8", when="@1.2:1.3")
-    depends_on("python@3.5:3.8", when="@1.0:1.1")
-    depends_on("python@2.7:3.7", when="@0.28:0")
+    # https://www.pygit2.org/install.html#version-numbers
+    depends_on("libgit2@1.6", when="@1.12")
+    depends_on("libgit2@1.5", when="@1.10:1.11")
+    depends_on("libgit2@1.4", when="@1.9")
+    depends_on("libgit2@1.3", when="@1.7:1.8")
+    depends_on("libgit2@1.1", when="@1.4:1.6")
+    depends_on("libgit2@1.0", when="@1.2:1.3")
+    depends_on("libgit2@0.99:1.0", when="@1.1")
+    depends_on("libgit2@0.28", when="@0.28:1.0")
+    depends_on("python@3.8:3.11", when="@1.11:1.12.1")
+    depends_on("python@:3.10", when="@1.7:1.10")
+    depends_on("python@:3.9", when="@1.4:1.6")
+    depends_on("python@:3.8", when="@1.0:1.3")
+    depends_on("python@:3.7", when="@0.28")
     depends_on("py-six", type=("build", "run"), when="@:0.28.2")
-    depends_on("py-cffi@1.4.0:", type=("build", "run"))
-    depends_on("py-cached-property", when="@1.1.0:1.5", type=("build", "run"))
-    depends_on("py-cached-property", when="@1.6.0: ^python@:3.7", type=("build", "run"))
+    depends_on("py-cffi@1.4.0:", when="@:1.5", type=("build", "run"))
+    depends_on("py-cffi@1.6.0:", when="@1.6:1.7", type=("build", "run"))
+    depends_on("py-cffi@1.9.1:", when="@1.8:", type=("build", "run"))
+    depends_on("py-cached-property", when="@1.1:1.5", type=("build", "run"))
+    depends_on("py-cached-property", when="@1.6: ^python@:3.7", type=("build", "run"))
 
     def setup_build_environment(self, env):
         spec = self.spec
diff --git a/var/spack/repos/builtin/packages/py-pygments/package.py b/var/spack/repos/builtin/packages/py-pygments/package.py
index e63a5d025d98a3..9c0cfbf90602e5 100644
--- a/var/spack/repos/builtin/packages/py-pygments/package.py
+++ b/var/spack/repos/builtin/packages/py-pygments/package.py
@@ -13,6 +13,8 @@ class PyPygments(PythonPackage):
     pypi = "Pygments/Pygments-2.4.2.tar.gz"
     git = "https://github.com/pygments/pygments.git"
 
+    version("2.16.1", sha256="1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29")
+    version("2.16.0", sha256="4f6df32f21dca07a54a0a130bda9a25d2241e9e0a206841d061c85a60cc96145")
     version("2.15.1", sha256="8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c")
     version("2.13.0", sha256="56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1")
     version("2.12.0", sha256="5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb")
diff --git a/var/spack/repos/builtin/packages/py-pyhdf/package.py b/var/spack/repos/builtin/packages/py-pyhdf/package.py
index 51271e782559f2..984081adc0e32e 100644
--- a/var/spack/repos/builtin/packages/py-pyhdf/package.py
+++ b/var/spack/repos/builtin/packages/py-pyhdf/package.py
@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from spack import *
+from spack.package import *
 
 
 class PyPyhdf(PythonPackage):
@@ -12,33 +12,33 @@ class PyPyhdf(PythonPackage):
     currently implemented. NetCDF files can also be read and modified."""
 
     homepage = "https://github.com/fhs/pyhdf"
-    pypi     = "pyhdf/pyhdf-0.10.4.tar.gz"
-    git      = "https://github.com/fhs/pyhdf.git"
-    maintainers('kgerheiser', 'climbfuji')
+    pypi = "pyhdf/pyhdf-0.10.4.tar.gz"
+    git = "https://github.com/fhs/pyhdf.git"
+    maintainers("kgerheiser", "climbfuji")
 
-    version('master', branch='master')
-    version('0.10.4', sha256='ea09b2bdafc9be0f7f43d72ff122d8efbde61881f4da3a659b33be5e29215f93')
+    version("master", branch="master")
+    version("0.10.4", sha256="ea09b2bdafc9be0f7f43d72ff122d8efbde61881f4da3a659b33be5e29215f93")
 
     # Python versions
-    depends_on('python@3.2:', type=('build', 'run'))
+    depends_on("python@3.2:", type=("build", "run"))
 
     # Dependencies
-    depends_on('zlib', type=('build', 'run'))
-    depends_on('hdf', type=('build', 'run'))
-    depends_on('py-numpy', type=('build', 'run'))
-    depends_on('jpeg', type=('build', 'run'))
+    depends_on("zlib", type=("build", "run"))
+    depends_on("hdf", type=("build", "run"))
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("jpeg", type=("build", "run"))
 
     def setup_build_environment(self, env):
         inc_dirs = []
         lib_dirs = []
         # Strip -I and -L from spec include_flags / search_flags
-        inc_dirs.append(self.spec['zlib'].headers.include_flags.lstrip('-I'))
-        inc_dirs.append(self.spec['hdf'].headers.include_flags.lstrip('-I'))
-        inc_dirs.append(self.spec['jpeg'].headers.include_flags.lstrip('-I'))
-        lib_dirs.append(self.spec['zlib'].libs.search_flags.lstrip('-L'))
-        lib_dirs.append(self.spec['hdf'].libs.search_flags.lstrip('-L'))
-        lib_dirs.append(self.spec['jpeg'].libs.search_flags.lstrip('-L'))
-        env.set('INCLUDE_DIRS', ':'.join(inc_dirs))
-        env.set('LIBRARY_DIRS', ':'.join(lib_dirs))
-        if self.spec['hdf'].satisfies('@:4.1'):
-            env.set('NO_COMPRESS', '1')
+        inc_dirs.append(self.spec["zlib"].headers.include_flags.lstrip("-I"))
+        inc_dirs.append(self.spec["hdf"].headers.include_flags.lstrip("-I"))
+        inc_dirs.append(self.spec["jpeg"].headers.include_flags.lstrip("-I"))
+        lib_dirs.append(self.spec["zlib"].libs.search_flags.lstrip("-L"))
+        lib_dirs.append(self.spec["hdf"].libs.search_flags.lstrip("-L"))
+        lib_dirs.append(self.spec["jpeg"].libs.search_flags.lstrip("-L"))
+        env.set("INCLUDE_DIRS", ":".join(inc_dirs))
+        env.set("LIBRARY_DIRS", ":".join(lib_dirs))
+        if self.spec["hdf"].satisfies("@:4.1"):
+            env.set("NO_COMPRESS", "1")
diff --git a/var/spack/repos/builtin/packages/py-pykwalify/package.py b/var/spack/repos/builtin/packages/py-pykwalify/package.py
index bbf8ec4713399c..73747d0dfe8a24 100644
--- a/var/spack/repos/builtin/packages/py-pykwalify/package.py
+++ b/var/spack/repos/builtin/packages/py-pykwalify/package.py
@@ -23,4 +23,4 @@ class PyPykwalify(PythonPackage):
     depends_on("py-python-dateutil@2.4.2:", type=("build", "run"))
     depends_on("py-pyyaml@3.11:", type=("build", "run"), when="@1.6.1")
 
-    conflicts("^py-ruamel@0.16.0:", when="@1.6.1")
+    conflicts("^py-ruamel-yaml@0.16.0:", when="@1.6.1")
diff --git a/var/spack/repos/builtin/packages/py-pyliblzma/package.py b/var/spack/repos/builtin/packages/py-pyliblzma/package.py
deleted file mode 100644
index c32f1e2098d55d..00000000000000
--- a/var/spack/repos/builtin/packages/py-pyliblzma/package.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-from spack.package import *
-
-
-class PyPyliblzma(PythonPackage):
-    """Python bindings for liblzma"""
-
-    homepage = "https://launchpad.net/pyliblzma"
-    pypi = "pyliblzma/pyliblzma-0.5.3.tar.bz2"
-
-    version("0.5.3", sha256="08d762f36d5e59fb9bb0e22e000c300b21f97e35b713321ee504cfb442667957")
-
-    depends_on("py-setuptools", type="build")
-    depends_on("lzma")
diff --git a/var/spack/repos/builtin/packages/py-pylint/package.py b/var/spack/repos/builtin/packages/py-pylint/package.py
index 1c7111b99b8aa8..e6e5d3ca70a6b8 100644
--- a/var/spack/repos/builtin/packages/py-pylint/package.py
+++ b/var/spack/repos/builtin/packages/py-pylint/package.py
@@ -58,7 +58,8 @@ class PyPylint(PythonPackage):
     depends_on("py-astroid@2.12.4:2.13", when="@2.15", type=("build", "run"))
     depends_on("py-astroid@2.14.2:2.15", when="@2.16:", type=("build", "run"))
     depends_on("py-isort@4.2.5:", type=("build", "run"))
-    depends_on("py-isort@4.2.5:5", when="@2.3.1:", type=("build", "run"))
+    depends_on("py-isort@4.2.5:4", when="@2.3.1:2.5", type=("build", "run"))
+    depends_on("py-isort@4.2.5:5", when="@2.6:", type=("build", "run"))
     depends_on("py-mccabe", type=("build", "run"))
     depends_on("py-mccabe@0.6.0:0.6", when="@2.3.1:2.11", type=("build", "run"))
     depends_on("py-mccabe@0.6.0:0.7", when="@2.13:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pymol/package.py b/var/spack/repos/builtin/packages/py-pymol/package.py
index 3f95868e045945..4d3b007eabb1ae 100644
--- a/var/spack/repos/builtin/packages/py-pymol/package.py
+++ b/var/spack/repos/builtin/packages/py-pymol/package.py
@@ -21,6 +21,8 @@ class PyPymol(PythonPackage):
 
     depends_on("python+tkinter@2.7:", type=("build", "link", "run"), when="@2.3.0:2.4.0")
     depends_on("python+tkinter@3.6:", type=("build", "link", "run"), when="@2.5.0:")
+    # in newer pip versions --install-option does not exist
+    depends_on("py-pip@:23.0", type="build")
     depends_on("gl")
     depends_on("glew")
     depends_on("libpng")
diff --git a/var/spack/repos/builtin/packages/py-pynio/package.py b/var/spack/repos/builtin/packages/py-pynio/package.py
index 70b0b7a0ad0abb..ba757b68d4bcf5 100644
--- a/var/spack/repos/builtin/packages/py-pynio/package.py
+++ b/var/spack/repos/builtin/packages/py-pynio/package.py
@@ -22,7 +22,7 @@ class PyPynio(PythonPackage):
     # The setup.py claims it requires these abolutely.
     depends_on("libpng")
     depends_on("jpeg")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # Spack does not currently have netcdf below 4.x, and 3.x is a
     # fundamentally different format. So, currently this is only providing
diff --git a/var/spack/repos/builtin/packages/py-pynucleus/package.py b/var/spack/repos/builtin/packages/py-pynucleus/package.py
index aa5636a2c3d4fa..c4f2f82b7a24ef 100644
--- a/var/spack/repos/builtin/packages/py-pynucleus/package.py
+++ b/var/spack/repos/builtin/packages/py-pynucleus/package.py
@@ -19,8 +19,9 @@ class PyPynucleus(PythonPackage):
     for ref in refs:
         version(ref, branch=ref)
 
+    depends_on("python@3.10:", type=("build", "run"))
     depends_on("py-mpi4py@2.0.0:", type=("build", "link", "run"))
-    depends_on("py-cython", type=("build", "run"))
+    depends_on("py-cython@0.29.32:", type=("build", "run"))
     depends_on("py-numpy", type=("build", "link", "run"))
     depends_on("py-scipy", type=("build", "link", "run"))
     depends_on("metis", type=("build", "link", "run"))
@@ -29,7 +30,7 @@ class PyPynucleus(PythonPackage):
     depends_on("py-h5py", type=("build", "run"))
     depends_on("py-tabulate", type=("build", "run"))
     depends_on("py-pyyaml", type=("build", "run"))
-    depends_on("py-matplotlib", type=("build", "run"))
+    depends_on("py-matplotlib+latex", type=("build", "run"))
     depends_on("py-scikit-sparse", type=("build", "run"))
     depends_on("py-modepy", type=("build", "run"))
     depends_on("py-meshpy", type=("build", "run"))
@@ -48,6 +49,9 @@ class PyPynucleus(PythonPackage):
         "PyNucleus-nl",
     ]
 
+    def setup_build_environment(self, env):
+        env.set("PYNUCLEUS_BUILD_PARALLELISM", make_jobs)
+
     @run_before("install")
     def install_python(self):
         prefix = self.prefix
diff --git a/var/spack/repos/builtin/packages/py-pyproj/package.py b/var/spack/repos/builtin/packages/py-pyproj/package.py
index 566c4064280435..16b561dc688f82 100644
--- a/var/spack/repos/builtin/packages/py-pyproj/package.py
+++ b/var/spack/repos/builtin/packages/py-pyproj/package.py
@@ -16,6 +16,7 @@ class PyPyproj(PythonPackage):
 
     maintainers("citibeth", "adamjstewart")
 
+    version("3.6.1", sha256="44aa7c704c2b7d8fb3d483bbf75af6cb2350d30a63b144279a09b75fead501bf")
     version("3.6.0", sha256="a5b111865b3f0f8b77b3983f2fbe4dd6248fc09d3730295949977c8dcd988062")
     version("3.5.0", sha256="9859d1591c1863414d875ae0759e72c2cffc01ab989dc64137fbac572cc81bf6")
     version("3.4.1", sha256="261eb29b1d55b1eb7f336127344d9b31284d950a9446d1e0d1c2411f7dd8e3ac")
@@ -35,7 +36,8 @@ class PyPyproj(PythonPackage):
     # In pyproject.toml
     depends_on("py-setuptools@61:", when="@3.4:", type="build")
     depends_on("py-setuptools", type="build")
-    depends_on("py-cython@0.28.4:", when="@2:", type="build")
+    depends_on("py-cython@3:", when="@3.6.1:", type="build")
+    depends_on("py-cython@0.28.4:2", when="@2:3.6.0", type="build")
     depends_on("python@3.9:", when="@3.6:", type=("build", "link", "run"))
     depends_on("python@3.8:", when="@3.3:", type=("build", "link", "run"))
     depends_on("py-certifi", when="@3:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pyproject-parser/package.py b/var/spack/repos/builtin/packages/py-pyproject-parser/package.py
new file mode 100644
index 00000000000000..63a164ce50aed7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-pyproject-parser/package.py
@@ -0,0 +1,31 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyPyprojectParser(PythonPackage):
+    """Parser for 'pyproject.toml'"""
+
+    homepage = "https://github.com/repo-helper/pyproject-parser"
+    pypi = "pyproject_parser/pyproject-parser-0.9.1.tar.gz"
+
+    version("0.9.1", sha256="fa0b2ff78bc95788b08d00e1aafa66d3f7f3ab693f19d9c2e23e20000a69fd9b")
+
+    depends_on("py-wheel@0.34.2:", type="build")
+    depends_on("py-setuptools@40.6:", type="build")
+    conflicts("^py-setuptools@61")
+    depends_on("py-apeye-core@1:", type=("build", "run"))
+    depends_on("py-attrs@20.3:", type=("build", "run"))
+    depends_on("py-dom-toml@0.4:", type=("build", "run"))
+    depends_on("py-domdf-python-tools@2.8:", type=("build", "run"))
+    depends_on("py-natsort@7.1.1:", type=("build", "run"))
+    depends_on("py-packaging@20.9:", type=("build", "run"))
+    depends_on("py-shippinglabel@1:", type=("build", "run"))
+    depends_on("py-toml@0.10.2:", type=("build", "run"))
+    depends_on("py-tomli@1.2.3:", type=("build", "run"), when="^python@:3.10")
+    depends_on("py-typing-extensions@3.7.4.3:", type=("build", "run"))
+    conflicts("^py-typing-extensions@4.7.0")
diff --git a/var/spack/repos/builtin/packages/py-pyqt-builder/package.py b/var/spack/repos/builtin/packages/py-pyqt-builder/package.py
index 25674c314212b6..74f31161d3d16b 100644
--- a/var/spack/repos/builtin/packages/py-pyqt-builder/package.py
+++ b/var/spack/repos/builtin/packages/py-pyqt-builder/package.py
@@ -12,9 +12,10 @@ class PyPyqtBuilder(PythonPackage):
     homepage = "https://www.riverbankcomputing.com/hg/PyQt-builder/"
     pypi = "PyQt-builder/PyQt-builder-1.12.2.tar.gz"
 
+    version("1.15.1", sha256="a2bd3cfbf952e959141dfe55b44b451aa945ca8916d1b773850bb2f9c0fa2985")
     version("1.12.2", sha256="f62bb688d70e0afd88c413a8d994bda824e6cebd12b612902d1945c5a67edcd7")
 
-    depends_on("python@3.5:", type=("build", "run"))
     depends_on("py-setuptools@30.3:", type="build")
     depends_on("py-packaging", type=("build", "run"))
+    depends_on("py-sip@6.7:6", when="@1.15:", type=("build", "run"))
     depends_on("py-sip@6.3:6", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pyqt4/package.py b/var/spack/repos/builtin/packages/py-pyqt4/package.py
index 6de10982ab90d7..4ce26a78290b24 100644
--- a/var/spack/repos/builtin/packages/py-pyqt4/package.py
+++ b/var/spack/repos/builtin/packages/py-pyqt4/package.py
@@ -13,11 +13,7 @@ class PyPyqt4(SIPPackage):
     against Qt v5."""
 
     homepage = "https://www.riverbankcomputing.com/software/pyqt/intro"
-    url = (
-        "http://sourceforge.net/projects/pyqt/files/PyQt4/PyQt-4.12.3/PyQt4_gpl_x11-4.12.3.tar.gz"
-    )
-
-    sip_module = "PyQt4.sip"
+    url = "https://www.riverbankcomputing.com/static/Downloads/PyQt4/4.12.3/PyQt4_gpl_x11-4.12.3.tar.gz"
 
     version("4.12.3", sha256="a00f5abef240a7b5852b7924fa5fdf5174569525dc076cd368a566619e56d472")
     version(
@@ -33,16 +29,28 @@ class PyPyqt4(SIPPackage):
     # Requires distutils
     depends_on("python@:3.11", type=("build", "link", "run"))
 
-    depends_on("qt@4")
-    depends_on("qt@4.1:", when="@4.12.3")
+    depends_on("qt@4.1:4")
+
+    # configure-ng.py
     depends_on("py-sip@4.19.12:4.19.18 module=PyQt4.sip")
 
-    # https://www.riverbankcomputing.com/static/Docs/PyQt4/installation.html
-    def configure_file(self):
-        return "configure-ng.py"
+    build_directory = "."
 
     def configure_args(self):
+        # https://www.riverbankcomputing.com/static/Docs/PyQt4/installation.html
         args = [
+            "--verbose",
+            "--confirm-license",
+            "--qmake",
+            self.spec["qt"].prefix.bin.qmake,
+            "--sip",
+            self.spec["py-sip"].prefix.bin.sip,
+            "--sip-incdir",
+            join_path(self.spec["py-sip"].prefix, self.spec["python"].package.include),
+            "--bindir",
+            self.prefix.bin,
+            "--destdir",
+            python_platlib,
             "--pyuic4-interpreter",
             self.spec["python"].command.path,
             "--sipdir",
@@ -53,3 +61,18 @@ def configure_args(self):
         if "+qsci_api" in self.spec:
             args.extend(["--qsci-api", "--qsci-api-destdir", self.prefix.share.qsci])
         return args
+
+    def configure(self, spec, prefix):
+        python("configure-ng.py", *self.configure_args())
+
+    @run_after("install")
+    def extend_path_setup(self):
+        # https://github.com/spack/spack/issues/14121
+        # https://github.com/spack/spack/pull/15297
+        # Same code comes by default with py-pyqt5 and py-pyqt6
+        text = """
+# Support PyQt4 sub-packages that have been created by setuptools.
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+"""
+        with open(join_path(python_platlib, "PyQt4", "__init__.py"), "a") as f:
+            f.write(text)
diff --git a/var/spack/repos/builtin/packages/py-pyqt5-sip/package.py b/var/spack/repos/builtin/packages/py-pyqt5-sip/package.py
index 89bd9833a307c8..04f46ccd0ab191 100644
--- a/var/spack/repos/builtin/packages/py-pyqt5-sip/package.py
+++ b/var/spack/repos/builtin/packages/py-pyqt5-sip/package.py
@@ -10,9 +10,9 @@ class PyPyqt5Sip(PythonPackage):
     """The sip module support for PyQt5."""
 
     homepage = "https://www.riverbankcomputing.com/software/sip/"
-    pypi = "PyQt5_sip/PyQt5_sip-12.9.0.tar.gz"
+    pypi = "PyQt5-sip/PyQt5_sip-12.9.0.tar.gz"
 
+    version("12.12.1", sha256="8fdc6e0148abd12d977a1d3828e7b79aae958e83c6cb5adae614916d888a6b10")
     version("12.9.0", sha256="d3e4489d7c2b0ece9d203ae66e573939f7f60d4d29e089c9f11daa17cfeaae32")
 
-    depends_on("python@3.5:", type=("build", "run"))
-    depends_on("py-setuptools", type="build")
+    depends_on("py-setuptools@30.3:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-pyqt5/package.py b/var/spack/repos/builtin/packages/py-pyqt5/package.py
index af8b33af46d477..55a63dce74a0ba 100644
--- a/var/spack/repos/builtin/packages/py-pyqt5/package.py
+++ b/var/spack/repos/builtin/packages/py-pyqt5/package.py
@@ -12,44 +12,36 @@ class PyPyqt5(SIPPackage):
     Windows, OS X, Linux, iOS and Android. PyQt5 supports Qt v5."""
 
     homepage = "https://www.riverbankcomputing.com/software/pyqt/intro"
-    url = (
-        "https://www.riverbankcomputing.com/static/Downloads/PyQt5/5.13.0/PyQt5_gpl-5.13.0.tar.gz"
+    url = "https://files.pythonhosted.org/packages/source/P/PyQt5/PyQt5-5.15.9.tar.gz"
+    list_url = "https://pypi.org/simple/PyQt5/"
+
+    version("5.15.9", sha256="dc41e8401a90dc3e2b692b411bd5492ab559ae27a27424eed4bd3915564ec4c0")
+    version(
+        "5.13.1",
+        sha256="54b7f456341b89eeb3930e786837762ea67f235e886512496c4152ebe106d4af",
+        deprecated=True,
+    )
+    version(
+        "5.13.0",
+        sha256="0cdbffe5135926527b61cc3692dd301cd0328dd87eeaf1313e610787c46faff9",
+        deprecated=True,
+    )
+    version(
+        "5.12.3",
+        sha256="0db0fa37debab147450f9e052286f7a530404e2aaddc438e97a7dcdf56292110",
+        deprecated=True,
     )
-    list_url = "https://www.riverbankcomputing.com/software/pyqt/download5"
-
-    sip_module = "PyQt5.sip"
 
-    version("5.13.1", sha256="54b7f456341b89eeb3930e786837762ea67f235e886512496c4152ebe106d4af")
-    version("5.13.0", sha256="0cdbffe5135926527b61cc3692dd301cd0328dd87eeaf1313e610787c46faff9")
-    version("5.12.3", sha256="0db0fa37debab147450f9e052286f7a530404e2aaddc438e97a7dcdf56292110")
+    # pyproject.toml
+    depends_on("py-sip@6.6.2:6", type="build")
+    depends_on("py-pyqt-builder@1.14.1:1", type="build")
 
-    # API files can be installed regardless if Qscintilla is installed or not
-    variant("qsci_api", default=False, description="Install PyQt API file for QScintilla")
+    # PKG-INFO
+    depends_on("py-pyqt5-sip@12.11:12", type=("build", "run"))
 
-    # Without opengl support, I got the following error:
-    # sip: QOpenGLFramebufferObject is undefined
-    depends_on("qt@5:+opengl")
-    depends_on("python@2.6:", type=("build", "run"))
-    depends_on("py-sip module=PyQt5.sip", type=("build", "run"))
-    depends_on("py-sip@:4.19.18 module=PyQt5.sip", type=("build", "run"), when="@:5.13.0")
+    # README
+    depends_on("qt@5+opengl")
 
-    # https://www.riverbankcomputing.com/static/Docs/PyQt5/installation.html
     def configure_args(self):
-        args = [
-            "--pyuic5-interpreter",
-            self.spec["python"].command.path,
-            "--sipdir",
-            self.prefix.share.sip.PyQt5,
-            "--designer-plugindir",
-            self.prefix.plugins.designer,
-            "--qml-plugindir",
-            self.prefix.plugins.PyQt5,
-            "--stubsdir",
-            join_path(python_platlib, "PyQt5"),
-        ]
-        if "+qsci_api" in self.spec:
-            args.extend(["--qsci-api", "--qsci-api-destdir", self.prefix.share.qsci])
-        return args
-
-    def setup_run_environment(self, env):
-        env.prepend_path("QT_PLUGIN_PATH", self.prefix.plugins)
+        # https://www.riverbankcomputing.com/static/Docs/PyQt5/installation.html
+        return ["--confirm-license", "--no-make", "--qmake", self.spec["qt"].prefix.bin.qmake]
diff --git a/var/spack/repos/builtin/packages/py-pyqt6-sip/package.py b/var/spack/repos/builtin/packages/py-pyqt6-sip/package.py
new file mode 100644
index 00000000000000..63d711eb92f18f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-pyqt6-sip/package.py
@@ -0,0 +1,17 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPyqt6Sip(PythonPackage):
+    """The sip module support for PyQt6."""
+
+    homepage = "https://www.riverbankcomputing.com/software/sip/"
+    pypi = "PyQt6-sip/PyQt6_sip-13.5.1.tar.gz"
+
+    version("13.5.1", sha256="d1e9141752966669576d04b37ba0b122abbc41cc9c35493751028d7d91c4dd49")
+
+    depends_on("py-setuptools@30.3:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-pyqt6/package.py b/var/spack/repos/builtin/packages/py-pyqt6/package.py
new file mode 100644
index 00000000000000..6791b50ff0e1ea
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-pyqt6/package.py
@@ -0,0 +1,37 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPyqt6(SIPPackage):
+    """PyQt6 is a comprehensive set of Python bindings for Qt v6."""
+
+    homepage = "https://www.riverbankcomputing.com/software/pyqt/"
+    url = "https://files.pythonhosted.org/packages/source/P/PyQt6/PyQt6-6.5.1.tar.gz"
+    list_url = "https://pypi.org/simple/PyQt6/"
+
+    version("6.5.2", sha256="1487ee7350f9ffb66d60ab4176519252c2b371762cbe8f8340fd951f63801280")
+    version("6.5.1", sha256="e166a0568c27bcc8db00271a5043936226690b6a4a74ce0a5caeb408040a97c3")
+
+    # pyproject.toml
+    depends_on("py-sip@6.5:6", type="build")
+    depends_on("py-pyqt-builder@1.15:1", type="build")
+
+    # PKG-INFO
+    depends_on("py-pyqt6-sip@13.4:13", type=("build", "run"))
+
+    # README
+    depends_on("qt-base@6")
+
+    def setup_build_environment(self, env):
+        # Detected system locale encoding (US-ASCII, locale "C") is not UTF-8.
+        # Qt shall use a UTF-8 locale ("UTF-8") instead. If this causes problems,
+        # reconfigure your locale. See the locale(1) manual for more information.
+        env.set("LC_ALL", "en_US.UTF-8")
+
+    def configure_args(self):
+        # https://www.riverbankcomputing.com/static/Docs/PyQt6/installation.html
+        return ["--confirm-license", "--no-make", "--qmake", self.spec["qt-base"].prefix.bin.qmake]
diff --git a/var/spack/repos/builtin/packages/py-pyside/package.py b/var/spack/repos/builtin/packages/py-pyside/package.py
index 5d5dd16acf3540..a7101518351111 100644
--- a/var/spack/repos/builtin/packages/py-pyside/package.py
+++ b/var/spack/repos/builtin/packages/py-pyside/package.py
@@ -26,19 +26,19 @@ class PyPyside(PythonPackage):
         "1.2.4", sha256="1421bc1bf612c396070de9e1ffe227c07c1f3129278bc7d30c754b5146be2433"
     )  # rpath problems
 
-    # v1.2.2 does not work with Python3
-    version(
-        "1.2.2",
-        sha256="53129fd85e133ef630144c0598d25c451eab72019cdcb1012f2aec773a3f25be",
-        preferred=True,
-    )
-
-    depends_on("cmake", type="build")
+    version("1.2.2", sha256="53129fd85e133ef630144c0598d25c451eab72019cdcb1012f2aec773a3f25be")
 
+    # to prevent error: 'PyTypeObject' {aka 'struct _typeobject'} has no member
+    # named 'tp_print'
+    depends_on("python@:3.8", type=("build", "run"))
     depends_on("py-setuptools", type="build")
+    # in newer pip versions --install-option does not exist
+    depends_on("py-pip@:23.0", type="build")
+    depends_on("cmake@2.6:", type="build")
+
     depends_on("py-sphinx", type=("build", "run"))
     depends_on("py-sphinx@:3.5.0", type=("build", "run"), when="@:1.2.2")
-    depends_on("qt@4.5:4.9")
+    depends_on("qt@4.6:4.8")
     depends_on("libxml2@2.6.32:")
     depends_on("libxslt@1.1.19:")
 
@@ -57,6 +57,9 @@ def patch(self):
         )
         filter_file("^    if subprocess.mswindows:", "    if mswindows:", "popenasync.py")
 
+        # Remove check for python version because the above patch adds support for newer versions
+        filter_file("^check_allowed_python_version()", "", "setup.py")
+
         # Add Spack's standard CMake args to the sub-builds.
         # They're called BY setup.py so we have to patch it.
         filter_file(
@@ -82,14 +85,5 @@ def patch(self):
         # PySide can't find the Shiboken library, even though it comes
         # bundled with it and is installed in the same directory.
 
-        # PySide does not provide official support for
-        # Python 3.5, but it should work fine
-        filter_file(
-            "'Programming Language :: Python :: 3.4'",
-            "'Programming Language :: Python :: 3.4',\r\n        "
-            "'Programming Language :: Python :: 3.5'",
-            "setup.py",
-        )
-
     def install_options(self, spec, prefix):
         return ["--jobs={0}".format(make_jobs)]
diff --git a/var/spack/repos/builtin/packages/py-pyside2/package.py b/var/spack/repos/builtin/packages/py-pyside2/package.py
index 3c9005d8a62fd0..b13f0c1aac4220 100644
--- a/var/spack/repos/builtin/packages/py-pyside2/package.py
+++ b/var/spack/repos/builtin/packages/py-pyside2/package.py
@@ -17,12 +17,30 @@ class PyPyside2(PythonPackage):
     # https://wiki.qt.io/Qt_for_Python_Development_Getting_Started
 
     version("develop", tag="dev")
-    version("5.15.2.1", tag="v5.15.2.1", submodules=True)
-    version("5.14.2.1", tag="v5.14.2.1", submodules=True)
-    version("5.13.2", tag="v5.13.2", submodules=True)
-    version("5.13.1", tag="v5.13.1", submodules=True)
-    version("5.13.0", tag="v5.13.0", submodules=True)
-    version("5.12.5", tag="v5.12.5", submodules=True)
+    version(
+        "5.15.2.1",
+        tag="v5.15.2.1",
+        commit="9282e03de471bb3772c8d3997159e49c113d7678",
+        submodules=True,
+    )
+    version(
+        "5.14.2.1",
+        tag="v5.14.2.1",
+        commit="6341c063dea6022c1e40cca28d3bbf0f52350dcb",
+        submodules=True,
+    )
+    version(
+        "5.13.2", tag="v5.13.2", commit="a1a94b43c5b277fd4e65c1389e24c4fbbb1c5641", submodules=True
+    )
+    version(
+        "5.13.1", tag="v5.13.1", commit="de1e75b55f6f59bba4bae5cd036d6c355c62986a", submodules=True
+    )
+    version(
+        "5.13.0", tag="v5.13.0", commit="208d0c8bc8595aebc2191dafd9d0e3ec719e2550", submodules=True
+    )
+    version(
+        "5.12.5", tag="v5.12.5", commit="af0953e0d261ab9b1fc498d63e8d790a329dd285", submodules=True
+    )
 
     variant(
         "doc",
@@ -36,12 +54,14 @@ class PyPyside2(PythonPackage):
     depends_on("cmake@3.1:", type="build")
     # libclang versioning from sources/shiboken2/doc/gettingstarted.rst
     depends_on("llvm@6", type="build", when="@5.12:5.13")
-    depends_on("llvm@10", type="build", when="@5.15")
+    depends_on("llvm@10:", type="build", when="@5.15:")
     depends_on("py-setuptools", type="build")
     depends_on("py-packaging", type="build")
     depends_on("py-wheel", type="build")
     # https://bugreports.qt.io/browse/PYSIDE-1385
     depends_on("py-wheel@:0.34", when="@:5.14", type="build")
+    # in newer pip versions --install-option does not exist
+    depends_on("py-pip@:23.0", type="build")
     depends_on("qt@5.11:+opengl")
 
     depends_on("graphviz", when="+doc", type="build")
@@ -49,6 +69,23 @@ class PyPyside2(PythonPackage):
     depends_on("libxslt@1.1.19:", when="+doc", type="build")
     depends_on("py-sphinx", when="+doc", type="build")
 
+    def patch(self):
+        filter_file(
+            "=${shiboken_include_dirs}",
+            ":".join(
+                [
+                    "=${shiboken_include_dirs}",
+                    self.spec["qt"]["glx"]["libglx"].prefix.include,
+                    self.spec["qt"]["libxcb"].prefix.include,
+                ]
+            ),
+            "sources/pyside2/cmake/Macros/PySideModules.cmake",
+            string=True,
+        )
+
+    def setup_build_environment(self, env):
+        env.set("LLVM_INSTALL_DIR", self.spec["llvm"].prefix)
+
     def install_options(self, spec, prefix):
         args = [
             "--parallel={0}".format(make_jobs),
diff --git a/var/spack/repos/builtin/packages/py-pytest-html/package.py b/var/spack/repos/builtin/packages/py-pytest-html/package.py
index bfc5a4f9ad7395..3809558543ac44 100644
--- a/var/spack/repos/builtin/packages/py-pytest-html/package.py
+++ b/var/spack/repos/builtin/packages/py-pytest-html/package.py
@@ -13,11 +13,29 @@ class PyPytestHtml(PythonPackage):
 
     homepage = "https://github.com/pytest-dev/pytest-html"
     pypi = "pytest-html/pytest-html-3.1.1.tar.gz"
+    git = "https://github.com/pytest-dev/pytest-html.git"
 
+    version("3.2.0", sha256="c4e2f4bb0bffc437f51ad2174a8a3e71df81bbc2f6894604e604af18fbe687c3")
     version("3.1.1", sha256="3ee1cf319c913d19fe53aeb0bc400e7b0bc2dbeb477553733db1dad12eb75ee3")
 
     depends_on("python@3.6:", type=("build", "run"))
-    depends_on("py-setuptools", type="build")
-    depends_on("py-setuptools-scm", type="build")
+    depends_on("py-setuptools@42:", type="build")
+    depends_on("py-setuptools-scm+toml@3.5.0:", type="build")
+    depends_on("py-setuptools-scm-git-archive@1.1:", type="build")
+    depends_on("py-wheel@0.33.6:", type="build")
     depends_on("py-pytest@5.0:5,6.0.1:", type=("build", "run"))
     depends_on("py-pytest-metadata", type=("build", "run"))
+
+    # https://github.com/spack/spack/pull/38989
+    # py-pytest@7.2 removed py-py dependency, but now py-pytest conflicts with py-py. And
+    # py-pytest-htm@:3 requires py-py.
+    # One workaround is to always add py-py *before* py-pytest in PYTHONPATH, but we cannot ensure
+    # that. So don't allow this configuration, pending py-pytest-html@4.
+    conflicts("^py-pytest@7.2:", when="@:3")
+
+    @run_after("install")
+    @on_package_attributes(run_tests=True)
+    def check_build(self):
+        # Simplest test: pytest will load pytest-html plugin
+        output = python("-m", "pytest", "-VV", output=str, error=str)
+        assert self.prefix in output, f"Missing pytest-html in {output!r}"
diff --git a/var/spack/repos/builtin/packages/py-python-certifi-win32/package.py b/var/spack/repos/builtin/packages/py-python-certifi-win32/package.py
index 0de85eaca31cae..76897d76cda768 100644
--- a/var/spack/repos/builtin/packages/py-python-certifi-win32/package.py
+++ b/var/spack/repos/builtin/packages/py-python-certifi-win32/package.py
@@ -14,7 +14,7 @@ class PyPythonCertifiWin32(PythonPackage):
     git = "https://gitlab.com/alelec/python-certifi-win32.git"
 
     # Tarball missing version information, need to use git checkout
-    version("1.6", tag="v1.6")
+    version("1.6", tag="v1.6", commit="8ef45c73e203024ed2e1df5151a23e27faff5b60")
 
     depends_on("py-setuptools", type="build")
     depends_on("py-setuptools-scm", type="build")
diff --git a/var/spack/repos/builtin/packages/py-python-fmask/package.py b/var/spack/repos/builtin/packages/py-python-fmask/package.py
index 2cb0ce8f1a6465..08a32be346a532 100644
--- a/var/spack/repos/builtin/packages/py-python-fmask/package.py
+++ b/var/spack/repos/builtin/packages/py-python-fmask/package.py
@@ -11,11 +11,16 @@ class PyPythonFmask(PythonPackage):
     the FMASK algorithm for Landsat and Sentinel-2"""
 
     homepage = "https://www.pythonfmask.org/en/latest/"
-    url = "https://github.com/ubarsc/python-fmask/archive/pythonfmask-0.5.4.tar.gz"
+    url = "https://github.com/ubarsc/python-fmask/releases/download/pythonfmask-0.5.8/python-fmask-0.5.8.tar.gz"
 
-    version("0.5.4", sha256="a216aa3108de837fec182602b2b4708442746be31fc1585906802437784a63fe")
+    version("0.5.8", sha256="d55f54d3fecde818374017fdbe0ad173c893ef74c79ba2a7bc1890b7ec416c2f")
+    version("0.5.7", sha256="da9dad1b977a50599d068dedaed007100b20322a79ca5d78f702712647c2c3f3")
+    version("0.5.6", sha256="a63abd12d36fb4ec010e618bcabd5e2f782a0479ebcbf40aec1bcef943c00c5c")
+    version("0.5.5", sha256="8257227d2527ea5fbd229f726d06d05986914beafd090acef05772a27dbbf062")
+    version("0.5.4", sha256="ed20776f6b63615f664da89a9e3951c79437b66c2bf88fe19a93c2cc7dc40c82")
+
+    # Note: Dependencies are listed here: https://github.com/ubarsc/python-fmask/blob/master/doc/source/index.rst#introduction
 
-    depends_on("python@2.7:2.8,3.4:", type=("build", "run"))
     # pip silently replaces distutils with setuptools
     depends_on("py-setuptools", type="build")
     depends_on("py-rios", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-python-hostlist/package.py b/var/spack/repos/builtin/packages/py-python-hostlist/package.py
new file mode 100644
index 00000000000000..3cd58dd7810a6d
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-python-hostlist/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPythonHostlist(PythonPackage):
+    """The hostlist.py module knows how to expand and collect hostlist expressions."""
+
+    homepage = "https://www.nsc.liu.se/~kent/python-hostlist/"
+    pypi = "python-hostlist/python-hostlist-1.23.0.tar.gz"
+    git = "git://www.nsc.liu.se/~kent/python-hostlist.git"
+
+    version("master", branch="master")
+    version("1.23.0", sha256="56e0156b501f792c078114f07324f34f37827041581ee5d1ffdce89cca533219")
+
+    # build dependencies
+    depends_on("py-setuptools", type=("build"))
diff --git a/var/spack/repos/builtin/packages/py-python-libsbml/package.py b/var/spack/repos/builtin/packages/py-python-libsbml/package.py
index ac473673624ee3..0c30d1632f8bb9 100644
--- a/var/spack/repos/builtin/packages/py-python-libsbml/package.py
+++ b/var/spack/repos/builtin/packages/py-python-libsbml/package.py
@@ -15,7 +15,9 @@ class PyPythonLibsbml(PythonPackage):
     pypi = "python-libsbml/python-libsbml-5.19.7.tar.gz"
 
     version("5.19.7", sha256="447b1fde7aceccd11a93dc9f589ffd9319ba854d7b7583f911259a8b0127ab7b")
-    version("5.19.5", tag="v5.19.5", submodules=True)
+    version(
+        "5.19.5", tag="v5.19.5", commit="6081d9e1b0aa2b3ff4198b39680b726094c47e85", submodules=True
+    )
 
     depends_on("py-setuptools", type="build")
 
diff --git a/var/spack/repos/builtin/packages/py-python-rapidjson/package.py b/var/spack/repos/builtin/packages/py-python-rapidjson/package.py
index d55a1479711849..ea2a0ae3209ec7 100644
--- a/var/spack/repos/builtin/packages/py-python-rapidjson/package.py
+++ b/var/spack/repos/builtin/packages/py-python-rapidjson/package.py
@@ -12,6 +12,8 @@ class PyPythonRapidjson(PythonPackage):
     homepage = "https://github.com/python-rapidjson/python-rapidjson"
     pypi = "python-rapidjson/python-rapidjson-0.9.1.tar.gz"
 
+    version("1.10", sha256="acfecbf5edb91ec72a20a125de7f56b8c2f6161eff4c65382c8ee6a2484d3540")
+    version("1.9", sha256="be7d351c7112dac608133a23f60e95395668d0981a07f4037f63e0e88afcf01a")
     version("1.8", sha256="170c2ff97d01735f67afd0e1cb0aaa690cb69ae6016e020c6afd5e0ab9b39899")
     version("1.5", sha256="04323e63cf57f7ed927fd9bcb1861ef5ecb0d4d7213f2755969d4a1ac3c2de6f")
     version("0.9.1", sha256="ad80bd7e4bb15d9705227630037a433e2e2a7982b54b51de2ebabdd1611394a1")
diff --git a/var/spack/repos/builtin/packages/py-pyyaml/package.py b/var/spack/repos/builtin/packages/py-pyyaml/package.py
index 9a530db7f5b90c..47138e1faf7dfa 100644
--- a/var/spack/repos/builtin/packages/py-pyyaml/package.py
+++ b/var/spack/repos/builtin/packages/py-pyyaml/package.py
@@ -34,6 +34,9 @@ class PyPyyaml(PythonPackage):
     # Includes "longintrepr.h" instead of Python.h
     conflicts("^python@3.11:", when="@:5.3")
 
+    # https://github.com/yaml/pyyaml/issues/601
+    conflicts("^py-cython@3:")
+
     @property
     def import_modules(self):
         modules = ["yaml"]
diff --git a/var/spack/repos/builtin/packages/py-pyzmq/package.py b/var/spack/repos/builtin/packages/py-pyzmq/package.py
index 6aec08c715d71c..bf60b4be57d91b 100644
--- a/var/spack/repos/builtin/packages/py-pyzmq/package.py
+++ b/var/spack/repos/builtin/packages/py-pyzmq/package.py
@@ -41,12 +41,13 @@ class PyPyzmq(PythonPackage):
     version("16.0.2", sha256="0322543fff5ab6f87d11a8a099c4c07dd8a1719040084b6ce9162bcdf5c45c9d")
     version("14.7.0", sha256="77994f80360488e7153e64e5959dc5471531d1648e3a4bff14a714d074a38cc2")
 
-    # Python 3.9 build issues
-    depends_on("python@2.7,3.3:3.8", type=("build", "run"), when="@16:18.0")
     depends_on("python@2.6:2.7,3.2:3.8", type=("build", "run"), when="@:14")
 
     # pyproject.toml
     depends_on("py-setuptools", type="build")
+    # https://github.com/zeromq/pyzmq/issues/1278
+    # https://github.com/zeromq/pyzmq/pull/1317
+    depends_on("py-setuptools@:59", when="@17:18.0", type="build")
     depends_on("py-packaging", type="build")
 
     # setup.py
@@ -62,6 +63,14 @@ class PyPyzmq(PythonPackage):
     # Undocumented dependencies
     depends_on("py-gevent", type=("build", "run"))
 
+    @run_before("install", when="@15:19")
+    def remove_cythonized_files(self):
+        # Before v20.0.0 an ancient cythonize API was used, for which we cannot
+        # force re-cythonization. Re-cythonizing v14.x fails in general, so
+        # restrict to 15:19
+        for f in find(".", "*.pyx"):
+            touch(f)
+
     @run_before("install")
     def setup(self):
         """Create config file listing dependency information."""
diff --git a/var/spack/repos/builtin/packages/py-rapidfuzz/package.py b/var/spack/repos/builtin/packages/py-rapidfuzz/package.py
index e5b4c9ae34a141..5bfd1563b5bb78 100644
--- a/var/spack/repos/builtin/packages/py-rapidfuzz/package.py
+++ b/var/spack/repos/builtin/packages/py-rapidfuzz/package.py
@@ -12,8 +12,14 @@ class PyRapidfuzz(PythonPackage):
     homepage = "https://github.com/maxbachmann/rapidfuzz"
     pypi = "rapidfuzz/rapidfuzz-1.8.2.tar.gz"
 
+    version("3.3.1", sha256="6783b3852f15ed7567688e2e358757a7b4f38683a915ba5edc6c64f1a3f0b450")
     version("1.8.2", sha256="d6efbb2b6b18b3a67d7bdfbcd9bb72732f55736852bbef823bdf210f9e0c6c90")
 
-    depends_on("python@2.7:", type=("build", "link", "run"))
+    depends_on("python", type=("build", "link", "run"))
+    depends_on("py-setuptools@42:", when="@3:", type="build")
     depends_on("py-setuptools", type="build")
-    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-scikit-build@0.17", when="@3:", type="build")
+
+    # CMakeLists.txt
+    depends_on("cmake@3.12:", type="build")
+    depends_on("ninja", type="build")
diff --git a/var/spack/repos/builtin/packages/py-rarfile/package.py b/var/spack/repos/builtin/packages/py-rarfile/package.py
index 120000469380d9..f7b2ff60f36f0f 100644
--- a/var/spack/repos/builtin/packages/py-rarfile/package.py
+++ b/var/spack/repos/builtin/packages/py-rarfile/package.py
@@ -12,8 +12,8 @@ class PyRarfile(PythonPackage):
     homepage = "https://github.com/markokr/rarfile"
     pypi = "rarfile/rarfile-4.0.tar.gz"
 
+    version("4.1", sha256="db60b3b5bc1c4bdeb941427d50b606d51df677353385255583847639473eda48")
     version("4.0", sha256="67548769229c5bda0827c1663dce3f54644f9dbfba4ae86d4da2b2afd3e602a1")
 
-    depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("unrar", type="run")
diff --git a/var/spack/repos/builtin/packages/py-rasterio/package.py b/var/spack/repos/builtin/packages/py-rasterio/package.py
index e837c282c1263f..83db363c9814c3 100644
--- a/var/spack/repos/builtin/packages/py-rasterio/package.py
+++ b/var/spack/repos/builtin/packages/py-rasterio/package.py
@@ -20,6 +20,7 @@ class PyRasterio(PythonPackage):
     maintainers("adamjstewart")
 
     version("master", branch="master")
+    version("1.3.9", sha256="fc6d0d290492fa1a5068711cfebb21cc936968891b7ed9da0690c8a7388885c5")
     version("1.3.8", sha256="ffdd18e78efdf8ad5861065fd812a66dd34264293317ff6540a078ea891cdef8")
     version("1.3.7", sha256="abfdcb8f10210b8fad939f40d545d6c47e9e3b5cf4a43773ca8dd11c58204304")
     version("1.3.6", sha256="c8b90eb10e16102d1ab0334a7436185f295de1c07f0d197e206d1c005fc33905")
@@ -37,6 +38,7 @@ class PyRasterio(PythonPackage):
     version("1.0a12", sha256="47d460326e04c64590ff56952271a184a6307f814efc34fb319c12e690585f3c")
 
     # From pyproject.toml
+    depends_on("py-setuptools@67.8:", when="@1.3.9:", type="build")
     depends_on("py-cython@0.29.29:", when="@1.3.3:", type="build")
     depends_on("py-cython@0.29.24:0.29", when="@1.3.0:1.3.2", type="build")
 
diff --git a/var/spack/repos/builtin/packages/py-rdflib/package.py b/var/spack/repos/builtin/packages/py-rdflib/package.py
index 2673bc3ad32cd2..e595319971489b 100644
--- a/var/spack/repos/builtin/packages/py-rdflib/package.py
+++ b/var/spack/repos/builtin/packages/py-rdflib/package.py
@@ -21,15 +21,22 @@ class PyRdflib(PythonPackage):
     homepage = "https://github.com/RDFLib/rdflib"
     pypi = "rdflib/rdflib-5.0.0.tar.gz"
 
+    version("6.3.2", sha256="72af591ff704f4caacea7ecc0c5a9056b8553e0489dd4f35a9bc52dbd41522e0")
     version("6.2.0", sha256="62dc3c86d1712db0f55785baf8047f63731fa59b2682be03219cb89262065942")
     version("6.0.2", sha256="6136ae056001474ee2aff5fc5b956e62a11c3a9c66bb0f3d9c0aaa5fbb56854e")
     version("5.0.0", sha256="78149dd49d385efec3b3adfbd61c87afaf1281c30d3fcaf1b323b34f603fb155")
 
-    depends_on("python@3.7:", when="@6:", type="build")
-    depends_on("py-setuptools", type="build")
+    depends_on("python@3.7:3", when="@6.3:", type=("build", "run"))
+    depends_on("py-poetry-core@1.4:", when="@6.3:", type="build")
 
+    depends_on("py-isodate@0.6", when="@6.3:", type=("build", "run"))
     depends_on("py-isodate", type=("build", "run"))
+    depends_on("py-pyparsing@2.1:3", when="@6.3:", type=("build", "run"))
     depends_on("py-pyparsing", type=("build", "run"))
-    depends_on("py-setuptools", when="@6:", type=("build", "run"))
+    depends_on("py-importlib-metadata@4", when="@6.3: ^python@:3.7", type=("build", "run"))
     depends_on("py-importlib-metadata", when="@6.1: ^python@:3.7", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-setuptools", when="@6:6.2", type=("build", "run"))
+    depends_on("py-setuptools", when="@:5", type="build")
     depends_on("py-six", when="@:5", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-requirements-parser/package.py b/var/spack/repos/builtin/packages/py-requirements-parser/package.py
new file mode 100644
index 00000000000000..d0a0cbb614e1a2
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-requirements-parser/package.py
@@ -0,0 +1,24 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+# ----------------------------------------------------------------------------
+from spack.package import *
+
+
+class PyRequirementsParser(PythonPackage):
+    """This is a small Python module for parsing Pip requirement files.
+    The goal is to parse everything in the Pip requirement file format spec."""
+
+    homepage = "https://github.com/madpah/requirements-parser"
+    pypi = "requirements-parser/requirements-parser-0.5.0.tar.gz"
+
+    maintainers("DaxLynch", "eugeneswalker")
+
+    version("0.5.0", sha256="3336f3a3ae23e06d3f0f88595e4052396e3adf91688787f637e5d2ca1a904069")
+
+    depends_on("python@3.6:3.99", type=("build", "run"))
+
+    depends_on("py-poetry-core@1:", type="build")
+
+    depends_on("py-types-setuptools@57:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-rios/package.py b/var/spack/repos/builtin/packages/py-rios/package.py
index de2f54c2ae2211..a2716f648e703e 100644
--- a/var/spack/repos/builtin/packages/py-rios/package.py
+++ b/var/spack/repos/builtin/packages/py-rios/package.py
@@ -15,11 +15,20 @@ class PyRios(PythonPackage):
     """
 
     homepage = "https://www.rioshome.org/en/latest/"
-    url = "https://github.com/ubarsc/rios/archive/rios-1.4.10.tar.gz"
+    url = "https://github.com/ubarsc/rios/releases/download/rios-1.4.16/rios-1.4.16.tar.gz"
 
-    version("1.4.10", sha256="7f11b54eb1f2ec551d7fc01c039b60bf2c67f0c2fc5b2946f8d986d6a9bc7063")
+    version("1.4.16", sha256="2f553d85ff4ff26bfda2a8c6bd3d9dcce5ace847f7d9bd2f072c8943f3758ded")
+    version("1.4.15", sha256="71670508dbffcd8f5d24fbb25e6a2b7e1d23b5e899ddc78c90d403bd65981cf4")
+    version("1.4.14", sha256="ea22fde3fe70004aa1ad46bd36fad58f3346e9c161ca44ac913518a6e4fcad82")
+    version("1.4.13", sha256="9f99f41f20ce769101e61bc8347aa96718e6e5ac37ccb47cb3e555dc4ca83427")
+    version("1.4.12", sha256="6d897488ce1ca77e470483472998afcb2eb3bb3307f392a924b85f88a16d73eb")
+    version("1.4.11", sha256="b7ae5311f987b32f1afe1fabc16f25586de8d15c17a69405d1950aeada7b748e")
+    version("1.4.10", sha256="6324acccc6018f9e06c40370bc366dc459890e8c09d26e0ebd245f6fd46dad71")
+
+    variant("parallel", default=True, description="Enables the parallel processing module")
 
     # pip silently replaces distutils with setuptools
     depends_on("py-setuptools", type="build")
     depends_on("py-numpy", type=("build", "run"))
     depends_on("gdal+python", type=("build", "run"))
+    depends_on("py-cloudpickle", type=("build", "run"), when="@1.4.16:+parallel")
diff --git a/var/spack/repos/builtin/packages/py-rseqc/package.py b/var/spack/repos/builtin/packages/py-rseqc/package.py
index 288977aa49d13b..231d663fb5b4e8 100644
--- a/var/spack/repos/builtin/packages/py-rseqc/package.py
+++ b/var/spack/repos/builtin/packages/py-rseqc/package.py
@@ -14,14 +14,16 @@ class PyRseqc(PythonPackage):
     homepage = "http://rseqc.sourceforge.net"
     pypi = "RSeQC/RSeQC-2.6.4.tar.gz"
 
+    version("5.0.1", sha256="3c7d458784861af352d8da3f4f1cc8941934b37643164e9b74f929a32bd9ca80")
+    version("4.0.1", sha256="6a16a3c56b73917082d3ac060a113c7d32cccf39f86efa87a6f1e6b52642210d")
     version("3.0.1", sha256="d5f4cb2c24a7348929f5c4947d84c5869e8cd2cba5ba5248d991ebb37c4c6b3d")
 
-    depends_on("python@3.5:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("py-nose@0.10.4:", type="build")
     depends_on("py-cython@0.17:", type=("build", "run"))
+    depends_on("py-pysam", type=("build", "run"))
     depends_on("py-bx-python", type=("build", "run"))
     depends_on("py-numpy", type=("build", "run"))
-    depends_on("py-pysam", type=("build", "run"))
     depends_on("py-pybigwig", type=("build", "run"))
+
     depends_on("r", type="run")
diff --git a/var/spack/repos/builtin/packages/py-rtree/package.py b/var/spack/repos/builtin/packages/py-rtree/package.py
index 7cfa444072f451..7caa6d4f78bb82 100644
--- a/var/spack/repos/builtin/packages/py-rtree/package.py
+++ b/var/spack/repos/builtin/packages/py-rtree/package.py
@@ -14,13 +14,14 @@ class PyRtree(PythonPackage):
 
     maintainers("adamjstewart", "hobu")
 
+    version("1.1.0", sha256="6f8ee504dde5d005b25b08aaf5be0b3404af3ad5fece6e1ddcde35908a798a95")
     version("1.0.1", sha256="222121699c303a64065d849bf7038b1ecabc37b65c7fa340bedb38ef0e805429")
     version("1.0.0", sha256="d0483482121346b093b9a42518d40f921adf445915b7aea307eb26768c839682")
     version("0.9.7", sha256="be8772ca34699a9ad3fb4cfe2cfb6629854e453c10b3328039301bbfc128ca3e")
     version("0.8.3", sha256="6cb9cf3000963ea6a3db777a597baee2bc55c4fc891e4f1967f262cc96148649")
 
-    depends_on("python@3.7:", when="@1:", type=("build", "run"))
-    depends_on("python@3:", when="@0.9.4:", type=("build", "run"))
+    depends_on("python@3.8:", when="@1.1:", type=("build", "run"))
+    depends_on("py-setuptools@61:", when="@1.1:", type="build")
     depends_on("py-setuptools@39.2:", when="@1:", type="build")
     depends_on("py-setuptools", type="build")
     depends_on("py-typing-extensions@3.7:", when="@1: ^python@:3.7", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-ruamel-yaml-clib/package.py b/var/spack/repos/builtin/packages/py-ruamel-yaml-clib/package.py
index 2a0389bbffe4f4..409208c2c1cb9e 100644
--- a/var/spack/repos/builtin/packages/py-ruamel-yaml-clib/package.py
+++ b/var/spack/repos/builtin/packages/py-ruamel-yaml-clib/package.py
@@ -22,3 +22,9 @@ class PyRuamelYamlClib(PythonPackage):
     # to prevent legacy-install-failure
     depends_on("python@:3.9", when="@0.2.0", type=("build", "link", "run"))
     depends_on("py-setuptools@28.7.0:", type="build")
+
+    def flag_handler(self, name, flags):
+        if name == "cflags":
+            if self.spec.satisfies("%oneapi"):
+                flags.append("-Wno-error=incompatible-function-pointer-types")
+        return (flags, None, None)
diff --git a/var/spack/repos/builtin/packages/py-ruamel-yaml-jinja2/package.py b/var/spack/repos/builtin/packages/py-ruamel-yaml-jinja2/package.py
new file mode 100644
index 00000000000000..0958c65b9ddbcc
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-ruamel-yaml-jinja2/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyRuamelYamlJinja2(PythonPackage):
+    """jinja2 pre and post-processor to update with YAML."""
+
+    homepage = "https://sourceforge.net/p/ruamel-yaml-jinja2/code/ci/default/tree"
+    pypi = "ruamel.yaml.jinja2/ruamel.yaml.jinja2-0.2.7.tar.gz"
+
+    version("0.2.7", sha256="8449be29d9a157fa92d1648adc161d718e469f0d38a6b21e0eabb76fd5b3e663")
+
+    depends_on("py-setuptools", type="build")
+
+    # __init__.py
+    depends_on("py-ruamel-yaml@0.16.1:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-s3cmd/package.py b/var/spack/repos/builtin/packages/py-s3cmd/package.py
index b4400927ee766d..532df1438219b5 100644
--- a/var/spack/repos/builtin/packages/py-s3cmd/package.py
+++ b/var/spack/repos/builtin/packages/py-s3cmd/package.py
@@ -18,6 +18,9 @@ class PyS3cmd(PythonPackage):
     homepage = "https://github.com/s3tools/s3cmd"
     url = "https://github.com/s3tools/s3cmd/releases/download/v2.0.2/s3cmd-2.0.2.tar.gz"
 
+    version("2.3.0", sha256="15330776e7ff993d8ae0ac213bf896f210719e9b91445f5f7626a8fa7e74e30b")
+    version("2.2.0", sha256="2a7d2afe09ce5aa9f2ce925b68c6e0c1903dd8d4e4a591cd7047da8e983a99c3")
+    version("2.1.0", sha256="966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03")
     version("2.0.2", sha256="9f244c0c10d58d0ccacbba3aa977463e32491bdd9d95109e27b67e4d46c5bd52")
     version("2.0.1", sha256="caf09f1473301c442fba6431c983c361c9af8bde503dac0953f0d2f8f2c53c8f")
     version("2.0.0", sha256="bf2a50802f1031cba83e99be488965803899d8ab0228c800c833b55c7269cd48")
diff --git a/var/spack/repos/builtin/packages/py-sacrebleu/package.py b/var/spack/repos/builtin/packages/py-sacrebleu/package.py
new file mode 100644
index 00000000000000..998faf14a82cc5
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-sacrebleu/package.py
@@ -0,0 +1,29 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PySacrebleu(PythonPackage):
+    """SacreBLEU is a standard BLEU implementation that downloads and manages
+    WMT datasets, produces scores on detokenized outputs, and reports a string
+    encapsulating BLEU parameters, facilitating the production of shareable,
+    comparable BLEU scores."""
+
+    homepage = "https://github.com/mjpost/sacrebleu"
+    pypi = "sacrebleu/sacrebleu-2.0.0.tar.gz"
+
+    version("2.0.0", sha256="51fb69b6683f1b9999cd180143bb6b21d7841744537c9aab235cfe676550f0cf")
+
+    depends_on("python@3.6.0:", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
+    depends_on("py-portalocker", type=("build", "run"))
+    depends_on("py-regex", type=("build", "run"))
+    depends_on("py-tabulate@0.8.9:", type=("build", "run"))
+    depends_on("py-numpy@1.17:", type=("build", "run"))
+    depends_on("py-colorama", type=("build", "run"))
+
+    def patch(self):
+        touch("CHANGELOG.md")
diff --git a/var/spack/repos/builtin/packages/py-scikit-build/package.py b/var/spack/repos/builtin/packages/py-scikit-build/package.py
index bfca8b7c77e7dc..a83a084fc179b9 100644
--- a/var/spack/repos/builtin/packages/py-scikit-build/package.py
+++ b/var/spack/repos/builtin/packages/py-scikit-build/package.py
@@ -16,18 +16,37 @@ class PyScikitBuild(PythonPackage):
     the setuptools Python module and CMake."""
 
     homepage = "https://scikit-build.readthedocs.io/en/latest/"
-    pypi = "scikit-build/scikit-build-0.15.0.tar.gz"
+    pypi = "scikit-build/scikit_build-0.17.6.tar.gz"
 
     maintainers("coreyjadams")
 
+    version("0.17.6", sha256="b51a51a36b37c42650994b5047912f59b22e3210b23e321f287611f9ef6e5c9d")
     version("0.15.0", sha256="e723cd0f3489a042370b9ea988bbb9cfd7725e8b25b20ca1c7981821fcf65fb9")
     version("0.12.0", sha256="f851382c469bcd9a8c98b1878bcfdd13b68556279d2fd9a329be41956ae5a7fe")
     version("0.11.1", sha256="da40dfd69b2456fad1349a894b90180b43712152b8a85d2a00f4ae2ce8ac9a5c")
     version("0.10.0", sha256="7342017cc82dd6178e3b19377389b8a8d1f8b429d9cdb315cfb1094e34a0f526")
 
-    depends_on("py-setuptools@28.0.0:", type=("build", "run"))
-    depends_on("py-setuptools@42.0.0:", when="@0.15.0:", type=("build", "run"))
-    depends_on("py-setuptools-scm+toml", when="@0.15.0:", type="build")
-    depends_on("py-packaging", type=("build", "run"))
-    depends_on("py-wheel@0.29.0:", type=("build", "run"))
+    depends_on("py-hatchling", when="@0.17:", type="build")
+    depends_on("py-hatch-fancy-pypi-readme", when="@0.17:", type="build")
+    depends_on("py-hatch-vcs", when="@0.17:", type="build")
     depends_on("py-distro", when="@0.11:", type=("build", "run"))
+    depends_on("py-packaging", type=("build", "run"))
+    depends_on("py-setuptools@42:", when="@0.15:", type=("build", "run"))
+    depends_on("py-setuptools@28:", type=("build", "run"))
+    depends_on("py-tomli", when="@0.17: ^python@:3.10", type=("build", "run"))
+    depends_on("py-typing-extensions@3.7:", when="@0.17: ^python@:3.7", type=("build", "run"))
+    depends_on("py-wheel@0.32:", when="@0.17:", type=("build", "run"))
+    depends_on("py-wheel@0.29:", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-setuptools-scm+toml", when="@0.15", type="build")
+
+    def url_for_version(self, version):
+        url = (
+            "https://files.pythonhosted.org/packages/source/s/scikit-build/scikit{}build-{}.tar.gz"
+        )
+        if version >= Version("0.17"):
+            separator = "_"
+        else:
+            separator = "-"
+        return url.format(separator, version)
diff --git a/var/spack/repos/builtin/packages/py-scikit-learn-extra/package.py b/var/spack/repos/builtin/packages/py-scikit-learn-extra/package.py
index e1871af60811d9..41843ed30201ce 100644
--- a/var/spack/repos/builtin/packages/py-scikit-learn-extra/package.py
+++ b/var/spack/repos/builtin/packages/py-scikit-learn-extra/package.py
@@ -19,7 +19,8 @@ class PyScikitLearnExtra(PythonPackage):
 
     version("0.2.0", sha256="3b1bb5fedde47920eb4b3fa0a0c18f80cc7359d9d0496720178788c6153b8019")
 
-    depends_on("python@3.6:", type=("build", "run"))
+    # For upperbound see https://github.com/scikit-learn-contrib/scikit-learn-extra/issues/164
+    depends_on("python@3.6:3.10", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("py-cython@0.28.5:", type="build")
     depends_on("py-numpy@1.13.3:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-scikit-learn/package.py b/var/spack/repos/builtin/packages/py-scikit-learn/package.py
index e5f90e9ee419ff..05f6d09b53952b 100644
--- a/var/spack/repos/builtin/packages/py-scikit-learn/package.py
+++ b/var/spack/repos/builtin/packages/py-scikit-learn/package.py
@@ -17,6 +17,8 @@ class PyScikitLearn(PythonPackage):
     maintainers("adamjstewart")
 
     version("master", branch="master")
+    version("1.3.2", sha256="a2f54c76accc15a34bfb9066e6c7a56c1e7235dda5762b990792330b52ccfb05")
+    version("1.3.1", sha256="1a231cced3ee3fa04756b4a7ab532dc9417acd581a330adff5f2c01ac2831fcf")
     version("1.3.0", sha256="8be549886f5eda46436b6e555b0e4873b4f10aa21c07df45c4bc1735afbccd7a")
     version("1.2.2", sha256="8429aea30ec24e7a8c7ed8a3fa6213adf3814a6efbea09e16e0a0c71e1a1a3d7")
     version("1.2.1", sha256="fbf8a5c893c9b4b99bcc7ed8fb3e8500957a113f4101860386d06635520f7cfb")
@@ -50,7 +52,8 @@ class PyScikitLearn(PythonPackage):
     variant("openmp", default=True, description="Build with OpenMP support")
 
     # Based on PyPI wheel availability
-    depends_on("python@3.8:3.11", when="@1.1.3:", type=("build", "run"))
+    depends_on("python@3.8:3.12", when="@1.3.1:", type=("build", "run"))
+    depends_on("python@3.8:3.11", when="@1.1.3:1.3.0", type=("build", "run"))
     depends_on("python@3.8:3.10", when="@1.1.0:1.1.2", type=("build", "run"))
     depends_on("python@:3.10", when="@1.0.2", type=("build", "run"))
     depends_on("python@:3.9", when="@0.24:1.0.1", type=("build", "run"))
@@ -60,6 +63,10 @@ class PyScikitLearn(PythonPackage):
     # pyproject.toml
     depends_on("py-setuptools", type="build")
     depends_on("py-setuptools@:59", when="@:1.2.1", type="build")
+    depends_on("py-cython@0.29.33:2", when="@1.3:", type="build")
+    depends_on("py-cython@0.29.24:2", when="@1.0.2:", type="build")
+    depends_on("py-cython@0.28.5:2", when="@0.21:", type="build")
+    depends_on("py-cython@0.23:2", type="build")
 
     # sklearn/_min_dependencies.py
     depends_on("py-numpy@1.17.3:", when="@1.1:", type=("build", "run"))
@@ -79,10 +86,6 @@ class PyScikitLearn(PythonPackage):
     depends_on("py-joblib@1:", when="@1.1:", type=("build", "run"))
     depends_on("py-joblib@0.11:", type=("build", "run"))
     depends_on("py-threadpoolctl@2.0.0:", when="@0.23:", type=("build", "run"))
-    depends_on("py-cython@0.29.33:", when="@1.3:", type="build")
-    depends_on("py-cython@0.29.24:", when="@1.0.2:", type="build")
-    depends_on("py-cython@0.28.5:", when="@0.21:", type="build")
-    depends_on("py-cython@0.23:", type="build")
     depends_on("llvm-openmp", when="@0.21: %apple-clang +openmp")
 
     # Test dependencies
diff --git a/var/spack/repos/builtin/packages/py-scikit-optimize/package.py b/var/spack/repos/builtin/packages/py-scikit-optimize/package.py
index 0c4bfec54f9f51..1bf5da83e1c043 100644
--- a/var/spack/repos/builtin/packages/py-scikit-optimize/package.py
+++ b/var/spack/repos/builtin/packages/py-scikit-optimize/package.py
@@ -21,17 +21,19 @@ class PyScikitOptimize(PythonPackage):
     maintainers("liuyangzhuan")
 
     version("master", branch="master")
+    version("0.9.0", sha256="77d8c9e64947fc9f5cc05bbc6aed7b8a9907871ae26fe11997fd67be90f26008")
     version("0.5.2", sha256="1d7657a4b8ef9aa6d81e49b369c677c584e83269f11710557741d3b3f8fa0a75")
-
     variant("plots", default=True, description="Build with plot support from py-matplotlib")
-    variant("gptune", default=False, description="Build with patches for GPTune")
 
     depends_on("py-setuptools", type="build")
     depends_on("py-numpy", type=("build", "run"))
-    depends_on("py-scipy@0.14.0:", type=("build", "run"))
+    depends_on("py-numpy@1.13.3:", when="@0.9:", type=("build", "run"))
+    depends_on("py-scipy@0.14:", type=("build", "run"))
+    depends_on("py-scipy@0.19.1:", when="@0.9:", type=("build", "run"))
     depends_on("py-scikit-learn@0.19.1:", type=("build", "run"))
-    depends_on("py-pyyaml", when="+gptune", type=("build", "run"))
+    depends_on("py-scikit-learn@0.20:", when="@0.9:", type=("build", "run"))
+    depends_on("py-pyaml@16.9:", when="@0.9:", type=("build", "run"))
+    depends_on("py-joblib@0.11:", when="@0.9:", type=("build", "run"))
 
     depends_on("py-matplotlib", when="+plots")
-
-    patch("space.patch", when="+gptune")
+    depends_on("py-matplotlib@2:", when="@0.9:+plots")
diff --git a/var/spack/repos/builtin/packages/py-scikit-sparse/package.py b/var/spack/repos/builtin/packages/py-scikit-sparse/package.py
index 51c4adb8367118..3f7533c6504821 100644
--- a/var/spack/repos/builtin/packages/py-scikit-sparse/package.py
+++ b/var/spack/repos/builtin/packages/py-scikit-sparse/package.py
@@ -15,11 +15,16 @@ class PyScikitSparse(PythonPackage):
 
     maintainers("cgcgcg")
 
+    version("0.4.12", sha256="e6502fea9ba561cfa5491eb222ed2c81c16263d8182a293950db20509c941166")
+    version("0.4.11", sha256="64c61a8777b7c7ba8e1f2bf76bc767f740e6426f1cce2d90f1324b177618e1ca")
     version("0.4.8", sha256="2a224c60da3ef951975242ea777478583d3265efc72db5cfb7861686521a4009")
 
-    depends_on("python@3.6:3.11", type=("build", "link", "run"))
+    depends_on("python@3.6:3.11", when="@0.4.12", type=("build", "link", "run"))
+    depends_on("python@3.6:3.11", when="@0.4.11", type=("build", "link", "run"))
+    depends_on("python@3.6:3.10", when="@0.4.8", type=("build", "link", "run"))
     depends_on("py-setuptools@40.8:", type="build")
-    depends_on("py-cython@0.22:", type="build")
+    depends_on("py-cython@0.22:", when="@0.4.12", type="build")
+    depends_on("py-cython@0.22:0.29", when="@:0.4.11", type="build")
     depends_on("py-numpy@1.13.3:", type=("build", "link", "run"))
     depends_on("py-scipy@0.19:", type="run")
     depends_on("suite-sparse", type=("build", "link", "run"))
diff --git a/var/spack/repos/builtin/packages/py-scikits-odes/package.py b/var/spack/repos/builtin/packages/py-scikits-odes/package.py
new file mode 100644
index 00000000000000..e70f5a671b3281
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-scikits-odes/package.py
@@ -0,0 +1,52 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyScikitsOdes(PythonPackage):
+    """Odes is a scikit toolkit for scipy to add extra ode solvers.
+    Specifically it interfaces the Sundials solvers cvode, cvodes, ida and
+    idas.  It this way it provides extra modern ode and dae solvers you can
+    use, extending the capabilities offered in scipy.integrade.ode."""
+
+    homepage = "https://github.com/bmcage/odes"
+
+    pypi = "scikits.odes/scikits.odes-2.7.0.tar.gz"
+    git = "https://github.com/bmcage/odes.git"
+
+    maintainers("omsai")
+
+    version("2.7.0", sha256="a71e19e1485893754ae8c050668232fcc694f17b83602e75fbebf7bf9f975e1e")
+
+    depends_on("py-setuptools@:64.0.0", type="build")
+
+    # Upstream incorrectly only lists py-numpy only as a build dependency even
+    # though it is directly imported.
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-scipy", type=("build", "run"))
+    depends_on("py-cython@:2", type="build")
+    # Although upstream's documentation claims support for sundials@5.1:
+    # py-scikit-odes@2.7.0 requires the header sundials/sundials_context.h
+    # which wasn't added until sundials@6
+    depends_on("sundials@6:")
+
+    depends_on("py-pytest", type="test")
+
+    # Remove numpy test runner imports to be compatible with py-numpy@1.25:
+    patch(
+        "https://github.com/bmcage/odes/pull/153.patch?full_index=1",
+        sha256="8d05d7bcc3582b7c482a4393bf5a8c0460a58eb62d1e3c86339c95a0d4ce30ac",
+    )
+
+    def setup_run_environment(self, env):
+        env.set("SUNDIALS_INST", self.spec["sundials"].prefix)
+
+    @run_after("install")
+    @on_package_attributes(run_tests=True)
+    def install_test(self):
+        with working_dir("spack-test", create=True):
+            pytest = which("pytest")
+            pytest(join_path(python_purelib, "scikits", "odes", "tests"))
diff --git a/var/spack/repos/builtin/packages/py-scipy/package.py b/var/spack/repos/builtin/packages/py-scipy/package.py
index e05b4aedeb88bf..6bcbac79122bda 100644
--- a/var/spack/repos/builtin/packages/py-scipy/package.py
+++ b/var/spack/repos/builtin/packages/py-scipy/package.py
@@ -3,16 +3,11 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import glob
-import os
-
 from spack.package import *
 
 
 class PyScipy(PythonPackage):
-    """SciPy (pronounced "Sigh Pie") is a Scientific Library for Python.
-    It provides many user-friendly and efficient numerical routines such
-    as routines for numerical integration and optimization."""
+    """Fundamental algorithms for scientific computing in Python."""
 
     homepage = "https://www.scipy.org/"
     pypi = "scipy/scipy-1.10.1.tar.gz"
@@ -20,7 +15,10 @@ class PyScipy(PythonPackage):
 
     maintainers("adamjstewart", "rgommers")
 
-    version("master", branch="master")
+    version("main", branch="main")
+    version("master", branch="master", deprecated=True)
+    version("1.11.3", sha256="bba4d955f54edd61899776bad459bf7326e14b9fa1c552181f0479cc60a568cd")
+    version("1.11.2", sha256="b29318a5e39bd200ca4381d80b065cdf3076c7d7281c5e36569e99273867f61d")
     version("1.11.1", sha256="fb5b492fa035334fd249f0973cc79ecad8b09c604b42a127a677b45a9a3d4289")
     version("1.11.0", sha256="f9b0248cb9d08eead44cde47cbf6339f1e9aa0dfde28f5fb27950743e317bd5d")
     version("1.10.1", sha256="2cf9dfb80a7b4589ba4c40ce7588986d6d5cebc5457cad2c2880f6bc2d42f3a5")
@@ -64,22 +62,18 @@ class PyScipy(PythonPackage):
     depends_on("python@:3.8", when="@1.3.2:1.5.3", type=("build", "link", "run"))
     depends_on("python@:3.7", when="@1.1:1.3.1", type=("build", "link", "run"))
 
-    # TODO: remove once pip build supports BLAS/LAPACK specification
-    # https://github.com/mesonbuild/meson-python/pull/167
-    depends_on("py-build", when="@1.9:", type="build")
-
-    depends_on("py-meson-python@0.12.1:0.13", when="@1.11:", type="build")
-    depends_on("py-meson-python@0.11:0.12", when="@1.10.1:1.10", type="build")
-    depends_on("py-meson-python@0.11", when="@1.10.0", type="build")
-    depends_on("py-meson-python@0.9:", when="@1.9.2:1.9", type="build")
-    depends_on("py-meson-python@0.8.1:", when="@1.9.1", type="build")
-    depends_on("py-meson-python@0.7", when="@1.9.0", type="build")
-    depends_on("meson@0.62.2", when="@1.9.0:1.9.1", type="build")
+    depends_on("py-meson-python@0.12.1:", when="@1.11:", type="build")
+    depends_on("py-meson-python@0.11:", when="@1.10:", type="build")
+    depends_on("py-meson-python@0.9:", when="@1.9.2:", type="build")
+    depends_on("py-meson-python@0.8.1:", when="@1.9.1:", type="build")
+    depends_on("py-meson-python@0.7:", when="@1.9:", type="build")
+    depends_on("meson", when="@1.9.0:1.9.1", type="build")
     depends_on("py-cython@0.29.35:2", when="@1.11:", type="build")
     depends_on("py-cython@0.29.32:2", when="@1.9.2:", type="build")
     depends_on("py-cython@0.29.21:2", when="@1.9:", type="build")
     depends_on("py-cython@0.29.18:2", when="@1.7:", type="build")
-    depends_on("py-pybind11@2.10.4:2.10", when="@1.11:", type=("build", "link"))
+    depends_on("py-pybind11@2.10.4:2.11.0", when="@1.11.3:", type=("build", "link"))
+    depends_on("py-pybind11@2.10.4:2.10", when="@1.11.0:1.11.2", type=("build", "link"))
     depends_on("py-pybind11@2.10.1", when="@1.10", type=("build", "link"))
     depends_on("py-pybind11@2.4.3:2.10", when="@1.9.1:1.9", type=("build", "link"))
     depends_on("py-pybind11@2.4.3:2.9", when="@1.9.0", type=("build", "link"))
@@ -89,14 +83,11 @@ class PyScipy(PythonPackage):
     depends_on("py-pybind11@2.4.3:", when="@1.5:1.6.1", type=("build", "link"))
     depends_on("py-pybind11@2.4.0:", when="@1.4.1:1.4", type=("build", "link"))
     depends_on("py-pybind11@2.2.4:", when="@1.4.0", type=("build", "link"))
-    depends_on("py-pythran@0.12:0.13", when="@1.11:", type="build")
-    depends_on("py-pythran@0.12", when="@1.10", type="build")
-    depends_on("py-pythran@0.9.12:0.12", when="@1.9.2:1.9", type="build")
-    depends_on("py-pythran@0.9.12:0.11", when="@1.9.0:1.9.1", type="build")
-    depends_on("py-pythran@0.10", when="@1.8", type="build")
-    depends_on("py-pythran@0.9.12:0.9", when="@1.7.2:1.7", type="build")
-    depends_on("py-pythran@0.9.11", when="@1.7.0:1.7.1", type="build")
-    depends_on("py-wheel@:0.40", when="@1.11:", type="build")
+    depends_on("py-pythran@0.12:", when="@1.10:", type="build")
+    depends_on("py-pythran@0.10:", when="@1.8", type="build")
+    depends_on("py-pythran@0.9.12:", when="@1.7.2:", type="build")
+    depends_on("py-pythran@0.9.11:", when="@1.7:", type="build")
+    depends_on("py-wheel@:0.40", when="@1.11.0:1.11.2", type="build")
     depends_on("py-wheel@:0.38", when="@1.10", type="build")
     depends_on("py-wheel@:0.37", when="@:1.9", type="build")
     depends_on("pkgconfig", when="@1.9:", type="build")
@@ -104,42 +95,52 @@ class PyScipy(PythonPackage):
     depends_on("py-setuptools@:59", when="@1.8", type="build")
     depends_on("py-setuptools@:57", when="@1.7", type="build")
     depends_on("py-setuptools@:51.0.0", when="@1.6", type="build")
-    depends_on("py-numpy@1.21.6:1.27+blas+lapack", when="@1.11:", type=("build", "link", "run"))
-    depends_on("py-numpy@1.19.5:1.26+blas+lapack", when="@1.10", type=("build", "link", "run"))
-    depends_on("py-numpy@1.18.5:1.25+blas+lapack", when="@1.9", type=("build", "link", "run"))
-    depends_on("py-numpy@1.17.3:1.24+blas+lapack", when="@1.8", type=("build", "link", "run"))
-    depends_on(
-        "py-numpy@1.16.5:1.22+blas+lapack", when="@1.6.2:1.7", type=("build", "link", "run")
-    )
-    depends_on("py-numpy@1.16.5:+blas+lapack", when="@1.6:1.6.1", type=("build", "link", "run"))
-    depends_on("py-numpy@1.14.5:+blas+lapack", when="@1.5.0:1.5", type=("build", "link", "run"))
-    depends_on("py-numpy@1.13.3:+blas+lapack", when="@1.3:1.4", type=("build", "link", "run"))
-    depends_on("py-numpy@1.8.2:+blas+lapack", when="@:1.2", type=("build", "link", "run"))
+    depends_on("py-numpy@1.21.6:1.27", when="@1.11:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.19.5:1.26", when="@1.10", type=("build", "link", "run"))
+    depends_on("py-numpy@1.18.5:1.25", when="@1.9", type=("build", "link", "run"))
+    depends_on("py-numpy@1.17.3:1.24", when="@1.8", type=("build", "link", "run"))
+    depends_on("py-numpy@1.16.5:1.22", when="@1.6:1.7", type=("build", "link", "run"))
+    depends_on("py-numpy@1.14.5:1.21", when="@1.5", type=("build", "link", "run"))
+    depends_on("py-numpy@1.13.3:1.21", when="@1.3:1.4", type=("build", "link", "run"))
+    depends_on("py-numpy@1.8.2:1.20", when="@:1.2", type=("build", "link", "run"))
     depends_on("py-pytest", type="test")
 
-    # NOTE: scipy should use the same BLAS/LAPACK as numpy.
-    # For scipy 1.8 and older, this is achieved by calling the set_blas_lapack()
-    # and setup_build_environment() from numpy in the scipy spec.
-    depends_on("blas")
-    depends_on("lapack")
+    # Required to use --config-settings
+    depends_on("py-pip@23.1:", when="@1.9:", type="build")
 
     # https://docs.scipy.org/doc/scipy/dev/toolchain.html#other-libraries
     depends_on("lapack@3.7.1:", when="@1.9:")
     depends_on("lapack@3.4.1:", when="@1.2:")
+    depends_on("lapack")
+    depends_on("blas")
 
+    # meson.build
     # https://docs.scipy.org/doc/scipy/dev/toolchain.html#compilers
-    conflicts("%gcc@:7", when="@1.10:")
-    conflicts("%gcc@:4.7", when="@:1.9")
-    conflicts("%apple-clang@:9", when="@1.10:")
-    conflicts("%msvc@:19.19", when="@1.10:")
+    conflicts("%gcc@:7", when="@1.10:", msg="SciPy requires GCC >= 8.0")
+    conflicts("%gcc@:4.7", when="@:1.9", msg="SciPy requires GCC >= 4.8")
+    conflicts(
+        "%msvc@:19.19",
+        when="@1.10:",
+        msg="SciPy requires at least vc142 (default with Visual Studio 2019) "
+        "when building with MSVC",
+    )
 
-    # https://github.com/scipy/scipy/pull/11324
-    conflicts("@1.4.0:1.4.1", when="target=ppc64le:")
+    # https://github.com/scipy/scipy/issues/19352
+    conflicts("^py-cython@3.0.3")
 
     # https://github.com/mesonbuild/meson/pull/10909#issuecomment-1282241479
     # Intel OneAPI ifx claims to support -fvisibility, but this does not work.
     # Meson adds this flag for all Python extensions which include Fortran code.
-    conflicts("%oneapi", when="@1.9:")
+    conflicts("%oneapi@:2023.0", when="@1.9:")
+
+    # error: expected unqualified-id (exact compiler versions unknown)
+    conflicts("%apple-clang@15:", when="@:1.9")
+
+    # https://docs.scipy.org/doc//scipy-1.10.1/release.1.7.3.html
+    conflicts("platform=darwin target=aarch64:", when="@:1.7.2")
+
+    # https://github.com/scipy/scipy/pull/11324
+    conflicts("@1.4.0:1.4.1", when="target=ppc64le:")
 
     # https://github.com/scipy/scipy/issues/12860
     patch(
@@ -165,12 +166,6 @@ class PyScipy(PythonPackage):
     def archive_files(self):
         return [join_path(self.stage.source_path, "build", "meson-logs", "meson-log.txt")]
 
-    @run_before("install")
-    def set_blas_lapack(self):
-        # Pick up BLAS/LAPACK from numpy
-        if self.spec.satisfies("@:1.8"):
-            self.spec["py-numpy"].package.set_blas_lapack()
-
     @run_before("install")
     def set_fortran_compiler(self):
         if self.compiler.f77 is None or self.compiler.fc is None:
@@ -215,53 +210,27 @@ def setup_build_environment(self, env):
         if self.spec.satisfies("@:1.8"):
             self.spec["py-numpy"].package.setup_build_environment(env)
 
-    # TODO: remove once pip build supports BLAS/LAPACK specification
-    # https://github.com/mesonbuild/meson-python/pull/167
-    @when("@1.9:")
-    def install(self, spec, prefix):
-        blas = spec["blas"].libs.names[0]
-        lapack = spec["lapack"].libs.names[0]
-        if spec["blas"].name in ["intel-mkl", "intel-parallel-studio", "intel-oneapi-mkl"]:
-            blas = "mkl-dynamic-lp64-seq"
-        if spec["lapack"].name in ["intel-mkl", "intel-parallel-studio", "intel-oneapi-mkl"]:
-            lapack = "mkl-dynamic-lp64-seq"
-        if spec["blas"].name in ["blis", "amdblis"]:
-            blas = "blis"
-        if "armpl" in blas:
-            if "_mp" in blas:
-                blas = "armpl-dynamic-lp64-omp"
-            else:
-                blas = "armpl-dynamic-lp64-seq"
-        if "armpl" in lapack:
-            if "_mp" in lapack:
-                lapack = "armpl-dynamic-lp64-omp"
-            else:
-                lapack = "armpl-dynamic-lp64-seq"
+        # https://github.com/scipy/scipy/issues/19357
+        if self.spec.satisfies("%apple-clang@15:"):
+            env.append_flags("LDFLAGS", "-Wl,-ld_classic")
 
-        args = [
-            "setup",
-            "build",
-            "-Dblas=" + blas,
-            "-Dlapack=" + lapack,
-            "--prefix=" + join_path(os.getcwd(), "build-install"),
-            "-Ddebug=false",
-            "-Doptimization=2",
-        ]
-        meson = which("meson")
-        meson(*args)
-        args = [
-            "-m",
-            "build",
-            "--wheel",
-            "-Cbuilddir=build",
-            "--no-isolation",
-            "--skip-dependency-check",
-            "-Ccompile-args=-j%s" % make_jobs,
-            ".",
-        ]
-        python(*args)
-        args = std_pip_args + ["--prefix=" + prefix, glob.glob(join_path("dist", "scipy*.whl"))[0]]
-        pip(*args)
+    @when("@1.9:")
+    def config_settings(self, spec, prefix):
+        blas, lapack = self.spec["py-numpy"].package.blas_lapack_pkg_config()
+        return {
+            "builddir": "build",
+            "compile-args": f"-j{make_jobs}",
+            "setup-args": {
+                # http://scipy.github.io/devdocs/building/blas_lapack.html
+                "-Dblas": blas,
+                "-Dlapack": lapack,
+            },
+        }
+
+    @when("@:1.8")
+    @run_before("install")
+    def set_blas_lapack(self):
+        self.spec["py-numpy"].package.blas_lapack_site_cfg()
 
     @run_after("install")
     @on_package_attributes(run_tests=True)
diff --git a/var/spack/repos/builtin/packages/py-screed/package.py b/var/spack/repos/builtin/packages/py-screed/package.py
new file mode 100644
index 00000000000000..3e31d87ea1f9ee
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-screed/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyScreed(PythonPackage):
+    """screed: a Python library for loading FASTA and FASTQ sequences."""
+
+    homepage = "https://screed.readthedocs.io/"
+    pypi = "screed/screed-1.1.2.tar.gz"
+
+    version("1.1.2", sha256="734ffa7a8a645286496d895b736f91d6b2988956e2fd42358123d93ec8519b6a")
+
+    depends_on("py-setuptools@48:", type="build")
+    depends_on("py-setuptools-scm@4:5", type="build")
+    depends_on("py-setuptools-scm-git-archive", type="build")
+    depends_on("py-wheel@0.29.0:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-semver/package.py b/var/spack/repos/builtin/packages/py-semver/package.py
index d3201097def6b0..59b05d9b64cca6 100644
--- a/var/spack/repos/builtin/packages/py-semver/package.py
+++ b/var/spack/repos/builtin/packages/py-semver/package.py
@@ -13,6 +13,8 @@ class PySemver(PythonPackage):
     homepage = "https://semver.org/"
     pypi = "semver/semver-2.8.1.tar.gz"
 
+    version("3.0.1", sha256="9ec78c5447883c67b97f98c3b6212796708191d22e4ad30f4570f840171cbce1")
     version("2.8.1", sha256="5b09010a66d9a3837211bb7ae5a20d10ba88f8cb49e92cb139a69ef90d5060d8")
 
     depends_on("py-setuptools", type="build")
+    depends_on("py-setuptools-scm", when="@3:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-setuptools-rust/package.py b/var/spack/repos/builtin/packages/py-setuptools-rust/package.py
index a21228b12e8fca..4a2663a59373f3 100644
--- a/var/spack/repos/builtin/packages/py-setuptools-rust/package.py
+++ b/var/spack/repos/builtin/packages/py-setuptools-rust/package.py
@@ -12,18 +12,24 @@ class PySetuptoolsRust(PythonPackage):
     homepage = "https://github.com/PyO3/setuptools-rust"
     pypi = "setuptools-rust/setuptools-rust-0.12.1.tar.gz"
 
+    version("1.6.0", sha256="c86e734deac330597998bfbc08da45187e6b27837e23bd91eadb320732392262")
     version("1.5.1", sha256="0e05e456645d59429cb1021370aede73c0760e9360bbfdaaefb5bced530eb9d7")
     version("1.4.1", sha256="18ff850831f58ee21d5783825c99fad632da21e47645e9427fd7dec048029e76")
     version("1.2.0", sha256="0a4ada479e8c7e3d8bd7cb56e1a29acc2b2bb98c2325051b0cdcb57d7f056de8")
     version("0.12.1", sha256="647009e924f0ae439c7f3e0141a184a69ad247ecb9044c511dabde232d3d570e")
 
+    variant("rust_bootstrap", default=False, description="Use prebuilt Rust")
+
     depends_on("py-setuptools@62.4:", when="@1.4.0:", type=("build", "run"))
     depends_on("py-setuptools@46.1:", type=("build", "run"))
     depends_on("py-setuptools", type=("build", "run"))
-    depends_on("py-setuptools-scm+toml@6.3.2:", when="@1.2.0:1.4.1", type="build")
-    depends_on("py-setuptools-scm+toml@3.4.3:", when="@:1.1", type="build")
     depends_on("py-semantic-version@2.8.2:2", when="@1.2.0:", type=("build", "run"))
     depends_on("py-semantic-version@2.6.0:", type=("build", "run"))
     depends_on("py-typing-extensions@3.7.4.3:", when="@1.2.0:", type=("build", "run"))
+    depends_on("rust", when="~rust_bootstrap", type="run")
+    depends_on("rust-bootstrap", when="+rust_bootstrap", type="run")
+
+    # Historical dependencies
+    depends_on("py-setuptools-scm+toml@6.3.2:", when="@1.2.0:1.4.1", type="build")
+    depends_on("py-setuptools-scm+toml@3.4.3:", when="@:1.1", type="build")
     depends_on("py-toml@0.9.0:", type=("build", "run"), when="@0.12.1")
-    depends_on("rust", type="run")
diff --git a/var/spack/repos/builtin/packages/py-setuptools-scm/package.py b/var/spack/repos/builtin/packages/py-setuptools-scm/package.py
index 807c0ced17f195..6b5412cf9b6a8c 100644
--- a/var/spack/repos/builtin/packages/py-setuptools-scm/package.py
+++ b/var/spack/repos/builtin/packages/py-setuptools-scm/package.py
@@ -42,3 +42,5 @@ class PySetuptoolsScm(PythonPackage):
     depends_on("py-tomli@1:", when="@7.1: ^python@:3.10", type=("build", "run"))
     depends_on("py-typing-extensions", when="@7:", type=("build", "run"))
     depends_on("py-importlib-metadata", when="@7: ^python@:3.7", type=("build", "run"))
+
+    depends_on("git", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-setuptools/package.py b/var/spack/repos/builtin/packages/py-setuptools/package.py
index 0d10ffb5f90b9c..36cc6de4ee38b7 100644
--- a/var/spack/repos/builtin/packages/py-setuptools/package.py
+++ b/var/spack/repos/builtin/packages/py-setuptools/package.py
@@ -14,6 +14,13 @@ class PySetuptools(Package, PythonExtension):
     url = "https://files.pythonhosted.org/packages/py3/s/setuptools/setuptools-62.3.2-py3-none-any.whl"
     list_url = "https://pypi.org/simple/setuptools/"
 
+    tags = ["build-tools"]
+
+    version(
+        "68.0.0",
+        sha256="11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f",
+        expand=False,
+    )
     version(
         "67.6.0",
         sha256="b78aaa36f6b90a074c1fa651168723acbf45d14cb1196b6f02c0fd07f17623b2",
@@ -186,11 +193,22 @@ class PySetuptools(Package, PythonExtension):
     )
 
     extends("python")
+
     depends_on("python@3.7:", when="@59.7:", type=("build", "run"))
     depends_on("python@3.6:", when="@51:", type=("build", "run"))
     depends_on("python@3.5:", when="@45:50", type=("build", "run"))
     depends_on("python@2.7:2.8,3.5:", when="@44", type=("build", "run"))
     depends_on("python@2.7:2.8,3.4:", when="@:43", type=("build", "run"))
+
+    # Uses HTMLParser.unescape
+    depends_on("python@:3.8", when="@:41.0", type=("build", "run"))
+
+    # Uses collections.MutableMapping
+    depends_on("python@:3.9", when="@:40.4.2", type=("build", "run"))
+
+    # https://github.com/pypa/setuptools/issues/3661
+    depends_on("python@:3.11", when="@:67", type=("build", "run"))
+
     depends_on("py-pip", type="build")
 
     def url_for_version(self, version):
diff --git a/var/spack/repos/builtin/packages/py-shapely/package.py b/var/spack/repos/builtin/packages/py-shapely/package.py
index 50163509177a65..a6282fc25e1a87 100644
--- a/var/spack/repos/builtin/packages/py-shapely/package.py
+++ b/var/spack/repos/builtin/packages/py-shapely/package.py
@@ -19,6 +19,7 @@ class PyShapely(PythonPackage):
     maintainers("adamjstewart")
 
     version("main", branch="main")
+    version("2.0.2", sha256="1713cc04c171baffc5b259ba8531c58acc2a301707b7f021d88a15ed090649e7")
     version("2.0.1", sha256="66a6b1a3e72ece97fc85536a281476f9b7794de2e646ca8a4517e2e3c1446893")
     version("2.0.0", sha256="11f1b1231a6c04213fb1226c6968d1b1b3b369ec42d1e9655066af87631860ea")
     version("1.8.5", sha256="e82b6d60ecfb124120c88fe106a478596bbeab142116d7e7f64a364dac902a92")
@@ -32,7 +33,8 @@ class PyShapely(PythonPackage):
     version("1.6.4", sha256="b10bc4199cfefcf1c0e5d932eac89369550320ca4bdf40559328d85f1ca4f655")
 
     # pyproject.toml
-    depends_on("py-cython@0.29:0", when="@2:", type="build")
+    depends_on("py-cython", when="@2.0.2:", type="build")
+    depends_on("py-cython@0.29:0", when="@2.0.0:2.0.1", type="build")
     depends_on("py-cython@0.29.24:2", when="@:1", type="build")
     depends_on("py-setuptools@61:", when="@2:", type="build")
     depends_on("py-setuptools@:63", when="@:1", type="build")
diff --git a/var/spack/repos/builtin/packages/py-shiboken/package.py b/var/spack/repos/builtin/packages/py-shiboken/package.py
index 057f19c4f6fc35..0baaa1d89b4161 100644
--- a/var/spack/repos/builtin/packages/py-shiboken/package.py
+++ b/var/spack/repos/builtin/packages/py-shiboken/package.py
@@ -16,12 +16,17 @@ class PyShiboken(PythonPackage):
 
     version("1.2.2", sha256="0baee03c6244ab56e42e4200d0cb5e234682b11cc296ed0a192fe457d054972f")
 
-    depends_on("cmake", type="build")
+    depends_on("cmake@2.6:", type="build")
 
+    # to prevent error: 'PyTypeObject' {aka 'struct _typeobject'} has no member
+    # named 'tp_print'
+    depends_on("python@:3.8", type=("build", "run"))
     depends_on("py-setuptools", type="build")
+    # in newer pip versions --install-option does not exist
+    depends_on("py-pip@:23.0", type="build")
     depends_on("py-sphinx@:3.4", type=("build", "run"))
     depends_on("libxml2")
-    depends_on("qt@:4.8")
+    depends_on("qt@4.6:4.8")
 
     # subprocess.mswindows was renamed to subprocess._mswindows in Python 3.5
     patch("python-3.5.patch", when="^python@3.5:")
diff --git a/var/spack/repos/builtin/packages/py-shippinglabel/package.py b/var/spack/repos/builtin/packages/py-shippinglabel/package.py
new file mode 100644
index 00000000000000..db9466c9d7064f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-shippinglabel/package.py
@@ -0,0 +1,27 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyShippinglabel(PythonPackage):
+    """Utilities for handling packages."""
+
+    homepage = "https://github.com/domdfcoding/shippinglabel"
+    pypi = "shippinglabel/shippinglabel-1.5.0.tar.gz"
+
+    version("1.5.0", sha256="b2332bf29853e29f877dab21f17f2a8632fd2b48f5f32a6fa0b254a6fdd0a451")
+
+    depends_on("py-wheel@0.34.2:", type="build")
+    depends_on("py-setuptools@40.6:", type="build")
+    conflicts("^py-setuptools@61")
+    depends_on("py-apeye@1:", type=("build", "run"))
+    depends_on("py-dist-meta@0.1.2:", type=("build", "run"))
+    depends_on("py-dom-toml@0.2.2:", type=("build", "run"))
+    depends_on("py-domdf-python-tools@3.1:", type=("build", "run"))
+    depends_on("py-packaging@20.9:", type=("build", "run"))
+    depends_on("py-platformdirs@2.3:", type=("build", "run"))
+    depends_on("py-typing-extensions@3.7.4.3:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-shroud/package.py b/var/spack/repos/builtin/packages/py-shroud/package.py
index 431feb2b65eeaf..de73dbeb00d73d 100644
--- a/var/spack/repos/builtin/packages/py-shroud/package.py
+++ b/var/spack/repos/builtin/packages/py-shroud/package.py
@@ -15,12 +15,12 @@ class PyShroud(PythonPackage):
 
     version("develop", branch="develop")
     version("master", branch="master")
-    version("0.12.2", tag="v0.12.2")
-    version("0.12.1", tag="v0.12.1")
-    version("0.11.0", tag="v0.11.0")
-    version("0.10.1", tag="v0.10.1")
-    version("0.9.0", tag="v0.9.0")
-    version("0.8.0", tag="v0.8.0")
+    version("0.12.2", tag="v0.12.2", commit="939ba0a3e8b5a885da3ddaebb92bf93cb12b0401")
+    version("0.12.1", tag="v0.12.1", commit="c09344655371885a42783f8c0ac8a31f2bbffc9f")
+    version("0.11.0", tag="v0.11.0", commit="503b852796d549199c5ab94b14e59ebd62988870")
+    version("0.10.1", tag="v0.10.1", commit="13a3c70bc5190e0e8531e17925928fbd7154acb5")
+    version("0.9.0", tag="v0.9.0", commit="94aa2831290d10b604df16cb87ee17aa722fb998")
+    version("0.8.0", tag="v0.8.0", commit="b58ac35f41514428d08849a578c45ad444bfddc9")
 
     depends_on("py-setuptools", type=("build", "run"))
     depends_on("py-pyyaml@4.2:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-simplejson/package.py b/var/spack/repos/builtin/packages/py-simplejson/package.py
index 413595cb971223..a5c52fc3bdcb5d 100644
--- a/var/spack/repos/builtin/packages/py-simplejson/package.py
+++ b/var/spack/repos/builtin/packages/py-simplejson/package.py
@@ -13,6 +13,7 @@ class PySimplejson(PythonPackage):
     homepage = "https://github.com/simplejson/simplejson"
     pypi = "simplejson/simplejson-3.10.0.tar.gz"
 
+    version("3.19.1", sha256="6277f60848a7d8319d27d2be767a7546bc965535b28070e310b3a9af90604a4c")
     version("3.18.0", sha256="58a429d2c2fa80834115b923ff689622de8f214cf0dc4afa9f59e824b444ab31")
     version("3.17.2", sha256="75ecc79f26d99222a084fbdd1ce5aad3ac3a8bd535cd9059528452da38b68841")
     version(
@@ -28,5 +29,4 @@ class PySimplejson(PythonPackage):
     version("3.8.0", sha256="217e4797da3a9a4a9fbe6722e0db98070b8443a88212d7acdbd241a7668141d9")
     version("3.3.0", sha256="7a8a6bd82e111976aeb06138316ab10847adf612925072eaff8512228bcf9a1f")
 
-    depends_on("python@2.5:2,3.3:", when="@3.16.0:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-sip/package.py b/var/spack/repos/builtin/packages/py-sip/package.py
index 031c04ca8b6c9c..e85f442127ec4d 100644
--- a/var/spack/repos/builtin/packages/py-sip/package.py
+++ b/var/spack/repos/builtin/packages/py-sip/package.py
@@ -14,9 +14,11 @@ class PySip(PythonPackage):
     homepage = "https://www.riverbankcomputing.com/software/sip"
     pypi = "sip/sip-6.4.0.tar.gz"
 
+    version("6.7.9", sha256="35d51fc10f599d3696abb50f29d068ad04763df7b77808c76b74597660f99b17")
     version("6.6.2", sha256="0e3efac1c5dfd8e525ae57140927df26993e13f58b89d1577c314f4105bfd90d")
     version("6.4.0", sha256="42ec368520b8da4a0987218510b1b520b4981e4405086c1be384733affc2bcb0")
     version("5.5.0", sha256="5d024c419b30fea8a6de8c71a560c7ab0bc3c221fbfb14d55a5b865bd58eaac5")
+    version("4.19.25", sha256="3d36986f7327b7b966bb6eacf22bcf6e4d0a3d24e392276ef92af89988818062")
     version("4.19.21", sha256="3bfd58e875a87471c00e008f25a01d8312885aa01efc4f688e5cac861c8676e4")
     version("4.19.20", sha256="475f85277a6601c406ade508b6c935b9f2a170c16fd3ae9dd4cdee7a4f7f340d")
     version("4.19.19", sha256="348cd6229b095a3090e851555814f5147bffcb601cec891f1038eb6b38c9d856")
@@ -33,27 +35,25 @@ class PySip(PythonPackage):
         multi=False,
     )
 
-    depends_on("python@3.7:", when="@6.6:", type=("build", "link", "run"))
-    depends_on("python@3.6:", when="@6:", type=("build", "link", "run"))
-
     depends_on("py-ply", when="@6.6:", type=("build", "run"))
 
     with when("@5:"):
-        depends_on("python@3.5.1:", type=("build", "link", "run"))
-        depends_on("py-packaging", type="build")
-        depends_on("py-setuptools@30.3:", type="build")
-        depends_on("py-toml", type="build")
+        depends_on("python", type=("build", "link", "run"))
+        depends_on("py-packaging", type=("build", "run"))
+        depends_on("py-setuptools@30.3:", type=("build", "run"))
+        depends_on("py-tomli", when="@6.7: ^python@:3.10", type=("build", "run"))
+        depends_on("py-toml", when="@:6.6", type=("build", "run"))
 
     with when("@:4"):
-        depends_on("python", type=("build", "link", "run"))
+        # Requires distutils
+        depends_on("python@:3.11", type=("build", "link", "run"))
         depends_on("flex", type="build")
         depends_on("bison", type="build")
 
     def url_for_version(self, version):
-        if version < Version("5.0.0"):
-            return "https://www.riverbankcomputing.com/hg/sip/archive/{0}.tar.gz".format(
-                version.dotted
-            )
+        if version < Version("5"):
+            url = "https://www.riverbankcomputing.com/hg/sip/archive/{0}.tar.gz"
+            return url.format(version.dotted)
         return super().url_for_version(version)
 
     @when("@:4")
@@ -75,12 +75,20 @@ def install(self, spec, prefix):
 
     @run_after("install")
     def extend_path_setup(self):
-        if self.spec.satisfies("@:4"):
-            # See github issue #14121 and PR #15297
-            module = self.spec.variants["module"].value
-            if module != "sip":
-                module = module.split(".")[0]
-                with working_dir(python_platlib):
-                    with open(os.path.join(module, "__init__.py"), "w") as f:
-                        f.write("from pkgutil import extend_path\n")
-                        f.write("__path__ = extend_path(__path__, __name__)\n")
+        # https://github.com/spack/spack/issues/14121
+        # https://github.com/spack/spack/pull/15297
+        # Same code comes by default with py-pyqt5 and py-pyqt6
+        if self.spec.satisfies("@5:"):
+            return
+
+        module = self.spec.variants["module"].value
+        if module == "sip":
+            return
+
+        module = module.split(".")[0]
+        text = f"""
+# Support {module} sub-packages that have been created by setuptools.
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+"""
+        with open(join_path(python_platlib, module, "__init__.py"), "w") as f:
+            f.write(text)
diff --git a/var/spack/repos/builtin/packages/py-slepc4py/package.py b/var/spack/repos/builtin/packages/py-slepc4py/package.py
index 7d0dcf52faa9ae..9bf3413fceba4e 100644
--- a/var/spack/repos/builtin/packages/py-slepc4py/package.py
+++ b/var/spack/repos/builtin/packages/py-slepc4py/package.py
@@ -16,6 +16,8 @@ class PySlepc4py(PythonPackage):
     maintainers("joseeroman", "balay")
 
     version("main", branch="main")
+    version("3.20.0", sha256="56cbea1f56746136e5a934bf4a481e566f35e475cb950c0a5bce7d5c3cc7690a")
+    version("3.19.2", sha256="da8b6a7aaaf5e4497b896b2e478c42dd9de4fb31da93eb294181bea3bb60c767")
     version("3.19.1", sha256="68303f4acef8efc0542ab288a19159d0e6cdf313726f573e0bea2edb3d2c9595")
     version("3.19.0", sha256="ae84d33cce259c1d6ff64308b2f819d1c0f7b018e048f9049ec6d5be15614ba5")
     version("3.18.3", sha256="93c978f115683900a575026111ff2abe6f3ce4de8c21eec53c07dfd97ea43c85")
@@ -45,25 +47,13 @@ class PySlepc4py(PythonPackage):
 
     depends_on("py-petsc4py", type=("build", "run"))
     depends_on("py-petsc4py@main", when="@main", type=("build", "run"))
-    depends_on("py-petsc4py@3.19.0:3.19", when="@3.19.0:3.19", type=("build", "run"))
-    depends_on("py-petsc4py@3.18.0:3.18", when="@3.18.0:3.18", type=("build", "run"))
-    depends_on("py-petsc4py@3.17.0:3.17", when="@3.17.0:3.17", type=("build", "run"))
-    depends_on("py-petsc4py@3.16.0:3.16", when="@3.16.0:3.16", type=("build", "run"))
-    depends_on("py-petsc4py@3.15.0:3.15", when="@3.15.0:3.15", type=("build", "run"))
-    depends_on("py-petsc4py@3.13.0:3.13", when="@3.13.0:3.13", type=("build", "run"))
-    depends_on("py-petsc4py@3.12.0:3.12", when="@3.12.0:3.12", type=("build", "run"))
-    depends_on("py-petsc4py@3.11.0:3.11", when="@3.11.0:3.11", type=("build", "run"))
+    for ver in ["3.20", "3.19", "3.18", "3.17", "3.16", "3.15", "3.13", "3.12", "3.11"]:
+        depends_on(f"py-petsc4py@{ver}", when=f"@{ver}", type=("build", "run"))
 
     depends_on("slepc")
     depends_on("slepc@main", when="@main")
-    depends_on("slepc@3.19.0:3.19", when="@3.19.0:3.19")
-    depends_on("slepc@3.18.0:3.18", when="@3.18.0:3.18")
-    depends_on("slepc@3.17.0:3.17", when="@3.17.0:3.17")
-    depends_on("slepc@3.16.0:3.16", when="@3.16.0:3.16")
-    depends_on("slepc@3.15.0:3.15", when="@3.15.0:3.15")
-    depends_on("slepc@3.13.0:3.13", when="@3.13.0:3.13")
-    depends_on("slepc@3.12.0:3.12", when="@3.12.0:3.12")
-    depends_on("slepc@3.11.0:3.11", when="@3.11.0:3.11")
+    for ver in ["3.20", "3.19", "3.18", "3.17", "3.16", "3.15", "3.13", "3.12", "3.11"]:
+        depends_on(f"slepc@{ver}", when=f"@{ver}")
 
     @property
     def build_directory(self):
diff --git a/var/spack/repos/builtin/packages/py-smartredis/package.py b/var/spack/repos/builtin/packages/py-smartredis/package.py
new file mode 100644
index 00000000000000..12969a04540551
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-smartredis/package.py
@@ -0,0 +1,52 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PySmartredis(PythonPackage):
+    """A Python Interface for the SmartRedis Library Client"""
+
+    homepage = "https://www.craylabs.org/docs/smartredis.html"
+    pypi = "smartredis/smartredis-0.4.0.tar.gz"
+    git = "https://github.com/CrayLabs/SmartRedis"
+
+    maintainers("MattToast")
+
+    version("0.4.1", sha256="fff16ed1eb09648ac3c3f845373beb37f3ffe7414d8745ae36af9daf585f8c5b")
+    version("0.4.0", sha256="d12779aa8bb038e837c25eac41b178aab9e16b729d50ee360b5af8f813d9f1dd")
+
+    depends_on("python@3.7:3.10", type=("build", "run"))
+    depends_on("py-setuptools@42:", type=("build",))
+
+    depends_on("cmake@3.13:", type=("build",))
+
+    # Documented dependencies
+    depends_on("hiredis@1.1:", type=("build", "link", "run"), when="@0.4.1")
+    depends_on("hiredis@1.0:", type=("build", "link", "run"), when="@0.4.0")
+    depends_on("redis-plus-plus@1.3.5: cxxstd=17", type=("build", "link"))
+
+    # Unlisted dependency needed to build the python client. The pybind requirement
+    # can be found:
+    #  - in the `build-scripts/build_deps.sh` for SmartRedis <= v0.4.0
+    #  - in the `Makefile` under the `pybind` target for SmartRedis >= v0.4.1
+    depends_on("py-pybind11", type=("build",))
+
+    depends_on("py-numpy@1.18.2:", type=("build", "run"))
+
+    # By default, the `setup.py` for SmartRedis <= v0.4.1 will fetch dependencies and
+    # use them to build the extension library; it does not allow users to supply
+    # their own previously obtained dependencies. These patches remove the 'autofetch'
+    # behavior and use the dependencies provided through spack.
+    patch("sr_0_4_1_no_deps.patch", when="@0.4.1")
+    patch("sr_0_4_0_no_deps.patch", when="@0.4.0")
+
+    def setup_build_environment(self, env):
+        spec = self.spec
+        env.set("REDISPP_LIB_DIR", spec["redis-plus-plus"].libs.directories[0])
+        env.set("REDISPP_INC_DIR", spec["redis-plus-plus"].headers.directories[0])
+        env.set("HIREDIS_LIB_DIR", spec["hiredis"].libs.directories[0])
+        env.set("HIREDIS_INC_DIR", spec["hiredis"].headers.directories[0])
+        env.set("PYBIND11_TOOLS", spec["py-pybind11"].prefix.share.cmake.pybind11)
diff --git a/var/spack/repos/builtin/packages/py-smartredis/sr_0_4_0_no_deps.patch b/var/spack/repos/builtin/packages/py-smartredis/sr_0_4_0_no_deps.patch
new file mode 100644
index 00000000000000..a22145eabec011
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-smartredis/sr_0_4_0_no_deps.patch
@@ -0,0 +1,101 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 7dc8931..658d823 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -59,8 +59,20 @@ if (COVERAGE)
+     endif()
+ endif()
+ 
+-find_library(REDISPP redis++ PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED)
+-find_library(HIREDIS hiredis PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED)
++set(REDISPP_LIB_DIR ${CMAKE_SOURCE_DIR}/install/lib CACHE PATH "path to redis++")
++set(HIREDIS_LIB_DIR ${CMAKE_SOURCE_DIR}/install/lib CACHE PATH "path to hiredis")
++
++set(REDISPP_INC_DIR ${CMAKE_SOURCE_DIR}/install/include CACHE PATH "path to redis++")
++set(HIREDIS_INC_DIR ${CMAKE_SOURCE_DIR}/install/include CACHE PATH "path to hiredis")
++
++message("USING RPP PATH: ${REDISPP_LIB_DIR}")
++message("USING HIR PATH: ${HIREDIS_LIB_DIR}")
++message("USING RPP IPATH: ${REDISPP_INC_DIR}")
++message("USING HIR IPATH: ${HIREDIS_INC_DIR}")
++
++
++find_library(REDISPP libredis++.a PATHS ${REDISPP_LIB_DIR}  NO_DEFAULT_PATH REQUIRED)
++find_library(HIREDIS hiredis PATHS ${HIREDIS_LIB_DIR} NO_DEFAULT_PATH REQUIRED)
+ find_package(Threads REQUIRED)
+ 
+ set(EXT_CLIENT_LIBRARIES ${REDISPP} ${HIREDIS})
+@@ -106,7 +118,8 @@ set(CLIENT_SRC
+ 
+ include_directories(SYSTEM
+     include
+-    install/include
++    ${REDISPP_INC_DIR}
++    ${HIREDIS_INC_DIR}
+ )
+ 
+ if (BUILD_FORTRAN)
+@@ -148,8 +161,9 @@ install(TARGETS smartredis
+ 
+ if(BUILD_PYTHON)
+ 	message("-- Python client build enabled")
+-	add_subdirectory(${CMAKE_SOURCE_DIR}/third-party/pybind
+-                     ${CMAKE_SOURCE_DIR}/third-party/pybind/build)
++
++    set(pybind11_DIR ${CMAKE_SOURCE_DIR}/third-party/pybind/tools)
++    find_package(pybind11)
+ 
+     add_library(smartredis_static STATIC ${CLIENT_SRC})
+ 
+diff --git a/setup.py b/setup.py
+index dd19c6c..4248aef 100644
+--- a/setup.py
++++ b/setup.py
+@@ -58,9 +58,20 @@ class CMakeBuild(build_ext):
+         build_directory = Path(self.build_temp).resolve()
+         cmake_args = [
+             '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + str(build_directory),
+-            '-DPYTHON_EXECUTABLE=' + sys.executable
++            '-DPYTHON_EXECUTABLE=' + sys.executable,
++            '-Dpybind11_DIR=' + str(os.getenv('PYBIND11_TOOLS')),
+         ]
+ 
++        for setting in [
++            "REDISPP_LIB_DIR",
++            "REDISPP_INC_DIR",
++            "HIREDIS_LIB_DIR",
++            "HIREDIS_INC_DIR",
++        ]:
++            val = os.getenv(setting)
++            if val is not None:
++                cmake_args.append(f"-D{setting}={val}")
++
+         cfg = 'Debug' if self.debug else 'Release'
+         build_args = ['--config', cfg]
+         build_args += ['--', f'-j{str(NPROC)}']
+@@ -78,15 +89,8 @@ class CMakeBuild(build_ext):
+         if not build_directory.is_dir():
+             os.makedirs(self.build_temp)
+ 
+-        print('-'*10, 'Building C dependencies', '-'*40)
+-        make_cmd = shutil.which("make")
+         setup_path = Path(os.path.abspath(os.path.dirname(__file__))).resolve()
+ 
+-        # build dependencies
+-        subprocess.check_call([f"{make_cmd} deps"],
+-                              cwd=setup_path,
+-                              shell=True)
+-
+         # run cmake prep step
+         print('-'*10, 'Running CMake prepare', '-'*40)
+         subprocess.check_call([self.cmake, setup_path] + cmake_args,
+@@ -99,9 +103,6 @@ class CMakeBuild(build_ext):
+         subprocess.check_call(cmake_cmd,
+                               cwd=build_directory)
+ 
+-        shutil.copytree(setup_path.joinpath("install"),
+-                        build_directory.joinpath("install"))
+-
+         # Move from build temp to final position
+         for ext in self.extensions:
+             self.move_output(ext)
diff --git a/var/spack/repos/builtin/packages/py-smartredis/sr_0_4_1_no_deps.patch b/var/spack/repos/builtin/packages/py-smartredis/sr_0_4_1_no_deps.patch
new file mode 100644
index 00000000000000..78fb4d21036392
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-smartredis/sr_0_4_1_no_deps.patch
@@ -0,0 +1,94 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index ca88a4d..30118fc 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -64,13 +64,27 @@ if (SR_PEDANTIC)
+     endif()
+ endif()
+ 
++set(REDISPP_LIB_DIR ${CMAKE_SOURCE_DIR}/install/lib CACHE PATH "path to redis++")
++set(REDISPP_INC_DIR ${CMAKE_SOURCE_DIR}/install/include CACHE PATH
++    "path to redis++ headers")
++
++set(HIREDIS_LIB_DIR ${CMAKE_SOURCE_DIR}/install/lib CACHE PATH "path to hiredis")
++set(HIREDIS_INC_DIR ${CMAKE_SOURCE_DIR}/install/include CACHE PATH
++    "path to hiredis headers")
++
++message("USING REDIS++ PATH: ${REDISPP_LIB_DIR}")
++message("USING REDIS++ INCLUDE PATH: ${REDISPP_INC_DIR}")
++
++message("USING HIREDIS PATH: ${HIREDIS_LIB_DIR}")
++message("USING HIREDIS INCLUDE PATH: ${HIREDIS_INC_DIR}")
++
+ # Bring in third-party libaries needed for the SmartRedis library
+ find_library(REDISPP redis++
+-    PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH
++    PATHS ${REDISPP_LIB_DIR} NO_DEFAULT_PATH
+     REQUIRED STATIC
+ )
+ find_library(HIREDIS hiredis
+-    PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH
++    PATHS ${HIREDIS_LIB_DIR} NO_DEFAULT_PATH
+     REQUIRED STATIC
+ )
+ find_package(Threads REQUIRED)
+@@ -121,7 +135,8 @@ set(CLIENT_SRC
+ # Define include directories for header files
+ include_directories(SYSTEM
+     include
+-    install/include
++    ${REDISPP_INC_DIR}
++    ${HIREDIS_INC_DIR}
+ )
+ 
+ # Build the Fortran library
+@@ -177,8 +192,8 @@ install(TARGETS smartredis
+ # Build the Python library for SmartRedis
+ if(SR_PYTHON)
+ 	message("-- Python client build enabled")
+-	add_subdirectory(${CMAKE_SOURCE_DIR}/third-party/pybind
+-                     ${CMAKE_SOURCE_DIR}/third-party/pybind/build)
++    set(pybind11_DIR ${CMAKE_SOURCE_DIR}/third-party/pybind/tools)
++    find_package(pybind11)
+ 
+ 	pybind11_add_module(smartredisPy
+ 	                    src/python/src/pyclient.cpp
+diff --git a/setup.py b/setup.py
+index 90493ee..dd075db 100644
+--- a/setup.py
++++ b/setup.py
+@@ -73,14 +73,6 @@ class CMakeBuild(build_ext):
+             env.get('CXXFLAGS', ''),
+             self.distribution.get_version())
+ 
+-        # Build dependencies
+-        print('-'*10, 'Building third-party dependencies', '-'*40)
+-        subprocess.check_call(
+-            [self.make, "deps"],
+-            cwd=source_directory,
+-            shell=False
+-        )
+-
+         # Run CMake config step
+         print('-'*10, 'Configuring build', '-'*40)
+         config_args = [
+@@ -90,7 +82,19 @@ class CMakeBuild(build_ext):
+             '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + str(build_directory),
+             '-DPYTHON_EXECUTABLE=' + sys.executable,
+             '-DSR_PYTHON=ON',
++            '-Dpybind11_DIR=' + str(os.getenv('PYBIND11_TOOLS')),
+         ]
++
++        for setting in [
++            "REDISPP_LIB_DIR",
++            "REDISPP_INC_DIR",
++            "HIREDIS_LIB_DIR",
++            "HIREDIS_INC_DIR",
++        ]:
++            val = os.getenv(setting)
++            if val is not None:
++                config_args.append(f"-D{setting}={val}")
++
+         subprocess.check_call(
+             [self.cmake] + config_args,
+             cwd=source_directory,
diff --git a/var/spack/repos/builtin/packages/py-smartsim/package.py b/var/spack/repos/builtin/packages/py-smartsim/package.py
new file mode 100644
index 00000000000000..4011e0c8dc521d
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-smartsim/package.py
@@ -0,0 +1,83 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+
+
+class PySmartsim(PythonPackage):
+    """A workflow library to easily integrate machine learning libraries with
+    high performance computing simulations and applications
+    """
+
+    homepage = "https://www.craylabs.org/docs/overview.html"
+    git = "https://github.com/CrayLabs/SmartSim"
+    pypi = "smartsim/smartsim-0.5.0.tar.gz"
+
+    maintainers("MattToast")
+
+    version("0.5.0", sha256="35b36243dc84af62261a7f772bae92f0b3502faf01401423899cb2a48339858c")
+
+    variant("torch", default=True, description="Build with the pytorch backend")
+
+    depends_on("python@3.8:3.10", type=("build", "run"))
+    depends_on("py-setuptools@39.2:", type=("build",))
+    depends_on("py-cmake@3.13:", type=("build",))
+
+    depends_on("py-psutil@5.7.2:", type=("build", "run"))
+    depends_on("py-coloredlogs@10:", type=("build", "run"))
+    depends_on("py-tabulate@0.8.9:", type=("build", "run"))
+    depends_on("py-redis@4.5:", type=("build", "run"))
+    depends_on("py-tqdm@4.50.2:", type=("build", "run"))
+    depends_on("py-filelock@3.4.2:", type=("build", "run"))
+    depends_on("py-protobuf@3.20:3", type=("build", "run"))
+
+    # Companion libs
+    depends_on("py-smartredis@0.4.1:", type=("build", "run"), when="@0.5.0")
+
+    # Backends
+    # SmartSim defines sensible and well tested lower bounds for redis
+    # for the CLI to fetch in the `smartsim._core._install.buildenv.Versioner`
+    # class (lower versions are unable to parse the default `redis.conf` shipped
+    # with SmartSim), but allows users to upgrade explicitly by setting
+    # environment variables.
+    depends_on("redis@7.0.5:", type=("build", "run"))
+    depends_on("redis-ai", type=("build", "run"))
+
+    # ML Deps
+    # The lower bound for these py-* deps can be found in the source code
+    # at `smartsim/_core/_install/buildenv.py`.
+    with when("+torch"):
+        depends_on("redis-ai+torch", type=("build", "run"))
+        depends_on("py-torch@1.11.0:", type=("build", "run"))
+        depends_on("py-torchvision@0.12.0:", type=("build", "run"))
+
+    # By default, the SmartSim `setup.py` will attempt to fetch and build
+    # its own copy of Redis. This should be patched out and the version of
+    # Redis retrieved through spack should be used instead.
+    patch("ss-dont-build-db.patch")
+
+    # SmartSim provides its own CLI to fetch and build its own
+    # copy of Redis, RedisAI, and ML deps. This functionality should be
+    # patched out so that users do not accidentally overwrite/break
+    # dependencies fetched though Spack
+    patch("ss-0-5-0-remove-cli-build-fns.patch")
+
+    def setup_build_environment(self, env):
+        env.set("BUILD_JOBS", make_jobs)
+
+    @run_after("install")
+    def symlink_bin_deps(self):
+        ss_core_path = join_path(python_purelib, "smartsim", "_core")
+        os.symlink(
+            self.spec["redis"].prefix.bin.join("redis-server"),
+            join_path(ss_core_path, "bin", "redis-server"),
+        )
+        os.symlink(
+            self.spec["redis"].prefix.bin.join("redis-cli"),
+            join_path(ss_core_path, "bin", "redis-cli"),
+        )
+        os.symlink(self.spec["redis-ai"].prefix, join_path(ss_core_path, "lib"))
diff --git a/var/spack/repos/builtin/packages/py-smartsim/ss-0-5-0-remove-cli-build-fns.patch b/var/spack/repos/builtin/packages/py-smartsim/ss-0-5-0-remove-cli-build-fns.patch
new file mode 100644
index 00000000000000..4dbb0f42733fef
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-smartsim/ss-0-5-0-remove-cli-build-fns.patch
@@ -0,0 +1,22 @@
+diff --git a/smartsim/_core/_cli/cli.py b/smartsim/_core/_cli/cli.py
+index 7dbb144..93da137 100644
+--- a/smartsim/_core/_cli/cli.py
++++ b/smartsim/_core/_cli/cli.py
+@@ -68,14 +68,17 @@ class SmartCli:
+         getattr(self, args.command)()
+ 
+     def build(self) -> None:
++        raise RuntimeError("`smart build` not supported through spack install")
+         Build()
+         sys.exit(0)
+ 
+     def clean(self) -> None:
++        raise RuntimeError("`smart clean` not supported through spack install")
+         Clean()
+         sys.exit(0)
+ 
+     def clobber(self) -> None:
++        raise RuntimeError("`smart clobber` not supported through spack install")
+         Clean(clean_all=True)
+         sys.exit(0)
+ 
diff --git a/var/spack/repos/builtin/packages/py-smartsim/ss-dont-build-db.patch b/var/spack/repos/builtin/packages/py-smartsim/ss-dont-build-db.patch
new file mode 100644
index 00000000000000..0553a499c56e90
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-smartsim/ss-dont-build-db.patch
@@ -0,0 +1,27 @@
+diff --git a/setup.py b/setup.py
+index 6758bec..499567e 100644
+--- a/setup.py
++++ b/setup.py
+@@ -134,14 +134,14 @@ class InstallPlatlib(install):
+ class SmartSimBuild(build_py):
+ 
+     def run(self):
+-        database_builder = builder.DatabaseBuilder(build_env(),
+-                                             build_env.MALLOC,
+-                                             build_env.JOBS)
+-        if not database_builder.is_built:
+-            database_builder.build_from_git(versions.REDIS_URL,
+-                                         versions.REDIS)
+-
+-            database_builder.cleanup()
++        # database_builder = builder.DatabaseBuilder(build_env(),
++        #                                      build_env.MALLOC,
++        #                                      build_env.JOBS)
++        # if not database_builder.is_built:
++        #     database_builder.build_from_git(versions.REDIS_URL,
++        #                                  versions.REDIS)
++        #
++        #     database_builder.cleanup()
+ 
+         # run original build_py command
+         super().run()
diff --git a/var/spack/repos/builtin/packages/py-soupsieve/package.py b/var/spack/repos/builtin/packages/py-soupsieve/package.py
index 7072a2f486f984..03da213c5e6316 100644
--- a/var/spack/repos/builtin/packages/py-soupsieve/package.py
+++ b/var/spack/repos/builtin/packages/py-soupsieve/package.py
@@ -12,6 +12,7 @@ class PySoupsieve(PythonPackage):
     homepage = "https://github.com/facelessuser/soupsieve"
     pypi = "soupsieve/soupsieve-1.9.3.tar.gz"
 
+    version("2.4.1", sha256="89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea")
     version(
         "2.3.2.post1", sha256="fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"
     )
@@ -19,7 +20,8 @@ class PySoupsieve(PythonPackage):
     version("1.9.6", sha256="7985bacc98c34923a439967c1a602dc4f1e15f923b6fcf02344184f86cc7efaa")
     version("1.9.3", sha256="8662843366b8d8779dec4e2f921bebec9afd856a5ff2e82cd419acc5054a1a92")
 
-    depends_on("python@3.6:", when="@2.2:", type=("build", "run"))
-    depends_on("py-setuptools", when="@:2.1", type="build")
-    depends_on("py-setuptools@42:", when="@2.2", type="build")
     depends_on("py-hatchling@0.21.1:", when="@2.3.2:", type="build")
+
+    # Historical dependencies
+    depends_on("py-setuptools@42:", when="@2.2", type="build")
+    depends_on("py-setuptools", when="@:2.1", type="build")
diff --git a/var/spack/repos/builtin/packages/py-sphinx-basic-ng/package.py b/var/spack/repos/builtin/packages/py-sphinx-basic-ng/package.py
new file mode 100644
index 00000000000000..34909e4725a19f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-sphinx-basic-ng/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PySphinxBasicNg(PythonPackage):
+    """A modern skeleton for Sphinx themes."""
+
+    homepage = "https://github.com/pradyunsg/sphinx-basic-ng"
+    pypi = "sphinx_basic_ng/sphinx_basic_ng-1.0.0b2.tar.gz"
+
+    version("1.0.0b2", sha256="9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-sphinx@4:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-sphinx-book-theme/package.py b/var/spack/repos/builtin/packages/py-sphinx-book-theme/package.py
new file mode 100644
index 00000000000000..d34efeb4aefc3f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-sphinx-book-theme/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PySphinxBookTheme(PythonPackage):
+    """Lightweight Sphinx theme designed to mimic the look-and-feel of an interactive book."""
+
+    homepage = "https://sphinx-book-theme.readthedocs.io/en/latest"
+    pypi = "sphinx_book_theme/sphinx_book_theme-1.0.1.tar.gz"
+
+    version("1.0.1", sha256="927b399a6906be067e49c11ef1a87472f1b1964075c9eea30fb82c64b20aedee")
+
+    depends_on("python@3.7:", type=("build", "run"))
+
+    depends_on("py-sphinx-theme-builder@0.2.0a7:", type="build")
+
+    depends_on("py-sphinx@4:6", type=("build", "run"))
+    depends_on("py-pydata-sphinx-theme@0.13.3:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-sphinx-rtd-theme/package.py b/var/spack/repos/builtin/packages/py-sphinx-rtd-theme/package.py
index 219417d23ef676..0e3c468b046a4c 100644
--- a/var/spack/repos/builtin/packages/py-sphinx-rtd-theme/package.py
+++ b/var/spack/repos/builtin/packages/py-sphinx-rtd-theme/package.py
@@ -12,6 +12,7 @@ class PySphinxRtdTheme(PythonPackage):
     homepage = "https://github.com/readthedocs/sphinx_rtd_theme"
     pypi = "sphinx-rtd-theme/sphinx_rtd_theme-0.5.1.tar.gz"
 
+    version("1.2.2", sha256="01c5c5a72e2d025bd23d1f06c59a4831b06e6ce6c01fdd5ebfe9986c0a880fc7")
     version("1.2.0", sha256="a0d8bd1a2ed52e0b338cbe19c4b2eef3c5e7a048769753dac6a9f059c7b641b8")
     version("1.0.0", sha256="eec6d497e4c2195fa0e8b2016b337532b8a699a68bcb22a512870e16925c6a5c")
     version("0.5.2", sha256="32bd3b5d13dc8186d7a42fc816a23d32e83a4827d7d9882948e7b837c232da5a")
@@ -20,13 +21,14 @@ class PySphinxRtdTheme(PythonPackage):
     version("0.4.3", sha256="728607e34d60456d736cc7991fd236afb828b21b82f956c5ea75f94c8414040a")
 
     depends_on("py-setuptools", type="build")
-    depends_on("py-sphinx", when="@0.4.1:", type=("build", "run"))
+
     depends_on("py-sphinx@1.6:6", when="@1:", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.4:", when="@1:", type=("build", "run"))
-    depends_on("py-docutils@:0.16", when="@0.5.2:0", type=("build", "run"))
-    depends_on("py-docutils@:0.17", when="@1:1.1", type=("build", "run"))
+    depends_on("py-sphinx@:6", when="@0", type=("build", "run"))
     depends_on("py-docutils@:0.18", when="@1.2:", type=("build", "run"))
-    depends_on("py-sphinxcontrib-jquery@2:", when="@1.2:", type=("build", "run"))
+    depends_on("py-docutils@:0.17", when="@1:1.1", type=("build", "run"))
+    depends_on("py-docutils@:0.16", when="@0.5.2:0", type=("build", "run"))
+    depends_on("py-sphinxcontrib-jquery@4", when="@1.2.2:", type=("build", "run"))
+    depends_on("py-sphinxcontrib-jquery@2:", when="@1.2:1.2.1", type=("build", "run"))
     conflicts("^py-sphinxcontrib-jquery@3.0.0")
 
     def setup_build_environment(self, env):
diff --git a/var/spack/repos/builtin/packages/py-sphinx-theme-builder/package.py b/var/spack/repos/builtin/packages/py-sphinx-theme-builder/package.py
new file mode 100644
index 00000000000000..ce873cac3f2f16
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-sphinx-theme-builder/package.py
@@ -0,0 +1,26 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PySphinxThemeBuilder(PythonPackage):
+    """A tool for authoring Sphinx themes with a simple (opinionated) workflow."""
+
+    homepage = "https://sphinx-theme-builder.readthedocs.io/en/latest"
+    pypi = "sphinx-theme-builder/sphinx-theme-builder-0.2.0b2.tar.gz"
+    git = "https://github.com/pradyunsg/sphinx-theme-builder"
+
+    version("0.2.0b2", sha256="e9cd98c2bb35bf414fe721469a043cdcc10f0808d1ffcf606acb4a6282a6f288")
+
+    depends_on("py-flit-core@3.2:3", type="build")
+
+    depends_on("py-pyproject-metadata", type=("build", "run"))
+    depends_on("py-packaging", type=("build", "run"))
+    depends_on("py-rich", type=("build", "run"))
+    depends_on("py-nodeenv", type=("build", "run"))
+    depends_on("py-setuptools", type=("build", "run"))
+    depends_on("py-tomli", when="^python@:3.10", type=("build", "run"))
+    depends_on("py-typing-extensions", when="^python@:3.7", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-sphinx/package.py b/var/spack/repos/builtin/packages/py-sphinx/package.py
index ae845d6171fbb7..9b0e1eb323070a 100644
--- a/var/spack/repos/builtin/packages/py-sphinx/package.py
+++ b/var/spack/repos/builtin/packages/py-sphinx/package.py
@@ -10,10 +10,17 @@ class PySphinx(PythonPackage):
     """Sphinx Documentation Generator."""
 
     homepage = "https://www.sphinx-doc.org/en/master/"
-    pypi = "Sphinx/Sphinx-3.2.0.tar.gz"
+    pypi = "Sphinx/sphinx-7.1.0.tar.gz"
 
     maintainers("adamjstewart")
 
+    version("7.2.3", sha256="ece68bb4d77b7dc090573825db45a6f9183e74098d1c21573485de250b1d1e3f")
+    version("7.2.2", sha256="1c0abe6d4de7a6b2c2b109a2e18387bf27b240742e1b34ea42ac3ed2ac99978c")
+    version("7.2.1", sha256="dad5e865dcdeb1486f70d8963cc9140561836bb243c311868cf11eb0f741497a")
+    version("7.2.0", sha256="da9a84f7456885622bb30ebac42467168396ac2e494182c60dd864aa27405ba3")
+    version("7.1.2", sha256="780f4d32f1d7d1126576e0e5ecc19dc32ab76cd24e950228dcf7b1f6d3d9e22f")
+    version("7.1.1", sha256="59b8e391f0768a96cd233e8300fe7f0a8dc2f64f83dc2a54336a9a84f428ff4e")
+    version("7.1.0", sha256="8f336d0221c3beb23006b3164ed1d46db9cebcce9cb41cdb9c5ecd4bcc509be0")
     version("7.0.1", sha256="61e025f788c5977d9412587e733733a289e2b9fdc2fef8868ddfbfc4ccfe881d")
     version("7.0.0", sha256="283c44aa28922bb4223777b44ac0d59af50a279ac7690dfe945bb2b9575dc41b")
 
@@ -70,9 +77,8 @@ class PySphinx(PythonPackage):
     version("1.3.1", sha256="1a6e5130c2b42d2de301693c299f78cc4bd3501e78b610c08e45efc70e2b5114")
 
     depends_on("py-flit-core@3.7:", when="@5.2:", type="build")
-    depends_on("py-setuptools", when="@4.4:5.1", type="build")
-    depends_on("py-setuptools", when="@:4.3", type=("build", "run"))
 
+    depends_on("python@3.9:", when="@7.2:", type=("build", "run"))
     depends_on("python@3.8:", when="@6:", type=("build", "run"))
 
     depends_on("py-sphinxcontrib-applehelp", when="@2:", type=("build", "run"))
@@ -80,12 +86,14 @@ class PySphinx(PythonPackage):
     depends_on("py-sphinxcontrib-jsmath", when="@2:", type=("build", "run"))
     depends_on("py-sphinxcontrib-htmlhelp@2:", when="@4.1.1:", type=("build", "run"))
     depends_on("py-sphinxcontrib-htmlhelp", when="@2:", type=("build", "run"))
+    depends_on("py-sphinxcontrib-serializinghtml@1.1.9:", when="@7.2.3:", type=("build", "run"))
     depends_on("py-sphinxcontrib-serializinghtml@1.1.5:", when="@4.1.1:", type=("build", "run"))
     depends_on("py-sphinxcontrib-serializinghtml", when="@2:", type=("build", "run"))
     depends_on("py-sphinxcontrib-qthelp", when="@2:", type=("build", "run"))
     depends_on("py-jinja2@3:", when="@5.2:", type=("build", "run"))
     depends_on("py-jinja2@2.3:2", when="@:4.0.1", type=("build", "run"))
     depends_on("py-jinja2@2.3:", type=("build", "run"))
+    depends_on("py-pygments@2.14:", when="@7.2:", type=("build", "run"))
     depends_on("py-pygments@2.13:", when="@6.0.1:", type=("build", "run"))
     depends_on("py-pygments@2.12:", when="@5.2:", type=("build", "run"))
     depends_on("py-pygments@2:", type=("build", "run"))
@@ -113,6 +121,17 @@ class PySphinx(PythonPackage):
     depends_on("py-colorama@0.4.5:", when="@5.2: platform=windows", type=("build", "run"))
     depends_on("py-colorama@0.3.5:", when="platform=windows", type=("build", "run"))
 
+    # Historical dependencies
+    depends_on("py-setuptools", when="@4.4:5.1", type="build")
+    depends_on("py-setuptools", when="@:4.3", type=("build", "run"))
     depends_on("py-sphinxcontrib-websupport", when="@1.6:1", type=("build", "run"))
     depends_on("py-six@1.5:", when="@:1", type=("build", "run"))
     depends_on("py-sphinx-rtd-theme@0.1:", when="@:1.3", type=("build", "run"))
+
+    def url_for_version(self, version):
+        url = "https://files.pythonhosted.org/packages/source/S/Sphinx/{}-{}.tar.gz"
+        if version >= Version("7.1"):
+            name = "sphinx"
+        else:
+            name = "Sphinx"
+        return url.format(name, version)
diff --git a/var/spack/repos/builtin/packages/py-sphinxcontrib-applehelp/package.py b/var/spack/repos/builtin/packages/py-sphinxcontrib-applehelp/package.py
index 821015d2e451bd..135ad47b86ee55 100644
--- a/var/spack/repos/builtin/packages/py-sphinxcontrib-applehelp/package.py
+++ b/var/spack/repos/builtin/packages/py-sphinxcontrib-applehelp/package.py
@@ -20,8 +20,10 @@ class PySphinxcontribApplehelp(PythonPackage):
     # import any modules for this package.
     import_modules: List[str] = []
 
+    version("1.0.4", sha256="828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e")
     version("1.0.2", sha256="a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58")
     version("1.0.1", sha256="edaa0ab2b2bc74403149cb0209d6775c96de797dfd5b5e2a71981309efab3897")
 
-    depends_on("python@3.5:", type=("build", "run"))
+    depends_on("python@3.8:", when="@1.0.3:", type=("build", "run"))
+    depends_on("py-setuptools@64:", when="@1.0.4:", type="build")
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-sphinxcontrib-htmlhelp/package.py b/var/spack/repos/builtin/packages/py-sphinxcontrib-htmlhelp/package.py
index 51e32170ef4809..9dd8b950813881 100644
--- a/var/spack/repos/builtin/packages/py-sphinxcontrib-htmlhelp/package.py
+++ b/var/spack/repos/builtin/packages/py-sphinxcontrib-htmlhelp/package.py
@@ -13,15 +13,17 @@ class PySphinxcontribHtmlhelp(PythonPackage):
 
     homepage = "http://sphinx-doc.org/"
     pypi = "sphinxcontrib-htmlhelp/sphinxcontrib-htmlhelp-1.0.2.tar.gz"
+    git = "https://github.com/sphinx-doc/sphinxcontrib-htmlhelp.git"
 
     # 'sphinx' requires 'sphinxcontrib-htmlhelp' at build-time, but
     # 'sphinxcontrib-htmlhelp' requires 'sphinx' at run-time. Don't bother trying to
     # import any modules.
     import_modules: List[str] = []
 
+    version("2.0.1", sha256="0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff")
     version("2.0.0", sha256="f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2")
     version("1.0.2", sha256="4670f99f8951bd78cd4ad2ab962f798f5618b17675c35c5ac3b2132a14ea8422")
 
-    depends_on("python@3.6:", when="@2:", type=("build", "run"))
-    depends_on("python@3.5:", type=("build", "run"))
+    depends_on("python@3.8:", when="@2.0.1:", type=("build", "run"))
+    depends_on("py-setuptools@64:", when="@2.0.1", type="build")
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-sphinxcontrib-jquery/package.py b/var/spack/repos/builtin/packages/py-sphinxcontrib-jquery/package.py
index bb5637a3dbd702..6a11a319989596 100644
--- a/var/spack/repos/builtin/packages/py-sphinxcontrib-jquery/package.py
+++ b/var/spack/repos/builtin/packages/py-sphinxcontrib-jquery/package.py
@@ -12,8 +12,11 @@ class PySphinxcontribJquery(PythonPackage):
     homepage = "https://github.com/sphinx-contrib/jquery"
     pypi = "sphinxcontrib-jquery/sphinxcontrib-jquery-2.0.0.tar.gz"
 
+    version("4.1", sha256="1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a")
     version("2.0.0", sha256="8fb65f6dba84bf7bcd1aea1f02ab3955ac34611d838bcc95d4983b805b234daa")
 
-    depends_on("python@3.5:", when="@2:", type=("build", "run"))
-    depends_on("py-setuptools@65:", type=("build", "run"))
     depends_on("py-flit-core@3.7:", when="@3:", type="build")
+    depends_on("py-sphinx@1.8:", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-setuptools@65:", when="@:3", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-sphinxcontrib-serializinghtml/package.py b/var/spack/repos/builtin/packages/py-sphinxcontrib-serializinghtml/package.py
index d85fba1a1ca10b..520470fb1dac78 100644
--- a/var/spack/repos/builtin/packages/py-sphinxcontrib-serializinghtml/package.py
+++ b/var/spack/repos/builtin/packages/py-sphinxcontrib-serializinghtml/package.py
@@ -12,15 +12,30 @@ class PySphinxcontribSerializinghtml(PythonPackage):
     "serialized" HTML files (json and pickle)."""
 
     homepage = "http://sphinx-doc.org/"
-    pypi = "sphinxcontrib-serializinghtml/sphinxcontrib-serializinghtml-1.1.3.tar.gz"
+    pypi = "sphinxcontrib-serializinghtml/sphinxcontrib_serializinghtml-1.1.3.tar.gz"
 
     # 'sphinx' requires 'sphinxcontrib-serializinghtml' at build-time, but
     # 'sphinxcontrib-serializinghtml' requires 'sphinx' at run-time. Don't bother trying
     # to import any modules.
     import_modules: List[str] = []
 
+    version("1.1.9", sha256="0c64ff898339e1fac29abd2bf5f11078f3ec413cfe9c046d3120d7ca65530b54")
     version("1.1.5", sha256="aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952")
     version("1.1.3", sha256="c0efb33f8052c04fd7a26c0a07f1678e8512e0faec19f4aa8f2473a8b81d5227")
 
-    depends_on("python@3.5:", type=("build", "run"))
-    depends_on("py-setuptools", type="build")
+    depends_on("python@3.9:", when="@1.1.6:", type=("build", "run"))
+    depends_on("py-flit-core@3.7:", when="@1.1.6:", type="build")
+
+    # Circular dependency
+    # depends_on("py-sphinx@5:", when="@1.1.6:", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-setuptools", when="@:1.1.5", type="build")
+
+    def url_for_version(self, version):
+        url = "https://files.pythonhosted.org/packages/source/s/sphinxcontrib-serializinghtml/sphinxcontrib{}serializinghtml-{}.tar.gz"
+        if version >= Version("1.1.6"):
+            separator = "_"
+        else:
+            separator = "-"
+        return url.format(separator, version)
diff --git a/var/spack/repos/builtin/packages/py-spython/package.py b/var/spack/repos/builtin/packages/py-spython/package.py
index d3c49ac9adb32f..41c5375563edf8 100644
--- a/var/spack/repos/builtin/packages/py-spython/package.py
+++ b/var/spack/repos/builtin/packages/py-spython/package.py
@@ -13,6 +13,7 @@ class PySpython(PythonPackage):
     homepage = "https://github.com/singularityhub/singularity-cli"
     pypi = "spython/spython-0.2.14.tar.gz"
 
+    version("0.3.1", sha256="143557849d636697ddd80e0ba95920efe4668351f5becce6bdc73a7651aa128d")
     version("0.2.14", sha256="49e22fbbdebe456b27ca17d30061489db8e0f95e62be3623267a23b85e3ce0f0")
 
     variant(
@@ -27,5 +28,4 @@ class PySpython(PythonPackage):
     depends_on("singularity@3.5.2:", when="runtime=singularity", type="run")
 
     depends_on("py-setuptools", type="build")
-
-    depends_on("py-semver@2.8.1:", type=("build", "run"))
+    depends_on("py-semver@2.8.1:", when="@:0.2", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-sqlalchemy/package.py b/var/spack/repos/builtin/packages/py-sqlalchemy/package.py
index 1480536e263b4e..de5d823e6d3755 100644
--- a/var/spack/repos/builtin/packages/py-sqlalchemy/package.py
+++ b/var/spack/repos/builtin/packages/py-sqlalchemy/package.py
@@ -13,6 +13,7 @@ class PySqlalchemy(PythonPackage):
     pypi = "SQLAlchemy/SQLAlchemy-1.3.9.tar.gz"
     git = "https://github.com/sqlalchemy/sqlalchemy.git"
 
+    version("2.0.19", sha256="77a14fa20264af73ddcdb1e2b9c5a829b8cc6b8304d0f093271980e36c200a3f")
     version("1.4.45", sha256="fd69850860093a3f69fefe0ab56d041edfdfe18510b53d9a2eaecba2f15fa795")
     version("1.4.44", sha256="2dda5f96719ae89b3ec0f1b79698d86eb9aecb1d54e990abb3fdd92c04b46a90")
     version("1.4.25", sha256="1adf3d25e2e33afbcd48cfad8076f9378793be43e7fec3e4334306cac6bec138")
@@ -24,22 +25,25 @@ class PySqlalchemy(PythonPackage):
     version("1.1.18", sha256="8b0ec71af9291191ba83a91c03d157b19ab3e7119e27da97932a4773a3f664a9")
     version("1.0.12", sha256="6679e20eae780b67ba136a4a76f83bb264debaac2542beefe02069d0206518d1")
 
-    depends_on("python@2.7:2.8,3.6:", when="@1.4:", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.4:", when="@:1.3", type=("build", "run"))
-    depends_on("py-setuptools", type="build")
-    depends_on("py-importlib-metadata", when="@1.4.0: ^python@:3.7", type="run")
-    depends_on("py-greenlet@:0.4.16,0.4.18:", when="@1.4.0:", type="run")
-
     variant(
         "backend",
         description="Python modules for database access",
-        values=any_combination_of("mysql", "pymysql", "postgresql"),
+        values=any_combination_of("mysql", "postgresql", "pymysql"),
     )
 
+    depends_on("py-setuptools@47:", when="@2:", type="build")
+    depends_on("py-setuptools", type="build")
+    depends_on("py-cython@0.29.24:", when="@2:", type="build")
+
+    depends_on("py-importlib-metadata", when="@1.4.0: ^python@:3.7", type=("build", "run"))
+    depends_on("py-greenlet", when="@1.4.0:", type=("build", "run"))
+    conflicts("^py-greenlet@0.4.17", when="@1.4.0:")
+    depends_on("py-typing-extensions@4.2.0", when="@2:", type=("build", "run"))
+
     # >=1.4.0
     depends_on("py-mysqlclient@1.4:", when="backend=mysql @1.4:", type=("build", "run"))
-    depends_on("py-pymysql", when="backend=pymysql @1.4:", type=("build", "run"))
     depends_on("py-psycopg2@2.7:", when="backend=postgresql @1.4:", type=("build", "run"))
+    depends_on("py-pymysql", when="backend=pymysql @1.4:", type=("build", "run"))
 
     # < 1.4.0
     depends_on("py-mysqlclient", when="backend=mysql @:1.3", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-srsly/package.py b/var/spack/repos/builtin/packages/py-srsly/package.py
index 2599893d5bec3f..63385405189c94 100644
--- a/var/spack/repos/builtin/packages/py-srsly/package.py
+++ b/var/spack/repos/builtin/packages/py-srsly/package.py
@@ -13,6 +13,7 @@ class PySrsly(PythonPackage):
     homepage = "https://github.com/explosion/srsly"
     pypi = "srsly/srsly-2.0.1.tar.gz"
 
+    version("2.4.6", sha256="47b41f323aba4c9c3311abf60e443c03a9efe9c69f65dc402d173c32f7744a6f")
     version("2.0.1", sha256="fa3c7375be8fe75f23c27feafbfb5f738d55ffdbf02964c6896fb7684f519a52")
     version("2.0.0", sha256="785b00e00406120dbef4ca82925051e6b60fe870c5f84f0d22b3632d574eb870")
     version("1.0.2", sha256="59258b81d567df207f8a0a33c4b5fa232afccf1d927c8ce3ba5395bfd64c0ed8")
@@ -21,6 +22,10 @@ class PySrsly(PythonPackage):
     depends_on("py-setuptools", type="build")
     depends_on("py-wheel", when="@2:", type="build")
     depends_on("py-cython@0.25:", when="@2:", type="build")
+    depends_on("py-catalogue@2.0.3:2.0", when="@2.4.6:", type=("build", "run"))
 
     # https://github.com/explosion/srsly/pull/24
     patch("subprocess.patch", when="@2.0.0:2.0.1")
+
+    # https://github.com/explosion/srsly/pull/97
+    conflicts("^py-cython@3:", when="@:2.4.6")
diff --git a/var/spack/repos/builtin/packages/py-stack-data/package.py b/var/spack/repos/builtin/packages/py-stack-data/package.py
index a1e9eb01074c1e..82be5afd92028e 100644
--- a/var/spack/repos/builtin/packages/py-stack-data/package.py
+++ b/var/spack/repos/builtin/packages/py-stack-data/package.py
@@ -13,11 +13,15 @@ class PyStackData(PythonPackage):
     homepage = "http://github.com/alexmojaki/stack_data"
     pypi = "stack_data/stack_data-0.2.0.tar.gz"
 
+    version("0.6.2", sha256="32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815")
     version("0.5.0", sha256="715c8855fbf5c43587b141e46cc9d9339cc0d1f8d6e0f98ed0d01c6cb974e29f")
     version("0.2.0", sha256="45692d41bd633a9503a5195552df22b583caf16f0b27c4e58c98d88c8b648e12")
 
     depends_on("py-setuptools@44:", type="build")
     depends_on("py-setuptools-scm+toml@3.4.3:", type="build")
+
+    depends_on("py-executing@1.2:", when="@0.6:", type=("build", "run"))
     depends_on("py-executing", type=("build", "run"))
+    depends_on("py-asttokens@2.1:", when="@0.6:", type=("build", "run"))
     depends_on("py-asttokens", type=("build", "run"))
     depends_on("py-pure-eval", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-statsmodels/package.py b/var/spack/repos/builtin/packages/py-statsmodels/package.py
index 10cc6c39cc923a..2fe227de4532cf 100644
--- a/var/spack/repos/builtin/packages/py-statsmodels/package.py
+++ b/var/spack/repos/builtin/packages/py-statsmodels/package.py
@@ -3,6 +3,9 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import glob
+import os
+
 from spack.package import *
 
 
@@ -11,7 +14,10 @@ class PyStatsmodels(PythonPackage):
 
     homepage = "https://www.statsmodels.org"
     pypi = "statsmodels/statsmodels-0.8.0.tar.gz"
+    git = "https://github.com/statsmodels/statsmodels.git"
 
+    version("0.14.0", sha256="6875c7d689e966d948f15eb816ab5616f4928706b180cf470fd5907ab6f647a4")
+    version("0.13.5", sha256="593526acae1c0fda0ea6c48439f67c3943094c542fe769f8b90fe9e6c6cc4871")
     version("0.13.2", sha256="77dc292c9939c036a476f1770f9d08976b05437daa229928da73231147cde7d4")
     version("0.13.1", sha256="006ec8d896d238873af8178d5475203844f2c391194ed8d42ddac37f5ff77a69")
     version("0.13.0", sha256="f2efc02011b7240a9e851acd76ab81150a07d35c97021cb0517887539a328f8a")
@@ -20,36 +26,52 @@ class PyStatsmodels(PythonPackage):
     version("0.10.2", sha256="9cd2194c6642a8754e85f9a6e6912cdf996bebf6ff715d3cc67f65dadfd37cc9")
     version("0.10.1", sha256="320659a80f916c2edf9dfbe83512d9004bb562b72eedb7d9374562038697fa10")
 
-    depends_on("python@2.7:2.8,3.4:", when="@0.10.1:", type=("build", "link", "run"))
-    depends_on("python@3.6:", when="@0.12.1:", type=("build", "link", "run"))
+    depends_on("python@3.8:", when="@0.14:", type=("build", "link", "run"))
+    depends_on("python", type=("build", "link", "run"))
 
-    # according to https://www.statsmodels.org/dev/install.html earlier versions
-    # might work.
+    depends_on("py-setuptools@59.2:", when="@0.13.3:", type="build")
     depends_on("py-setuptools@0.6c5:", type="build")
-    depends_on("py-cython@0.29:", type="build")
-    depends_on("py-cython@0.29.14:", type="build", when="@0.12.0:")
-    depends_on("py-cython@0.29.22:", type="build", when="@0.13.0:")
+
+    # pyproject.toml
+    depends_on("py-cython@0.29.26:2", when="@0.14:", type="build")
+    depends_on("py-cython@0.29.32:2", when="@0.13.5:0.13", type="build")
+    depends_on("py-cython@0.29.22:2", when="@0.13:", type="build")
+    depends_on("py-cython@0.29.14:2", when="@0.12:", type="build")
+    depends_on("py-cython@0.29:2", type="build")
+    depends_on("py-setuptools-scm+toml@7.0", when="@0.13.3:", type="build")
 
     # patsy@0.5.1 works around a Python change
     #    https://github.com/statsmodels/statsmodels/issues/5343 and
     #    https://github.com/pydata/patsy/pull/131
 
-    depends_on("py-numpy@1.11.0:", type=("build", "link", "run"), when="@0.10.1:")
-    depends_on("py-numpy@1.15.0:", type=("build", "link", "run"), when="@0.12.1:")
-    depends_on("py-numpy@1.17.0:", type=("build", "link", "run"), when="@0.13.0:")
-    depends_on("py-pandas@0.19:", type=("build", "run"), when="@0.10.1:")
-    depends_on("py-pandas@0.23:", type=("build", "run"), when="@0.12.0:")
-    depends_on("py-pandas@0.25:", type=("build", "run"), when="@0.13.0:")
-    depends_on("py-patsy@0.4.0:", type=("build", "run"), when="@0.10.1:")
-    depends_on("py-patsy@0.5.1:", type=("build", "run"), when="@0.12.0:")
-    depends_on("py-patsy@0.5.2:", type=("build", "run"), when="@0.13.0:")
-    depends_on("py-scipy@0.18:", type=("build", "run"), when="@0.10.1:")
-    depends_on("py-scipy@1.2:", type=("build", "run"), when="@0.12.0:")
-    depends_on("py-scipy@1.3:", type=("build", "run"), when="@0.13.0:")
-    depends_on("py-packaging@21.3:", type=("build", "run"), when="@0.13.2:")
+    # requirements.txt
+    depends_on("py-numpy@1.18:", when="@0.14:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.17:", when="@0.13:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.15:", when="@0.12.1:", type=("build", "link", "run"))
+    depends_on("py-numpy@1.11:", when="@0.10.1:", type=("build", "link", "run"))
+    depends_on("py-scipy@1.4:", when="@0.13.5:", type=("build", "run"))
+    conflicts("^py-scipy@1.9.2")
+    depends_on("py-scipy@1.3:", when="@0.13:", type=("build", "run"))
+    depends_on("py-scipy@1.2:", when="@0.12:", type=("build", "run"))
+    depends_on("py-scipy@0.18:", when="@0.10.1:", type=("build", "run"))
+    depends_on("py-pandas@1:", when="@0.14:", type=("build", "run"))
+    depends_on("py-pandas@0.25:", when="@0.13:", type=("build", "run"))
+    depends_on("py-pandas@0.23:", when="@0.12:", type=("build", "run"))
+    depends_on("py-pandas@0.19:", when="@0.10.1:", type=("build", "run"))
+    depends_on("py-patsy@0.5.2:", when="@0.13:", type=("build", "run"))
+    depends_on("py-patsy@0.5.1:", when="@0.12:", type=("build", "run"))
+    depends_on("py-patsy@0.4:", when="@0.10.1:", type=("build", "run"))
+    depends_on("py-packaging@21.3:", when="@0.13.2:", type=("build", "run"))
 
     depends_on("py-pytest", type="test")
 
+    @run_before("install")
+    def remove_generated_sources(self):
+        # Automatic recythonization doesn't work here, because cythonize is called
+        # with force=False by default, so remove generated C files manually.
+        for f in find(".", "*.c"):
+            os.unlink(f)
+
     @run_after("install")
     @on_package_attributes(run_tests=True)
     def build_test(self):
diff --git a/var/spack/repos/builtin/packages/py-tables/package.py b/var/spack/repos/builtin/packages/py-tables/package.py
index d6b7e9c61edc18..84cd48bc5843d3 100644
--- a/var/spack/repos/builtin/packages/py-tables/package.py
+++ b/var/spack/repos/builtin/packages/py-tables/package.py
@@ -12,7 +12,11 @@ class PyTables(PythonPackage):
 
     homepage = "https://www.pytables.org/"
     pypi = "tables/tables-3.6.1.tar.gz"
+    git = "https://github.com/PyTables/PyTables.git"
 
+    version("master", branch="master")
+    version("3.9.0", sha256="27c9ca14c359d875caf945a6a527c12690e017650402dd17d8eb8b6caf6687d5")
+    version("3.8.0", sha256="34f3fa2366ce20b18f1df573a77c1d27306ce1f2a41d9f9eff621b5192ea8788")
     version("3.7.0", sha256="e92a887ad6f2a983e564a69902de4a7645c30069fc01abd353ec5da255c5e1fe")
     version("3.6.1", sha256="49a972b8a7c27a8a173aeb05f67acb45fe608b64cd8e9fa667c0962a60b71b49")
     version("3.6.0", sha256="db3488214864fb313a611fca68bf1c9019afe4e7877be54d0e61c84416603d4d")
@@ -25,30 +29,44 @@ class PyTables(PythonPackage):
     variant("bzip2", default=False, description="Support for bzip2 compression")
     variant("lzo", default=False, description="Support for lzo compression")
 
-    # requirements.txt
-    depends_on("python@3.6:", when="@3.7.0:", type=("build", "run"))
-    depends_on("python@3.5:", when="@3.6.1:", type=("build", "run"))
-    depends_on("python@3.4:", when="@3.6.0:", type=("build", "run"))
-    depends_on("python@2.6:", type=("build", "run"))
-
-    depends_on("py-setuptools", type=("build", "run"))
-    depends_on("py-setuptools@42.0:", when="@3.7.0:", type=("build", "run"))
+    # pyproject.toml
+    depends_on("py-setuptools@61:", when="@3.9:", type="build")
+    depends_on("py-setuptools@42:", when="@3.7:", type="build")
+    depends_on("py-setuptools", type="build")
+    depends_on("py-cython@0.29.32:", when="@3.9:", type="build")
+    depends_on("py-cython@0.29.21:", when="@3.7:3.8", type=("build", "run"))
     depends_on("py-cython@0.21:", type="build")
-    depends_on("py-cython@0.29.21:", when="@3.7.0:", type="build")
+
+    # setup.py
+    depends_on("python@3.8:", when="@3.8:", type=("build", "run"))
+
+    # requirements.txt
+    depends_on("py-numpy@1.19:", when="@3.8:", type=("build", "run"))
     depends_on("py-numpy@1.9.3:", type=("build", "run"))
     depends_on("py-numexpr@2.6.2:", type=("build", "run"))
-    depends_on("py-six@1.9.0:", when="@:3.5", type=("build", "run"))
-    depends_on("py-packaging", when="@3.7.0:", type=("build", "run"))
+    depends_on("py-packaging", when="@3.7:", type=("build", "run"))
+    depends_on("py-py-cpuinfo", when="@3.8:", type=("build", "run"))
+    depends_on("py-blosc2@2.2.8:", when="@3.9:", type=("build", "run"))
+    depends_on("py-blosc2@2.0", when="@3.8", type=("build", "run"))
+
     # tables/req_versions.py
+    depends_on("hdf5@1.10.5:", when="@3.8:")
+    depends_on("hdf5@1.8.4:", when="@3.4:")
     depends_on("hdf5@1.8.4:1.8", when="@:3.3")
-    depends_on("hdf5@1.8.4:", when="@3.4.0:")
     # Versions prior to 3.3 must build with the internal blosc due to a lock
     # problem in a multithreaded environment.
-    depends_on("c-blosc@1.4.1:", when="@3.3.0:")
-    depends_on("zlib", when="+zlib")
+    depends_on("c-blosc@1.11.1:", when="@3.8:")
+    depends_on("c-blosc@1.4.1:", when="@3.3:")
+
+    depends_on("zlib-api", when="+zlib")
     depends_on("bzip2", when="+bzip2")
     depends_on("lzo", when="+lzo")
 
+    conflicts("%apple-clang@15:", when="@:3.8")
+
+    # Historical dependencies
+    depends_on("py-six@1.9:", when="@:3.5", type=("build", "run"))
+
     def setup_build_environment(self, env):
         env.set("HDF5_DIR", self.spec["hdf5"].prefix)
         if "+bzip2" in self.spec:
diff --git a/var/spack/repos/builtin/packages/py-tensorboard/package.py b/var/spack/repos/builtin/packages/py-tensorboard/package.py
index ca98f8ced27014..f1a9b03bc16ee9 100644
--- a/var/spack/repos/builtin/packages/py-tensorboard/package.py
+++ b/var/spack/repos/builtin/packages/py-tensorboard/package.py
@@ -7,9 +7,8 @@
 
 
 class PyTensorboard(PythonPackage):
-    """TensorBoard is a suite of web applications for
-    inspecting and understanding your TensorFlow runs and
-    graphs."""
+    """TensorBoard is a suite of web applications for inspecting and understanding
+    your TensorFlow runs and graphs."""
 
     homepage = "https://github.com/tensorflow/tensorboard"
     url = "https://files.pythonhosted.org/packages/py3/t/tensorboard/tensorboard-2.9.1-py3-none-any.whl"
@@ -17,6 +16,36 @@ class PyTensorboard(PythonPackage):
 
     maintainers("aweits")
 
+    version(
+        "2.14.1",
+        sha256="3db108fb58f023b6439880e177743c5f1e703e9eeb5fb7d597871f949f85fd58",
+        expand=False,
+    )
+    version(
+        "2.14.0",
+        sha256="3667f9745d99280836ad673022362c840f60ed8fefd5a3e30bf071f5a8fd0017",
+        expand=False,
+    )
+    version(
+        "2.13.0",
+        sha256="ab69961ebddbddc83f5fa2ff9233572bdad5b883778c35e4fe94bf1798bd8481",
+        expand=False,
+    )
+    version(
+        "2.12.3",
+        sha256="b4a69366784bc347e02fbe7d847e01896a649ca52f8948a11005e205dcf724fb",
+        expand=False,
+    )
+    version(
+        "2.12.2",
+        sha256="811ab0d27a139445836db9fd4f974424602c3dce12379364d379bcba7c783a68",
+        expand=False,
+    )
+    version(
+        "2.12.1",
+        sha256="58f1c2a25b4829b9c48d2b1ec951dedc9325dcd1ea4b0f601d241d2887d0ed65",
+        expand=False,
+    )
     version(
         "2.12.0",
         sha256="3cbdc32448d7a28dc1bf0b1754760c08b8e0e2e37c451027ebd5ff4896613012",
@@ -98,6 +127,7 @@ class PyTensorboard(PythonPackage):
         expand=False,
     )
 
+    depends_on("python@3.9:", type=("build", "run"), when="@2.14:")
     depends_on("python@3.8:", type=("build", "run"), when="@2.12:")
     depends_on("py-absl-py@0.4:", type=("build", "run"))
     depends_on("py-grpcio@1.48.2:", type=("build", "run"), when="@2.12:")
@@ -105,7 +135,8 @@ class PyTensorboard(PythonPackage):
     depends_on("py-grpcio@1.23.3:", type=("build", "run"), when="@2.2")
     depends_on("py-google-auth@1.6.3:2", type=("build", "run"), when="@2.7:")
     depends_on("py-google-auth@1.6.3:1", type=("build", "run"), when="@:2.6")
-    depends_on("py-google-auth-oauthlib@0.4.1:0.4", type=("build", "run"))
+    depends_on("py-google-auth-oauthlib@0.5:1.0", type=("build", "run"), when="@2.13:")
+    depends_on("py-google-auth-oauthlib@0.4.1:0.4", type=("build", "run"), when="@:2.12")
     depends_on("py-markdown@2.6.8:", type=("build", "run"))
     depends_on("py-numpy@1.12.0:", type=("build", "run"))
     depends_on("py-protobuf@3.19.6:", type=("build", "run"), when="@2.12:")
@@ -114,10 +145,10 @@ class PyTensorboard(PythonPackage):
     depends_on("py-protobuf@3.6.0:3.19", type=("build", "run"), when="@:2.8")
     depends_on("py-requests@2.21.0:2", type=("build", "run"))
     depends_on("py-setuptools@41.0.0:", type=("build", "run"))
+    depends_on("py-six@1.10.0:", type=("build", "run"), when="@:2.4,2.14:")
     depends_on("py-tensorboard-data-server@0.7", type=("build", "run"), when="@2.12:")
     depends_on("py-tensorboard-data-server@0.6", type=("build", "run"), when="@2.5:2.11")
-    depends_on("py-tensorboard-plugin-wit@1.6.0:", type=("build", "run"))
+    depends_on("py-tensorboard-plugin-wit@1.6.0:", type=("build", "run"), when="@:2.13")
     depends_on("py-werkzeug@1.0.1:", type=("build", "run"), when="@2.9:")
     depends_on("py-werkzeug@0.11.15:", type=("build", "run"))
-    depends_on("py-wheel@0.26:", type="build")
-    depends_on("py-six@1.10.0:", type=("build", "run"), when="@:2.4")
+    depends_on("py-wheel@0.26:", type="build", when="@:2.13")
diff --git a/var/spack/repos/builtin/packages/py-tensorboardx/package.py b/var/spack/repos/builtin/packages/py-tensorboardx/package.py
index 5c901eedb468e0..3bcd5f17545fee 100644
--- a/var/spack/repos/builtin/packages/py-tensorboardx/package.py
+++ b/var/spack/repos/builtin/packages/py-tensorboardx/package.py
@@ -17,6 +17,7 @@ class PyTensorboardx(PythonPackage):
     homepage = "https://github.com/lanpa/tensorboardX"
     pypi = "tensorboardx/tensorboardX-1.8.tar.gz"
 
+    version("2.6.2.2", sha256="c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666")
     version("2.5.1", sha256="ea85a3446f22ce8a917fe4fa4d8a7a96222ef84ac835267d038c34bb99f6d61b")
     version("2.1", sha256="9e8907cf2ab900542d6cb72bf91aa87b43005a7f0aa43126268697e3727872f9")
     version("2.0", sha256="835d85db0aef2c6768f07c35e69a74e3dcb122d6afceaf2b8504d7d16c7209a5")
@@ -24,7 +25,10 @@ class PyTensorboardx(PythonPackage):
     version("1.8", sha256="13fe0abba27f407778a7321937190eedaf12bc8c544d9a4e294fcf0ba177fd76")
 
     depends_on("py-setuptools", type="build")
+    depends_on("py-setuptools-scm", when="@2.6.2.2:", type="build")
     depends_on("py-numpy", type=("build", "run"))
-    depends_on("py-protobuf@3.8.0:3.20.1", type=("build", "run"))
+    depends_on("py-protobuf@3.8.0:3.20.1", when="@:2.5.1", type=("build", "run"))
+    depends_on("py-protobuf@3.20:", when="@2.6.2.2:", type=("build", "run"))
+    depends_on("py-packaging", when="@2.6.2.2:", type=("build", "run"))
 
     depends_on("py-six", when="@:2.1", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-tensorflow-estimator/package.py b/var/spack/repos/builtin/packages/py-tensorflow-estimator/package.py
index 7a6b5967cfa7b0..f0bde513db7259 100644
--- a/var/spack/repos/builtin/packages/py-tensorflow-estimator/package.py
+++ b/var/spack/repos/builtin/packages/py-tensorflow-estimator/package.py
@@ -74,14 +74,7 @@ def install(self, spec, prefix):
             "--jobs={0}".format(make_jobs),
             # Enable verbose output for failures
             "--verbose_failures",
-            # Show (formatted) subcommands being executed
-            "--subcommands=pretty_print",
             "--spawn_strategy=local",
-            # Ask bazel to explain what it's up to
-            # Needs a filename as argument
-            "--explain=explainlogfile.txt",
-            # Increase verbosity of explanation,
-            "--verbose_explanations",
             # bazel uses system PYTHONPATH instead of spack paths
             "--action_env",
             "PYTHONPATH={0}".format(env["PYTHONPATH"]),
diff --git a/var/spack/repos/builtin/packages/py-tensorflow-hub/package.py b/var/spack/repos/builtin/packages/py-tensorflow-hub/package.py
index 1ea9b79f90a969..6c59391a54210b 100644
--- a/var/spack/repos/builtin/packages/py-tensorflow-hub/package.py
+++ b/var/spack/repos/builtin/packages/py-tensorflow-hub/package.py
@@ -49,14 +49,7 @@ def install(self, spec, prefix):
             "--jobs={0}".format(make_jobs),
             # Enable verbose output for failures
             "--verbose_failures",
-            # Show (formatted) subcommands being executed
-            "--subcommands=pretty_print",
             "--spawn_strategy=local",
-            # Ask bazel to explain what it's up to
-            # Needs a filename as argument
-            "--explain=explainlogfile.txt",
-            # Increase verbosity of explanation,
-            "--verbose_explanations",
             # bazel uses system PYTHONPATH instead of spack paths
             "--action_env",
             "PYTHONPATH={0}".format(env["PYTHONPATH"]),
diff --git a/var/spack/repos/builtin/packages/py-tensorflow-probability/package.py b/var/spack/repos/builtin/packages/py-tensorflow-probability/package.py
index 0ef321b1df9b39..553fe250b36767 100644
--- a/var/spack/repos/builtin/packages/py-tensorflow-probability/package.py
+++ b/var/spack/repos/builtin/packages/py-tensorflow-probability/package.py
@@ -69,14 +69,7 @@ def install(self, spec, prefix):
             "--jobs={0}".format(make_jobs),
             # Enable verbose output for failures
             "--verbose_failures",
-            # Show (formatted) subcommands being executed
-            "--subcommands=pretty_print",
             "--spawn_strategy=local",
-            # Ask bazel to explain what it's up to
-            # Needs a filename as argument
-            "--explain=explainlogfile.txt",
-            # Increase verbosity of explanation,
-            "--verbose_explanations",
             # bazel uses system PYTHONPATH instead of spack paths
             "--action_env",
             "PYTHONPATH={0}".format(env["PYTHONPATH"]),
diff --git a/var/spack/repos/builtin/packages/py-tensorflow/package.py b/var/spack/repos/builtin/packages/py-tensorflow/package.py
index 35f33eb9190d25..c85f078689fe7d 100644
--- a/var/spack/repos/builtin/packages/py-tensorflow/package.py
+++ b/var/spack/repos/builtin/packages/py-tensorflow/package.py
@@ -6,23 +6,11 @@
 import sys
 import tempfile
 
-from spack.operating_systems.mac_os import macos_version
 from spack.package import *
 
 
 class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
-    """An Open Source Machine Learning Framework for Everyone.
-
-    TensorFlow is an end-to-end open source platform for machine learning. It has a
-    comprehensive, flexible ecosystem of tools, libraries, and community resources that
-    lets researchers push the state-of-the-art in ML and developers easily build and
-    deploy ML-powered applications.
-
-    TensorFlow was originally developed by researchers and engineers working on the
-    Google Brain team within Google's Machine Intelligence Research organization to
-    conduct machine learning and deep neural networks research. The system is general
-    enough to be applicable in a wide variety of other domains, as well.
-    """
+    """TensorFlow is an open source machine learning framework for everyone."""
 
     homepage = "https://www.tensorflow.org"
     url = "https://github.com/tensorflow/tensorflow/archive/v2.3.1.tar.gz"
@@ -30,6 +18,10 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     maintainers("adamjstewart", "aweits")
     import_modules = ["tensorflow"]
 
+    version("2.14.0", sha256="ce357fd0728f0d1b0831d1653f475591662ec5bca736a94ff789e6b1944df19f")
+    version("2.13.1", sha256="89c07aebd4f41fbe0d08cc88aef00305542134f2f16d3b62918dc3c1182f33e2")
+    version("2.13.0", sha256="e58c939079588623e6fa1d054aec2f90f95018266e0a970fd353a5244f5173dc")
+    version("2.12.1", sha256="6bc4600cc0b88e9e40f1800096f5bddbbd3b6e5527a030dea631b87f2ae46b5b")
     version("2.12.0", sha256="c030cb1905bff1d2446615992aad8d8d85cbe90c4fb625cee458c63bf466bc8e")
     version("2.11.1", sha256="624ed1cc170cdcc19e8a15d8cdde989a9a1c6b0534c90b38a6b2f06fb2963e5f")
     version("2.11.0", sha256="99c732b92b1b37fc243a559e02f9aef5671771e272758aa4aec7f34dc92dac48")
@@ -156,16 +148,16 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     extends("python")
 
     # Python support based on wheel availability
-    depends_on("python@3.8:3.11", when="@2.12:", type=("build", "run"))
-    depends_on("python@3.7:3.10", when="@2.8:2.11", type=("build", "run"))
-    depends_on("python@3.7:3.9", when="@2.7", type=("build", "run"))
-    depends_on("python@3.6:3.9", when="@2.5:2.6", type=("build", "run"))
-    depends_on("python@3.6:3.8", when="@2.4", type=("build", "run"))
-    depends_on("python@3.5:3.8", when="@2.2:2.3", type=("build", "run"))
-    depends_on("python@2.7,3.5:3.7", when="@:2.1", type=("build", "run"))
+    depends_on("python@3.9:3.11", when="@2.14:", type=("build", "run"))
+    depends_on("python@3.8:3.11", when="@2.12:2.13", type=("build", "run"))
+    depends_on("python@:3.10", when="@2.8:2.11", type=("build", "run"))
+    depends_on("python@:3.9", when="@2.5:2.7", type=("build", "run"))
+    depends_on("python@:3.8", when="@2.2:2.4", type=("build", "run"))
+    depends_on("python@:3.7", when="@:2.1", type=("build", "run"))
 
     # See .bazelversion
-    depends_on("bazel@5.3.0", type="build", when="@2.11:")
+    depends_on("bazel@6.1.0", type="build", when="@2.14:")
+    depends_on("bazel@5.3.0", type="build", when="@2.11:2.13")
     depends_on("bazel@5.1.1", type="build", when="@2.10")
     # See _TF_MIN_BAZEL_VERSION and _TF_MAX_BAZEL_VERSION in configure.py
     depends_on("bazel@4.2.2:5.99.0", type="build", when="@2.9")
@@ -202,14 +194,15 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     depends_on("py-astunparse@1.6:", type=("build", "run"), when="@2.7:")
     depends_on("py-astunparse@1.6.3:1.6", type=("build", "run"), when="@2.4:2.6")
     depends_on("py-astunparse@1.6.3", type=("build", "run"), when="@2.2:2.3")
-    depends_on("py-flatbuffers@2:", type=("build", "run"), when="@2.10:")
+    depends_on("py-flatbuffers@23.5.26:", type=("build", "run"), when="@2.14:")
+    depends_on("py-flatbuffers@23.1.21:", type=("build", "run"), when="@2.13")
+    depends_on("py-flatbuffers@2:", type=("build", "run"), when="@2.10:2.12")
     depends_on("py-flatbuffers@1.12:1", type=("build", "run"), when="@2.9")
     depends_on("py-flatbuffers@1.12:", type=("build", "run"), when="@2.8")
     depends_on("py-flatbuffers@1.12:2", type=("build", "run"), when="@2.7")
     depends_on("py-flatbuffers@1.12", type=("build", "run"), when="@2.4:2.6")
-    # Unpin overly strict dependency requirements to ease concretization of TensorFlow
-    depends_on("py-gast@0.2.1:", type=("build", "run"), when="@2.9:")
-    # depends_on("py-gast@0.2.1:0.4.0", type=("build", "run"), when="@2.9:")
+    depends_on("py-gast@0.2.1:0.4,0.5.3:", type=("build", "run"), when="@2.14:")
+    depends_on("py-gast@0.2.1:0.4.0", type=("build", "run"), when="@2.9:2.13")
     depends_on("py-gast@0.2.1:", type=("build", "run"), when="@2.8")
     depends_on("py-gast@0.2.1:0.4", type=("build", "run"), when="@2.7")
     depends_on("py-gast@0.4.0", type=("build", "run"), when="@2.5:2.6")
@@ -230,10 +223,12 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     depends_on("py-h5py~mpi", type=("build", "run"), when="@1.15.5,2.0.4,2.1.3:~mpi")
     depends_on("hdf5+mpi", type="build", when="@1.15.5,2.0.4,2.1.3:+mpi")
     depends_on("hdf5~mpi", type="build", when="@1.15.5,2.0.4,2.1.3:~mpi")
-    depends_on("py-jax@0.3.15:", type=("build", "run"), when="@2.12:")
     depends_on("py-libclang@13:", type=("build", "run"), when="@2.9:")
     depends_on("py-libclang@9.0.1:", type=("build", "run"), when="@2.7:2.8")
-    depends_on("py-numpy@1.22:1.23", type=("build", "run"), when="@2.12:")
+    depends_on("py-ml-dtypes@0.2.0", type=("build", "run"), when="@2.14:")
+    depends_on("py-numpy@1.23.5:", type=("build", "run"), when="@2.14:")
+    depends_on("py-numpy@1.22:1.24.3", type=("build", "run"), when="@2.13:")
+    depends_on("py-numpy@1.22:1.23", type=("build", "run"), when="@2.12")
     depends_on("py-numpy@1.20:", type=("build", "run"), when="@2.8:2.11")
     depends_on("py-numpy@1.14.5:", type=("build", "run"), when="@2.7")
     depends_on("py-numpy@1.19.2:1.19", type=("build", "run"), when="@2.4:2.6")
@@ -261,44 +256,25 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     # https://github.com/protocolbuffers/protobuf/issues/10051
     # https://github.com/tensorflow/tensorflow/issues/56266
     depends_on("py-protobuf@:3.19", type=("build", "run"), when="@:2.11")
-    depends_on("py-protobuf+cpp", type=("build", "run"))
-    depends_on("protobuf@:3.21.9", when="@:2.12")
-    depends_on("protobuf@:3.19", when="@:2.11")
-    depends_on("protobuf@:3.17", when="@:2.11")
-    depends_on("protobuf@:3.12", when="@:2.4")
-    depends_on("protobuf", type=("build", "run"))
+    # Must be matching versions of py-protobuf and protobuf
+    conflicts("^py-protobuf~cpp")
     depends_on("py-setuptools", type=("build", "run"))
     depends_on("py-six@1.12:", type=("build", "run"), when="@2.1:2.3,2.7:")
     depends_on("py-six@1.15", type=("build", "run"), when="@2.4:2.6")
     depends_on("py-six@1.10:", type=("build", "run"), when="@:2.0")
     depends_on("py-termcolor@1.1:", type=("build", "run"), when="@1.6:2.3,2.7:")
     depends_on("py-termcolor@1.1", type=("build", "run"), when="@2.4:2.6")
-    depends_on("py-typing-extensions@3.6.6:", type=("build", "run"), when="@2.7:")
+    depends_on("py-typing-extensions@3.6.6:", type=("build", "run"), when="@2.7:2.12,2.14:")
+    depends_on("py-typing-extensions@3.6.6:4.5", type=("build", "run"), when="@2.13")
     depends_on("py-typing-extensions@3.7.4:3.7", type=("build", "run"), when="@2.4:2.6")
-    depends_on("py-wrapt@1.11:1.14", type=("build", "run"), when="@2.12:")
-    depends_on("py-wrapt@1.11:", type=("build", "run"), when="@2.7:2.11")
+    depends_on("py-wrapt@1.11:1.14", type=("build", "run"), when="@2.12,2.14:")
+    depends_on("py-wrapt@1.11:", type=("build", "run"), when="@2.7:2.11,2.13")
     depends_on("py-wrapt@1.12.1:1.12", type=("build", "run"), when="@2.4:2.6")
     depends_on("py-wrapt@1.11.1:", type=("build", "run"), when="@1.12.1,1.14:2.3")
-    # TODO: add package for this dependency
-    # depends_on('py-tensorflow-io-gcs-filesystem@0.23.1:',
-    #            type=('build', 'run'), when='@2.8:')
-    # depends_on('py-tensorflow-io-gcs-filesystem@0.21:',
-    #            type=('build', 'run'), when='@2.7')
-    with when("+rocm"):
-        depends_on("hip")
-        depends_on("rocrand")
-        depends_on("rocblas")
-        depends_on("rocfft")
-        depends_on("hipfft")
-        depends_on("rccl", when="+nccl")
-        depends_on("hipsparse")
-        depends_on("hipcub")
-        depends_on("rocsolver")
-        depends_on("rocprim")
-        depends_on("miopen-hip")
-        depends_on("llvm-amdgpu")
-        depends_on("hsa-rocr-dev")
-        depends_on("rocminfo")
+
+    # TODO: add packages for these dependencies
+    # depends_on('py-tensorflow-io-gcs-filesystem@0.23.1:', type=('build', 'run'), when='@2.8:')
+    # depends_on('py-tensorflow-io-gcs-filesystem@0.21:', type=('build', 'run'), when='@2.7')
 
     if sys.byteorder == "little":
         # Only builds correctly on little-endian machines
@@ -308,7 +284,7 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
         depends_on("py-grpcio@1.32", type=("build", "run"), when="@2.4")
         depends_on("py-grpcio@1.8.6:", type=("build", "run"), when="@1.6:2.3")
 
-    for minor_ver in range(5, 13):
+    for minor_ver in range(5, 15):
         depends_on(
             "py-tensorboard@2.{}".format(minor_ver),
             type=("build", "run"),
@@ -321,10 +297,13 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
 
     # Undocumented dependencies
     depends_on("py-requests", type=("build", "run"))
+    # https://github.com/tensorflow/tensorflow/issues/60179#issuecomment-1491238631
+    depends_on("coreutils", when="@2.13: platform=darwin", type="build")
 
     # No longer a dependency in latest versions
     depends_on("py-astor@0.6:", type=("build", "run"), when="@1.6:2.1")
     depends_on("py-backports-weakref@1.0rc1", type=("build", "run"), when="@1.2")
+    depends_on("py-jax@0.3.15:", type=("build", "run"), when="@2.12")
     depends_on("py-keras-applications@1.0.8:", type=("build", "run"), when="@1.15:2.1")
     depends_on("py-keras-applications@1.0.6:", type=("build", "run"), when="@1.12:1.14")
     depends_on("py-keras-applications@1.0.5:", type=("build", "run"), when="@1.11")
@@ -357,6 +336,22 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     # depends_on('android-ndk@10:18', when='+android')
     # depends_on('android-sdk', when='+android')
 
+    with when("+rocm"):
+        depends_on("hip")
+        depends_on("rocrand")
+        depends_on("rocblas")
+        depends_on("rocfft")
+        depends_on("hipfft")
+        depends_on("rccl", when="+nccl")
+        depends_on("hipsparse")
+        depends_on("hipcub")
+        depends_on("rocsolver")
+        depends_on("rocprim")
+        depends_on("miopen-hip")
+        depends_on("llvm-amdgpu")
+        depends_on("hsa-rocr-dev")
+        depends_on("rocminfo")
+
     # Check configure and configure.py to see when these variants are supported
     conflicts("+mkl", when="@:1.0")
     conflicts("+mkl", when="platform=darwin", msg="Darwin is not yet supported")
@@ -376,7 +371,13 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     conflicts("+gdr", when="@:1.3")
     conflicts("+verbs", when="@:1.1")
     conflicts("+ngraph", when="@:1.10")
+    conflicts("+opencl", when="platform=windows")
     conflicts("+computecpp", when="~opencl")
+    conflicts(
+        "+cuda",
+        when="+rocm",
+        msg="CUDA / ROCm are mututally exclusive. At most 1 GPU platform can be configured",
+    )
     conflicts("+cuda", when="platform=darwin", msg="There is no GPU support for macOS")
     conflicts(
         "cuda_arch=none",
@@ -423,6 +424,7 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     conflicts(
         "+nccl", when="platform=cray", msg="Currently NCCL is only supported on Linux platform"
     )
+    conflicts("+mpi", when="platform=windows")
     conflicts("+mpi", when="@:1.2")
     conflicts("+android", when="@:1.4")
     conflicts("+ios", when="@:1.12.0,1.12.2:1.13")
@@ -438,6 +440,9 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     conflicts("~rocm", when="@2.7.4-rocm-enhanced")
     conflicts("+rocm", when="@:2.7.4-a,2.7.4.0:")
 
+    # wheel 0.40 upgrades vendored packaging, trips over tensorflow-io-gcs-filesystem identifier
+    conflicts("^py-wheel@0.40:", when="@2.11:2.13")
+
     # https://www.tensorflow.org/install/source#tested_build_configurations
     conflicts("%gcc@:9.3.0", when="@2.9:")
     conflicts("%gcc@:7.3.0", when="@1.15:")
@@ -513,6 +518,7 @@ def setup_build_environment(self, env):
 
         # Please input the desired Python library path to use
         env.set("PYTHON_LIB_PATH", python_platlib)
+        env.set("TF_PYTHON_VERSION", spec["python"].version.up_to(2))
 
         # Ensure swig is in PATH or set SWIG_PATH
         env.set("SWIG_PATH", spec["swig"].prefix.bin.swig)
@@ -694,6 +700,12 @@ def setup_build_environment(self, env):
         else:
             env.set("TF_NEED_CUDA", "0")
 
+        # Do you want to use Clang to build TensorFlow?
+        if "%clang" in spec:
+            env.set("TF_NEED_CLANG", "1")
+        else:
+            env.set("TF_NEED_CLANG", "0")
+
         # Do you wish to download a fresh release of clang? (Experimental)
         env.set("TF_DOWNLOAD_CLANG", "0")
 
@@ -753,67 +765,6 @@ def setup_build_environment(self, env):
         tmp_path = tempfile.mkdtemp(prefix="spack")
         env.set("TEST_TMPDIR", tmp_path)
 
-        env.set("TF_SYSTEM_LIBS", "com_google_protobuf")
-        if spec.satisfies("@:2.3"):
-            # NOTE: INCLUDEDIR is not just relevant to protobuf
-            # see third_party/systemlibs/jsoncpp.BUILD
-            env.set("INCLUDEDIR", spec["protobuf"].prefix.include)
-
-    def patch(self):
-        filter_file(
-            '"-U_FORTIFY_SOURCE",',
-            '"-U_FORTIFY_SOURCE", "-I%s",' % self.spec["protobuf"].prefix.include,
-            "third_party/gpus/crosstool/BUILD.rocm.tpl",
-        )
-        if self.spec.satisfies("@2.12:"):
-            filter_file(
-                'genproto_deps.append("@com_google_protobuf//:well_known_types_py_pb2_genproto")',
-                "pass",
-                "tensorflow/tsl/platform/default/build_config.bzl",
-                string=True,
-            )
-        if self.spec.satisfies("@2.11:"):
-            filter_file(
-                "deps = protodeps + well_known_proto_libs(),",
-                "deps = protodeps,",
-                "tensorflow/tsl/platform/default/build_config.bzl",
-                string=True,
-            )
-        if self.spec.satisfies("@2.3:2.10"):
-            filter_file(
-                "deps = protodeps + well_known_proto_libs(),",
-                "deps = protodeps,",
-                "tensorflow/core/platform/default/build_config.bzl",
-                string=True,
-            )
-        if self.spec.satisfies("@2.4.0:2.5"):
-            text = """
-def protobuf_deps():
-    pass
-"""
-            with open("third_party/systemlibs/protobuf_deps.bzl", "w") as f:
-                f.write(text)
-
-            if self.spec.satisfies("@2.5.0"):
-                file_to_patch = "tensorflow/workspace2.bzl"
-            else:
-                file_to_patch = "tensorflow/workspace.bzl"
-
-            filter_file(
-                '"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",',
-                '"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",\n'
-                '"//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl",',  # noqa: E501
-                file_to_patch,
-                string=True,
-            )
-
-        # Set protobuf path
-        filter_file(
-            r"(^build:linux --define=PROTOBUF_INCLUDE_PATH=).*",
-            r"\1{0}".format(self.spec["protobuf"].prefix.include),
-            ".bazelrc",
-        )
-
     def configure(self, spec, prefix):
         # NOTE: configure script is interactive. If you set the appropriate
         # environment variables, this interactivity is skipped. If you don't,
@@ -848,14 +799,6 @@ def post_configure_fixes(self):
                 "tensorflow/workspace.bzl",
             )
 
-            # starting with tensorflow 1.3, tensorboard becomes a dependency
-            # -> remove from list of required packages
-            filter_file(
-                r"'tensorflow-tensorboard",
-                r"#'tensorflow-tensorboard",
-                "tensorflow/tools/pip_package/setup.py",
-            )
-
         if spec.satisfies("@1.5.0: ~gcp"):
             # google cloud support seems to be installed on default, leading
             # to boringssl error manually set the flag to false to avoid
@@ -867,15 +810,6 @@ def post_configure_fixes(self):
                 ".tf_configure.bazelrc",
             )
 
-        if spec.satisfies("@1.6.0:2.1"):
-            # tensorboard name changed
-            # there are no corresponding versions of these in spack
-            filter_file(
-                r"(^\s*)'tensorboard (>=|~=)",
-                r"\1#'tensorboard \2",
-                "tensorflow/tools/pip_package/setup.py",
-            )
-
         if spec.satisfies("@1.8.0: ~opencl"):
             # 1.8.0 and 1.9.0 aborts with numpy import error during python_api
             # generation somehow the wrong PYTHONPATH is used...
@@ -885,64 +819,6 @@ def post_configure_fixes(self):
                 f.write("build --distinct_host_configuration=false\n")
                 f.write('build --action_env PYTHONPATH="{0}"\n'.format(env["PYTHONPATH"]))
 
-        if spec.satisfies("@1.13.1:"):
-            # tensorflow_estimator is an API for tensorflow
-            # tensorflow-estimator imports tensorflow during build, so
-            # tensorflow has to be set up first
-            filter_file(
-                r"(^\s*)'tensorflow_estimator (>=|~=)",
-                r"\1#'tensorflow_estimator \2",
-                "tensorflow/tools/pip_package/setup.py",
-            )
-
-        if spec.satisfies("@2.5"):
-            filter_file(
-                r"(^\s*)'keras-nightly (>=|~=)",
-                r"\1#'keras-nightly \2",
-                "tensorflow/tools/pip_package/setup.py",
-            )
-
-        if spec.satisfies("@2.6:"):
-            filter_file(
-                r"(^\s*)'keras (>=|~=)", r"\1#'keras \2", "tensorflow/tools/pip_package/setup.py"
-            )
-
-        if spec.satisfies("@2.6"):
-            filter_file(
-                r"(^\s*)'clang (>=|~=)", r"\1#'clang \2", "tensorflow/tools/pip_package/setup.py"
-            )
-
-        # TODO: add support for tensorflow-io-gcs-filesystem
-        if spec.satisfies("@2.7:"):
-            filter_file(
-                r"(^\s*)'tensorflow-io-gcs-filesystem (>=|~=)",
-                r"\1#'tensorflow-io-gcs-filesystem \2",
-                "tensorflow/tools/pip_package/setup.py",
-            )
-
-        if spec.satisfies("@2.0.0:"):
-            # now it depends on the nightly versions...
-            filter_file(
-                r"REQUIRED_PACKAGES\[i\] = 'tb-nightly (>=|~=)",
-                r"pass #REQUIRED_PACKAGES[i] = 'tb-nightly \1",
-                "tensorflow/tools/pip_package/setup.py",
-            )
-            filter_file(
-                r"REQUIRED_PACKAGES\[i\] = 'tensorflow-estimator-2.0-preview",
-                r"pass #REQUIRED_PACKAGES[i] = 'tensorflow-estimator-2.0-preview",
-                "tensorflow/tools/pip_package/setup.py",
-            )
-            filter_file(
-                r"REQUIRED_PACKAGES\[i\] = 'tf-estimator-nightly (>=|~=)",
-                r"pass #REQUIRED_PACKAGES[i] = 'tf-estimator-nightly \1",
-                "tensorflow/tools/pip_package/setup.py",
-            )
-            filter_file(
-                r"REQUIRED_PACKAGES\[i\] = 'keras-nightly (>=|~=)",
-                r"pass #REQUIRED_PACKAGES[i] = 'keras-nightly \1",
-                "tensorflow/tools/pip_package/setup.py",
-            )
-
         if spec.satisfies("@1.13.1 +nccl"):
             filter_file(
                 r"^build --action_env NCCL_INSTALL_PATH=.*",
@@ -996,22 +872,12 @@ def build(self, spec, prefix):
             "--config=opt",
             # Enable verbose output for failures
             "--verbose_failures",
-            # Show (formatted) subcommands being executed
-            "--subcommands=pretty_print",
-            # Ask bazel to explain what it's up to
-            # Needs a filename as argument
-            "--explain=explainlogfile.txt",
-            # Increase verbosity of explanation,
-            "--verbose_explanations",
         ]
 
         if spec.satisfies("^bazel@:3.5"):
             # removed in bazel 3.6
             args.append("--incompatible_no_support_tools_in_action_inputs=false")
 
-        if spec.satisfies("@2.9: platform=darwin"):
-            args.append("--macos_sdk_version={}".format(macos_version()))
-
         # See .bazelrc for when each config flag is supported
         if spec.satisfies("@1.12.1:"):
             if "+mkl" in spec:
diff --git a/var/spack/repos/builtin/packages/py-terminado/package.py b/var/spack/repos/builtin/packages/py-terminado/package.py
index 8f2dd5cfdbda1f..826910a159b208 100644
--- a/var/spack/repos/builtin/packages/py-terminado/package.py
+++ b/var/spack/repos/builtin/packages/py-terminado/package.py
@@ -10,8 +10,10 @@
 class PyTerminado(PythonPackage):
     """Terminals served to term.js using Tornado websockets"""
 
+    homepage = "https://github.com/jupyter/terminado"
     pypi = "terminado/terminado-0.8.3.tar.gz"
 
+    version("0.17.1", sha256="6ccbbcd3a4f8a25a5ec04991f39a0b8db52dfcd487ea0e578d977e6752380333")
     version("0.15.0", sha256="ab4eeedccfcc1e6134bfee86106af90852c69d602884ea3a1e8ca6d4486e9bfe")
     version("0.12.1", sha256="b20fd93cc57c1678c799799d117874367cc07a3d2d55be95205b1a88fa08393f")
     version("0.8.3", sha256="4804a774f802306a7d9af7322193c5390f1da0abb429e082a10ef1d46e6fb2c2")
@@ -19,13 +21,16 @@ class PyTerminado(PythonPackage):
     version("0.8.1", sha256="55abf9ade563b8f9be1f34e4233c7b7bde726059947a593322e8a553cc4c067a")
     version("0.6", sha256="2c0ba1f624067dccaaead7d2247cfe029806355cef124dc2ccb53c83229f0126")
 
-    depends_on("python@3.7:", when="@0.15.0:", type=("build", "run"))
-    depends_on("python@3.6:", when="@0.12.1:", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.4:", when="@0.8.2:", type=("build", "run"))
+    depends_on("py-hatchling@1.5:", when="@0.16:", type="build")
+    depends_on("py-hatchling@0.25:", when="@0.15:", type="build")
+
+    depends_on("py-ptyprocess", type=("build", "run"))
+    depends_on("py-tornado@6.1.0:", when="@0.15:", type=("build", "run"))
+    depends_on("py-tornado@4:", type=("build", "run"))
+    # not yet in spack
+    # depends_on("py-pywinpty@1.1:", when="platform=windows", type=("build", "run"))
+
+    # Historical dependencies
     depends_on("py-setuptools@40.8.0:", when="@0.12.1", type="build")
-    depends_on("py-setuptools", when="@:0.6", type="build")
     depends_on("py-flit", when="@0.8", type="build")
-    depends_on("py-hatchling@0.25:", when="@0.15.0:", type="build")
-    depends_on("py-tornado@4:", type=("build", "run"))
-    depends_on("py-tornado@6.1.0:", when="@0.15.0:", type=("build", "run"))
-    depends_on("py-ptyprocess", type=("build", "run"))
+    depends_on("py-setuptools", when="@:0.6", type="build")
diff --git a/var/spack/repos/builtin/packages/py-tfdlpack/package.py b/var/spack/repos/builtin/packages/py-tfdlpack/package.py
index ccafcb151462a1..96ab92df54dd65 100644
--- a/var/spack/repos/builtin/packages/py-tfdlpack/package.py
+++ b/var/spack/repos/builtin/packages/py-tfdlpack/package.py
@@ -16,7 +16,9 @@ class PyTfdlpack(CMakePackage, PythonExtension):
     maintainers("adamjstewart")
 
     version("master", branch="master", submodules=True)
-    version("0.1.1", tag="v0.1.1", submodules=True)
+    version(
+        "0.1.1", tag="v0.1.1", commit="a1fdb53096158c2ec9189bb1ff46c92c6f571bbe", submodules=True
+    )
 
     variant("cuda", default=True, description="Build with CUDA support")
 
diff --git a/var/spack/repos/builtin/packages/py-tidynamics/package.py b/var/spack/repos/builtin/packages/py-tidynamics/package.py
index 5afbf5d70eac74..58571fc337356c 100644
--- a/var/spack/repos/builtin/packages/py-tidynamics/package.py
+++ b/var/spack/repos/builtin/packages/py-tidynamics/package.py
@@ -14,8 +14,10 @@ class PyTidynamics(PythonPackage):
 
     maintainers("RMeli")
 
+    version("1.1.2", sha256="103874edd79dc64a0c7b765f51200926822e74df63703acb6c630a8167dbcfa2")
     version("1.0.0", sha256="b7bd669d380b0f469f3a8aedfbc0e5fa967fe8dc44e196f54baf0edb59846976")
 
     depends_on("py-setuptools", type="build")
+    depends_on("py-setuptools-scm", when="@1.1.2:", type="build")
 
     depends_on("py-numpy", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-tifffile/package.py b/var/spack/repos/builtin/packages/py-tifffile/package.py
index 6faeca3b04c86d..a82e64e259e7f1 100644
--- a/var/spack/repos/builtin/packages/py-tifffile/package.py
+++ b/var/spack/repos/builtin/packages/py-tifffile/package.py
@@ -12,6 +12,7 @@ class PyTifffile(PythonPackage):
     homepage = "https://github.com/cgohlke/tifffile"
     pypi = "tifffile/tifffile-0.12.1.tar.gz"
 
+    version("2023.8.30", sha256="6a8c53b012a286b75d09a1498ab32f202f24cc6270a105b5d5911dc4426f162a")
     version(
         "2022.10.10", sha256="50b61ba943b866d191295bc38a00191c9fdab23ece063544c7f1a264e3f6aa8e"
     )
@@ -19,9 +20,12 @@ class PyTifffile(PythonPackage):
     version("2020.10.1", sha256="799feeccc91965b69e1288c51a1d1118faec7f40b2eb89ad2979591b85324830")
     version("0.12.1", sha256="802367effe86b0d1e64cb5c2ed886771f677fa63260b945e51a27acccdc08fa1")
 
+    depends_on("python@3.9:", when="@2023.7.18:", type=("build", "run"))
     depends_on("python@3.8:", when="@2022.2.2:", type=("build", "run"))
-    depends_on("python@3.7:", when="@2020.10.1:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
+
+    # py-tifffile@2023.1.23: don't have a lower bound on py-numpy anymore
+    # -> leave it in nonetheless
     depends_on("py-numpy@1.19.2:", when="@2022.2.2:", type=("build", "run"))
     depends_on("py-numpy@1.15.1:", when="@2020.10.1:", type=("build", "run"))
     depends_on("py-numpy@1.8.2:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-tiktoken/package.py b/var/spack/repos/builtin/packages/py-tiktoken/package.py
new file mode 100644
index 00000000000000..cd5515ec06d7c5
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-tiktoken/package.py
@@ -0,0 +1,27 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyTiktoken(PythonPackage):
+    """tiktoken is a fast BPE tokeniser for use with OpenAI's models."""
+
+    homepage = "https://github.com/openai/tiktoken"
+    pypi = "tiktoken/tiktoken-0.4.0.tar.gz"
+
+    maintainers("meyersbs")
+
+    version("0.4.0", sha256="59b20a819969735b48161ced9b92f05dc4519c17be4015cfb73b65270a243620")
+    version("0.3.1", sha256="8295912429374f5f3c6c6bf053a091ce1de8c1792a62e3b30d4ad36f47fa8b52")
+
+    # From pyproject.toml
+    depends_on("py-setuptools@62.4:", type="build")
+    depends_on("py-setuptools-rust@1.5.2:", type="build")
+    depends_on("py-wheel", type="build")
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("py-regex@2022.1.18:", type=("build", "run"))
+    depends_on("py-requests@2.26.0:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-timm/package.py b/var/spack/repos/builtin/packages/py-timm/package.py
index a021cdb6d7d323..2be19562d47329 100644
--- a/var/spack/repos/builtin/packages/py-timm/package.py
+++ b/var/spack/repos/builtin/packages/py-timm/package.py
@@ -14,6 +14,7 @@ class PyTimm(PythonPackage):
 
     maintainers("adamjstewart")
 
+    version("0.9.5", sha256="669835f0030cfb2412c464b7b563bb240d4d41a141226afbbf1b457e4f18cff1")
     version("0.9.2", sha256="d0977cc5e02c69bda979fca8b52aa315a5f2cb64ebf8ad2c4631b1e452762c14")
     version("0.9.1", sha256="171420ac499e7999d38fb8b08fffa5ca3950b38db23bba84763cd92621ca80a2")
     version("0.9.0", sha256="f0159bbeea5c8d11551ac3077752ee77008d2638578571303296054b5ffddad4")
diff --git a/var/spack/repos/builtin/packages/py-tinyarray/package.py b/var/spack/repos/builtin/packages/py-tinyarray/package.py
index 524419c1a1a1ba..61cb28763b2e3d 100644
--- a/var/spack/repos/builtin/packages/py-tinyarray/package.py
+++ b/var/spack/repos/builtin/packages/py-tinyarray/package.py
@@ -22,8 +22,12 @@ class PyTinyarray(PythonPackage):
     # package is updated
     maintainers("payerle")
 
+    version("1.2.4", sha256="ecd3428fd8a48b61fc5f0a413ede03e27db3a1dd53fcd49e24a36d11a8a29aba")
     version("1.2.3", sha256="47a06f801ed4b3d438f4f7098e244cd0c6d7db09428b1bc5ee813e52234dee9f")
     version("1.2.2", sha256="660d6d8532e1db5efbebae2861e5733a7082486fbdeb47d57d84b8f477d697e4")
     version("1.2.1", sha256="47a06f801ed4b3d438f4f7098e244cd0c6d7db09428b1bc5ee813e52234dee9f")
 
     depends_on("py-setuptools", type="build")
+
+    # See https://gitlab.kwant-project.org/kwant/tinyarray/-/merge_requests/14
+    conflicts("^python@3.10:", when="@:1.2.3", msg="py-tinyarray must be @1.2.4: for python@3.10")
diff --git a/var/spack/repos/builtin/packages/py-tinycss2/package.py b/var/spack/repos/builtin/packages/py-tinycss2/package.py
index 64d2972c159dca..823d7879aa50d8 100644
--- a/var/spack/repos/builtin/packages/py-tinycss2/package.py
+++ b/var/spack/repos/builtin/packages/py-tinycss2/package.py
@@ -13,9 +13,10 @@ class PyTinycss2(PythonPackage):
 
     homepage = "https://www.courtbouillon.org/tinycss2"
     pypi = "tinycss2/tinycss2-1.1.1.tar.gz"
+    git = "https://github.com/Kozea/tinycss2.git"
 
+    version("1.2.1", sha256="8cff3a8f066c2ec677c06dbc7b45619804a6938478d9d73c284b29d14ecb0627")
     version("1.1.1", sha256="b2e44dd8883c360c35dd0d1b5aad0b610e5156c2cb3b33434634e539ead9d8bf")
 
-    depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-flit-core@3.2:3", type="build")
     depends_on("py-webencodings@0.4:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-tokenizers/package.py b/var/spack/repos/builtin/packages/py-tokenizers/package.py
index c45f301c20baf8..5555fcdb087e4c 100644
--- a/var/spack/repos/builtin/packages/py-tokenizers/package.py
+++ b/var/spack/repos/builtin/packages/py-tokenizers/package.py
@@ -13,6 +13,7 @@ class PyTokenizers(PythonPackage):
     homepage = "https://github.com/huggingface/tokenizers"
     pypi = "tokenizers/tokenizers-0.6.0.tar.gz"
 
+    version("0.13.3", sha256="2e546dbb68b623008a5442353137fbb0123d311a6d7ba52f2667c8862a75af2e")
     version("0.13.1", sha256="3333d1cee5c8f47c96362ea0abc1f81c77c9b92c6c3d11cbf1d01985f0d5cf1d")
     version("0.10.3", sha256="1a5d3b596c6d3a237e1ad7f46c472d467b0246be7fd1a364f12576eb8db8f7e6")
     version("0.6.0", sha256="1da11fbfb4f73be695bed0d655576097d09a137a16dceab2f66399716afaffac")
diff --git a/var/spack/repos/builtin/packages/py-tomlkit/package.py b/var/spack/repos/builtin/packages/py-tomlkit/package.py
index 81d8c69ce2e1a0..8e7486d36aaf39 100644
--- a/var/spack/repos/builtin/packages/py-tomlkit/package.py
+++ b/var/spack/repos/builtin/packages/py-tomlkit/package.py
@@ -12,11 +12,11 @@ class PyTomlkit(PythonPackage):
     homepage = "https://github.com/sdispater/tomlkit"
     pypi = "tomlkit/tomlkit-0.7.0.tar.gz"
 
+    version("0.12.1", sha256="38e1ff8edb991273ec9f6181244a6a391ac30e9f5098e7535640ea6be97a7c86")
     version("0.11.4", sha256="3235a9010fae54323e727c3ac06fb720752fe6635b3426e379daec60fbd44a83")
     version("0.11.0", sha256="71ceb10c0eefd8b8f11fe34e8a51ad07812cb1dc3de23247425fbc9ddc47b9dd")
     version("0.7.2", sha256="d7a454f319a7e9bd2e249f239168729327e4dd2d27b17dc68be264ad1ce36754")
     version("0.7.0", sha256="ac57f29693fab3e309ea789252fcce3061e19110085aa31af5446ca749325618")
 
-    depends_on("python@2.7,3.5:", type=("build", "run"))
-    depends_on("python@3.6:3", when="@0.11.0:", type=("build", "run"))
+    depends_on("python@3.6:3", when="@0.11.0:0.11.5", type=("build", "run"))
     depends_on("py-poetry-core@1:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-tomopy/package.py b/var/spack/repos/builtin/packages/py-tomopy/package.py
index b99e60ef91cb3e..59a1c0f1b32d49 100644
--- a/var/spack/repos/builtin/packages/py-tomopy/package.py
+++ b/var/spack/repos/builtin/packages/py-tomopy/package.py
@@ -34,7 +34,7 @@ class PyTomopy(PythonPackage):
     # Note: The module name of py-scikit-build is skbuild:
     depends_on("py-scikit-build", type=("build"))
     depends_on("py-scikit-image@0.17:", type=("build", "run"))
-    depends_on("py-numpy+blas", type=("build", "run"))
+    depends_on("py-numpy", type=("build", "run"))
     depends_on("py-pyfftw", type=("build", "run"), when="@1.0:1.9")
     depends_on("py-scipy", type=("build", "run"))
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-topiary-asr/package.py b/var/spack/repos/builtin/packages/py-topiary-asr/package.py
index af6f5766cb8b5f..6e5f14e11a5b9e 100644
--- a/var/spack/repos/builtin/packages/py-topiary-asr/package.py
+++ b/var/spack/repos/builtin/packages/py-topiary-asr/package.py
@@ -47,7 +47,7 @@ class PyTopiaryAsr(PythonPackage):
     depends_on("mpi", type="run")
     depends_on("openmpi+legacylaunchers", type="run", when="^openmpi schedulers=slurm")
 
-    conflicts("mpich")
+    conflicts("^mpich")
 
     def patch(self):
         if self.spec.satisfies("^raxml-ng+mpi"):
diff --git a/var/spack/repos/builtin/packages/py-torch-sparse/package.py b/var/spack/repos/builtin/packages/py-torch-sparse/package.py
index 4d4495301adb96..b74a7bed549f24 100644
--- a/var/spack/repos/builtin/packages/py-torch-sparse/package.py
+++ b/var/spack/repos/builtin/packages/py-torch-sparse/package.py
@@ -13,18 +13,19 @@ class PyTorchSparse(PythonPackage):
     homepage = "https://github.com/rusty1s/pytorch_sparse/"
     url = "https://github.com/rusty1s/pytorch_sparse/archive/0.6.7.tar.gz"
 
+    version("0.6.17", sha256="c964a70ed978bff65009250eb12fae96317c60c9a04d7d1b07f0beee8b4b9c22")
     version("0.6.8", sha256="98f7ff1f0f9cd5031bc81c70c11970c3864545ae33677025a6efd2466a97e6f9")
     version("0.6.7", sha256="0d038a1502548692972a085cd0496460b5d2050bb7328427add990f081d6c44d")
 
     variant("cuda", default=False, description="Enable CUDA support")
 
-    depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
-    depends_on("py-pytest-runner", type="build")
+    depends_on("py-pytest-runner", when="@:0.6.8", type="build")
     depends_on("py-scipy", type=("build", "run"))
     depends_on("py-torch", type=("build", "run"))
     depends_on("py-torch-scatter+cuda", when="+cuda")
     depends_on("py-torch-scatter~cuda", when="~cuda")
+    depends_on("parallel-hashmap", when="@0.6.17:")
 
     def setup_build_environment(self, env):
         if "+cuda" in self.spec:
diff --git a/var/spack/repos/builtin/packages/py-torch/package.py b/var/spack/repos/builtin/packages/py-torch/package.py
index ae59d416449250..d2edd9453837bf 100644
--- a/var/spack/repos/builtin/packages/py-torch/package.py
+++ b/var/spack/repos/builtin/packages/py-torch/package.py
@@ -11,11 +11,11 @@
 
 
 class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
-    """Tensors and Dynamic neural networks in Python
-    with strong GPU acceleration."""
+    """Tensors and Dynamic neural networks in Python with strong GPU acceleration."""
 
     homepage = "https://pytorch.org/"
     git = "https://github.com/pytorch/pytorch.git"
+    submodules = True
 
     maintainers("adamjstewart")
 
@@ -23,34 +23,36 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
     # core libraries to ensure that the package was successfully installed.
     import_modules = ["torch", "torch.autograd", "torch.nn", "torch.utils"]
 
-    version("master", branch="master", submodules=True)
-    version("2.0.1", tag="v2.0.1", submodules=True)
-    version("2.0.0", tag="v2.0.0", submodules=True)
-    version("1.13.1", tag="v1.13.1", submodules=True)
-    version("1.13.0", tag="v1.13.0", submodules=True)
-    version("1.12.1", tag="v1.12.1", submodules=True)
-    version("1.12.0", tag="v1.12.0", submodules=True)
-    version("1.11.0", tag="v1.11.0", submodules=True)
-    version("1.10.2", tag="v1.10.2", submodules=True)
-    version("1.10.1", tag="v1.10.1", submodules=True)
-    version("1.10.0", tag="v1.10.0", submodules=True)
-    version("1.9.1", tag="v1.9.1", submodules=True)
-    version("1.9.0", tag="v1.9.0", submodules=True)
-    version("1.8.2", tag="v1.8.2", submodules=True)
-    version("1.8.1", tag="v1.8.1", submodules=True)
-    version("1.8.0", tag="v1.8.0", submodules=True)
-    version("1.7.1", tag="v1.7.1", submodules=True)
-    version("1.7.0", tag="v1.7.0", submodules=True)
-    version("1.6.0", tag="v1.6.0", submodules=True)
-    version("1.5.1", tag="v1.5.1", submodules=True)
-    version("1.5.0", tag="v1.5.0", submodules=True)
-    version("1.4.1", tag="v1.4.1", submodules=True)
-    version("1.3.1", tag="v1.3.1", submodules=True)
-    version("1.3.0", tag="v1.3.0", submodules=True)
-    version("1.2.0", tag="v1.2.0", submodules=True)
-    version("1.1.0", tag="v1.1.0", submodules=True)
-    version("1.0.1", tag="v1.0.1", submodules=True)
-    version("1.0.0", tag="v1.0.0", submodules=True)
+    version("main", branch="main")
+    version("master", branch="main", deprecated=True)
+    version("2.1.0", tag="v2.1.0", commit="7bcf7da3a268b435777fe87c7794c382f444e86d")
+    version("2.0.1", tag="v2.0.1", commit="e9ebda29d87ce0916ab08c06ab26fd3766a870e5")
+    version("2.0.0", tag="v2.0.0", commit="c263bd43e8e8502d4726643bc6fd046f0130ac0e")
+    version("1.13.1", tag="v1.13.1", commit="49444c3e546bf240bed24a101e747422d1f8a0ee")
+    version("1.13.0", tag="v1.13.0", commit="7c98e70d44abc7a1aead68b6ea6c8adc8c554db5")
+    version("1.12.1", tag="v1.12.1", commit="664058fa83f1d8eede5d66418abff6e20bd76ca8")
+    version("1.12.0", tag="v1.12.0", commit="67ece03c8cd632cce9523cd96efde6f2d1cc8121")
+    version("1.11.0", tag="v1.11.0", commit="bc2c6edaf163b1a1330e37a6e34caf8c553e4755")
+    version("1.10.2", tag="v1.10.2", commit="71f889c7d265b9636b93ede9d651c0a9c4bee191")
+    version("1.10.1", tag="v1.10.1", commit="302ee7bfb604ebef384602c56e3853efed262030")
+    version("1.10.0", tag="v1.10.0", commit="36449ea93134574c2a22b87baad3de0bf8d64d42")
+    version("1.9.1", tag="v1.9.1", commit="dfbd030854359207cb3040b864614affeace11ce")
+    version("1.9.0", tag="v1.9.0", commit="d69c22dd61a2f006dcfe1e3ea8468a3ecaf931aa")
+    version("1.8.2", tag="v1.8.2", commit="e0495a7aa104471d95dc85a1b8f6473fbcc427a8")
+    version("1.8.1", tag="v1.8.1", commit="56b43f4fec1f76953f15a627694d4bba34588969")
+    version("1.8.0", tag="v1.8.0", commit="37c1f4a7fef115d719104e871d0cf39434aa9d56")
+    version("1.7.1", tag="v1.7.1", commit="57bffc3a8e4fee0cce31e1ff1f662ccf7b16db57")
+    version("1.7.0", tag="v1.7.0", commit="e85d494707b835c12165976b8442af54b9afcb26")
+    version("1.6.0", tag="v1.6.0", commit="b31f58de6fa8bbda5353b3c77d9be4914399724d")
+    version("1.5.1", tag="v1.5.1", commit="3c31d73c875d9a4a6ea8a843b9a0d1b19fbe36f3")
+    version("1.5.0", tag="v1.5.0", commit="4ff3872a2099993bf7e8c588f7182f3df777205b")
+    version("1.4.1", tag="v1.4.1", commit="74044638f755cd8667bedc73da4dbda4aa64c948")
+    version("1.3.1", tag="v1.3.1", commit="ee77ccbb6da4e2efd83673e798acf7081bc03564")
+    version("1.3.0", tag="v1.3.0", commit="de394b672d0346f2f387a8bb1a1280d5d2eaf9cb")
+    version("1.2.0", tag="v1.2.0", commit="8554416a199c4cec01c60c7015d8301d2bb39b64")
+    version("1.1.0", tag="v1.1.0", commit="142c973f4179e768164cd578951489e89021b29c")
+    version("1.0.1", tag="v1.0.1", commit="83221655a8237ca80f9673dad06a98d34c43e546")
+    version("1.0.0", tag="v1.0.0", commit="db5d3131d16f57abd4f13d3f4b885d5f67bf6644")
 
     is_darwin = sys.platform == "darwin"
 
@@ -101,7 +103,7 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
         "breakpad",
         default=True,
         description="Enable breakpad crash dump library",
-        when="@1.9:1.11",
+        when="@1.10:1.11",
     )
 
     conflicts("+cuda+rocm")
@@ -122,6 +124,9 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
         msg="TensorPipe must be enabled with +distributed",
     )
 
+    # https://github.com/pytorch/pytorch/issues/100991
+    conflicts("%apple-clang@14:", when="@:1")
+
     conflicts(
         "cuda_arch=none",
         when="+cuda",
@@ -133,12 +138,10 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
     # See python_min_version in setup.py
     # Upper bounds come from wheel availability on PyPI
     depends_on("python@3.8:3.11", when="@2:", type=("build", "link", "run"))
-    depends_on("python@3.7:3.10", when="@1.11:1", type=("build", "link", "run"))
-    depends_on("python@3.6.2:3.9", when="@1.7.1:1.10", type=("build", "link", "run"))
-    depends_on("python@3.6.1:3.8", when="@1.6:1.7.0", type=("build", "link", "run"))
-    depends_on("python@3.5:3.8", when="@1.5", type=("build", "link", "run"))
-    depends_on("python@2.7:2,3.5:3.8", when="@1.4", type=("build", "link", "run"))
-    depends_on("python@2.7:2,3.5:3.7", when="@:1.3", type=("build", "link", "run"))
+    depends_on("python@:3.10", when="@1.11:1", type=("build", "link", "run"))
+    depends_on("python@:3.9", when="@1.7.1:1.10", type=("build", "link", "run"))
+    depends_on("python@:3.8", when="@1.4:1.7.0", type=("build", "link", "run"))
+    depends_on("python@:3.7", when="@:1.3", type=("build", "link", "run"))
 
     # CMakelists.txt
     depends_on("cmake@3.18:", when="@2:", type="build")
@@ -163,6 +166,7 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
     depends_on("py-sympy", when="@2:", type=("build", "run"))
     depends_on("py-networkx", when="@2:", type=("build", "run"))
     depends_on("py-jinja2", when="@2:", type=("build", "run"))
+    depends_on("py-fsspec", when="@2.1:", type=("build", "run"))
 
     # Undocumented dependencies
     depends_on("py-tqdm", type="run")
@@ -170,7 +174,8 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
     depends_on("lapack")
 
     # third_party
-    depends_on("py-pybind11@2.10.1", when="@2:", type=("build", "link", "run"))
+    depends_on("py-pybind11@2.11.0", when="@2.1:", type=("build", "link", "run"))
+    depends_on("py-pybind11@2.10.1", when="@2.0", type=("build", "link", "run"))
     depends_on("py-pybind11@2.10.0", when="@1.13:1", type=("build", "link", "run"))
     depends_on("py-pybind11@2.6.2", when="@1.8:1.12", type=("build", "link", "run"))
     depends_on("py-pybind11@2.3.0", when="@1.1:1.7", type=("build", "link", "run"))
@@ -185,15 +190,13 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
     depends_on("protobuf@:3", type=("build", "run"))
     depends_on("eigen")
     # https://github.com/pytorch/pytorch/issues/60329
-    # depends_on("cpuinfo@2022-08-19", when="@1.13:")
+    # depends_on("cpuinfo@2023-01-13", when="@2.1:")
+    # depends_on("cpuinfo@2022-08-19", when="@1.13:2.0")
     # depends_on("cpuinfo@2020-12-17", when="@1.8:1.12")
     # depends_on("cpuinfo@2020-06-11", when="@1.6:1.7")
-    # https://github.com/shibatch/sleef/issues/427
-    # depends_on("sleef@3.5.1_2020-12-22", when="@1.8:")
-    # https://github.com/pytorch/pytorch/issues/60334
-    # depends_on("sleef@3.4.0_2019-07-30", when="@1.6:1.7")
-    # https://github.com/Maratyszcza/FP16/issues/18
-    # depends_on("fp16@2020-05-14", when="@1.6:")
+    depends_on("sleef@3.5.1_2020-12-22", when="@1.8:")
+    depends_on("sleef@3.4.0_2019-07-30", when="@1.6:1.7")
+    depends_on("fp16@2020-05-14", when="@1.6:")
     depends_on("pthreadpool@2021-04-13", when="@1.9:")
     depends_on("pthreadpool@2020-10-05", when="@1.8")
     depends_on("pthreadpool@2020-06-15", when="@1.6:1.7")
@@ -241,15 +244,17 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
     # depends_on("xnnpack@2021-02-22", when="@1.8:1.9+xnnpack")
     # depends_on("xnnpack@2020-03-23", when="@1.6:1.7+xnnpack")
     depends_on("mpi", when="+mpi")
-    # https://github.com/pytorch/pytorch/issues/60270
-    # depends_on("gloo@2023-01-17", when="@2:+gloo")
-    # depends_on("gloo@2022-05-18", when="@1.13:1+gloo")
-    # depends_on("gloo@2021-05-21", when="@1.10:1.12+gloo")
-    # depends_on("gloo@2021-05-04", when="@1.9+gloo")
-    # depends_on("gloo@2020-09-18", when="@1.7:1.8+gloo")
-    # depends_on("gloo@2020-03-17", when="@1.6+gloo")
+    depends_on("gloo@2023-05-19", when="@2.1:+gloo")
+    depends_on("gloo@2023-01-17", when="@2.0+gloo")
+    depends_on("gloo@2022-05-18", when="@1.13:1+gloo")
+    depends_on("gloo@2021-05-21", when="@1.10:1.12+gloo")
+    depends_on("gloo@2021-05-04", when="@1.9+gloo")
+    depends_on("gloo@2020-09-18", when="@1.7:1.8+gloo")
+    depends_on("gloo@2020-03-17", when="@1.6+gloo")
+    depends_on("gloo+cuda", when="@1.6:+gloo+cuda")
     # https://github.com/pytorch/pytorch/issues/60331
-    # depends_on("onnx@1.13.1", when="@2:+onnx_ml")
+    # depends_on("onnx@1.14.1", when="@2.1:+onnx_ml")
+    # depends_on("onnx@1.13.1", when="@2.0+onnx_ml")
     # depends_on("onnx@1.12.0", when="@1.13:1+onnx_ml")
     # depends_on("onnx@1.11.0", when="@1.12+onnx_ml")
     # depends_on("onnx@1.10.1_2021-10-08", when="@1.11+onnx_ml")
@@ -263,6 +268,13 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
     depends_on("py-six", type="test")
     depends_on("py-psutil", type="test")
 
+    # https://github.com/pytorch/pytorch/issues/90448
+    patch(
+        "https://github.com/pytorch/pytorch/pull/97270.patch?full_index=1",
+        sha256="beb3fb57746cf8443f5caa6e08b2f8f4d4822c1e11e0c912134bd166c6a0ade7",
+        when="@1.10:2.0",
+    )
+
     # Fix BLAS being overridden by MKL
     # https://github.com/pytorch/pytorch/issues/60328
     patch(
@@ -278,6 +290,14 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
         when="@1.1:1.8.1",
     )
 
+    # https://github.com/pytorch/pytorch/issues/70297
+    patch(
+        "https://github.com/google/breakpad/commit/605c51ed96ad44b34c457bbca320e74e194c317e.patch?full_index=1",
+        sha256="694d83db3a2147d543357f22ba5c8d5683d0ed43e693d42bca8f24ec50080f98",
+        when="+breakpad",
+        working_dir="third_party/breakpad",
+    )
+
     # Fixes CMake configuration error when XNNPACK is disabled
     # https://github.com/pytorch/pytorch/pull/35607
     # https://github.com/pytorch/pytorch/pull/37865
@@ -286,11 +306,6 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
     # Fixes build error when ROCm is enabled for pytorch-1.5 release
     patch("rocm.patch", when="@1.5+rocm")
 
-    # Fixes fatal error: sleef.h: No such file or directory
-    # https://github.com/pytorch/pytorch/pull/35359
-    # https://github.com/pytorch/pytorch/issues/26555
-    # patch("sleef.patch", when="@:1.5")
-
     # Fixes compilation with Clang 9.0.0 and Apple Clang 11.0.3
     # https://github.com/pytorch/pytorch/pull/37086
     patch(
@@ -362,7 +377,22 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
         sha256="a54db63640b90e5833cc1099c0935572f5297d2d8625f62f01ac1fda79ed4569",
         when="@1.13 arch=ppc64le:",
     )
-    conflicts("arch=ppc64le:", when="@:1.9,2:")
+    patch(
+        "https://github.com/open-ce/pytorch-feedstock/raw/open-ce-v1.9.0/pytorch-2.0/recipe/0309-fallback-to-cpu_kernel-with-VSX.patch",
+        sha256="27f41c8d6cb61e69e761be62f03dc1ce023cbca34926e3ba559996821a7ce726",
+        when="@2.0 arch=ppc64le:",
+    )
+    patch(
+        "https://github.com/open-ce/pytorch-feedstock/raw/open-ce-v1.9.0/pytorch-2.0/recipe/0310-PR100149.patch",
+        sha256="1adbd38a9cc1611f1caaa325614695f4349d9ffd236332e0d8f0de5a3880f4dd",
+        when="@2.0 arch=ppc64le:",
+    )
+    patch(
+        "https://github.com/open-ce/pytorch-feedstock/raw/open-ce-v1.10.0/pytorch-2.0/recipe/0311-PR104956.patch",
+        sha256="be27c906924a21be198a3ea6c459739a1daa8b8b89045af339dafa4cd6f90d6c",
+        when="@2.0 arch=ppc64le:",
+    )
+    conflicts("arch=ppc64le:", when="@:1.9")
 
     # Cherry-pick a patch to allow earlier versions of PyTorch to work with CUDA 11.4
     patch(
@@ -545,7 +575,7 @@ def enable_or_disable(variant, keyword="USE", var=None, newer=False):
         elif "~onnx_ml" in self.spec:
             env.set("ONNX_ML", "OFF")
 
-        if not self.spec.satisfies("@master"):
+        if not self.spec.satisfies("@main,master"):
             env.set("PYTORCH_BUILD_VERSION", self.version)
             env.set("PYTORCH_BUILD_NUMBER", 0)
 
@@ -591,17 +621,13 @@ def enable_or_disable(variant, keyword="USE", var=None, newer=False):
         env.set("pybind11_INCLUDE_DIR", self.spec["py-pybind11"].prefix.include)
         if self.spec.satisfies("@1.10:"):
             env.set("USE_SYSTEM_PYBIND11", "ON")
-        # https://github.com/pytorch/pytorch/issues/60334
-        # if self.spec.satisfies("@1.8:"):
-        #     env.set("USE_SYSTEM_SLEEF", "ON")
         if self.spec.satisfies("@1.6:"):
             # env.set("USE_SYSTEM_LIBS", "ON")
             # https://github.com/pytorch/pytorch/issues/60329
             # env.set("USE_SYSTEM_CPUINFO", "ON")
-            # https://github.com/pytorch/pytorch/issues/60270
-            # env.set("USE_SYSTEM_GLOO", "ON")
-            # https://github.com/Maratyszcza/FP16/issues/18
-            # env.set("USE_SYSTEM_FP16", "ON")
+            env.set("USE_SYSTEM_SLEEF", "ON")
+            env.set("USE_SYSTEM_GLOO", "ON")
+            env.set("USE_SYSTEM_FP16", "ON")
             env.set("USE_SYSTEM_PTHREADPOOL", "ON")
             env.set("USE_SYSTEM_PSIMD", "ON")
             env.set("USE_SYSTEM_FXDIV", "ON")
@@ -611,6 +637,10 @@ def enable_or_disable(variant, keyword="USE", var=None, newer=False):
             # https://github.com/pytorch/pytorch/issues/60332
             # env.set("USE_SYSTEM_XNNPACK", "ON")
 
+        # https://github.com/pytorch/pytorch/issues/111086
+        if self.spec.satisfies("%apple-clang@15:"):
+            env.append_flags("LDFLAGS", "-Wl,-ld_classic")
+
     @run_before("install")
     def build_amd(self):
         if "+rocm" in self.spec:
diff --git a/var/spack/repos/builtin/packages/py-torch/sleef.patch b/var/spack/repos/builtin/packages/py-torch/sleef.patch
deleted file mode 100644
index 67f0234162d1a1..00000000000000
--- a/var/spack/repos/builtin/packages/py-torch/sleef.patch
+++ /dev/null
@@ -1,12 +0,0 @@
-diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
-index 8025a7de3c..2e5cdbb5c9 100644
---- a/caffe2/CMakeLists.txt
-+++ b/caffe2/CMakeLists.txt
-@@ -1232,6 +1232,7 @@ if (BUILD_TEST)
-     add_executable(${test_name} "${test_src}")
-     target_link_libraries(${test_name} ${Caffe2_MAIN_LIBS} gtest_main)
-     target_include_directories(${test_name} PRIVATE $)
-+    target_include_directories(${test_name} PRIVATE $)
-     target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
-     add_test(NAME ${test_name} COMMAND $)
-     if (INSTALL_TEST)
diff --git a/var/spack/repos/builtin/packages/py-torchaudio/package.py b/var/spack/repos/builtin/packages/py-torchaudio/package.py
index c3698635b2cf13..d07ce1de2182c7 100644
--- a/var/spack/repos/builtin/packages/py-torchaudio/package.py
+++ b/var/spack/repos/builtin/packages/py-torchaudio/package.py
@@ -8,51 +8,41 @@
 
 
 class PyTorchaudio(PythonPackage):
-    """The aim of torchaudio is to apply PyTorch to the audio
-    domain. By supporting PyTorch, torchaudio follows the same
-    philosophy of providing strong GPU acceleration, having a focus on
-    trainable features through the autograd system, and having
-    consistent style (tensor names and dimension names). Therefore, it
-    is primarily a machine learning library and not a general signal
-    processing library. The benefits of Pytorch is be seen in
-    torchaudio through having all the computations be through Pytorch
-    operations which makes it easy to use and feel like a natural
-    extension."""
+    """An audio package for PyTorch."""
 
     homepage = "https://github.com/pytorch/audio"
     git = "https://github.com/pytorch/audio.git"
+    submodules = True
 
-    version("main", branch="main", submodules=True)
-    version("2.0.2", tag="v2.0.2", submodules=True)
-    version("2.0.1", tag="v2.0.1", submodules=True)
-    version("0.13.1", tag="v0.13.1", submodules=True)
-    version("0.13.0", tag="v0.13.0", submodules=True)
-    version("0.12.1", tag="v0.12.1", submodules=True)
-    version("0.12.0", tag="v0.12.0", submodules=True)
-    version("0.11.0", tag="v0.11.0", submodules=True)
-    version("0.10.2", tag="v0.10.2", submodules=True)
-    version("0.10.1", tag="v0.10.1", submodules=True)
-    version("0.10.0", tag="v0.10.0", submodules=True)
-    version("0.9.1", tag="v0.9.1", submodules=True)
-    version("0.9.0", tag="v0.9.0", submodules=True)
-    version("0.8.2", tag="v0.8.2", submodules=True)
-    version("0.8.1", tag="v0.8.1", submodules=True)
-    version("0.8.0", tag="v0.8.0", submodules=True)
-    version("0.7.2", tag="v0.7.2", submodules=True)
-    version("0.7.0", tag="v0.7.0", submodules=True)
-    version("0.6.0", tag="v0.6.0", submodules=True)
-    version("0.5.1", tag="v0.5.1", submodules=True)
-    version("0.5.0", tag="v0.5.0", submodules=True)
-    version("0.4.0", tag="v0.4.0", submodules=True)
+    version("main", branch="main")
+    version("2.1.0", tag="v2.1.0", commit="6ea1133706801ec6e81bb29142da2e21a8583a0a")
+    version("2.0.2", tag="v2.0.2", commit="31de77dad5c89274451b3f5c4bcb630be12787c4")
+    version("2.0.1", tag="v2.0.1", commit="3b40834aca41957002dfe074175e900cf8906237")
+    version("0.13.1", tag="v0.13.1", commit="b90d79882c3521fb3882833320b4b85df3b622f4")
+    version("0.13.0", tag="v0.13.0", commit="bc8640b4722abf6587fb4cc2521da45aeb55a711")
+    version("0.12.1", tag="v0.12.1", commit="58da31733e08438f9d1816f55f54756e53872a92")
+    version("0.12.0", tag="v0.12.0", commit="2e1388401c434011e9f044b40bc8374f2ddfc414")
+    version("0.11.0", tag="v0.11.0", commit="820b383b3b21fc06e91631a5b1e6ea1557836216")
+    version("0.10.2", tag="v0.10.2", commit="6f539cf3edc4224b51798e962ca28519e5479ffb")
+    version("0.10.1", tag="v0.10.1", commit="4b64f80bef85bd951ea35048c461c8304e7fc4c4")
+    version("0.10.0", tag="v0.10.0", commit="d2634d866603c1e2fc8e44cd6e9aea7ddd21fe29")
+    version("0.9.1", tag="v0.9.1", commit="a85b2398722182dd87e76d9ffcbbbf7e227b83ce")
+    version("0.9.0", tag="v0.9.0", commit="33b2469744955e2129c6367457dffe9bb4b05dea")
+    version("0.8.2", tag="v0.8.2", commit="d254d547d183e7203e455de6b99e56d3ffdd4499")
+    version("0.8.1", tag="v0.8.1", commit="e4e171a51714b2b2bd79e1aea199c3f658eddf9a")
+    version("0.8.0", tag="v0.8.0", commit="099d7883c6b7af1d1c3b416191e5f3edf492e104")
+    version("0.7.2", tag="v0.7.2", commit="a853dff25de36cc637b1f02029343790d2dd0199")
+    version("0.7.0", tag="v0.7.0", commit="ac17b64f4daedd45d0495e2512e22eaa6e5b7eeb")
+    version("0.6.0", tag="v0.6.0", commit="f17ae39ff9da0df8f795fef2fcc192f298f81268")
+    version("0.5.1", tag="v0.5.1", commit="71434798460a4ceca9d42004567ef419c62a612e")
+    version("0.5.0", tag="v0.5.0", commit="09494ea545738538f9db2dceeffe10d421060ee5")
+    version("0.4.0", tag="v0.4.0", commit="8afed303af3de41f3586007079c0534543c8f663")
 
-    # https://github.com/pytorch/audio#dependencies
+    # https://pytorch.org/audio/main/installation.html#dependencies
     depends_on("python@3.8:3.11", when="@2:", type=("build", "link", "run"))
-    depends_on("python@3.7:3.10", when="@0.12:0", type=("build", "link", "run"))
-    depends_on("python@3.7:3.9", when="@0.11", type=("build", "link", "run"))
-    depends_on("python@3.6:3.9", when="@0.7.2:0.10", type=("build", "link", "run"))
-    depends_on("python@3.6:3.8", when="@0.6:0.7.0", type=("build", "link", "run"))
-    depends_on("python@3.5:3.8", when="@0.5", type=("build", "link", "run"))
-    depends_on("python@2.7,3.5:3.8", when="@0.4", type=("build", "link", "run"))
+    depends_on("python@:3.10", when="@0.12:0", type=("build", "link", "run"))
+    depends_on("python@:3.9", when="@0.7.2:0.11", type=("build", "link", "run"))
+    depends_on("python@:3.8", when="@:0.7.0", type=("build", "link", "run"))
 
     # CMakelists.txt
     depends_on("cmake@3.18:", when="@0.10:", type="build")
@@ -65,8 +55,8 @@ class PyTorchaudio(PythonPackage):
     depends_on("pkgconfig", type="build")
     depends_on("sox")
 
-    # https://github.com/pytorch/audio#dependencies
-    depends_on("py-torch@master", when="@main", type=("build", "link", "run"))
+    depends_on("py-torch@main", when="@main", type=("build", "link", "run"))
+    depends_on("py-torch@2.1.0", when="@2.1.0", type=("build", "link", "run"))
     depends_on("py-torch@2.0.1", when="@2.0.2", type=("build", "link", "run"))
     depends_on("py-torch@2.0.0", when="@2.0.1", type=("build", "link", "run"))
     depends_on("py-torch@1.13.1", when="@0.13.1", type=("build", "link", "run"))
diff --git a/var/spack/repos/builtin/packages/py-torchdata/package.py b/var/spack/repos/builtin/packages/py-torchdata/package.py
index a207d7f7b4192b..fd9367f31d0179 100644
--- a/var/spack/repos/builtin/packages/py-torchdata/package.py
+++ b/var/spack/repos/builtin/packages/py-torchdata/package.py
@@ -16,6 +16,7 @@ class PyTorchdata(PythonPackage):
     maintainers("adamjstewart")
 
     version("main", branch="main")
+    version("0.7.0", sha256="0b444719c3abc67201ed0fea92ea9c4100e7f36551ba0d19a09446cc11154eb3")
     version("0.6.1", sha256="c596db251c5e6550db3f00e4308ee7112585cca4d6a1c82a433478fd86693257")
     version("0.6.0", sha256="048dea12ee96c0ea1525097959fee811d7b38c2ed05f44a90f35f8961895fb5b")
     version("0.5.1", sha256="69d80bd33ce8f08e7cfeeb71cefddfc29cede25a85881e33dbae47576b96ed29")
@@ -36,7 +37,8 @@ class PyTorchdata(PythonPackage):
     depends_on("ninja", when="@0.4:", type="build")
 
     # https://github.com/pytorch/data#version-compatibility
-    depends_on("py-torch@master", when="@main", type=("build", "run"))
+    depends_on("py-torch@main", when="@main", type=("build", "run"))
+    depends_on("py-torch@2.1.0", when="@0.7.0", type=("build", "run"))
     depends_on("py-torch@2.0.1", when="@0.6.1", type=("build", "run"))
     depends_on("py-torch@2.0.0", when="@0.6.0", type=("build", "run"))
     depends_on("py-torch@1.13.1", when="@0.5.1", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-torchgeo/package.py b/var/spack/repos/builtin/packages/py-torchgeo/package.py
index d67c928b0901f4..f5ef2ddc4bc783 100644
--- a/var/spack/repos/builtin/packages/py-torchgeo/package.py
+++ b/var/spack/repos/builtin/packages/py-torchgeo/package.py
@@ -7,11 +7,7 @@
 
 
 class PyTorchgeo(PythonPackage):
-    """TorchGeo: datasets, samplers, transforms, and pre-trained models for geospatial data.
-
-    TorchGeo is a PyTorch domain library, similar to torchvision, providing datasets, samplers,
-    transforms, and pre-trained models specific to geospatial data.
-    """
+    """TorchGeo: datasets, samplers, transforms, and pre-trained models for geospatial data."""
 
     homepage = "https://github.com/microsoft/torchgeo"
     pypi = "torchgeo/torchgeo-0.1.0.tar.gz"
@@ -20,6 +16,7 @@ class PyTorchgeo(PythonPackage):
     maintainers("adamjstewart", "calebrob6")
 
     version("main", branch="main")
+    version("0.5.0", sha256="2bc2f9c4a19a569790cb3396499fdec17496632b0e52b86be390a2cc7a1a7033")
     version("0.4.1", sha256="a3692436bf63df8d2f9b76d16eea5ee309dd1bd74e0fde6e64456abfdb2a5b58")
     version("0.4.0", sha256="a0812487205aa2db7bc92119d896ae4bf4f1014e6fdc0ce0f75bcb24fada6613")
     version("0.3.1", sha256="ba7a716843575d173abab383c6cc2d5fc8faf5834472f16a4abe1b932040ece5")
@@ -34,114 +31,127 @@ class PyTorchgeo(PythonPackage):
     variant("style", default=False, description="Install style checking tools")
     variant("tests", default=False, description="Install testing tools")
 
+    # NOTE: historically, dependencies had upper bounds based on semantic version compatibility.
+    # However, these were removed to improve maintainability and flexibility of the recipe.
+
     # Required dependencies
-    depends_on("python@3.7:3+bz2", when="@0.3:", type=("build", "run"))
-    depends_on("python@3.6:3+bz2", when="@:0.2", type=("build", "run"))
-    depends_on("py-setuptools@42:67", when="@0.4.1:", type="build")
-    depends_on("py-setuptools@42:66", when="@0.4.0", type="build")
-    depends_on("py-setuptools@42:65", when="@0.3.1", type="build")
-    depends_on("py-setuptools@42:63", when="@:0.3.0", type="build")
-    depends_on("py-einops@0.3:0.6", when="@0.4:", type=("build", "run"))
-    depends_on("py-einops@0.3:0.4", when="@:0.3", type=("build", "run"))
-    depends_on("py-fiona@1.8:1", when="@0.3:", type=("build", "run"))
-    depends_on("py-fiona@1.5:1", when="@:0.2", type=("build", "run"))
-    depends_on("py-kornia@0.6.5:0.6", when="@0.4.1:", type=("build", "run"))
+    depends_on("python@3.9:", when="@0.5:", type=("build", "run"))
+    # COWC dataset requires unpacking .bz2 files.
+    depends_on("python+bz2", type=("build", "run"))
+    depends_on("py-setuptools@61:", when="@0.5:", type="build")
+    depends_on("py-setuptools@42:", type="build")
+    depends_on("py-einops@0.3:", type=("build", "run"))
+    depends_on("py-fiona@1.8.19:", when="@0.5:", type=("build", "run"))
+    depends_on("py-fiona@1.8:", when="@0.3:", type=("build", "run"))
+    depends_on("py-fiona@1.5:", type=("build", "run"))
+    # Only part of lightning[pytorch-extra] we actually require.
+    depends_on("py-jsonargparse@4.18:+signatures", when="@0.5:", type=("build", "run"))
+    depends_on("py-kornia@0.6.9:", when="@0.5:", type=("build", "run"))
+    depends_on("py-kornia@0.6.5:", when="@0.4.1:", type=("build", "run"))
+    # https://github.com/microsoft/torchgeo/pull/1123
     depends_on("py-kornia@0.6.5:0.6.9", when="@0.4.0", type=("build", "run"))
     depends_on("py-kornia@0.6.4:0.6.9", when="@0.3", type=("build", "run"))
     depends_on("py-kornia@0.5.11:0.6.9", when="@0.2", type=("build", "run"))
     depends_on("py-kornia@0.5.4:0.6.9", when="@0.1", type=("build", "run"))
-    depends_on("py-lightning@1.8:1", when="@0.4.1:", type=("build", "run"))
-    depends_on("py-matplotlib@3.3:3", type=("build", "run"))
-    depends_on("py-numpy@1.17.2:1", type=("build", "run"))
-    depends_on("py-omegaconf@2.1:2", when="@:0.4.0", type=("build", "run"))
-    depends_on("py-packaging@17:21", when="@0.3", type=("build", "run"))
-    depends_on("pil@6.2:9+zlib+jpeg+tiff", type=("build", "run"))
-    depends_on("py-pyproj@2.2:3", type=("build", "run"))
-    depends_on("py-pytorch-lightning@1.5.1:1", when="@0.4.0", type=("build", "run"))
-    depends_on("py-pytorch-lightning@1.5.1:1", when="@0.3.1", type=("build", "run"))
-    depends_on("py-pytorch-lightning@1.5.1:1.8", when="@0.3.0", type=("build", "run"))
-    depends_on("py-pytorch-lightning@1.3:1.8", when="@:0.2", type=("build", "run"))
-    depends_on("py-rasterio@1.0.20:1", when="@0.3:", type=("build", "run"))
-    depends_on("py-rasterio@1.0.16:1", when="@:0.2", type=("build", "run"))
-    depends_on("py-rtree@1", when="@0.3:", type=("build", "run"))
-    depends_on("py-rtree@0.9.4:1", when="@0.2.1", type=("build", "run"))
-    depends_on("py-rtree@0.5:1", when="@:0.2.0", type=("build", "run"))
-    depends_on("py-scikit-learn@0.21:1", when="@0.3:", type=("build", "run"))
-    depends_on("py-scikit-learn@0.18:1", when="@:0.2", type=("build", "run"))
-    depends_on("py-segmentation-models-pytorch@0.2:0.3", when="@0.3.1:", type=("build", "run"))
-    depends_on("py-segmentation-models-pytorch@0.2", when="@:0.3.0", type=("build", "run"))
-    depends_on("py-shapely@1.3:2", when="@0.4:", type=("build", "run"))
-    depends_on("py-shapely@1.3:1", when="@:0.3", type=("build", "run"))
-    depends_on("py-timm@0.4.12:0.6", when="@0.4:", type=("build", "run"))
-    depends_on("py-timm@0.4.12:0.4", when="@:0.3", type=("build", "run"))
-    depends_on("py-torch@1.12:2", when="@0.4.1:", type=("build", "run"))
-    depends_on("py-torch@1.12:1", when="@0.4.0", type=("build", "run"))
-    depends_on("py-torch@1.9:1", when="@0.2:0.3", type=("build", "run"))
-    depends_on("py-torch@1.7:1", when="@0.1", type=("build", "run"))
-    depends_on("py-torchmetrics@0.10:0.11", when="@0.4:", type=("build", "run"))
-    depends_on("py-torchmetrics@0.7:0.9", when="@0.3", type=("build", "run"))
-    depends_on("py-torchmetrics@0.7:0.8", when="@0.2.1", type=("build", "run"))
-    depends_on("py-torchmetrics@0.7", when="@:0.2.0", type=("build", "run"))
-    depends_on("py-torchvision@0.13:0.15", when="@0.4.1:", type=("build", "run"))
-    depends_on("py-torchvision@0.13:0.14", when="@0.4.0", type=("build", "run"))
-    depends_on("py-torchvision@0.10:0.13", when="@0.3", type=("build", "run"))
-    depends_on("py-torchvision@0.10:0.12", when="@0.2", type=("build", "run"))
-    depends_on("py-torchvision@0.3:0.12", when="@0.1", type=("build", "run"))
+    depends_on("py-lightly@1.4.4:", when="@0.5:", type=("build", "run"))
+    depends_on("py-lightning@2:", when="@0.5:", type=("build", "run"))
+    depends_on("py-lightning@1.8:", when="@0.4.1:", type=("build", "run"))
+    depends_on("py-matplotlib@3.3.3:", when="@0.5:", type=("build", "run"))
+    depends_on("py-matplotlib@3.3:", type=("build", "run"))
+    depends_on("py-numpy@1.19.3:", when="@0.5:", type=("build", "run"))
+    depends_on("py-numpy@1.17.2:", type=("build", "run"))
+    depends_on("py-pandas@1.1.3:", when="@0.5:", type=("build", "run"))
+    depends_on("pil@8:", when="@0.5:", type=("build", "run"))
+    depends_on("pil@6.2:", type=("build", "run"))
+    # JPEG, TIFF, and compressed PNG support required for file I/O in several datasets.
+    depends_on("pil+jpeg+tiff+zlib", type=("build", "run"))
+    depends_on("py-pyproj@3:", when="@0.5:", type=("build", "run"))
+    depends_on("py-pyproj@2.2:", type=("build", "run"))
+    depends_on("py-rasterio@1.2:", when="@0.5:", type=("build", "run"))
+    depends_on("py-rasterio@1.0.20:", when="@0.3:", type=("build", "run"))
+    depends_on("py-rasterio@1.0.16:", type=("build", "run"))
+    depends_on("py-rtree@1:", when="@0.3:", type=("build", "run"))
+    depends_on("py-rtree@0.9.4:", when="@0.2.1:", type=("build", "run"))
+    depends_on("py-rtree@0.5:", type=("build", "run"))
+    depends_on("py-segmentation-models-pytorch@0.2:", type=("build", "run"))
+    depends_on("py-shapely@1.7.1:", when="@0.5:", type=("build", "run"))
+    depends_on("py-shapely@1.3:", type=("build", "run"))
+    depends_on("py-timm@0.4.12:", type=("build", "run"))
+    depends_on("py-torch@1.12:", when="@0.4:", type=("build", "run"))
+    depends_on("py-torch@1.9:", when="@0.2:", type=("build", "run"))
+    depends_on("py-torch@1.7:", type=("build", "run"))
+    depends_on("py-torchmetrics@0.10:", when="@0.4:", type=("build", "run"))
+    depends_on("py-torchmetrics@0.7:", type=("build", "run"))
+    depends_on("py-torchvision@0.13:", when="@0.4:", type=("build", "run"))
+    depends_on("py-torchvision@0.10:", when="@0.2:", type=("build", "run"))
+    depends_on("py-torchvision@0.3:", type=("build", "run"))
 
     # Optional dependencies
     with when("+datasets"):
-        depends_on("py-h5py@2.6:3", type="run")
-        depends_on("py-laspy@2", when="@0.2:", type="run")
+        # GDAL and libtiff are both dependencies of rasterio.
+        # Sentinel 2 dataset requires OpenJPEG to read .jp2 files.
         depends_on("gdal+openjpeg", when="@0.3.1:", type="run")
-        depends_on("libtiff+jpeg+zlib", type="run")
-        depends_on("open3d@0.11.2:0.14+python", when="@0.2:0.3", type="run")
-        depends_on("opencv@3.4.2.17:4+python3+imgcodecs+tiff+jpeg+png", type="run")
-        depends_on("py-pandas@0.23.2:2", when="@0.4.1:", type="run")
-        depends_on("py-pandas@0.23.2:1", when="@0.3:0.4.0", type="run")
-        depends_on("py-pandas@0.19.1:1", when="@0.2", type="run")
-        depends_on("py-pycocotools@2", type="run")
-        depends_on("py-pyvista@0.20:0.38", when="@0.4.1:", type="run")
-        depends_on("py-pyvista@0.20:0.37", when="@0.4.0", type="run")
-        depends_on("py-radiant-mlhub@0.3:0.5", when="@0.4.1:", type="run")
+        # JPEG required for GDAL to read JPEG files
+        # LIBDEFLATE, ZLIB, and ZSTD required for compressed file I/O.
+        depends_on("libtiff+jpeg+libdeflate+zlib+zstd", type="run")
+        depends_on("py-h5py@3:", when="@0.5:", type="run")
+        depends_on("py-h5py@2.6:", type="run")
+        depends_on("py-laspy@2:", when="@0.2:", type="run")
+        depends_on("opencv@4.4.0.46:", when="@0.5:", type="run")
+        depends_on("opencv@3.4.2.17:", type="run")
+        # LandCover.ai dataset requires ability to read .tif and write .jpg and .png files.
+        # Doing this from Python requires both imgcodecs and Python bindings.
+        depends_on("opencv+imgcodecs+jpeg+png+python3+tiff", type="run")
+        depends_on("py-pycocotools@2.0.5:", when="@0.5:", type="run")
+        depends_on("py-pycocotools@2:", type="run")
+        depends_on("py-pyvista@0.34.2:", when="@0.5:", type="run")
+        depends_on("py-pyvista@0.20:", when="@0.4:", type="run")
+        depends_on("py-radiant-mlhub@0.3:", when="@0.4.1:", type="run")
         depends_on("py-radiant-mlhub@0.2.1:0.4", when="@:0.4.0", type="run")
-        depends_on("py-rarfile@3:4", type="run")
-        depends_on("py-scikit-image@0.18:0.20", when="@0.4.1:", type="run")
-        depends_on("py-scikit-image@0.18:0.19", when="@0.4.0", type="run")
-        depends_on("py-scipy@1.6.2:1", when="@0.4:", type="run")
-        depends_on("py-scipy@1.2:1", when="@0.3", type="run")
-        depends_on("py-scipy@0.9:1", when="@:0.2", type="run")
-        depends_on("py-zipfile-deflate64@0.2", when="@0.2.1:", type="run")
+        depends_on("py-rarfile@4:", when="@0.5:", type="run")
+        depends_on("py-rarfile@3:", type="run")
+        depends_on("py-scikit-image@0.18:", when="@0.4:", type="run")
+        depends_on("py-scipy@1.6.2:", when="@0.4:", type="run")
+        depends_on("py-scipy@1.2:", when="@0.3:", type="run")
+        depends_on("py-scipy@0.9:", type="run")
+        depends_on("py-zipfile-deflate64@0.2:", when="@0.2.1:", type="run")
 
     with when("+docs"):
-        depends_on("py-ipywidgets@7:8", when="@0.3.1:", type="run")
-        depends_on("py-ipywidgets@7", when="@:0.3.0", type="run")
-        depends_on("py-nbsphinx@0.8.5:0.9", when="@0.4.1:", type="run")
-        depends_on("py-nbsphinx@0.8.5:0.8", when="@:0.4.0", type="run")
+        depends_on("py-ipywidgets@7:", type="run")
+        depends_on("py-nbsphinx@0.8.5:", type="run")
         depends_on("py-pytorch-sphinx-theme", type="run")
         depends_on("py-sphinx@4:5", type="run")
 
     with when("+style"):
-        depends_on("py-black@21.8:23+jupyter", when="@0.4.1:", type="run")
-        depends_on("py-black@21.8:22+jupyter", when="@0.3:0.4.0", type="run")
-        depends_on("py-black@21:22", when="@:0.2", type="run")
-        depends_on("py-flake8@3.8:6", when="@0.4:", type="run")
-        depends_on("py-flake8@3.8:5", when="@0.3.1", type="run")
-        depends_on("py-flake8@3.8:4", when="@:0.3.0", type="run")
-        depends_on("py-isort@5.8:5+colors", type="run")
-        depends_on("py-pydocstyle@6.1:6+toml", type="run")
-        depends_on("py-pyupgrade@1.24:3", when="@0.4:", type="run")
-        depends_on("py-pyupgrade@1.24:2", when="@0.3", type="run")
+        depends_on("py-black@21.8:+jupyter", when="@0.3:", type="run")
+        depends_on("py-black@21:", type="run")
+        depends_on("py-flake8@3.8:", type="run")
+        depends_on("py-isort@5.8:+colors", type="run")
+        depends_on("py-pydocstyle@6.1:+toml", type="run")
+        depends_on("py-pyupgrade@2.8:", when="@0.5:", type="run")
+        depends_on("py-pyupgrade@1.24:", when="@0.3:", type="run")
 
     with when("+tests"):
-        depends_on("py-mypy@0.900:1", when="@0.4.1:", type="run")
-        depends_on("py-mypy@0.900:0.991", when="@0.4.0", type="run")
-        depends_on("py-mypy@0.900:0.971", when="@0.3.1", type="run")
-        depends_on("py-mypy@0.900:0.961", when="@:0.3.0", type="run")
-        depends_on("py-nbmake@1.3.3:1", when="@0.4.1:", type="run")
-        depends_on("py-nbmake@0.1:1", when="@0.3.1:0.4.0", type="run")
+        depends_on("py-mypy@0.900:", type="run")
+        depends_on("py-nbmake@1.3.3:", when="@0.4.1:", type="run")
+        depends_on("py-nbmake@0.1:", when="@0.3.1:", type="run")
         depends_on("py-nbmake@0.1:1.1", when="@:0.3.0", type="run")
-        depends_on("py-omegaconf@2.1:2", when="@0.4.1:", type="run")
-        depends_on("py-pytest@6.1.2:7", type="run")
-        depends_on("py-pytest-cov@2.4:4", when="@0.4:", type="run")
-        depends_on("py-pytest-cov@2.4:3", when="@:0.3", type="run")
-        depends_on("py-tensorboard@2.9.1:2", when="@0.4.1:", type="run")
+        depends_on("py-pytest@6.2:", when="@0.5:", type="run")
+        depends_on("py-pytest@6.1.2:", type="run")
+        depends_on("py-pytest-cov@2.4:", type="run")
+
+    # Historical dependencies
+    depends_on("py-omegaconf@2.1:", when="@:0.4.0", type=("build", "run"))
+    depends_on("py-packaging@17:", when="@0.3", type=("build", "run"))
+    depends_on("py-pytorch-lightning@1.5.1:", when="@0.3.1:0.4.0", type=("build", "run"))
+    # https://github.com/microsoft/torchgeo/pull/697
+    depends_on("py-pytorch-lightning@1.5.1:1.8", when="@0.3.0", type=("build", "run"))
+    depends_on("py-pytorch-lightning@1.3:1.8", when="@:0.2", type=("build", "run"))
+    depends_on("py-scikit-learn@0.21:", when="@0.3:0.4", type=("build", "run"))
+    depends_on("py-scikit-learn@0.18:", when="@:0.2", type=("build", "run"))
+    depends_on("open3d@0.11.2:+python", when="@0.2:0.3+datasets", type="run")
+    # https://github.com/microsoft/torchgeo/pull/1537
+    depends_on("py-pandas@0.23.2:2.0", when="@0.3:0.4+datasets", type="run")
+    depends_on("py-pandas@0.19.1:2.0", when="@0.2+datasets", type="run")
+    depends_on("py-omegaconf@2.1:", when="@0.4.1+tests", type="run")
+    depends_on("py-tensorboard@2.9.1:", when="@0.4.1+tests", type="run")
diff --git a/var/spack/repos/builtin/packages/py-torchmetrics/package.py b/var/spack/repos/builtin/packages/py-torchmetrics/package.py
index 042b93aa8db1f2..7b5d0672461fac 100644
--- a/var/spack/repos/builtin/packages/py-torchmetrics/package.py
+++ b/var/spack/repos/builtin/packages/py-torchmetrics/package.py
@@ -14,6 +14,11 @@ class PyTorchmetrics(PythonPackage):
 
     maintainers("adamjstewart")
 
+    version("1.2.0", sha256="7eb28340bde45e13187a9ad54a4a7010a50417815d8181a5df6131f116ffe1b7")
+    version("1.1.1", sha256="65ea34205c0506eecfd06b98f63f4d2a2c5c0e17367cf324e1747adc854c80a5")
+    version("1.1.0", sha256="94b51aeb3d5ff55503fa47086bbc2af9e26efabb840e2d3c2381db9623dda5fd")
+    version("1.0.3", sha256="1c20ea2f0db434334e88da6c015ddf936d43379bfb403e9dc2a7272b0eab453c")
+    version("1.0.2", sha256="537989d02337814e621a45232eeb1eacfd4700a66c2b5161d19ca2158246e075")
     version("0.11.4", sha256="1fe45a14b44dd65d90199017dd5a4b5a128d56a8a311da7916c402c18c671494")
     version("0.11.3", sha256="6a2bcc17361f0e4c1668c92595b12ef30ccf9ef1d03263bee7c6136a882afe30")
     version("0.11.2", sha256="5a1f5fff9b1fb695bbcab6442d768e2e9b9535d00f4b4dea9ce03d40c866be07")
@@ -32,11 +37,20 @@ class PyTorchmetrics(PythonPackage):
     version("0.3.1", sha256="78f4057db53f7c219fdf9ec9eed151adad18dd43488a44e5c780806d218e3f1d")
     version("0.2.0", sha256="481a28759acd2d77cc088acba6bc7dc4a356c7cb767da2e1495e91e612e2d548")
 
+    # setup.py
+    depends_on("python@3.8:", when="@1:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
+
+    # requirements.txt (upper bound is removed during processing)
+    depends_on("py-numpy@1.20.1:", when="@1:", type=("build", "run"))
     depends_on("py-numpy@1.17.2:", when="@0.4:", type=("build", "run"))
     depends_on("py-numpy", when="@0.3:", type=("build", "run"))
     depends_on("py-torch@1.8.1:", when="@0.11:", type=("build", "run"))
     depends_on("py-torch@1.3.1:", type=("build", "run"))
-    depends_on("py-pydeprecate@0.3", when="@0.7:0.8", type=("build", "run"))
-    depends_on("py-packaging", when="@0.3:", type=("build", "run"))
     depends_on("py-typing-extensions", when="@0.9: ^python@:3.8", type=("build", "run"))
+    depends_on("py-lightning-utilities@0.8:", when="@1.1:", type=("build", "run"))
+    depends_on("py-lightning-utilities@0.7:", when="@1:", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-packaging", when="@0.3:1.1", type=("build", "run"))
+    depends_on("py-pydeprecate@0.3", when="@0.7:0.8", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-torchtext/package.py b/var/spack/repos/builtin/packages/py-torchtext/package.py
index ff02c597c4d590..180b555069bc5e 100644
--- a/var/spack/repos/builtin/packages/py-torchtext/package.py
+++ b/var/spack/repos/builtin/packages/py-torchtext/package.py
@@ -8,37 +8,37 @@
 
 
 class PyTorchtext(PythonPackage):
-    """Text utilities and datasets for PyTorch."""
+    """Text utilities, models, transforms, and datasets for PyTorch."""
 
     homepage = "https://github.com/pytorch/text"
     git = "https://github.com/pytorch/text.git"
+    submodules = True
 
     maintainers("adamjstewart")
 
-    version("main", branch="main", submodules=True)
-    version("0.15.2", tag="v0.15.2", submodules=True)
-    version("0.15.1", tag="v0.15.1", submodules=True)
-    version("0.14.1", tag="v0.14.1", submodules=True)
-    version("0.14.0", tag="v0.14.0", submodules=True)
-    version("0.13.1", tag="v0.13.1", submodules=True)
-    version("0.13.0", tag="v0.13.0", submodules=True)
-    version("0.12.0", tag="v0.12.0", submodules=True)
-    version("0.11.2", tag="v0.11.2", submodules=True)
-    version("0.11.1", tag="v0.11.1", submodules=True)
-    version("0.10.1", tag="v0.10.1", submodules=True)
-    version("0.10.0", tag="v0.10.0", submodules=True)
-    version("0.9.2", tag="v0.9.2", submodules=True)
-    version("0.8.1", tag="v0.8.1", submodules=True)
-    version("0.6.0", tag="0.6.0", submodules=True)
-    version("0.5.0", tag="0.5.0", submodules=True)
+    version("main", branch="main")
+    version("0.16.0", tag="v0.16.0", commit="4e255c95c76b1ccde4f6650391c0bc30650d6dbe")
+    version("0.15.2", tag="v0.15.2", commit="4571036cf66c539e50625218aeb99a288d79f3e1")
+    version("0.15.1", tag="v0.15.1", commit="c696895e524c61fd2b8b26916dd006411c5f3ba5")
+    version("0.14.1", tag="v0.14.1", commit="e1e969d4947bb3dd01ea927af2f8ac9a2d778c39")
+    version("0.14.0", tag="v0.14.0", commit="e2b27f9b06ca71d55c2fcf6d47c60866ee936f40")
+    version("0.13.1", tag="v0.13.1", commit="330201f1132dcd0981180c19bc6843a19d310ff0")
+    version("0.13.0", tag="v0.13.0", commit="35298c43f3ce908fe06c177ecbd8ef1503a1292b")
+    version("0.12.0", tag="v0.12.0", commit="d7a34d6ae0f4e36a52777854d0163b9e85f1576b")
+    version("0.11.2", tag="v0.11.2", commit="92f4d158d8cbe9136896befa2d4234ea8b8e2795")
+    version("0.11.1", tag="v0.11.1", commit="5c65ec05d7c1eba5b0ea2d7ee170ccf977d9674f")
+    version("0.10.1", tag="v0.10.1", commit="0d670e03c1eee7e30e032bb96df4c12b785a15ff")
+    version("0.10.0", tag="v0.10.0", commit="4da1de36247aa06622088e78508e0e38a4392e38")
+    version("0.9.2", tag="v0.9.2", commit="22e5ee7548a85190eee78e8ed6c8911ec2c53035")
+    version("0.8.1", tag="v0.8.1", commit="0f911ec35ab020983efbf36b8c14415651e98618")
+    version("0.6.0", tag="0.6.0", commit="3a54c7f52584f201c17ca7489b52b812152612dc")
+    version("0.5.0", tag="0.5.0", commit="0169cde2f1d446ae886ef0be07e9a673585ed256")
 
     # https://github.com/pytorch/text#installation
-    depends_on("python@3.8:3.11", when="@2:", type=("build", "link", "run"))
-    depends_on("python@3.7:3.10", when="@0.13:1", type=("build", "link", "run"))
-    depends_on("python@3.6:3.9", when="@0.8.1:0.12", type=("build", "link", "run"))
-    depends_on("python@3.6:3.8", when="@0.7:0.8.0", type=("build", "link", "run"))
-    depends_on("python@3.5:3.8", when="@0.6", type=("build", "link", "run"))
-    depends_on("python@2.7,3.5:3.8", when="@:0.5", type=("build", "link", "run"))
+    depends_on("python@3.8:3.11", when="@0.15:", type=("build", "link", "run"))
+    depends_on("python@:3.10", when="@0.13:0.14", type=("build", "link", "run"))
+    depends_on("python@:3.9", when="@0.8.1:0.12", type=("build", "link", "run"))
+    depends_on("python@:3.8", when="@:0.8.0", type=("build", "link", "run"))
 
     # CMakelists.txt
     depends_on("cmake@3.18:", when="@0.13:", type="build")
@@ -49,13 +49,16 @@ class PyTorchtext(PythonPackage):
     depends_on("py-tqdm", type=("build", "run"))
     depends_on("py-requests", type=("build", "run"))
     depends_on("py-numpy", type=("build", "run"))
-    depends_on("py-torchdata@0.6:", when="@0.15:", type=("build", "run"))
+    depends_on("py-torchdata@0.7.0", when="@0.16.0", type=("build", "run"))
+    depends_on("py-torchdata@0.6.1", when="@0.15.2", type=("build", "run"))
+    depends_on("py-torchdata@0.6.0", when="@0.15.1", type=("build", "run"))
     depends_on("py-pybind11", when="@0.8:", type=("build", "link"))
     depends_on("py-six", when="@:0.6", type=("build", "run"))
     depends_on("py-sentencepiece", when="@:0.7", type=("build", "run"))
 
     # https://github.com/pytorch/text#installation
-    depends_on("py-torch@master", when="@main", type=("build", "link", "run"))
+    depends_on("py-torch@main", when="@main", type=("build", "link", "run"))
+    depends_on("py-torch@2.1.0", when="@0.16.0", type=("build", "link", "run"))
     depends_on("py-torch@2.0.1", when="@0.15.2", type=("build", "link", "run"))
     depends_on("py-torch@2.0.0", when="@0.15.1", type=("build", "link", "run"))
     depends_on("py-torch@1.13.1", when="@0.14.1", type=("build", "link", "run"))
diff --git a/var/spack/repos/builtin/packages/py-torchvision/package.py b/var/spack/repos/builtin/packages/py-torchvision/package.py
index c49f35d9876f77..5aef4c6aef8a29 100644
--- a/var/spack/repos/builtin/packages/py-torchvision/package.py
+++ b/var/spack/repos/builtin/packages/py-torchvision/package.py
@@ -8,8 +8,7 @@
 
 
 class PyTorchvision(PythonPackage):
-    """The torchvision package consists of popular datasets, model
-    architectures, and common image transformations for computer vision."""
+    """Image and video datasets and models for torch deep learning."""
 
     homepage = "https://github.com/pytorch/vision"
     url = "https://github.com/pytorch/vision/archive/v0.8.2.tar.gz"
@@ -18,6 +17,7 @@ class PyTorchvision(PythonPackage):
     maintainers("adamjstewart")
 
     version("main", branch="main")
+    version("0.16.0", sha256="79b30b082237e3ead21e74587cedf4a4d832f977cf7dfeccfb65f67988b12ceb")
     version("0.15.2", sha256="1efcb80e0a6e42c54f07ee16167839b4d302aeeecc12839cc47c74b06a2c20d4")
     version("0.15.1", sha256="689d23d4ebb0c7e54e8651c89b17155b64341c14ae4444a04ca7dc6f2b6a0a43")
     version("0.14.1", sha256="ced67e1cf1f97e168cdf271851a4d0b6d382ab7936e7bcbb39aaa87239c324b6")
@@ -46,38 +46,23 @@ class PyTorchvision(PythonPackage):
     version("0.4.0", sha256="c270d74e568bad4559fed4544f6dd1e22e2eb1c60b088e04a5bd5787c4150589")
     version("0.3.0", sha256="c205f0618c268c6ed2f8abb869ef6eb83e5339c1336c243ad321a2f2a85195f0")
 
-    # https://github.com/pytorch/vision#image-backend
-    variant(
-        "backend",
-        default="pil",
-        description="Image backend",
-        values=[
-            "pil",
-            "accimage",
-            conditional("png", when="@0.8:"),
-            conditional("jpeg", when="@0.8:"),
-        ],
-        multi=False,
-    )
+    desc = "Enable support for native encoding/decoding of {} formats in torchvision.io"
+    variant("png", default=False, description=desc.format("PNG"))
+    variant("jpeg", default=False, description=desc.format("JPEG"))
+    variant("nvjpeg", default=False, description=desc.format("JPEG"))
+    variant("ffmpeg", default=False, description=desc.format("FFMPEG"))
+    variant("video_codec", default=False, description=desc.format("video_codec"))
 
     # https://github.com/pytorch/vision#installation
     depends_on("python@3.8:3.11", when="@0.15:", type=("build", "link", "run"))
-    depends_on("python@3.7:3.10", when="@0.12:0.14", type=("build", "link", "run"))
-    depends_on("python@3.6:3.9", when="@0.8.2:0.11", type=("build", "link", "run"))
-    depends_on("python@3.6:3.8", when="@0.7:0.8.1", type=("build", "link", "run"))
-    depends_on("python@3.5:3.8", when="@0.6", type=("build", "link", "run"))
-    depends_on("python@2.7,3.5:3.8", when="@0.5", type=("build", "link", "run"))
-    depends_on("python@2.7,3.5:3.7", when="@:0.4", type=("build", "link", "run"))
-
-    depends_on("py-setuptools", type="build")
-    depends_on("ninja", type="build")
-    depends_on("py-typing-extensions", when="@0.12:", type=("build", "run"))
-    depends_on("py-numpy", type=("build", "run"))
-    depends_on("py-requests", when="@0.12:", type=("build", "run"))
-    depends_on("py-six", when="@:0.5", type=("build", "run"))
+    depends_on("python@:3.10", when="@0.12:0.14", type=("build", "link", "run"))
+    depends_on("python@:3.9", when="@0.8.2:0.11", type=("build", "link", "run"))
+    depends_on("python@:3.8", when="@0.5:0.8.1", type=("build", "link", "run"))
+    depends_on("python@:3.7", when="@:0.4", type=("build", "link", "run"))
 
     # https://github.com/pytorch/vision#installation
-    depends_on("py-torch@master", when="@main", type=("build", "link", "run"))
+    depends_on("py-torch@main", when="@main", type=("build", "link", "run"))
+    depends_on("py-torch@2.1.0", when="@0.16.0", type=("build", "link", "run"))
     depends_on("py-torch@2.0.1", when="@0.15.2", type=("build", "link", "run"))
     depends_on("py-torch@2.0.0", when="@0.15.1", type=("build", "link", "run"))
     depends_on("py-torch@1.13.1", when="@0.14.1", type=("build", "link", "run"))
@@ -107,54 +92,84 @@ class PyTorchvision(PythonPackage):
     depends_on("py-torch@1.1.0", when="@0.3.0", type=("build", "link", "run"))
     depends_on("py-torch@:1.0.1", when="@0.2.2", type=("build", "link", "run"))
 
+    depends_on("ninja", type="build")
+
+    # setup.py
+    depends_on("py-setuptools", type="build")
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-requests", when="@0.12:", type=("build", "run"))
+    depends_on("pil@5.3:", when="@0.10:", type=("build", "run"))
+    depends_on("pil@4.1.1:", type=("build", "run"))
+
+    # Extensions
+    depends_on("libpng@1.6:", when="+png")
+    depends_on("jpeg", when="+jpeg")
+    depends_on("cuda", when="+nvjpeg")
+    depends_on("ffmpeg@3.1:", when="+ffmpeg")
+    depends_on("cuda", when="+video_codec")
+
+    # Historical dependencies
+    depends_on("py-typing-extensions", when="@0.12:0.14", type=("build", "run"))
+    depends_on("py-six", when="@:0.5", type=("build", "run"))
+
     # https://github.com/pytorch/vision/pull/5898
-    depends_on("pil@5.3:8.2,8.4:", when="@0.13: backend=pil", type=("build", "run"))
+    conflicts("^pil@10:", when="@:0.12")
+    # https://github.com/pytorch/vision/issues/1712
+    conflicts("^pil@7:", when="@:0.4")
     # https://github.com/pytorch/vision/issues/4146
     # https://github.com/pytorch/vision/issues/4934
-    depends_on("pil@5.3:8.2,8.4:9", when="@0.10:0.12 backend=pil", type=("build", "run"))
-    depends_on("pil@4.1.1:9", when="@0.5: backend=pil", type=("build", "run"))
-    # https://github.com/pytorch/vision/issues/1712
-    depends_on("pil@4.1.1:6", when="@:0.4 backend=pil", type=("build", "run"))
-    depends_on("py-accimage", when="backend=accimage", type=("build", "run"))
-    depends_on("libpng@1.6.0:", when="backend=png")
-    depends_on("jpeg")  # seems to be required for all backends
+    conflicts("^pil@8.3")
+    # https://github.com/pytorch/pytorch/issues/65000
+    conflicts("+ffmpeg", when="platform=darwin")
+    # https://github.com/pytorch/vision/issues/3367
+    conflicts("+ffmpeg", when="^python@3.9")
     # https://github.com/pytorch/vision/pull/7378
-    depends_on("ffmpeg@3.1:5", when="@0.13:")
-    depends_on("ffmpeg@3.1:4.4", when="@0.4.2:0.12")
+    conflicts("^ffmpeg@6:")
+    # https://github.com/pytorch/vision/issues/5616
+    # https://github.com/pytorch/vision/pull/5644
+    conflicts("^ffmpeg@5:", when="@:0.12")
 
     # Many of the datasets require additional dependencies to use.
     # These can be installed after the fact.
 
     def setup_build_environment(self, env):
-        include = []
-        library = []
-        for dep in self.spec.dependencies(deptype="link"):
-            query = self.spec[dep.name]
-            include.extend(query.headers.directories)
-            library.extend(query.libs.directories)
-
-        # README says to use TORCHVISION_INCLUDE and TORCHVISION_LIBRARY,
-        # but these do not work for older releases. Build uses a mix of
-        # Spack's compiler wrapper and the actual compiler, so this is
-        # needed to get parts of the build working.
-        # See https://github.com/pytorch/vision/issues/2591
-        env.set("TORCHVISION_INCLUDE", ":".join(include))
-        env.set("TORCHVISION_LIBRARY", ":".join(library))
-        env.set("CPATH", ":".join(include))
-        env.set("LIBRARY_PATH", ":".join(library))
+        # The only documentation on building is what is found in setup.py and:
+        # https://github.com/pytorch/vision/blob/main/CONTRIBUTING.md#development-installation
 
         # By default, version is read from `version.txt`, but this includes an `a0`
         # suffix used for alpha builds. Override the version for stable releases.
         if not self.spec.satisfies("@main"):
             env.set("BUILD_VERSION", self.version)
 
-        if "+cuda" in self.spec["py-torch"]:
-            env.set("FORCE_CUDA", 1)
+        # Used by ninja
+        env.set("MAX_JOBS", make_jobs)
+
+        if "^cuda" in self.spec:
             env.set("CUDA_HOME", self.spec["cuda"].prefix)
             torch_cuda_arch_list = ";".join(
                 "{0:.1f}".format(float(i) / 10.0)
                 for i in self.spec["py-torch"].variants["cuda_arch"].value
             )
             env.set("TORCH_CUDA_ARCH_LIST", torch_cuda_arch_list)
-        else:
-            env.set("FORCE_CUDA", 0)
+
+        for gpu in ["cuda", "mps"]:
+            env.set(f"FORCE_{gpu.upper()}", int(f"+{gpu}" in self.spec["py-torch"]))
+
+        for extension in ["png", "jpeg", "nvjpeg", "ffmpeg", "video_codec"]:
+            env.set(f"TORCHVISION_USE_{extension.upper()}", int(f"+{extension}" in self.spec))
+
+        include = []
+        library = []
+        for dep in self.spec.dependencies(deptype="link"):
+            query = self.spec[dep.name]
+            include.extend(query.headers.directories)
+            library.extend(query.libs.directories)
+
+        # CONTRIBUTING.md says to use TORCHVISION_INCLUDE and TORCHVISION_LIBRARY, but
+        # these do not work for older releases. Build uses a mix of Spack's compiler wrapper
+        # and the actual compiler, so this is needed to get parts of the build working.
+        # See https://github.com/pytorch/vision/issues/2591
+        env.set("TORCHVISION_INCLUDE", ":".join(include))
+        env.set("TORCHVISION_LIBRARY", ":".join(library))
+        env.set("CPATH", ":".join(include))
+        env.set("LIBRARY_PATH", ":".join(library))
diff --git a/var/spack/repos/builtin/packages/py-tornado/package.py b/var/spack/repos/builtin/packages/py-tornado/package.py
index 6a98594f310fb2..b93b2f03318e5b 100644
--- a/var/spack/repos/builtin/packages/py-tornado/package.py
+++ b/var/spack/repos/builtin/packages/py-tornado/package.py
@@ -13,14 +13,12 @@ class PyTornado(PythonPackage):
     homepage = "https://github.com/tornadoweb/tornado"
     pypi = "tornado/tornado-6.1.tar.gz"
 
+    version("6.3.3", sha256="e7d8db41c0181c80d76c982aacc442c0783a2c54d6400fe028954201a2e032fe")
     version("6.2", sha256="9b630419bde84ec666bfd7ea0a4cb2a8a651c2d5cccdbdd1972a0c859dfc3c13")
     version("6.1", sha256="33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791")
     version("6.0.3", sha256="c845db36ba616912074c5b1ee897f8e0124df269468f25e4fe21fe72f6edd7a9")
     version("5.1.1", sha256="4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409")
     version("4.4", sha256="3176545b6cb2966870db4def4f646da6ab7a0c19400576969c57279a7561ab02")
 
+    depends_on("python@3.8:", when="@6.3:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
-
-    depends_on("python@3.7:", when="@6.2:", type=("build", "run"))
-    depends_on("python@3.5.2:", when="@6:", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.4:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-tqdm/package.py b/var/spack/repos/builtin/packages/py-tqdm/package.py
index 1b1e35c7474bf5..61801b5d918940 100644
--- a/var/spack/repos/builtin/packages/py-tqdm/package.py
+++ b/var/spack/repos/builtin/packages/py-tqdm/package.py
@@ -12,6 +12,7 @@ class PyTqdm(PythonPackage):
     homepage = "https://github.com/tqdm/tqdm"
     pypi = "tqdm/tqdm-4.45.0.tar.gz"
 
+    version("4.66.1", sha256="d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7")
     version("4.65.0", sha256="1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5")
     version("4.64.1", sha256="5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4")
     version("4.64.0", sha256="40be55d30e200777a307a7585aee69e4eabb46b4ec6a4b4a5f2d9f11e7d5408d")
@@ -30,5 +31,6 @@ class PyTqdm(PythonPackage):
     depends_on("py-setuptools@42:", type=("build", "run"))
     depends_on("py-setuptools-scm@3.4:+toml", type="build")
     depends_on("py-colorama", when="platform=windows", type=("build", "run"))
+
     depends_on("py-requests", when="+telegram", type=("build", "run"))
     depends_on("py-ipywidgets@6:", when="+notebook", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-traits/package.py b/var/spack/repos/builtin/packages/py-traits/package.py
index a31f360eabfac7..665e7498568919 100644
--- a/var/spack/repos/builtin/packages/py-traits/package.py
+++ b/var/spack/repos/builtin/packages/py-traits/package.py
@@ -14,11 +14,10 @@ class PyTraits(PythonPackage):
     pypi = "traits/traits-6.0.0.tar.gz"
     git = "https://github.com/enthought/traits.git"
 
+    version("6.4.2", sha256="5be7cc5fb7a99cba7e9014786373e3ad2f75efb445eeced094654bbaf3b0fa82")
     version("6.4.1", sha256="78bb2ccafd60aff606515aac46de64668a0a81cb5c54c650b9877a841aa9e812")
     version("6.3.1", sha256="ebdd9b067a262045840a85e3ff34e1567ce4e9b6548c716cdcc82b5884ed9100")
     version("6.2.0", sha256="16fa1518b0778fd53bf0547e6a562b1787bf68c8f6b7995a13bd1902529fdb0c")
     version("6.0.0", sha256="dbcd70166feca434130a1193284d5819ca72ffbc8dbce8deeecc0cebb41a3bfb")
 
-    depends_on("python@3.6:", type=("build", "run"), when="@6.2.0:")
-    depends_on("python@3.5:", type=("build", "run"), when="@:6.1")
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-transformers/package.py b/var/spack/repos/builtin/packages/py-transformers/package.py
index f89a9ec9d40264..5380f44a78684d 100644
--- a/var/spack/repos/builtin/packages/py-transformers/package.py
+++ b/var/spack/repos/builtin/packages/py-transformers/package.py
@@ -16,15 +16,18 @@ class PyTransformers(PythonPackage):
 
     maintainers("adamjstewart")
 
+    version("4.31.0", sha256="4302fba920a1c24d3a429a29efff6a63eac03f3f3cf55b55927fc795d01cb273")
     version("4.24.0", sha256="486f353a8e594002e48be0e2aba723d96eda839e63bfe274702a4b5eda85559b")
     version("4.6.1", sha256="83dbff763b7e7dc57cbef1a6b849655d4fcab6bffdd955c5e8bea12a4f76dc10")
     version("2.8.0", sha256="b9f29cdfd39c28f29e0806c321270dea337d6174a7aa60daf9625bf83dbb12ee")
 
+    depends_on("python@3.8:", when="@4.31:", type=("build", "run"))
     depends_on("python@3.7:", when="@4.24:", type=("build", "run"))
     depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("py-importlib-metadata", when="@4.6: ^python@:3.7", type=("build", "run"))
     depends_on("py-filelock", type=("build", "run"))
+    depends_on("py-huggingface-hub@0.14.1:0", when="@4.26:", type=("build", "run"))
     depends_on("py-huggingface-hub@0.10:0", when="@4.24:", type=("build", "run"))
     depends_on("py-huggingface-hub@0.0.8", when="@4.6.1", type=("build", "run"))
     depends_on("py-numpy@1.17:", when="@4.6:", type=("build", "run"))
@@ -34,6 +37,7 @@ class PyTransformers(PythonPackage):
     depends_on("py-pyyaml@5.1:", when="@4.24:", type=("build", "run"))
     depends_on("py-regex@:2019.12.16,2019.12.18:", type=("build", "run"))
     depends_on("py-requests", type=("build", "run"))
+    depends_on("py-safetensors@0.3.1:", when="@4.31:", type=("build", "run"))
     depends_on("py-tokenizers@0.11.1:0.11.2,0.11.4:0.13", when="@4.24:", type=("build", "run"))
     depends_on("py-tokenizers@0.10.1:0.10", when="@4.6.1", type=("build", "run"))
     depends_on("py-tokenizers@0.5.2", when="@2.8.0", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-trove-classifiers/package.py b/var/spack/repos/builtin/packages/py-trove-classifiers/package.py
index b7f22692dc0457..06fbfb7cda5bd3 100644
--- a/var/spack/repos/builtin/packages/py-trove-classifiers/package.py
+++ b/var/spack/repos/builtin/packages/py-trove-classifiers/package.py
@@ -13,6 +13,7 @@ class PyTroveClassifiers(PythonPackage):
     homepage = "https://github.com/pypa/trove-classifiers"
     pypi = "trove-classifiers/trove-classifiers-2023.3.9.tar.gz"
 
+    version("2023.8.7", sha256="c9f2a0a85d545e5362e967e4f069f56fddfd91215e22ffa48c66fb283521319a")
     version("2023.3.9", sha256="ee42f2f8c1d4bcfe35f746e472f07633570d485fab45407effc0379270a3bb03")
 
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-tuiview/package.py b/var/spack/repos/builtin/packages/py-tuiview/package.py
index 7824f834e704d3..40c94a6fe1394e 100644
--- a/var/spack/repos/builtin/packages/py-tuiview/package.py
+++ b/var/spack/repos/builtin/packages/py-tuiview/package.py
@@ -12,14 +12,28 @@ class PyTuiview(PythonPackage):
     """
 
     homepage = "https://github.com/ubarsc/tuiview"
-    url = "https://github.com/ubarsc/tuiview/releases/download/tuiview-1.2.6/tuiview-1.2.6.tar.gz"
+    url = (
+        "https://github.com/ubarsc/tuiview/releases/download/tuiview-1.2.13/TuiView-1.2.13.tar.gz"
+    )
 
+    version("1.2.13", sha256="48c8d4175c324f70941dc49c5a119882c9d501bd20bc13c76bc2455dee5236a5")
+    version("1.2.12", sha256="3f0c1673f2f861db01726f3d7f6f1dde4a42ec57894a79b89457c398768dd25f")
+    version("1.2.11", sha256="81f870ad98ec1e3175f25028d261135b6198fa85038bfaa900789e04e3cf8517")
+    version("1.2.10", sha256="5ea777a4e89780488b03b346f00b586b46a0bd4c8a994e6def46a6494fa486ef")
+    version("1.2.9", sha256="b5d11e9501cf61cf62f1223416dfe408cf604ae48c06d697589dfc0a606ad6a9")
+    version("1.2.8", sha256="e75950908a2d1f7c7216dfeead82483e1d3b0267fff9561549d85ca00725456b")
+    version("1.2.7", sha256="35dfeb79b2bb57dfb5b8c90c3edf8c8a0a3f89cef85c33f9935e4a4add282aaf")
     version("1.2.6", sha256="61b136fa31c949d7a7a4dbf8562e6fc677d5b1845b152ec39e337f4eb2e91662")
-    version("1.1.7", sha256="fbf0bf29cc775357dad4f8a2f0c2ffa98bbf69d603a96353e75b321adef67573")
+    version(
+        "1.1.7",
+        sha256="fbf0bf29cc775357dad4f8a2f0c2ffa98bbf69d603a96353e75b321adef67573",
+        deprecated=True,
+    )
 
     # pip silently replaces distutils with setuptools
     depends_on("py-setuptools", type="build")
     depends_on("py-pyqt4", type=("build", "run"), when="@:1.1")
     depends_on("py-pyqt5", type=("build", "run"), when="@1.2.0:")
-    depends_on("py-numpy", type=("build", "run"))
-    depends_on("gdal@1.11.0:+python")
+    depends_on("py-numpy", type=("build", "link", "run"))
+    depends_on("gdal+geos+python", type=("build", "run"), when="@1.2.0:")
+    depends_on("gdal@1.11.0:+python", when="@:1.1")
diff --git a/var/spack/repos/builtin/packages/py-twine/package.py b/var/spack/repos/builtin/packages/py-twine/package.py
index a5f2f9d3f7883c..d08b4e8601ebb4 100644
--- a/var/spack/repos/builtin/packages/py-twine/package.py
+++ b/var/spack/repos/builtin/packages/py-twine/package.py
@@ -13,11 +13,10 @@ class PyTwine(PythonPackage):
     pypi = "twine/twine-2.0.0.tar.gz"
     git = "https://github.com/pypa/twine.git"
 
+    version("4.0.2", sha256="9e102ef5fdd5a20661eb88fad46338806c3bd32cf1db729603fe3697b1bc83c8")
     version("4.0.1", sha256="96b1cf12f7ae611a4a40b6ae8e9570215daff0611828f5fe1f37a16255ab24a0")
     version("2.0.0", sha256="9fe7091715c7576df166df8ef6654e61bada39571783f2fd415bdcba867c6993")
 
-    depends_on("python@3.7:", when="@4:", type=("build", "run"))
-    depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-setuptools@45:", when="@3.4.1:", type="build")
     depends_on("py-setuptools@0.7.0:", type="build")
     depends_on("py-setuptools-scm+toml@6:", when="@3.4.2:", type="build")
@@ -33,4 +32,6 @@ class PyTwine(PythonPackage):
     depends_on("py-keyring@15.1:", when="@3:", type=("build", "run"))
     depends_on("py-rfc3986@1.4:", when="@3.2:", type=("build", "run"))
     depends_on("py-rich@12:", when="@4:", type=("build", "run"))
+
+    # Historical Dependencies
     depends_on("py-tqdm@4.14:", when="@:3", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-types-psutil/package.py b/var/spack/repos/builtin/packages/py-types-psutil/package.py
index 4800f3ca17097d..f6b06254f926a4 100644
--- a/var/spack/repos/builtin/packages/py-types-psutil/package.py
+++ b/var/spack/repos/builtin/packages/py-types-psutil/package.py
@@ -12,6 +12,7 @@ class PyTypesPsutil(PythonPackage):
     homepage = "https://github.com/python/typeshed"
     pypi = "types-psutil/types-psutil-5.9.5.5.tar.gz"
 
+    version("5.9.5.16", sha256="4e9b219efb625d3d04f6bf106934f87cab49aa41a94b0a3b3089403f47a79228")
     version("5.9.5.5", sha256="4f26fdb2cb064b274cbc6359fba4abf3b3a2993d7d4abc336ad0947568212c62")
 
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-types-python-dateutil/package.py b/var/spack/repos/builtin/packages/py-types-python-dateutil/package.py
index a76ef43fff9b72..62a878e4153b88 100644
--- a/var/spack/repos/builtin/packages/py-types-python-dateutil/package.py
+++ b/var/spack/repos/builtin/packages/py-types-python-dateutil/package.py
@@ -12,7 +12,7 @@ class PyTypesPythonDateutil(PythonPackage):
     homepage = "https://github.com/python/typeshed"
     pypi = "types-python-dateutil/types-python-dateutil-2.8.19.tar.gz"
 
+    version("2.8.19.14", sha256="1f4f10ac98bb8b16ade9dbee3518d9ace017821d94b057a425b069f834737f4b")
     version("2.8.19", sha256="bfd3eb39c7253aea4ba23b10f69b017d30b013662bb4be4ab48b20bbd763f309")
 
-    depends_on("python@3.7:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-types-pytz/package.py b/var/spack/repos/builtin/packages/py-types-pytz/package.py
new file mode 100644
index 00000000000000..58f0d9c37ee8df
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-types-pytz/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyTypesPytz(PythonPackage):
+    """This is a PEP 561 type stub package for the pytz package. It can be used
+    by type-checking tools like mypy, pyright, pytype, PyCharm, etc. to check
+    code that uses pytz."""
+
+    homepage = "https://github.com/python/typeshed"
+    pypi = "types-pytz/types-pytz-2023.3.0.0.tar.gz"
+
+    version(
+        "2023.3.0.0", sha256="ecdc70d543aaf3616a7e48631543a884f74205f284cefd6649ddf44c6a820aac"
+    )
+
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-types-requests/package.py b/var/spack/repos/builtin/packages/py-types-requests/package.py
index 4c6e79bc48bc1e..17db94f04ed33f 100644
--- a/var/spack/repos/builtin/packages/py-types-requests/package.py
+++ b/var/spack/repos/builtin/packages/py-types-requests/package.py
@@ -12,9 +12,10 @@ class PyTypesRequests(PythonPackage):
     homepage = "https://github.com/python/typeshed"
     pypi = "types-requests/types-requests-2.28.10.tar.gz"
 
+    version("2.31.0.2", sha256="6aa3f7faf0ea52d728bb18c0a0d1522d9bfd8c72d26ff6f61bfc3d06a411cf40")
     version("2.28.10", sha256="97d8f40aa1ffe1e58c3726c77d63c182daea9a72d9f1fa2cafdea756b2a19f2c")
 
-    depends_on("python@3.7:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
 
-    depends_on("py-types-urllib3@:1.26", type=("build", "run"))
+    depends_on("py-types-urllib3", type=("build", "run"))
+    depends_on("py-types-urllib3@:1.26", when="@:2.29", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-types-setuptools/package.py b/var/spack/repos/builtin/packages/py-types-setuptools/package.py
index b4e2b063423888..60eef117873b92 100644
--- a/var/spack/repos/builtin/packages/py-types-setuptools/package.py
+++ b/var/spack/repos/builtin/packages/py-types-setuptools/package.py
@@ -12,6 +12,7 @@ class PyTypesSetuptools(PythonPackage):
     homepage = "https://github.com/python/typeshed"
     pypi = "types-setuptools/types-setuptools-65.5.0.3.tar.gz"
 
+    version("68.2.0.0", sha256="a4216f1e2ef29d089877b3af3ab2acf489eb869ccaf905125c69d2dc3932fd85")
     version("65.5.0.3", sha256="17769171f5f2a2dc69b25c0d3106552a5cda767bbf6b36cb6212b26dae5aa9fc")
 
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-types-typed-ast/package.py b/var/spack/repos/builtin/packages/py-types-typed-ast/package.py
index 8e05d83a67e6d8..f89a17599ae651 100644
--- a/var/spack/repos/builtin/packages/py-types-typed-ast/package.py
+++ b/var/spack/repos/builtin/packages/py-types-typed-ast/package.py
@@ -12,6 +12,7 @@ class PyTypesTypedAst(PythonPackage):
     homepage = "https://github.com/python/typeshed"
     pypi = "types-typed-ast/types-typed-ast-1.5.8.3.tar.gz"
 
+    version("1.5.8.7", sha256="f7795f6f9d597b35212314040b993f6613b51d81738edce3c1e3a3e9ef655124")
     version("1.5.8.3", sha256="3a62bc25168f8b44ce74e1114f9fbc2ee87d6e96e3880cbef39aad9522555b4e")
 
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-types-urllib3/package.py b/var/spack/repos/builtin/packages/py-types-urllib3/package.py
index 722556ed8044d7..7d10082df5f3a2 100644
--- a/var/spack/repos/builtin/packages/py-types-urllib3/package.py
+++ b/var/spack/repos/builtin/packages/py-types-urllib3/package.py
@@ -12,7 +12,9 @@ class PyTypesUrllib3(PythonPackage):
     homepage = "https://github.com/python/typeshed"
     pypi = "types-urllib3/types-urllib3-1.26.24.tar.gz"
 
+    version(
+        "1.26.25.14", sha256="229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"
+    )
     version("1.26.24", sha256="a1b3aaea7dda3eb1b51699ee723aadd235488e4dc4648e030f09bc429ecff42f")
 
-    depends_on("python@3.7:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-typing-extensions/package.py b/var/spack/repos/builtin/packages/py-typing-extensions/package.py
index 31181cae7a2cd1..189b97f7461b0c 100644
--- a/var/spack/repos/builtin/packages/py-typing-extensions/package.py
+++ b/var/spack/repos/builtin/packages/py-typing-extensions/package.py
@@ -12,9 +12,10 @@ class PyTypingExtensions(PythonPackage):
     added to the typing module, such as Protocol (see PEP 544 for
     details about protocols and static duck typing)."""
 
-    homepage = "https://github.com/python/typing/tree/master/typing_extensions"
+    homepage = "https://github.com/python/typing_extensions"
     pypi = "typing_extensions/typing_extensions-3.7.4.tar.gz"
 
+    version("4.8.0", sha256="df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef")
     version("4.6.3", sha256="d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5")
     version("4.5.0", sha256="5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb")
     version("4.3.0", sha256="e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6")
@@ -27,7 +28,8 @@ class PyTypingExtensions(PythonPackage):
     version("3.7.2", sha256="fb2cd053238d33a8ec939190f30cfd736c00653a85a2919415cecf7dc3d9da71")
     version("3.6.6", sha256="51e7b7f3dcabf9ad22eed61490f3b8d23d9922af400fe6656cb08e66656b701f")
 
-    # typing-extensions 4+ uses flit
+    depends_on("python@3.8:", when="@4.8:", type=("build", "run"))
+    # Needed to ensure that Spack can bootstrap with Python 3.6
     depends_on("python@3.7:", when="@4.2:", type=("build", "run"))
     depends_on("py-flit-core@3.4:3", when="@4:", type="build")
 
diff --git a/var/spack/repos/builtin/packages/py-uc-micro-py/package.py b/var/spack/repos/builtin/packages/py-uc-micro-py/package.py
new file mode 100644
index 00000000000000..d6f601f71b35f1
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-uc-micro-py/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyUcMicroPy(PythonPackage):
+    """Micro subset of unicode data files for linkify-it-py projects."""
+
+    homepage = "https://github.com/tsutsu3/uc.micro-py"
+    pypi = "uc-micro-py/uc-micro-py-1.0.2.tar.gz"
+
+    version("1.0.2", sha256="30ae2ac9c49f39ac6dce743bd187fcd2b574b16ca095fa74cd9396795c954c54")
+
+    depends_on("python@3.7:", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/builtin/packages/py-uhi/package.py b/var/spack/repos/builtin/packages/py-uhi/package.py
index 90937732ee9e69..7cf536c66046f7 100644
--- a/var/spack/repos/builtin/packages/py-uhi/package.py
+++ b/var/spack/repos/builtin/packages/py-uhi/package.py
@@ -13,11 +13,15 @@ class PyUhi(PythonPackage):
     homepage = "https://github.com/Scikit-HEP/uhi"
     pypi = "uhi/uhi-0.3.0.tar.gz"
 
+    version("0.3.3", sha256="800caf3a5f1273b08bcc3bb4b49228fe003942e23423812b0110546aad9a24be")
+    version("0.3.2", sha256="fd6ed2ae8ce68ba6be37b872de86e7775b45d54f858768c8fdaba162b6452ab2")
     version("0.3.1", sha256="6f1ebcadd1d0628337a30b012184325618047abc01c3539538b1655c69101d91")
     version("0.3.0", sha256="3f441bfa89fae11aa762ae1ef1b1b454362d228e9084477773ffb82d6e9f5d2c")
 
     depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-numpy@1.13.3:", type=("build", "run"))
     depends_on("py-typing-extensions@3.7:", type=("build", "run"), when="^python@:3.7")
+    depends_on("py-hatchling", when="@0.3.2:", type="build")
+    depends_on("py-hatch-vcs", when="@0.3.3:", type="build")
+    depends_on("py-flit-core@3.2:", when="@0.3.1", type="build")
     depends_on("py-poetry-core@1:", when="@:0.3.0", type="build")
-    depends_on("py-flit-core@3.2:", when="@0.3.1:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-urllib3/package.py b/var/spack/repos/builtin/packages/py-urllib3/package.py
index 236ae0f14d9036..d4b061fa0be8fa 100644
--- a/var/spack/repos/builtin/packages/py-urllib3/package.py
+++ b/var/spack/repos/builtin/packages/py-urllib3/package.py
@@ -12,7 +12,10 @@ class PyUrllib3(PythonPackage):
 
     homepage = "https://urllib3.readthedocs.io/"
     pypi = "urllib3/urllib3-1.25.6.tar.gz"
+    git = "https://github.com/urllib3/urllib3.git"
 
+    version("2.0.6", sha256="b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564")
+    version("2.0.5", sha256="13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594")
     version("1.26.12", sha256="3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e")
     version("1.26.6", sha256="f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f")
     version("1.25.9", sha256="3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527")
@@ -23,22 +26,29 @@ class PyUrllib3(PythonPackage):
     version("1.20", sha256="97ef2b6e2878d84c0126b9f4e608e37a951ca7848e4855a7f7f4437d5c34a72f")
     version("1.14", sha256="dd4fb13a4ce50b18338c7e4d665b21fd38632c5d4b1d9f1a1379276bd3c08d37")
 
-    variant("socks", default=False, description="SOCKS and HTTP proxy support")
+    variant("brotli", default=False, when="@1.25:", description="Add Brotli support")
     variant("secure", default=False, description="Add SSL/TLS support")
-    variant("brotli", default=False, description="Add Brotli support")
+    variant("socks", default=False, when="@1.15:", description="SOCKS and HTTP proxy support")
 
-    depends_on("python@2.7:2.8,3.4:", when="@:1.25", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.5:", when="@1.26.6", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.6:3", when="@1.26.12:", type=("build", "run"))
+    depends_on("py-hatchling@1.6:1", when="@2:", type="build")
 
-    depends_on("py-setuptools", type="build")
+    with when("+brotli"):
+        depends_on("py-brotli@1.0.9:", when="@1.26.9:", type=("build", "run"))
 
-    depends_on("py-pyopenssl@0.14:", when="+secure")
-    depends_on("py-cryptography@1.3.4:", when="+secure")
-    depends_on("py-idna@2:", when="+secure")
-    depends_on("py-certifi", when="+secure")
-    depends_on("py-urllib3-secure-extra", when="+secure @1.26.12:")
+        # Historical dependencies
+        depends_on("py-brotlipy@0.6:", when="@:1.26.8", type=("build", "run"))
 
-    depends_on("py-pysocks@1.5.6,1.5.8:1", when="+socks")
+    with when("+secure"):
+        depends_on("py-pyopenssl@17.1:", when="@2:", type=("build", "run"))
+        depends_on("py-pyopenssl@0.14:", when="@1", type=("build", "run"))
+        depends_on("py-cryptography@1.9:", when="@2:", type=("build", "run"))
+        depends_on("py-cryptography@1.3.4:", when="@1", type=("build", "run"))
+        depends_on("py-idna@2:", type=("build", "run"))
+        depends_on("py-certifi", type=("build", "run"))
+        depends_on("py-urllib3-secure-extra", when="@1.26.12:", type=("build", "run"))
 
-    depends_on("py-brotlipy@0.6:", when="+brotli")
+    depends_on("py-pysocks@1.5.6,1.5.8:1", when="+socks", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-setuptools", when="@1", type="build")
+    depends_on("python@3.6:3", when="@1.26.12:1", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-vcrpy/package.py b/var/spack/repos/builtin/packages/py-vcrpy/package.py
index d7039241803e3a..b11e67b91215b4 100644
--- a/var/spack/repos/builtin/packages/py-vcrpy/package.py
+++ b/var/spack/repos/builtin/packages/py-vcrpy/package.py
@@ -12,14 +12,17 @@ class PyVcrpy(PythonPackage):
     homepage = "https://github.com/kevin1024/vcrpy"
     pypi = "vcrpy/vcrpy-4.1.1.tar.gz"
 
+    version("5.1.0", sha256="bbf1532f2618a04f11bce2a99af3a9647a32c880957293ff91e0a5f187b6b3d2")
     version("4.2.1", sha256="7cd3e81a2c492e01c281f180bcc2a86b520b173d2b656cb5d89d99475423e013")
     version("4.1.1", sha256="57095bf22fc0a2d99ee9674cdafebed0f3ba763018582450706f7d3a74fff599")
 
-    depends_on("python@3.7:", when="@4.2:", type=("build", "run"))
-    depends_on("python@3.5:", type=("build", "run"))
+    depends_on("python@3.8:", when="@5:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
 
     depends_on("py-pyyaml", type=("build", "run"))
     depends_on("py-wrapt", type=("build", "run"))
-    depends_on("py-six@1.5:", type=("build", "run"))
     depends_on("py-yarl", type=("build", "run"))
+    depends_on("py-urllib3@:1", when="@4.3.1: ^python@:3.9", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-six@1.5:", when="@:5.0", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-versioneer/package.py b/var/spack/repos/builtin/packages/py-versioneer/package.py
index 123d984f71abbd..915b40ff369374 100644
--- a/var/spack/repos/builtin/packages/py-versioneer/package.py
+++ b/var/spack/repos/builtin/packages/py-versioneer/package.py
@@ -16,6 +16,7 @@ class PyVersioneer(PythonPackage):
 
     maintainers("scemama")
 
+    version("0.29", sha256="5ab283b9857211d61b53318b7c792cf68e798e765ee17c27ade9f6c924235731")
     version("0.28", sha256="7175ca8e7bb4dd0e3c9779dd2745e5b4a6036304af3f5e50bd896f10196586d6")
     version("0.27", sha256="452e0130658e9d3f0ba3e8a70cf34ef23c0ff6cbf743555b3e73a6c11d0161a3")
     version("0.26", sha256="84fc729aa296d1d26645a8f62f178019885ff6f9a1073b29a4a228270ac5257b")
@@ -23,8 +24,6 @@ class PyVersioneer(PythonPackage):
 
     variant("toml", default=True, description="Install TOML support", when="@0.26:")
 
-    depends_on("python@3.7:", when="@0.23:", type=("build", "run"))
-    depends_on("python@3.6:", when="@0.19:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("py-tomli", when="@0.28: ^python@:3.10", type="build")
     depends_on("py-tomli", when="@0.27", type="build")
diff --git a/var/spack/repos/builtin/packages/py-virtualenv/package.py b/var/spack/repos/builtin/packages/py-virtualenv/package.py
index 7b3790b82e90d7..4eec5ac359b883 100644
--- a/var/spack/repos/builtin/packages/py-virtualenv/package.py
+++ b/var/spack/repos/builtin/packages/py-virtualenv/package.py
@@ -13,6 +13,7 @@ class PyVirtualenv(PythonPackage):
     pypi = "virtualenv/virtualenv-16.7.6.tar.gz"
     git = "https://github.com/pypa/virtualenv.git"
 
+    version("20.24.5", sha256="e8361967f6da6fbdf1426483bfe9fca8287c242ac0bc30429905721cefbff752")
     version("20.22.0", sha256="278753c47aaef1a0f14e6db8a4c5e1e040e90aea654d0fc1dc7e0d8a42616cc3")
     version("20.17.1", sha256="f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058")
     version("20.16.4", sha256="014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782")
@@ -26,30 +27,21 @@ class PyVirtualenv(PythonPackage):
     version("1.11.6", sha256="3e7a4c151e2ee97f51db0215bfd2a073b04a91e9786df6cb67c916f16abe04f7")
 
     depends_on("py-hatch-vcs@0.3:", when="@20.18:", type="build")
+    depends_on("py-hatchling@1.17.1:", when="@20.23.1:", type="build")
     depends_on("py-hatchling@1.14:", when="@20.22:", type="build")
     depends_on("py-hatchling@1.12.2:", when="@20.18:", type="build")
 
-    with when("@:20.17"):
-        # not just build-time, requires pkg_resources
-        depends_on("py-setuptools@59.6:", when="@20.16.3:", type=("build", "run"))
-        depends_on("py-setuptools@41.0.0:", when="@20.0.0:20.16.2", type=("build", "run"))
-        depends_on("py-setuptools@40.6.3:", when="@16.1.2:16", type=("build", "run"))
-        depends_on("py-setuptools@40.0.4:", when="@16.1.0", type=("build", "run"))
-        depends_on("py-setuptools", type=("build", "run"))
-        depends_on("py-setuptools-scm@6.4.2:", when="@20.16.3:", type="build")
-        depends_on("py-setuptools-scm@2:", when="@20.0.5:20.16.2", type="build")
-        depends_on("py-setuptools-scm+toml@3.4:", when="@20.0.0:20.0.4", type="build")
-        depends_on("py-wheel@0.30:", when="@20.0.0:20.16.2", type="build")
-        depends_on("py-wheel@0.29:", when="@16.1:16", type="build")
-
-    depends_on("py-distlib@0.3.6:0", when="@20.16.6:", type=("build", "run"))
+    depends_on("py-distlib@0.3.7:0", when="@20.24.2:", type=("build", "run"))
+    depends_on("py-distlib@0.3.6:0", when="@20.16.6:20.24.1", type=("build", "run"))
     depends_on("py-distlib@0.3.5:0", when="@20.16.3:20.16.5", type=("build", "run"))
     depends_on("py-distlib@0.3.1:0", when="@20.0.26:20.16.2", type=("build", "run"))
     depends_on("py-distlib@0.3.0:0", when="@20.0.0:20.0.25", type=("build", "run"))
-    depends_on("py-filelock@3.11:3", when="@20.22:", type=("build", "run"))
-    depends_on("py-filelock@3.4.1:3", when="@20.16.3:", type=("build", "run"))
+    depends_on("py-filelock@3.12.2:3", when="@20.24.2:", type=("build", "run"))
+    depends_on("py-filelock@3.11:3", when="@20.22:20.23.0", type=("build", "run"))
+    depends_on("py-filelock@3.4.1:3", when="@20.16.3:20.21", type=("build", "run"))
     depends_on("py-filelock@3.2:3", when="@20.9:20.16.2", type=("build", "run"))
     depends_on("py-filelock@3.0.0:3", when="@20.0:20.8", type=("build", "run"))
+    depends_on("py-importlib-metadata@6.6:", when="@20.23.1: ^python@:3.7", type=("build", "run"))
     depends_on("py-importlib-metadata@6.4.1:", when="@20.22: ^python@:3.7", type=("build", "run"))
     depends_on(
         "py-importlib-metadata@4.8.3:", when="@20.16.3: ^python@:3.7", type=("build", "run")
@@ -58,11 +50,25 @@ class PyVirtualenv(PythonPackage):
     depends_on(
         "py-importlib-metadata@0.12:3", when="@20.0.0:20.2.0 ^python@:3.7", type=("build", "run")
     )
-    depends_on("py-platformdirs@3.2:3", when="@20.22:", type=("build", "run"))
+    depends_on("py-platformdirs@3.9.1:3", when="@20.24.1:", type=("build", "run"))
+    depends_on("py-platformdirs@3.2:3", when="@20.22:20.23.0", type=("build", "run"))
     depends_on("py-platformdirs@2.4:2", when="@20.16.3:20.21", type=("build", "run"))
     depends_on("py-platformdirs@2:2", when="@20.5:20.16.2", type=("build", "run"))
 
-    # dependencies of old versions
+    # Historical dependencies
+    with when("@:20.17"):
+        # not just build-time, requires pkg_resources
+        depends_on("py-setuptools@59.6:", when="@20.16.3:", type=("build", "run"))
+        depends_on("py-setuptools@41.0.0:", when="@20.0.0:20.16.2", type=("build", "run"))
+        depends_on("py-setuptools@40.6.3:", when="@16.1.2:16", type=("build", "run"))
+        depends_on("py-setuptools@40.0.4:", when="@16.1.0", type=("build", "run"))
+        depends_on("py-setuptools", type=("build", "run"))
+        depends_on("py-setuptools-scm@6.4.2:", when="@20.16.3:", type="build")
+        depends_on("py-setuptools-scm@2:", when="@20.0.5:20.16.2", type="build")
+        depends_on("py-setuptools-scm+toml@3.4:", when="@20.0.0:20.0.4", type="build")
+        depends_on("py-wheel@0.30:", when="@20.0.0:20.16.2", type="build")
+        depends_on("py-wheel@0.29:", when="@16.1:16", type="build")
+
     depends_on(
         "py-backports-entry-points-selectable @1.0.4:", when="@20.5:20.10", type=("build", "run")
     )
diff --git a/var/spack/repos/builtin/packages/py-wand/package.py b/var/spack/repos/builtin/packages/py-wand/package.py
index 6c92620287ecb9..07ebb2247f3408 100644
--- a/var/spack/repos/builtin/packages/py-wand/package.py
+++ b/var/spack/repos/builtin/packages/py-wand/package.py
@@ -12,14 +12,13 @@ class PyWand(PythonPackage):
     homepage = "https://docs.wand-py.org"
     pypi = "Wand/Wand-0.5.6.tar.gz"
 
+    version("0.6.11", sha256="b661700da9f8f1e931e52726e4fc643a565b9514f5883d41b773e3c37c9fa995")
     version("0.5.6", sha256="d06b59f36454024ce952488956319eb542d5dc65f1e1b00fead71df94dbfcf88")
     version("0.4.2", sha256="a0ded99a9824ddd82617a4b449164e2c5c93853aaff96f9e0bab8b405d62ca7c")
 
-    variant("docs", default=False, description="Build docs")
-
     depends_on("py-setuptools", type="build")
     # provides libmagickwand
     depends_on("imagemagick")
-    depends_on("python@2.7:2.8,3.3:", type=("build", "run"))
 
-    depends_on("py-sphinx@1:", type="build", when="+docs")
+    def setup_build_environment(self, env):
+        env.set("MAGICK_HOME", self.spec["imagemagick"].prefix)
diff --git a/var/spack/repos/builtin/packages/py-warpx/package.py b/var/spack/repos/builtin/packages/py-warpx/package.py
index a08ef02bd04b58..1b3fb9f788eb62 100644
--- a/var/spack/repos/builtin/packages/py-warpx/package.py
+++ b/var/spack/repos/builtin/packages/py-warpx/package.py
@@ -18,7 +18,7 @@ class PyWarpx(PythonPackage):
     """
 
     homepage = "https://ecp-warpx.github.io"
-    url = "https://github.com/ECP-WarpX/WarpX/archive/refs/tags/23.06.tar.gz"
+    url = "https://github.com/ECP-WarpX/WarpX/archive/refs/tags/23.08.tar.gz"
     git = "https://github.com/ECP-WarpX/WarpX.git"
 
     maintainers("ax3l", "dpgrote", "RemiLehe")
@@ -27,6 +27,8 @@ class PyWarpx(PythonPackage):
 
     # NOTE: if you update the versions here, also see warpx
     version("develop", branch="development")
+    version("23.08", sha256="67695ff04b83d1823ea621c19488e54ebaf268532b0e5eb4ea8ad293d7ab3ddc")
+    version("23.07", sha256="511633f94c0d0205013609bde5bbf92a29c2e69f6e69b461b80d09dc25602945")
     version("23.06", sha256="75fcac949220c44dce04de581860c9a2caa31a0eee8aa7d49455fa5fc928514b")
     version("23.05", sha256="34306a98fdb1f5f44ab4fb92f35966bfccdcf1680a722aa773af2b59a3060d73")
     version("23.04", sha256="e5b285c73e13a0d922eba5d83760c168d4fd388e54a519830003b2e692dab823")
@@ -45,19 +47,12 @@ class PyWarpx(PythonPackage):
     version("22.03", sha256="ddbef760c8000f2f827dfb097ca3359e7aecbea8766bec5c3a91ee28d3641564")
     version("22.02", sha256="d74b593d6f396e037970c5fbe10c2e5d71d557a99c97d40e4255226bc6c26e42")
     version("22.01", sha256="e465ffadabb7dc360c63c4d3862dc08082b5b0e77923d3fb05570408748b0d28")
-    version("21.12", sha256="847c98aac20c73d94c823378803c82be9a14139f1c14ea483757229b452ce4c1")
-    version("21.11", sha256="ce60377771c732033a77351cd3500b24b5d14b54a5adc7a622767b9251c10d0b")
-    version("21.10", sha256="d372c573f0360094d5982d64eceeb0149d6620eb75e8fdbfdc6777f3328fb454")
-    version("21.09", sha256="861a65f11846541c803564db133c8678b9e8779e69902ef1637b21399d257eab")
-    version("21.08", sha256="6128a32cfd075bc63d08eebea6d4f62d33ce0570f4fd72330a71023ceacccc86")
-    version("21.07", sha256="a8740316d813c365715f7471201499905798b50bd94950d33f1bd91478d49561")
-    version("21.06", sha256="a26039dc4061da45e779dd5002467c67a533fc08d30841e01e7abb3a890fbe30")
-    version("21.05", sha256="f835f0ae6c5702550d23191aa0bb0722f981abb1460410e3d8952bc3d945a9fc")
-    version("21.04", sha256="51d2d8b4542eada96216e8b128c0545c4b7527addc2038efebe586c32c4020a0")
 
     variant("mpi", default=True, description="Enable MPI support")
 
     for v in [
+        "23.08",
+        "23.07",
         "23.06",
         "23.05",
         "23.04",
@@ -76,38 +71,27 @@ class PyWarpx(PythonPackage):
         "22.03",
         "22.02",
         "22.01",
-        "21.12",
-        "21.11",
-        "21.10",
-        "21.09",
-        "21.08",
-        "21.07",
-        "21.06",
-        "21.05",
-        "21.04",
         "develop",
     ]:
         depends_on("warpx@{0}".format(v), when="@{0}".format(v), type=["build", "link"])
 
-    depends_on("python@3.6:3.9", type=("build", "run"), when="@:21.12")
-    depends_on("python@3.6:", type=("build", "run"), when="@22.01:")
+    depends_on("python@3.7:", type=("build", "run"))
+    depends_on("python@3.8:", type=("build", "run"), when="@23.09:")
     depends_on("py-numpy@1.15.0:1", type=("build", "run"))
     depends_on("py-mpi4py@2.1.0:", type=("build", "run"), when="+mpi")
     depends_on("py-periodictable@1.5:1", type=("build", "run"))
-    depends_on("py-picmistandard@0.0.14", type=("build", "run"), when="@21.03:21.11")
-    depends_on("py-picmistandard@0.0.16", type=("build", "run"), when="@21.12")
-    depends_on("py-picmistandard@0.0.18", type=("build", "run"), when="@22.01")
-    depends_on("py-picmistandard@0.0.19", type=("build", "run"), when="@22.02:22.09")
-    depends_on("py-picmistandard@0.0.20", type=("build", "run"), when="@22.10:22.11")
-    depends_on("py-picmistandard@0.0.22", type=("build", "run"), when="@22.12:23.03")
+    depends_on("py-picmistandard@0.25.0", type=("build", "run"), when="@23.07:")
+    depends_on("py-picmistandard@0.24.0", type=("build", "run"), when="@23.06")
     depends_on("py-picmistandard@0.23.2", type=("build", "run"), when="@23.04:23.05")
-    depends_on("py-picmistandard@0.24.0", type=("build", "run"), when="@23.06:")
+    depends_on("py-picmistandard@0.0.22", type=("build", "run"), when="@22.12:23.03")
+    depends_on("py-picmistandard@0.0.20", type=("build", "run"), when="@22.10:22.11")
+    depends_on("py-picmistandard@0.0.19", type=("build", "run"), when="@22.02:22.09")
+    depends_on("py-picmistandard@0.0.18", type=("build", "run"), when="@22.01")
     depends_on("py-setuptools@42:", type="build")
     # Since we use PYWARPX_LIB_DIR to pull binaries out of the
     # 'warpx' spack package, we don't need py-cmake as declared
     # depends_on('py-cmake@3.15:3', type='build')
     # depends_on('py-cmake@3.18:3', type='build', when='@22.01:')
-    depends_on("py-wheel", type="build")
     depends_on("warpx +lib ~mpi +shared", type=("build", "link"), when="~mpi")
     depends_on("warpx +lib +mpi +shared", type=("build", "link"), when="+mpi")
 
diff --git a/var/spack/repos/builtin/packages/py-wasabi/package.py b/var/spack/repos/builtin/packages/py-wasabi/package.py
index 3babe894bf4caa..ea2c5fd466242c 100644
--- a/var/spack/repos/builtin/packages/py-wasabi/package.py
+++ b/var/spack/repos/builtin/packages/py-wasabi/package.py
@@ -10,9 +10,14 @@
 class PyWasabi(PythonPackage):
     """wasabi: A lightweight console printing and formatting toolkit."""
 
-    homepage = "https://ines.io/"
+    homepage = "https://github.com/explosion/wasabi"
     pypi = "wasabi/wasabi-0.6.0.tar.gz"
 
+    version("1.1.2", sha256="1aaef3aceaa32edb9c91330d29d3936c0c39fdb965743549c173cb54b16c30b5")
     version("0.6.0", sha256="b8dd3e963cd693fde1eb6bfbecf51790171aa3534fa299faf35cf269f2fd6063")
 
     depends_on("py-setuptools", type="build")
+    depends_on(
+        "py-typing-extensions@3.7.4.1:4.4", type=("build", "run"), when="@1.1.2:^python@:3.7"
+    )
+    depends_on("py-colorama@0.4.6:", type=("build", "run"), when="@1.1.2:platform=windows")
diff --git a/var/spack/repos/builtin/packages/py-wcwidth/package.py b/var/spack/repos/builtin/packages/py-wcwidth/package.py
index e0c4ad93ac876e..8120bab72bb568 100644
--- a/var/spack/repos/builtin/packages/py-wcwidth/package.py
+++ b/var/spack/repos/builtin/packages/py-wcwidth/package.py
@@ -9,8 +9,10 @@
 class PyWcwidth(PythonPackage):
     """Measures number of Terminal column cells of wide-character codes"""
 
+    homepage = "https://github.com/jquast/wcwidth"
     pypi = "wcwidth/wcwidth-0.1.7.tar.gz"
 
+    version("0.2.7", sha256="1b6d30a98ddd5ce9bbdb33658191fd2423fc9da203fe3ef1855407dcb7ee4e26")
     version("0.2.5", sha256="c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83")
     version("0.1.7", sha256="3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e")
 
diff --git a/var/spack/repos/builtin/packages/py-websocket-client/package.py b/var/spack/repos/builtin/packages/py-websocket-client/package.py
index 295108f0e45913..09a11e3f49289b 100644
--- a/var/spack/repos/builtin/packages/py-websocket-client/package.py
+++ b/var/spack/repos/builtin/packages/py-websocket-client/package.py
@@ -13,27 +13,24 @@ class PyWebsocketClient(PythonPackage):
     homepage = "https://github.com/websocket-client/websocket-client.git"
     pypi = "websocket-client/websocket-client-0.57.0.tar.gz"
 
+    version("1.6.3", sha256="3aad25d31284266bcfcfd1fd8a743f63282305a364b8d0948a43bd606acc652f")
     version("1.5.1", sha256="3f09e6d8230892547132177f575a4e3e73cfdf06526e20cc02aa1c3b47184d40")
     version("1.4.1", sha256="f9611eb65c8241a67fb373bef040b3cf8ad377a9f6546a12b620b6511e8ea9ef")
     version("1.2.1", sha256="8dfb715d8a992f5712fff8c843adae94e22b22a99b2c5e6b0ec4a1a981cc4e0d")
-    version(
-        "0.57.0",
-        sha256="d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010",
-        url="https://files.pythonhosted.org/packages/source/w/websocket_client/websocket_client-0.57.0.tar.gz",
-    )
-    version(
-        "0.56.0",
-        sha256="1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a",
-        url="https://files.pythonhosted.org/packages/source/w/websocket_client/websocket_client-0.56.0.tar.gz",
-    )
-    version(
-        "0.48.0",
-        sha256="18f1170e6a1b5463986739d9fd45c4308b0d025c1b2f9b88788d8f69e8a5eb4a",
-        url="https://files.pythonhosted.org/packages/source/w/websocket_client/websocket_client-0.48.0.tar.gz",
-    )
-
-    depends_on("python@2.6:2.8,3.4:", type=("build", "run"))
-    depends_on("python@3.6:", type=("build", "run"), when="@1.2.1:")
-    depends_on("python@3.7:", type=("build", "run"), when="@1.4.1:")
+    version("0.57.0", sha256="d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010")
+    version("0.56.0", sha256="1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a")
+    version("0.48.0", sha256="18f1170e6a1b5463986739d9fd45c4308b0d025c1b2f9b88788d8f69e8a5eb4a")
+
+    depends_on("python@3.8:", when="@1.6.2:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
+
+    # Historical dependencies
     depends_on("py-six", type=("build", "run"), when="@:1.2.0")
+
+    def url_for_version(self, version):
+        url = "https://files.pythonhosted.org/packages/source/w/{0}/{0}-{1}.tar.gz"
+        if version >= Version("0.59.0"):
+            letter = "websocket-client"
+        else:
+            letter = "websocket_client"
+        return url.format(letter, version)
diff --git a/var/spack/repos/builtin/packages/py-werkzeug/package.py b/var/spack/repos/builtin/packages/py-werkzeug/package.py
index 444276c6dc2f8c..e4099e015156c5 100644
--- a/var/spack/repos/builtin/packages/py-werkzeug/package.py
+++ b/var/spack/repos/builtin/packages/py-werkzeug/package.py
@@ -10,9 +10,11 @@ class PyWerkzeug(PythonPackage):
     """The Swiss Army knife of Python web development"""
 
     homepage = "https://palletsprojects.com/p/werkzeug"
-    pypi = "Werkzeug/Werkzeug-0.16.0.tar.gz"
+    pypi = "werkzeug/werkzeug-3.0.0.tar.gz"
     git = "https://github.com/pallets/werkzeug.git"
 
+    version("3.0.0", sha256="3ffff4dcc32db52ef3cc94dff3000a3c2846890f3a5a51800a27b909c5e770f0")
+    version("2.3.7", sha256="2b8c0e447b4b9dbcc85dd97b6eeb4dcbaf6c8b6c3be0bd654e25553e0a2157d8")
     version("2.3.4", sha256="1d5a58e0377d1fe39d061a5de4469e414e78ccb1e1e59c0f5ad6fa1c36c52b76")
     version("2.2.2", sha256="7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f")
     version("2.0.2", sha256="aa2bb6fc8dee8d6c504c0ac1e7f5f7dc5810a9903e793b6f715a9f015bdadb9a")
@@ -24,9 +26,22 @@ class PyWerkzeug(PythonPackage):
     version("0.15.2", sha256="0a73e8bb2ff2feecfc5d56e6f458f5b99290ef34f565ffb2665801ff7de6af7a")
     version("0.15.1", sha256="ca5c2dcd367d6c0df87185b9082929d255358f5391923269335782b213d52655")
     version("0.15.0", sha256="590abe38f8be026d78457fe3b5200895b3543e58ac3fc1dd792c6333ea11af64")
+    version("0.12.2", sha256="903a7b87b74635244548b30d30db4c8947fe64c5198f58899ddcd3a13c23bb26")
     version("0.11.15", sha256="455d7798ac263266dbd38d4841f7534dd35ca9c3da4a8df303f8488f38f3bcc0")
     version("0.11.11", sha256="e72c46bc14405cba7a26bd2ce28df734471bc9016bc8b4cb69466c2c14c2f7e5")
 
     depends_on("python@3.8:", when="@2.3:", type=("build", "run"))
-    depends_on("py-setuptools", type="build")
+    depends_on("python@:3.9", when="@:0.12", type=("build", "run"))
+    depends_on("py-flit-core@:3", when="@2.3.7:", type="build")
     depends_on("py-markupsafe@2.1.1:", when="@2.2:", type=("build", "run"))
+
+    # Historical dependencies
+    depends_on("py-setuptools", when="@:2.3.6", type="build")
+
+    def url_for_version(self, version):
+        url = "https://files.pythonhosted.org/packages/source/w/werkzeug/{0}-{1}.tar.gz"
+        if version >= Version("2.3.7"):
+            letter = "werkzeug"
+        else:
+            letter = "Werkzeug"
+        return url.format(letter, version)
diff --git a/var/spack/repos/builtin/packages/py-wheel/package.py b/var/spack/repos/builtin/packages/py-wheel/package.py
index 770397b09feb9b..66192db3298d57 100644
--- a/var/spack/repos/builtin/packages/py-wheel/package.py
+++ b/var/spack/repos/builtin/packages/py-wheel/package.py
@@ -10,11 +10,14 @@ class PyWheel(Package, PythonExtension):
     """A built-package format for Python."""
 
     homepage = "https://github.com/pypa/wheel"
-    url = (
-        "https://files.pythonhosted.org/packages/py2.py3/w/wheel/wheel-0.34.2-py2.py3-none-any.whl"
-    )
+    url = "https://files.pythonhosted.org/packages/py3/w/wheel/wheel-0.41.2-py3-none-any.whl"
     list_url = "https://pypi.org/simple/wheel/"
 
+    version(
+        "0.41.2",
+        sha256="75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8",
+        expand=False,
+    )
     version(
         "0.37.1",
         sha256="4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a",
@@ -73,11 +76,17 @@ class PyWheel(Package, PythonExtension):
 
     extends("python")
     depends_on("python +ctypes", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.5:", when="@0.34:", type=("build", "run"))
-    depends_on("python@2.7:2.8,3.4:", when="@0.30:", type=("build", "run"))
-    depends_on("python@2.6:2.8,3.2:", type=("build", "run"))
+    depends_on("python@3.7:", when="@0.38:", type=("build", "run"))
     depends_on("py-pip", type="build")
 
+    def url_for_version(self, version):
+        url = "https://files.pythonhosted.org/packages/{0}/w/wheel/wheel-{1}-{0}-none-any.whl"
+        if version >= Version("0.38"):
+            python = "py3"
+        else:
+            python = "py2.py3"
+        return url.format(python, version)
+
     def install(self, spec, prefix):
         # To build wheel from source, you need setuptools and wheel already installed.
         # We get around this by using a pre-built wheel, see:
diff --git a/var/spack/repos/builtin/packages/py-whey/package.py b/var/spack/repos/builtin/packages/py-whey/package.py
new file mode 100644
index 00000000000000..ac97b551b5c967
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-whey/package.py
@@ -0,0 +1,32 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class PyWhey(PythonPackage):
+    """A simple Python wheel builder for simple projects."""
+
+    homepage = "https://github.com/repo-helper/whey"
+    pypi = "whey/whey-0.0.24.tar.gz"
+
+    version("0.0.24", sha256="411905d85aa8aa239733818894e08dc20b682f0a3614f942aa35b430db568aa2")
+
+    depends_on("py-wheel@0.34.2", type="build")
+    depends_on("py-setuptools@40.6:", type="build")
+
+    conflicts("^py-setuptools@61")
+
+    depends_on("py-click@7.1.2:", type=("build", "run"))
+    depends_on("py-consolekit@1.4.1:", type=("build", "run"))
+    depends_on("py-dist-meta@0.1:", type=("build", "run"))
+    depends_on("py-dom-toml@0.4:", type=("build", "run"))
+    depends_on("py-domdf-python-tools@2.8:", type=("build", "run"))
+    depends_on("py-handy-archives@0.1:", type=("build", "run"))
+    depends_on("py-natsort@7.1.1:", type=("build", "run"))
+    depends_on("py-packaging@20.9:", type=("build", "run"))
+    depends_on("py-pyproject-parser@0.6:", type=("build", "run"))
+    depends_on("py-shippinglabel@0.16:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-widgetsnbextension/package.py b/var/spack/repos/builtin/packages/py-widgetsnbextension/package.py
index ba9a0ab5f171ed..ee53aac4e7b5ca 100644
--- a/var/spack/repos/builtin/packages/py-widgetsnbextension/package.py
+++ b/var/spack/repos/builtin/packages/py-widgetsnbextension/package.py
@@ -20,8 +20,7 @@ class PyWidgetsnbextension(PythonPackage):
     version("1.2.6", sha256="c618cfb32978c9517caf0b4ef3aec312f8dd138577745e7b0d4abfcc7315ce51")
 
     depends_on("py-setuptools", type="build")
-    # TODO: replace this after concretizer learns how to concretize separate build deps
-    depends_on("py-jupyter-packaging11", when="@4.0.3:", type="build")
+    depends_on("py-jupyter-packaging@0.10:0", when="@4.0.3:", type="build")
 
     depends_on("python@2.7:2.8,3.3:", type=("build", "run"))
     depends_on("python@3.7:", when="@4.0.3:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-wrapt/package.py b/var/spack/repos/builtin/packages/py-wrapt/package.py
index 2f0b92bf5952ee..8c6266dffe5f32 100644
--- a/var/spack/repos/builtin/packages/py-wrapt/package.py
+++ b/var/spack/repos/builtin/packages/py-wrapt/package.py
@@ -12,6 +12,7 @@ class PyWrapt(PythonPackage):
     homepage = "https://github.com/GrahamDumpleton/wrapt"
     pypi = "wrapt/wrapt-1.11.2.tar.gz"
 
+    version("1.15.0", sha256="d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a")
     version("1.14.1", sha256="380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d")
     version("1.13.3", sha256="1fea9cd438686e6682271d36f3481a9f3636195578bab9ca3382e2f5f01fc185")
     version("1.12.1", sha256="b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7")
@@ -19,6 +20,4 @@ class PyWrapt(PythonPackage):
     version("1.11.1", sha256="4aea003270831cceb8a90ff27c4031da6ead7ec1886023b80ce0dfe0adf61533")
     version("1.10.10", sha256="42160c91b77f1bc64a955890038e02f2f72986c01d462d53cb6cb039b995cdd9")
 
-    depends_on("python@2.7:2,3.5:", when="@1.13.1:", type=("build", "run"))
-    depends_on("python@2.7:2,3.3:", when="@1.13:", type=("build", "run"))
     depends_on("py-setuptools@38.3:", type="build")
diff --git a/var/spack/repos/builtin/packages/py-xarray/package.py b/var/spack/repos/builtin/packages/py-xarray/package.py
index 9b9064568ed781..ceebb19ea3d7a9 100644
--- a/var/spack/repos/builtin/packages/py-xarray/package.py
+++ b/var/spack/repos/builtin/packages/py-xarray/package.py
@@ -23,6 +23,7 @@ class PyXarray(PythonPackage):
         "xarray.coding",
     ]
 
+    version("2023.7.0", sha256="dace2fdbf1b7ff185d9c1226a24bf83c2ae52f3253dbfe80e17d1162600d055c")
     version("2022.3.0", sha256="398344bf7d170477aaceff70210e11ebd69af6b156fe13978054d25c48729440")
     version("0.18.2", sha256="5d2e72a228286fcf60f66e16876bd27629a1a70bf64822c565f16515c4d10284")
     version("0.17.0", sha256="9c2edad2a4e588f9117c666a4249920b9717fb75703b96998cf65fcd4f60551f")
@@ -41,8 +42,9 @@ class PyXarray(PythonPackage):
     depends_on("py-setuptools@38.4:", when="@0.16:", type=("build", "run"))
     depends_on("py-setuptools@42:", when="@0.17:", type=("build", "run"))
     depends_on("py-setuptools-scm", when="@0.15:", type="build")
-    depends_on("py-setuptools-scm@3.4:+toml", when="@0.17:", type="build")
-    depends_on("py-setuptools-scm-git-archive", when="@0.17:", type="build")
+    depends_on("py-setuptools-scm@7:", when="@2023.7.0:", type="build")
+    depends_on("py-setuptools-scm@3.4:+toml", when="@0.17:2022.3.0", type="build")
+    depends_on("py-setuptools-scm-git-archive", when="@0.17:2022.3.0", type="build")
 
     # setup.cfg
     depends_on("python@2.7,3.5:", when="@0.11:", type=("build", "run"))
@@ -51,6 +53,7 @@ class PyXarray(PythonPackage):
     depends_on("python@3.6:", when="@0.14:", type=("build", "run"))
     depends_on("python@3.7:", when="@0.17:", type=("build", "run"))
     depends_on("python@3.8:", when="@0.21:", type=("build", "run"))
+    depends_on("python@3.9:", when="@2023.7.0:", type=("build", "run"))
 
     depends_on("py-numpy@1.7:", when="@0.9.1", type=("build", "run"))
     depends_on("py-numpy@1.12:", when="@0.11:0.13", type=("build", "run"))
@@ -58,6 +61,7 @@ class PyXarray(PythonPackage):
     depends_on("py-numpy@1.15:", when="@0.15:", type=("build", "run"))
     depends_on("py-numpy@1.17:", when="@0.18:", type=("build", "run"))
     depends_on("py-numpy@1.18:", when="@0.20:", type=("build", "run"))
+    depends_on("py-numpy@1.21:", when="@2023.7.0:", type=("build", "run"))
 
     depends_on("py-pandas@0.15.0:", when="@0.9.1", type=("build", "run"))
     depends_on("py-pandas@0.19.2:", when="@0.11:0.13", type=("build", "run"))
@@ -65,18 +69,20 @@ class PyXarray(PythonPackage):
     depends_on("py-pandas@0.25:", when="@0.15:", type=("build", "run"))
     depends_on("py-pandas@1:", when="@0.18:", type=("build", "run"))
     depends_on("py-pandas@1.1:", when="@0.20:", type=("build", "run"))
+    depends_on("py-pandas@1.4:", when="@2023.7.0:", type=("build", "run"))
 
     depends_on("py-packaging@20:", when="@0.21:", type=("build", "run"))
+    depends_on("py-packaging@21.3:", when="@2023.7.0:", type=("build", "run"))
 
     depends_on("py-netcdf4", when="+io", type=("build", "run"))
     depends_on("py-h5netcdf", when="+io", type=("build", "run"))
     depends_on("py-scipy", when="+io", type=("build", "run"))
-    depends_on("py-pydap", when="+io", type=("build", "run"))
+    depends_on("py-pydap", when="+io ^python@:3.9", type=("build", "run"))
     depends_on("py-zarr", when="+io", type=("build", "run"))
     depends_on("py-fsspec", when="+io", type=("build", "run"))
     depends_on("py-cftime", when="+io", type=("build", "run"))
-    depends_on("py-rasterio", when="+io", type=("build", "run"))
-    depends_on("py-cfgrib", when="+io", type=("build", "run"))
+    depends_on("py-rasterio", when="@:2022.3.0 +io", type=("build", "run"))
+    depends_on("py-cfgrib", when="@:2022.3.0 +io", type=("build", "run"))
     depends_on("py-pooch", when="+io", type=("build", "run"))
     depends_on(
         "py-dask+array+dataframe+distributed+diagnostics+delayed",
diff --git a/var/spack/repos/builtin/packages/py-yarl/package.py b/var/spack/repos/builtin/packages/py-yarl/package.py
index b9dd00ca4b189d..bad47e340bfad6 100644
--- a/var/spack/repos/builtin/packages/py-yarl/package.py
+++ b/var/spack/repos/builtin/packages/py-yarl/package.py
@@ -13,19 +13,19 @@ class PyYarl(PythonPackage):
     homepage = "https://github.com/aio-libs/yarl"
     pypi = "yarl/yarl-1.4.2.tar.gz"
 
+    version("1.9.2", sha256="04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571")
     version("1.8.1", sha256="af887845b8c2e060eb5605ff72b6f2dd2aab7a761379373fd89d314f4752abbf")
     version("1.7.2", sha256="45399b46d60c253327a460e99856752009fcee5f5d3c80b2f7c0cae1c38d56dd")
     version("1.4.2", sha256="58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b")
     version("1.3.0", sha256="024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9")
 
-    depends_on("python@3.5:", type=("build", "run"))
-    depends_on("python@3.7:", when="@1.8.1:", type=("build", "run"))
-    depends_on("py-setuptools", type="build")
     depends_on("py-setuptools@40:", type="build", when="@1.7.2:")
+    depends_on("py-setuptools", type="build")
     depends_on("py-cython", type="build")
+
     depends_on("py-multidict@4.0:", type=("build", "run"))
     depends_on("py-idna@2.0:", type=("build", "run"))
-    depends_on("py-typing-extensions@3.7.4:", type=("build", "run"), when="@1.7.2: ^python@:3.7")
+    depends_on("py-typing-extensions@3.7.4:", when="@1.7.2: ^python@:3.7", type=("build", "run"))
 
     @run_before("install")
     def fix_cython(self):
diff --git a/var/spack/repos/builtin/packages/py-zipp/package.py b/var/spack/repos/builtin/packages/py-zipp/package.py
index 50a9abe35ae154..113506c3ea03c5 100644
--- a/var/spack/repos/builtin/packages/py-zipp/package.py
+++ b/var/spack/repos/builtin/packages/py-zipp/package.py
@@ -12,16 +12,19 @@ class PyZipp(PythonPackage):
     homepage = "https://github.com/jaraco/zipp"
     pypi = "zipp/zipp-0.6.0.tar.gz"
 
+    version("3.17.0", sha256="84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0")
     version("3.8.1", sha256="05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2")
     version("3.6.0", sha256="71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832")
     version("0.6.0", sha256="3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e")
     version("0.5.1", sha256="ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3")
 
-    depends_on("python@2.7:", type=("build", "run"))
-    depends_on("python@3.6:", type=("build", "run"), when="@2.0.0:")
-    depends_on("python@3.7:", type=("build", "run"), when="@3.8.1:")
-    depends_on("py-setuptools@34.4:", type="build", when="@0.3.3:")
-    depends_on("py-setuptools@56:", type="build", when="@3.5.1:")
+    depends_on("python@3.8:", when="@3.16:", type=("build", "run"))
+    # needed for spack bootstrap as spack itself supports python 3.6
+    depends_on("python@3.7:", when="@3.8.1:", type=("build", "run"))
+    depends_on("py-setuptools@56:", when="@3.5.1:", type="build")
+    depends_on("py-setuptools@34.4:", when="@0.3.3:", type="build")
+    depends_on("py-setuptools-scm@3.4.1: +toml", when="@2.0.1:", type="build")
     depends_on("py-setuptools-scm@1.15.0:", type="build")
-    depends_on("py-setuptools-scm@3.4.1: +toml", type="build", when="@2.0.1:")
+
+    # Historical dependencies
     depends_on("py-more-itertools", type=("build", "run"), when="@0.6.0:2.1.0")
diff --git a/var/spack/repos/builtin/packages/pypy-bootstrap/package.py b/var/spack/repos/builtin/packages/pypy-bootstrap/package.py
new file mode 100644
index 00000000000000..be10d920acec98
--- /dev/null
+++ b/var/spack/repos/builtin/packages/pypy-bootstrap/package.py
@@ -0,0 +1,63 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import platform
+
+from spack.package import *
+
+
+class PypyBootstrap(Package):
+    """Binary build of PyPy 2 for bootstrapping source build of PyPy 3."""
+
+    homepage = "https://www.pypy.org/"
+    url = "https://downloads.python.org/pypy/pypy2.7-v7.3.12-linux64.tar.bz2"
+
+    maintainers("adamjstewart")
+
+    if platform.system() == "Linux":
+        if platform.machine() == "x86_64":
+            version(
+                "2.7-v7.3.12", "1a61a2574b79466f606010f2999a2b995bd96cd085f91a78ebdd3d5c2c40e81d"
+            )
+        elif platform.machine() == "aarch64":
+            version(
+                "2.7-v7.3.12", "e04dcb6286a7b4724ec3f0e50d3cc1ba8583301dd1658c06d7f37599e4201c59"
+            )
+    elif platform.system() == "Darwin":
+        if platform.machine() == "arm64":
+            version(
+                "2.7-v7.3.12", "6b747aa076ae8597e49603c5dec4ca5935a1a0a132d7404a559be96a260d9bf7"
+            )
+        elif platform.machine() == "x86_64":
+            version(
+                "2.7-v7.3.12", "6e89ffdd15537ce4ffce3145b65ee57c2e9c952892bd95b934012d2f009f503b"
+            )
+    elif platform.system() == "Windows":
+        version("2.7-v7.3.12", "84cd3b98812d47a1ddb36f3417cc96b3dbdfa32c2b4e16438f205e1253f7ccea")
+
+    def url_for_version(self, version):
+        url = "https://downloads.python.org/pypy/pypy{}-{}.{}"
+        ext = "tar.bz2"
+        if platform.system() == "Linux":
+            if platform.machine() == "x86_64":
+                arch = "linux64"
+            elif platform.machine() == "aarch64":
+                arch = "aarch64"
+        elif platform.system() == "Darwin":
+            arch = "macos_" + platform.machine()
+        elif platform.system() == "Windows":
+            arch = "win64"
+            ext = "zip"
+        return url.format(version, arch, ext)
+
+    def install(self, spec, prefix):
+        install_tree(".", prefix)
+
+    @property
+    def command(self):
+        return Executable(self.prefix.bin.pypy)
+
+    def setup_dependent_package(self, module, dependent_spec):
+        module.pypy = self.command
diff --git a/var/spack/repos/builtin/packages/pypy/package.py b/var/spack/repos/builtin/packages/pypy/package.py
new file mode 100644
index 00000000000000..3f1ec58764a62b
--- /dev/null
+++ b/var/spack/repos/builtin/packages/pypy/package.py
@@ -0,0 +1,171 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Pypy(Package):
+    """A fast, compliant alternative implementation of Python."""
+
+    homepage = "https://www.pypy.org/"
+    url = "https://downloads.python.org/pypy/pypy3.10-v7.3.12-src.tar.bz2"
+    hg = "https://foss.heptapod.net/pypy/pypy"
+
+    maintainers("adamjstewart")
+
+    version(
+        "3.10-v7.3.12", sha256="86e4e4eacc36046c6182f43018796537fe33a60e1d2a2cc6b8e7f91a5dcb3e42"
+    )
+
+    variant("ctypes", default=True, description="Build ctypes module")
+    variant("zlib", default=True, description="Build zlib module")
+    variant("bz2", default=True, description="Build bz2 module")
+    variant("pyexpat", default=True, description="Build pyexpat module")
+    variant("sqlite3", default=True, description="Build sqlite3 module")
+    variant("ssl", default=True, description="Build ssl module")
+    variant("curses", default=True, description="Build curses module")
+    variant("dbm", default=True, description="Build dbm module")
+    variant("tkinter", default=False, description="Build tkinter module")
+    variant("lzma", default=True, description="Build lzma module")
+
+    # https://doc.pypy.org/en/latest/build.html#install-build-time-dependencies
+    depends_on("pypy-bootstrap", type="build")  # any Python 2 executable
+    # depends_on("py-cffi", type="build")  # only for CPython
+
+    depends_on("libffi", when="+ctypes")
+    depends_on("pkgconfig", when="+ctypes", type="build")
+    depends_on("zlib-api", when="+zlib")
+    depends_on("bzip2", when="+bz2")
+    depends_on("expat", when="+pyexpat")
+    depends_on("sqlite", when="+sqlite3")
+    depends_on("openssl", when="+ssl")
+    depends_on("ncurses", when="+curses")
+    depends_on("gdbm", when="+dbm")
+    depends_on("tcl", when="+tkinter")
+    depends_on("tk", when="+tkinter")
+    depends_on("libx11", when="+tkinter")
+    depends_on("xz", when="+lzma")
+
+    # TODO: add testing
+    # https://doc.pypy.org/en/latest/contributing.html#testing
+    # depends_on("bdw-gc@7.4:", type="test")
+    # depends_on("openssl", type="test")
+    # depends_on("py-pycparser", type="test")
+    # depends_on("py-hypothesis", type="test")
+
+    phases = ["translate", "build", "package", "install"]
+
+    @property
+    def build_directory(self):
+        return join_path(self.stage.source_path, "pypy", "goal")
+
+    def patch(self):
+        # Fix detection of tcl/tk
+        tklib_build = FileFilter(join_path("lib_pypy", "_tkinter", "tklib_build.py"))
+        if "+tkinter" in self.spec:
+            incs = self.spec["tcl"].headers + self.spec["tk"].headers
+            libs = self.spec["tcl"].libs + self.spec["tk"].libs
+            tklib_build.filter("incdirs = .*", f"incdirs = {incs.directories}")
+            tklib_build.filter("linklibs = .*", f"linklibs = {libs.names}")
+            tklib_build.filter("libdirs = .*", f"libdirs = {libs.directories}")
+
+    def setup_build_environment(self, env):
+        # https://doc.pypy.org/en/latest/build.html#set-environment-variables-that-will-affect-translation
+        env.set("PYPY_USESSION_DIR", self.stage.source_path)
+        env.prepend_path("PYTHONPATH", self.stage.source_path)
+
+    def translate_args(self):
+        args = ["--opt=jit", "--shared", "--make-jobs", str(make_jobs), "targetpypystandalone.py"]
+
+        variant_to_flag = {
+            "ctypes": "_cffi_backend",
+            "bz2": "bz2",
+            "zlib": "zlib",
+            "pyexpat": "pyexpat",
+        }
+
+        for variant, flag in variant_to_flag.items():
+            if f"+{variant}" in self.spec:
+                args.append(f"--withmod-{flag}")
+            else:
+                args.append(f"--withoutmod-{flag}")
+
+        return args
+
+    def translate(self, spec, prefix):
+        # https://doc.pypy.org/en/latest/build.html#run-the-translation
+        rpython = join_path(self.stage.source_path, "rpython", "bin", "rpython")
+        with working_dir(self.build_directory):
+            pypy(rpython, *self.translate_args())
+
+    def build_args(self):
+        modules = ["audioop", "syslog", "grp", "resource", "_posixshmem"]
+
+        if "+ctypes" in self.spec:
+            modules.extend(["_ctypes._ctypes_cffi", "_pypy_util_cffi_inner"])
+
+        if "+ssl" in self.spec:
+            modules.extend(["_blake2", "_ssl", "_sha3"])
+
+        if "+sqlite3" in self.spec:
+            modules.append("sqlite3")
+
+        if "+tkinter" in self.spec:
+            modules.append("_tkinter")
+
+        if "+curses" in self.spec:
+            modules.append("curses")
+
+        if "+dbm" in self.spec:
+            modules.append("_gdbm")
+
+        if "+lzma" in self.spec:
+            modules.append("lzma")
+
+        return ["--only=" + ",".join(modules)]
+
+    def build(self, spec, prefix):
+        # https://doc.pypy.org/en/latest/build.html#build-cffi-import-libraries-for-the-stdlib
+        build_cffi_imports = join_path(
+            self.stage.source_path, "lib_pypy", "pypy_tools", "build_cffi_imports.py"
+        )
+        pypy_c = Executable(join_path(self.build_directory, f"pypy{self.version.up_to(2)}-c"))
+        with working_dir(self.build_directory):
+            pypy_c(build_cffi_imports, *self.build_args())
+
+    def package_args(self):
+        args = [
+            f"--builddir={self.build_directory}",
+            "--no-embedded-dependencies",
+            "--no-make-portable",
+        ]
+
+        variant_to_flag = {
+            "dbm": "_gdbm",
+            "ssl": "_ssl",
+            "curses": "curses",
+            "lzma": "lzma",
+            "sqlite3": "sqlite3",
+            "tkinter": "_tkinter",
+            "ctypes": "cffi",
+        }
+
+        for variant, flag in variant_to_flag.items():
+            if f"~{variant}" in self.spec:
+                args.append(f"--without-{flag}")
+
+        return args
+
+    def package(self, spec, prefix):
+        # https://doc.pypy.org/en/latest/build.html#packaging-preparing-for-installation
+        package = join_path(self.stage.source_path, "pypy", "tool", "release", "package.py")
+        pypy(package, *self.package_args())
+
+    def install(self, spec, prefix):
+        # https://doc.pypy.org/en/latest/build.html#installation
+        with working_dir(join_path(self.build_directory, "pypy-nightly")):
+            install_tree("bin", prefix.bin)
+            install_tree("include", prefix.include)
+            install_tree("lib", prefix.lib)
diff --git a/var/spack/repos/builtin/packages/pythia8/package.py b/var/spack/repos/builtin/packages/pythia8/package.py
index af355588a891f2..f7ef4ad30bdf5d 100644
--- a/var/spack/repos/builtin/packages/pythia8/package.py
+++ b/var/spack/repos/builtin/packages/pythia8/package.py
@@ -131,16 +131,16 @@ def configure_args(self):
                 args.append("--with-boost=" + self.spec["boost"].prefix)
 
         if "+madgraph5amc" in self.spec:
-            args += "--with-mg5mes=" + self.spec["madgraph5amc"].prefix
+            args.append("--with-mg5mes=" + self.spec["madgraph5amc"].prefix)
         else:
-            args += "--without-mg5mes"
+            args.append("--without-mg5mes")
 
         args += self.with_or_without("hepmc3", activation_value="prefix")
 
         if "+fastjet" in self.spec:
-            args += "--with-fastjet3=" + self.spec["fastjet"].prefix
+            args.append("--with-fastjet3=" + self.spec["fastjet"].prefix)
         else:
-            args += "--without-fastjet3"
+            args.append("--without-fastjet3")
 
         args += self.with_or_without("evtgen", activation_value="prefix")
         args += self.with_or_without("root", activation_value="prefix")
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index 0cf74cfba6b437..73a5043ffbd5d6 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -32,7 +32,7 @@ class Python(Package):
     list_depth = 1
     tags = ["windows"]
 
-    maintainers("adamjstewart", "skosukhin", "scheibelp", "pradyunsg")
+    maintainers("adamjstewart", "skosukhin", "scheibelp")
 
     phases = ["configure", "build", "install"]
 
@@ -40,16 +40,20 @@ class Python(Package):
     install_targets = ["install"]
     build_targets: List[str] = []
 
+    version("3.12.0", sha256="51412956d24a1ef7c97f1cb5f70e185c13e3de1f50d131c0aac6338080687afb")
+    version(
+        "3.11.6",
+        sha256="c049bf317e877cbf9fce8c3af902436774ecef5249a29d10984ca3a37f7f4736",
+        preferred=True,
+    )
+    version("3.11.5", sha256="a12a0a013a30b846c786c010f2c19dd36b7298d888f7c4bd1581d90ce18b5e58")
     version("3.11.4", sha256="85c37a265e5c9dd9f75b35f954e31fbfc10383162417285e30ad25cc073a0d63")
     version("3.11.3", sha256="1a79f3df32265d9e6625f1a0b31c28eb1594df911403d11f3320ee1da1b3e048")
     version("3.11.2", sha256="2411c74bda5bbcfcddaf4531f66d1adc73f247f529aee981b029513aefdbf849")
     version("3.11.1", sha256="baed518e26b337d4d8105679caf68c5c32630d702614fc174e98cb95c46bdfa4")
     version("3.11.0", sha256="64424e96e2457abbac899b90f9530985b51eef2905951febd935f0e73414caeb")
-    version(
-        "3.10.12",
-        sha256="a43cd383f3999a6f4a7db2062b2fc9594fefa73e175b3aedafa295a51a7bb65c",
-        preferred=True,
-    )
+    version("3.10.13", sha256="698ec55234c1363bd813b460ed53b0f108877c7a133d48bde9a50a1eb57b7e65")
+    version("3.10.12", sha256="a43cd383f3999a6f4a7db2062b2fc9594fefa73e175b3aedafa295a51a7bb65c")
     version("3.10.11", sha256="f3db31b668efa983508bd67b5712898aa4247899a346f2eb745734699ccd3859")
     version("3.10.10", sha256="fba64559dde21ebdc953e4565e731573bb61159de8e4d4cedee70fb1196f610d")
     version("3.10.9", sha256="4ccd7e46c8898f4c7862910a1703aa0e63525913a519abb2f55e26220a914d88")
@@ -62,6 +66,7 @@ class Python(Package):
     version("3.10.2", sha256="3c0ede893011319f9b0a56b44953a3d52c7abf9657c23fb4bc9ced93b86e9c97")
     version("3.10.1", sha256="b76117670e7c5064344b9c138e141a377e686b9063f3a8a620ff674fa8ec90d3")
     version("3.10.0", sha256="c4e0cbad57c90690cb813fb4663ef670b4d0f587d8171e2c42bd4c9245bd2758")
+    version("3.9.18", sha256="504ce8cfd59addc04c22f590377c6be454ae7406cb1ebf6f5a350149225a9354")
     version("3.9.17", sha256="8ead58f669f7e19d777c3556b62fae29a81d7f06a7122ff9bc57f7dd82d7e014")
     version("3.9.16", sha256="1ad539e9dbd2b42df714b69726e0693bc6b9d2d2c8e91c2e43204026605140c5")
     version("3.9.15", sha256="48d1ccb29d5fbaf1fb8f912271d09f7450e426d4dfe95978ef6aaada70ece4d8")
@@ -80,6 +85,7 @@ class Python(Package):
     version("3.9.2", sha256="7899e8a6f7946748830d66739f2d8f2b30214dad956e56b9ba216b3de5581519")
     version("3.9.1", sha256="29cb91ba038346da0bd9ab84a0a55a845d872c341a4da6879f462e94c741f117")
     version("3.9.0", sha256="df796b2dc8ef085edae2597a41c1c0a63625ebd92487adaef2fed22b567873e8")
+    version("3.8.18", sha256="7c5df68bab1be81a52dea0cc2e2705ea00553b67107a301188383d7b57320b16")
     version("3.8.17", sha256="def428fa6cf61b66bcde72e3d9f7d07d33b2e4226f04f9d6fce8384c055113ae")
     version("3.8.16", sha256="71ca9d935637ed2feb59e90a368361dc91eca472a90acb1d344a2e8178ccaf10")
     version("3.8.15", sha256="924d46999df82aa2eaa1de5ca51d6800ffb56b4bf52486a28f40634e3362abc4")
@@ -249,7 +255,7 @@ class Python(Package):
         depends_on("sqlite@3.7.15:", when="@3.10:+sqlite3")
         depends_on("gdbm", when="+dbm")  # alternatively ndbm or berkeley-db
         depends_on("libnsl", when="+nis")
-        depends_on("zlib@1.1.3:", when="+zlib")
+        depends_on("zlib-api", when="+zlib")
         depends_on("bzip2", when="+bz2")
         depends_on("xz libs=shared", when="+lzma")
         depends_on("expat", when="+pyexpat")
@@ -276,7 +282,7 @@ class Python(Package):
     patch("python-3.7.2-distutils-C++.patch", when="@3.7.2")
     patch("python-3.7.3-distutils-C++.patch", when="@3.7.3")
     patch("python-3.7.4+-distutils-C++.patch", when="@3.7.4:3.10")
-    patch("python-3.7.4+-distutils-C++-testsuite.patch", when="@3.7.4:")
+    patch("python-3.7.4+-distutils-C++-testsuite.patch", when="@3.7.4:3.11")
     patch("python-3.11-distutils-C++.patch", when="@3.11.0:3.11")
     patch("cpython-windows-externals.patch", when="@:3.9.6 platform=windows")
     patch("tkinter-3.7.patch", when="@3.7 platform=darwin")
@@ -287,11 +293,11 @@ class Python(Package):
     patch("tkinter-3.11.patch", when="@3.11.0:3.11 ~tkinter")
 
     # Ensure that distutils chooses correct compiler option for RPATH:
-    patch("rpath-non-gcc.patch")
+    patch("rpath-non-gcc.patch", when="@:3.11")
 
     # Ensure that distutils chooses correct compiler option for RPATH on fj:
     patch("fj-rpath-3.1.patch", when="@:3.9.7,3.10.0 %fj")
-    patch("fj-rpath-3.9.patch", when="@3.9.8:3.9,3.10.1: %fj")
+    patch("fj-rpath-3.9.patch", when="@3.9.8:3.9,3.10.1:3.11 %fj")
 
     # Fixes build with the Intel compilers
     # https://github.com/python/cpython/pull/16717
@@ -308,11 +314,11 @@ class Python(Package):
     conflicts("%nvhpc")
 
     # https://bugs.python.org/issue45405
-    conflicts("@:3.7.2,3.8.0:3.8.12,3.9.0:3.9.10,3.10.0:3.10.2", when="%apple-clang@13:")
+    conflicts("@:3.7.12,3.8.0:3.8.12,3.9.0:3.9.7,3.10.0", when="%apple-clang@13:")
 
     # See https://github.com/python/cpython/issues/106424
     # datetime.now(timezone.utc) segfaults
-    conflicts("@3.9:", when="%oneapi@2022.2.1:")
+    conflicts("@3.9:", when="%oneapi@2022.2.1:2023")
 
     # Used to cache various attributes that are expensive to compute
     _config_vars: Dict[str, Dict[str, str]] = {}
@@ -320,7 +326,7 @@ class Python(Package):
     # An in-source build with --enable-optimizations fails for python@3.X
     build_directory = "spack-build"
 
-    executables = [r"^python[\d.]*[mw]?$"]
+    executables = [r"^python\d?$"]
 
     @classmethod
     def determine_version(cls, exe):
@@ -435,6 +441,11 @@ def setup_build_environment(self, env):
             if spec.satisfies("%gcc") or spec.satisfies("%fj"):
                 env.unset("LC_ALL")
 
+        # https://github.com/python/cpython/issues/87275
+        if spec.satisfies("@:3.9.5 +optimizations %apple-clang"):
+            xcrun = Executable("/usr/bin/xcrun")
+            env.set("LLVM_AR", xcrun("-find", "ar", output=str).strip())
+
     def flag_handler(self, name, flags):
         # python 3.8 requires -fwrapv when compiled with intel
         if self.spec.satisfies("@3.8: %intel"):
@@ -885,6 +896,9 @@ def config_vars(self):
                 "INCLUDEPY": self.prefix.include.join("python{}").format(version),
                 "LIBDEST": self.prefix.lib.join("python{}").format(version),
                 "LIBDIR": self.prefix.lib,
+                "LIBPL": self.prefix.lib.join("python{0}")
+                .join("config-{0}-{1}")
+                .format(version, sys.platform),
                 "LDLIBRARY": "{}python{}.{}".format(lib_prefix, version, dso_suffix),
                 "LIBRARY": "{}python{}.{}".format(lib_prefix, version, stat_suffix),
                 "LDSHARED": "cc",
@@ -954,6 +968,10 @@ def find_library(self, library):
         # in either lib or lib64, so we need to ask Python where its LIBDIR is.
         libdir = self.config_vars["LIBDIR"]
 
+        # Debian and derivatives use a triplet subdir under /usr/lib, LIBPL can be used
+        # to get the Python library directory
+        libpldir = self.config_vars["LIBPL"]
+
         # The system Python installation on macOS and Homebrew installations
         # install libraries into a Frameworks directory
         frameworkprefix = self.config_vars["PYTHONFRAMEWORKPREFIX"]
@@ -969,7 +987,14 @@ def find_library(self, library):
         win_bin_dir = self.config_vars["BINDIR"]
         win_root_dir = self.config_vars["prefix"]
 
-        directories = [libdir, frameworkprefix, macos_developerdir, win_bin_dir, win_root_dir]
+        directories = [
+            libdir,
+            libpldir,
+            frameworkprefix,
+            macos_developerdir,
+            win_bin_dir,
+            win_root_dir,
+        ]
 
         # The Python shipped with Xcode command line tools isn't in any of these locations
         for subdir in ["lib", "lib64"]:
@@ -1028,8 +1053,8 @@ def libs(self):
                 return lib
 
         raise spack.error.NoLibrariesError(
-            "Unable to find {} libraries with the following names:".format(self.name),
-            "\n".join(candidates),
+            "Unable to find {} libraries with the following names:\n\n* ".format(self.name)
+            + "\n* ".join(candidates)
         )
 
     @property
@@ -1054,8 +1079,8 @@ def headers(self):
                 break
         else:
             raise spack.error.NoHeadersError(
-                "Unable to locate {} headers in any of these locations:".format(self.name),
-                "\n".join(candidates),
+                "Unable to locate {} headers in any of these locations:\n\n* ".format(self.name)
+                + "\n* ".join(candidates)
             )
 
         headers.directories = [os.path.dirname(config_h)]
@@ -1227,12 +1252,11 @@ def setup_dependent_run_environment(self, env, dependent_spec):
         """Set PYTHONPATH to include the site-packages directory for the
         extension and any other python extensions it depends on.
         """
-        for d in dependent_spec.traverse(deptype=("run"), root=True):
-            if d.package.extends(self.spec):
-                # Packages may be installed in platform-specific or platform-independent
-                # site-packages directories
-                for directory in {self.platlib, self.purelib}:
-                    env.prepend_path("PYTHONPATH", os.path.join(d.prefix, directory))
+        if dependent_spec.package.extends(self.spec):
+            # Packages may be installed in platform-specific or platform-independent
+            # site-packages directories
+            for directory in {self.platlib, self.purelib}:
+                env.prepend_path("PYTHONPATH", os.path.join(dependent_spec.prefix, directory))
 
     def setup_dependent_package(self, module, dependent_spec):
         """Called before python modules' install() methods."""
diff --git a/var/spack/repos/builtin/packages/q-e-sirius/package.py b/var/spack/repos/builtin/packages/q-e-sirius/package.py
index b3ef5761fd6209..1605e4e37d5703 100644
--- a/var/spack/repos/builtin/packages/q-e-sirius/package.py
+++ b/var/spack/repos/builtin/packages/q-e-sirius/package.py
@@ -17,35 +17,63 @@ class QESirius(CMakePackage):
 
     maintainers("simonpintarelli")
 
-    version("develop-ristretto", branch="ristretto", submodules=True)
-    version("6.7-rc1-sirius", tag="v6.7-rc1-sirius", submodules=True)
+    version("develop-ristretto", branch="ristretto", preferred=True, submodules=True)
+    version(
+        "6.7-rc1-sirius",
+        tag="v6.7-rc1-sirius",
+        commit="b1c79e30a2f9351316a90ca296f98cffef1f35c3",
+        submodules=True,
+    )
 
     variant("mpi", default=True, description="Builds with MPI support")
     variant("openmp", default=True, description="Enables OpenMP support")
-    variant("scalapack", default=False, description="Enables SCALAPACK support")
-    variant("elpa", default=False, description="Uses ELPA as an eigenvalue solver")
     variant("libxc", default=False, description="Support functionals through libxc")
-    variant("hdf5", default=False, description="Enables HDF5 support")
     variant("sirius_apps", default=False, description="Build SIRIUS standalone binaries")
+    # Support for HDF5 has been added starting in QE 6.1.0 and is
+    # still experimental
+    variant(
+        "hdf5",
+        default="none",
+        description="Orbital and density data I/O with HDF5",
+        values=("parallel", "serial", "none"),
+        multi=False,
+    )
 
     depends_on("sirius +fortran")
     depends_on("sirius +apps", when="+sirius_apps")
     depends_on("sirius ~apps", when="~sirius_apps")
     depends_on("sirius +openmp", when="+openmp")
-    depends_on("sirius@develop", when="@develop")
+    depends_on("sirius@develop", when="@develop-ristretto")
 
     depends_on("mpi", when="+mpi")
-    depends_on("scalapack", when="+scalapack")
     depends_on("elpa", when="+elpa")
     depends_on("libxc", when="+libxc")
-    depends_on("hdf5", when="+hdf5")
-
+    depends_on("fftw-api@3")
+    depends_on("blas")
+    depends_on("lapack")
     depends_on("git", type="build")
     depends_on("pkgconfig", type="build")
 
     conflicts("~mpi", when="+scalapack", msg="SCALAPACK requires MPI support")
     conflicts("~scalapack", when="+elpa", msg="ELPA requires SCALAPACK support")
 
+    with when("+mpi"):
+        depends_on("mpi")
+        variant("scalapack", default=True, description="Enables scalapack support")
+
+    with when("+scalapack"):
+        depends_on("scalapack")
+        variant("elpa", default=False, description="Uses elpa as an eigenvalue solver")
+
+    # Versions of HDF5 prior to 1.8.16 lead to QE runtime errors
+    depends_on("hdf5@1.8.16:+fortran+hl+mpi", when="hdf5=parallel")
+    depends_on("hdf5@1.8.16:+fortran+hl~mpi", when="hdf5=serial")
+
+    with when("+openmp"):
+        depends_on("fftw+openmp", when="^fftw")
+        depends_on("openblas threads=openmp", when="^openblas")
+        depends_on("intel-mkl threads=openmp", when="^intel-mkl")
+
     def cmake_args(self):
         args = [
             "-DQE_ENABLE_SIRIUS=ON",
@@ -56,14 +84,19 @@ def cmake_args(self):
             self.define_from_variant("QE_ENABLE_OPENMP", "openmp"),
             self.define_from_variant("QE_ENABLE_ELPA", "elpa"),
             self.define_from_variant("QE_ENABLE_LIBXC", "libxc"),
-            self.define_from_variant("QE_ENABLE_HDF5", "hdf5"),
             self.define_from_variant("QE_ENABLE_SCALAPACK", "scalapack"),
         ]
 
+        if not self.spec.satisfies("hdf5=none"):
+            args.append(self.define("QE_ENABLE_HDF5", True))
+
         # Work around spack issue #19970 where spack sets
         # rpaths for MKL just during make, but cmake removes
         # them during make install.
-        if "^mkl" in self.spec:
+        if self.spec["lapack"].name in INTEL_MATH_LIBRARIES:
             args.append("-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON")
+        spec = self.spec
+        args.append(self.define("BLAS_LIBRARIES", spec["blas"].libs.joined(";")))
+        args.append(self.define("LAPACK_LIBRARIES", spec["lapack"].libs.joined(";")))
 
         return args
diff --git a/var/spack/repos/builtin/packages/qd/package.py b/var/spack/repos/builtin/packages/qd/package.py
index 30a631e193aa4a..9a5c90a74bbd8b 100644
--- a/var/spack/repos/builtin/packages/qd/package.py
+++ b/var/spack/repos/builtin/packages/qd/package.py
@@ -16,7 +16,7 @@ class Qd(AutotoolsPackage):
 
     tags = ["hep"]
 
-    version("2.3.13", commit="a57dde9")
+    version("2.3.13", commit="a57dde96b3255b80f7f39cd80217c213bf78d949")
 
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
diff --git a/var/spack/repos/builtin/packages/qgis/package.py b/var/spack/repos/builtin/packages/qgis/package.py
index fb9d22a244b3eb..8daf503c98b4c7 100644
--- a/var/spack/repos/builtin/packages/qgis/package.py
+++ b/var/spack/repos/builtin/packages/qgis/package.py
@@ -17,15 +17,20 @@ class Qgis(CMakePackage):
 
     maintainers("adamjstewart", "Sinan81")
 
-    version("3.28.3", sha256="a09124f46465a520f6d735306ba3954c339b84aa396d6f52b476b82edcc4fe0e")
+    version("3.34.0", sha256="348a2df4c4520813a319b7f72546b3823e044cacd28646ba189b56a49c7d1b5f")
     # Prefer latest long term release
     version(
-        "3.22.16",
-        sha256="dbd1f8a639291bb2492eea61e4ef96079d7b27d3dfa538dab8cd98f31429254a",
+        "3.28.12",
+        sha256="d6d0ea39ed3433d553f8b83324dc14cfa90f8caaf766fa484791df9169800f25",
         preferred=True,
     )
+    version("3.28.11", sha256="c5eb703893c7f98de051c45d677c4a34b40f986db51782a4930ddefad4e193b4")
+    version("3.28.10", sha256="cff867e97909bbc2facce6343770dcb1b61fc6e4855f57783e30bf63d51c5218")
+    version("3.28.3", sha256="a09124f46465a520f6d735306ba3954c339b84aa396d6f52b476b82edcc4fe0e")
+    version("3.22.16", sha256="dbd1f8a639291bb2492eea61e4ef96079d7b27d3dfa538dab8cd98f31429254a")
     version("3.22.0", sha256="cf0c169863f332aab67d8c4943e14b73a564f0254bf54015f5826c6427e6785b")
     version("3.18.2", sha256="1913e4d5596bbc8b7d143f3defb18bf376f750a71f334f69d76af5deca7ecc5d")
+    version("3.16.16", sha256="ccd2f404534fcb00b5e17863375462090c9575e68b32ce50b2e7e925d1e01a49")
     version("3.16.12", sha256="65e9634b5c885c98f3555cf77bc2e3fae5e19279aa17e3f6626ff5d7455fd2b9")
     version("3.16.5", sha256="525f469ad6e40dd7a8f09ebab5eb6a2dffc45939b99b7d937750cc04ed78d61c")
     version("3.14.16", sha256="c9915c2e577f1812a2b35b678b123c58407e07824d73e5ec0dda13db7ca75c04")
@@ -60,6 +65,7 @@ class Qgis(CMakePackage):
         "custom_widgets", default=False, description="Build QGIS custom widgets for Qt Designer"
     )
     variant("desktop", default=True, description="Build QGIS desktop")
+    # variant("draco", default=True, description="Build with DRACO support") #TODO
     variant("georeferencer", default=True, description="Build GeoReferencer plugin")
     variant("globe", default=False, description="Build Globe plugin")
     variant("grass7", default=False, description="Build with GRASS providers and plugin")
@@ -74,6 +80,7 @@ class Qgis(CMakePackage):
     )
     variant("oauth2_plugin", default=True, description="Build OAuth2 authentication method plugin")
     variant("oracle", default=False, description="Build with Oracle support")
+    # variant("pdal", default=False, description="Build with PDAL support") #TODO
     variant("postgresql", default=True, description="Build with PostreSQL support")
     variant(
         "py_compile",
@@ -110,9 +117,12 @@ class Qgis(CMakePackage):
     depends_on("proj@4.4.0:")
     depends_on("proj@4.9.3:", when="@3.8.2:")
     depends_on("proj@7.2:", when="@3.28:")
+    depends_on("proj@:8", when="@3.28")  # build fails with proj@9
     depends_on("py-psycopg2", type=("build", "run"))  # TODO: is build dependency necessary?
     depends_on("py-pyqt4", when="@2")
     depends_on("py-pyqt5@5.3:", when="@3")
+    depends_on("py-sip", type="build")
+    depends_on("py-pyqt-builder", type="build", when="^py-sip@5:")
     depends_on("py-requests", type=("build", "run"))  # TODO: is build dependency necessary?
     depends_on("python@3.0.0:", type=("build", "run"), when="@3")
     depends_on("python@3.6:", type=("build", "run"), when="@3.18:")
@@ -120,14 +130,15 @@ class Qgis(CMakePackage):
     depends_on("qca@2.2.1:")
     depends_on("qjson")
     depends_on("qscintilla +python")
-    depends_on("qt+dbus")
-    depends_on("qt+dbus@5.12.0:", when="@3.20:")
-    depends_on("qt+dbus@5.14.0:", when="@3.28:")
+    depends_on("qt+dbus+location")
+    depends_on("qt+dbus+location@5.12.0:", when="@3.20:")
+    depends_on("qt+dbus+location@5.14.0:", when="@3.28:")
     depends_on("qtkeychain@0.5:", when="@3:")
     depends_on("qwt@5:")
     depends_on("qwtpolar")
     depends_on("sqlite@3.0.0: +column_metadata")
     depends_on("protobuf", when="@3.16.4:")
+    depends_on("protobuf@:3.21", when="@:3.28")
     depends_on("zstd", when="@3.22:")
 
     # Runtime python dependencies, not mentioned in install instructions
@@ -161,13 +172,40 @@ class Qgis(CMakePackage):
     depends_on("qt@5.9.0:", when="@3.10.0:")
     depends_on("qtkeychain@:1.5", when="^qt@4")
     depends_on("qt@:4", when="@2")
-    # Help concretizer
-    # +qsci_api is implied by qscintilla+python dependency
-    depends_on("py-pyqt5@5.3: +qsci_api", when="@3")
 
     patch("pyqt5.patch", when="@:3.14 ^qt@5")
-    patch("pyqt5_3165x.patch", when="@3.16.5:3.21 ^qt@5")
-    patch("pyqt5_322x.patch", when="@3.22: ^qt@5")
+    patch("pyqt5_3165x.patch", when="@3.16.5:3.21 ^qt@5 ^py-sip@4")
+    patch("pyqt5_322x.patch", when="@3.22: ^qt@5 ^py-sip@4")
+
+    @run_before("cmake", when="^py-pyqt5")
+    def fix_pyqt5_cmake(self):
+        cmfile = FileFilter(join_path("cmake", "FindPyQt5.cmake"))
+        pyqtpath = join_path(
+            self.spec["py-pyqt5"].prefix, self.spec["python"].package.platlib, "PyQt5"
+        )
+        cmfile.filter(
+            'SET(PYQT5_MOD_DIR "${Python_SITEARCH}/PyQt5")',
+            'SET(PYQT5_MOD_DIR "' + pyqtpath + '")',
+            string=True,
+        )
+        cmfile.filter(
+            'SET(PYQT5_SIP_DIR "${Python_SITEARCH}/PyQt5/bindings")',
+            'SET(PYQT5_SIP_DIR "' + pyqtpath + '/bindings")',
+            string=True,
+        )
+
+    @run_before("build")
+    def fix_qsci_sip(self):
+        if "^py-pyqt5" in self.spec:
+            pyqtx = "PyQt5"
+        elif "^py-pyqt6" in self.spec:
+            pyqtx = "PyQt6"
+
+        sip_inc_dir = join_path(
+            self.spec["qscintilla"].prefix, self.spec["python"].package.platlib, pyqtx, "bindings"
+        )
+        with open(join_path("python", "gui", "pyproject.toml.in"), "a") as tomlfile:
+            tomlfile.write(f'\n[tool.sip.project]\nsip-include-dirs = ["{sip_inc_dir}"]\n')
 
     def cmake_args(self):
         spec = self.spec
@@ -188,6 +226,7 @@ def cmake_args(self):
                 "-DLIBZIP_INCLUDE_DIR=" + self.spec["libzip"].prefix.include,
                 "-DLIBZIP_CONF_INCLUDE_DIR=" + self.spec["libzip"].prefix.lib.libzip.include,
                 "-DGDAL_CONFIG_PREFER_PATH=" + self.spec["gdal"].prefix.bin,
+                "-DGDAL_CONFIG=" + join_path(self.spec["gdal"].prefix.bin, "gdal-config"),
                 "-DGEOS_CONFIG_PREFER_PATH=" + self.spec["geos"].prefix.bin,
                 "-DGSL_CONFIG_PREFER_PATH=" + self.spec["gsl"].prefix.bin,
                 "-DPOSTGRES_CONFIG_PREFER_PATH=" + self.spec["postgresql"].prefix.bin,
@@ -209,6 +248,7 @@ def cmake_args(self):
                     "TRUE" if "+custom_widgets" in spec else "FALSE"
                 ),
                 "-DWITH_DESKTOP={0}".format("TRUE" if "+desktop" in spec else "FALSE"),
+                "-DWITH_DRACO={0}".format("TRUE" if "+draco" in spec else "FALSE"),
                 "-DWITH_GEOREFERENCER={0}".format("TRUE" if "+georeferencer" in spec else "FALSE"),
                 "-DWITH_GLOBE={0}".format("TRUE" if "+globe" in spec else "FALSE"),
                 "-DWITH_GUI={0}".format("TRUE" if "+gui" in spec else "FALSE"),
@@ -216,6 +256,7 @@ def cmake_args(self):
                 self.define_from_variant("WITH_INTERNAL_O2", "internal_o2"),
                 "-DWITH_OAUTH2_PLUGIN={0}".format("TRUE" if "+oauth2_plugin" in spec else "FALSE"),
                 "-DWITH_ORACLE={0}".format("TRUE" if "+oracle" in spec else "FALSE"),
+                "-DWITH_PDAL={0}".format("TRUE" if "+pdal" in spec else "FALSE"),
                 "-DWITH_POSTGRESQL={0}".format("TRUE" if "+postgresql" in spec else "FALSE"),
                 "-DWITH_PY_COMPILE={0}".format("TRUE" if "+py_compile" in spec else "FALSE"),
                 "-DWITH_QSCIAPI={0}".format("TRUE" if "+qsciapi" in spec else "FALSE"),
diff --git a/var/spack/repos/builtin/packages/qmcpack/package.py b/var/spack/repos/builtin/packages/qmcpack/package.py
index 0190ba73d2c5fe..99612cedf9650b 100644
--- a/var/spack/repos/builtin/packages/qmcpack/package.py
+++ b/var/spack/repos/builtin/packages/qmcpack/package.py
@@ -23,25 +23,27 @@ class Qmcpack(CMakePackage, CudaPackage):
     # can occasionally change.
     # NOTE: 12/19/2017 QMCPACK 3.0.0 does not build properly with Spack.
     version("develop")
-    version("3.16.0", tag="v3.16.0")
-    version("3.15.0", tag="v3.15.0")
-    version("3.14.0", tag="v3.14.0")
-    version("3.13.0", tag="v3.13.0")
-    version("3.12.0", tag="v3.12.0")
-    version("3.11.0", tag="v3.11.0")
-    version("3.10.0", tag="v3.10.0")
-    version("3.9.2", tag="v3.9.2")
-    version("3.9.1", tag="v3.9.1")
-    version("3.9.0", tag="v3.9.0")
-    version("3.8.0", tag="v3.8.0")
-    version("3.7.0", tag="v3.7.0")
-    version("3.6.0", tag="v3.6.0")
-    version("3.5.0", tag="v3.5.0")
-    version("3.4.0", tag="v3.4.0")
-    version("3.3.0", tag="v3.3.0")
-    version("3.2.0", tag="v3.2.0")
-    version("3.1.1", tag="v3.1.1")
-    version("3.1.0", tag="v3.1.0")
+    version("3.17.1", tag="v3.17.1", commit="9d0d968139fc33f71dbf9159f526dd7b47f10a3b")
+    version("3.17.0", tag="v3.17.0", commit="9049a90626d1fe3c431f55c56a7197f8a13d5fc6")
+    version("3.16.0", tag="v3.16.0", commit="5b7544c40be105b0aafa1602601ccb0cf23ea547")
+    version("3.15.0", tag="v3.15.0", commit="690350ae148a1345b57e610f240d800384514d69")
+    version("3.14.0", tag="v3.14.0", commit="dbe4cebbd5f88ce42bf2deab5e571e64398e07ed")
+    version("3.13.0", tag="v3.13.0", commit="5f4ba013eceaf437530d5792dac5c08760d51c96")
+    version("3.12.0", tag="v3.12.0", commit="262d1915135aaf5786d9481765f79f48d3309533")
+    version("3.11.0", tag="v3.11.0", commit="129b7d88497b27de9efd4f2cf56d83b97541f0f0")
+    version("3.10.0", tag="v3.10.0", commit="f775d65b313b16b4ab22ecad40a968c0a6675ddc")
+    version("3.9.2", tag="v3.9.2", commit="8064ebfec9a8dd88748e8864013168c9aa7e05dd")
+    version("3.9.1", tag="v3.9.1", commit="e53d0b905ca2870c8c58b14437165addc6a6296f")
+    version("3.9.0", tag="v3.9.0", commit="a5c74d4327e4ba84c097fd904ce2b696bf2735ae")
+    version("3.8.0", tag="v3.8.0", commit="6abbff0b825dae52097f7965bdb549f84aed75cb")
+    version("3.7.0", tag="v3.7.0", commit="bc47a435682751c26c2695a005684d95f700bebb")
+    version("3.6.0", tag="v3.6.0", commit="91aca74ecb4e64bdccc7e05b28eda4653f8c07ff")
+    version("3.5.0", tag="v3.5.0", commit="1009b07ad6fa873d71a0755747e6a7f610eb169b")
+    version("3.4.0", tag="v3.4.0", commit="2d4b3d01224f7a4868da71d715e831341b2e7e5e")
+    version("3.3.0", tag="v3.3.0", commit="ff1599015f0695a75436f2942f257d3a29b8b752")
+    version("3.2.0", tag="v3.2.0", commit="d531f6b35fbab9ab2b80e9222f694b66e08bdd9d")
+    version("3.1.1", tag="v3.1.1", commit="07611637f823187ac5133d6e2249cdb86b92b04d")
+    version("3.1.0", tag="v3.1.0", commit="146d920cf33590eac6a7a976f88871c1fe6418a6")
 
     # These defaults match those in the QMCPACK manual
     variant(
@@ -374,7 +376,7 @@ def cmake_args(self):
         # Next two environment variables were introduced in QMCPACK 3.5.0
         # Prior to v3.5.0, these lines should be benign but CMake
         # may issue a warning.
-        if "^mkl" in spec:
+        if spec["lapack"].name in INTEL_MATH_LIBRARIES:
             args.append("-DENABLE_MKL=1")
             args.append("-DMKL_ROOT=%s" % env["MKLROOT"])
         else:
diff --git a/var/spack/repos/builtin/packages/qscintilla/package.py b/var/spack/repos/builtin/packages/qscintilla/package.py
index dea8ae886cf5c9..2048e8318eae10 100644
--- a/var/spack/repos/builtin/packages/qscintilla/package.py
+++ b/var/spack/repos/builtin/packages/qscintilla/package.py
@@ -16,49 +16,39 @@ class Qscintilla(QMakePackage):
     homepage = "https://www.riverbankcomputing.com/software/qscintilla/intro"
     url = "https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.12.0/QScintilla_src-2.12.0.tar.gz"
 
-    # Directory structure is changed in latest release, logic is lost
-    version(
-        "2.12.0",
-        sha256="a4cc9e7d2130ecfcdb18afb43b813ef122473f6f35deff747415fbc2fe0c60ed",
-        url="https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.12.0/QScintilla_src-2.12.0.tar.gz",
-    )
-
-    # Last standard release dates back to 2021/11/23
-    version(
-        "2.11.6",
-        sha256="e7346057db47d2fb384467fafccfcb13aa0741373c5d593bc72b55b2f0dd20a7",
-        preferred=True,
-        url="https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.11.6/QScintilla-2.11.6.tar.gz",
-    )
-    version(
-        "2.11.2",
-        sha256="029bdc476a069fda2cea3cd937ba19cc7fa614fb90578caef98ed703b658f4a1",
-        url="https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.11.2/QScintilla_gpl-2.11.2.tar.gz",
-    )
-    version(
-        "2.10.2",
-        sha256="14b31d20717eed95ea9bea4cd16e5e1b72cee7ebac647cba878e0f6db6a65ed0",
-        url="https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.10.2/QScintilla-2.10.2.tar.gz",
-    )
+    version("2.14.1", sha256="dfe13c6acc9d85dfcba76ccc8061e71a223957a6c02f3c343b30a9d43a4cdd4d")
+    version("2.14.0", sha256="449353928340300804c47b3785c3e62096f918a723d5eed8a5439764e6507f4c")
+    version("2.13.4", sha256="890c261f31e116f426b0ea03a136d44fc89551ebfd126d7b0bdf8a7197879986")
+    version("2.13.3", sha256="711d28e37c8fccaa8229e8e39a5b3b2d97f3fffc63da10b71c71b84fa3649398")
+    version("2.12.0", sha256="2116181cce3076aa4897e36182532d0e6768081fb0cf6dcdd5be720519ab1434")
 
     variant("designer", default=False, description="Enable pluging for Qt-Designer")
     variant("python", default=False, description="Build python bindings")
 
-    depends_on("qt+opengl", when="+python")
-    depends_on("qt")
-    depends_on("py-pyqt5 +qsci_api", type=("build", "run"), when="+python ^qt@5")
-    depends_on("py-pyqt4 +qsci_api", type=("build", "run"), when="+python ^qt@4")
+    depends_on("qmake")
+    depends_on("qmake+opengl", when="+python")
+    depends_on("py-pyqt6", type=("build", "run"), when="+python ^qt-base")
+    depends_on("py-pyqt-builder", type="build", when="+python")
+    depends_on("py-pyqt5", type=("build", "run"), when="+python ^qt@5")
     depends_on("python", type=("build", "run"), when="+python")
     # adter install inquires py-sip variant : so we need to have it
     depends_on("py-sip", type="build", when="~python")
 
     extends("python", when="+python")
-    build_directory = "Qt4Qt5"
+
+    # https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.12.0/ChangeLog
+    conflicts("^qt@4", when="@2.12:")
+
+    build_directory = "src"  # was Qt4Qt5 before 2.12.0
 
     def qmake_args(self):
         # below, DEFINES ... gets rid of ...regex...errors during build
         # although, there shouldn't be such errors since we use '-std=c++11'
         args = ["CONFIG+=-std=c++11", "DEFINES+=NO_CXX11_REGEX=1"]
+        # by default, the package tries to build with accessibility support, and fails
+        # possibly there's a bug somewhere that needs to be fixed
+        if "^qt-base" in self.spec:
+            args.append("DEFINES+=QT_NO_ACCESSIBILITY")
         return args
 
     # When INSTALL_ROOT is unset, qscintilla is installed under qt_prefix
@@ -72,92 +62,66 @@ def setup_run_environment(self, env):
     # Fix install prefix
     @run_after("qmake")
     def fix_install_path(self):
-        makefile = FileFilter(join_path("Qt4Qt5", "Makefile"))
-        makefile.filter(r"\$\(INSTALL_ROOT\)" + self.spec["qt"].prefix, "$(INSTALL_ROOT)")
+        makefile = FileFilter(join_path(self.build_directory, "Makefile"))
+        makefile.filter(
+            "$(INSTALL_ROOT)" + self.spec["qmake"].prefix, "$(INSTALL_ROOT)", string=True
+        )
 
-    @run_after("install")
-    def postinstall(self):
+    @run_after("install", when="+designer")
+    def make_designer(self):
         # Make designer plugin
-        if "+designer" in self.spec:
-            with working_dir(os.path.join(self.stage.source_path, "designer-Qt4Qt5")):
-                qscipro = FileFilter("designer.pro")
-                qscipro.filter("TEMPLATE = lib", "TEMPLATE = lib\nINCLUDEPATH += ../Qt4Qt5\n")
-
-                qmake()
-                make()
-                makefile = FileFilter("Makefile")
-                makefile.filter(r"\$\(INSTALL_ROOT\)" + self.spec["qt"].prefix, "$(INSTALL_ROOT)")
-                make("install")
-
-    @run_after("install")
-    def make_qsci(self):
-        if "+python" in self.spec:
-            if "^py-pyqt4" in self.spec:
-                py_pyqtx = "py-pyqt4"
-                pyqtx = "PyQt4"
-            elif "^py-pyqt5" in self.spec:
-                py_pyqtx = "py-pyqt5"
-                pyqtx = "PyQt5"
-
-            with working_dir(join_path(self.stage.source_path, "Python")):
-                pydir = join_path(python_platlib, pyqtx)
-                mkdirp(os.path.join(self.prefix.share.sip, pyqtx))
-                python = self.spec["python"].command
-                python(
-                    "configure.py",
-                    "--pyqt=" + pyqtx,
-                    "--sip=" + self.spec["py-sip"].prefix.bin.sip,
-                    "--qsci-incdir=" + self.spec.prefix.include,
-                    "--qsci-libdir=" + self.spec.prefix.lib,
-                    "--qsci-sipdir=" + os.path.join(self.prefix.share.sip, pyqtx),
-                    "--apidir=" + self.prefix.share.qsci,
-                    "--destdir=" + pydir,
-                    "--pyqt-sipdir=" + os.path.join(self.spec[py_pyqtx].prefix.share.sip, pyqtx),
-                    "--sip-incdir="
-                    + join_path(
-                        self.spec["py-sip"].prefix.include,
-                        "python" + str(self.spec["python"].version.up_to(2)),
-                    ),
-                    "--stubsdir=" + pydir,
+        with working_dir(os.path.join(self.stage.source_path, "designer")):
+            # TODO: qmake fails with qt6
+            qmake("designer.pro", "INCLUDEPATH+=../src")
+            make()
+            makefile = FileFilter("Makefile")
+            makefile.filter(
+                "$(INSTALL_ROOT)" + self.spec["qmake"].prefix, "$(INSTALL_ROOT)", string=True
+            )
+            make("install")
+
+    @run_after("install", when="+python")
+    def make_qsci_python(self):
+        if "^py-pyqt5" in self.spec:
+            py_pyqtx = "py-pyqt5"
+            pyqtx = "PyQt5"
+            ftoml = "pyproject-qt5.toml"
+        elif "^py-pyqt6" in self.spec:
+            py_pyqtx = "py-pyqt6"
+            pyqtx = "PyQt6"
+            ftoml = "pyproject-qt6.toml"
+
+        with working_dir(join_path(self.stage.source_path, "Python")):
+            copy(ftoml, "pyproject.toml")
+            sip_inc_dir = join_path(
+                self.spec[py_pyqtx].prefix, self.spec["python"].package.platlib, pyqtx, "bindings"
+            )
+
+            with open("pyproject.toml", "a") as tomlfile:
+                tomlfile.write(f'\n[tool.sip.project]\nsip-include-dirs = ["{sip_inc_dir}"]\n')
+            mkdirp(os.path.join(self.prefix.share.sip, pyqtx))
+
+            if "^py-pyqt5" in self.spec:
+                # QT += widgets and QT += printsupport need to be added to Qsci.pro file
+                # to be generated via project.py
+                qsciproj = FileFilter(join_path("project.py"))
+                ptrn = "super().__init__(project, 'Qsci', qmake_CONFIG=qmake_CONFIG"
+                qsciproj.filter(
+                    ptrn + ")", ptrn + ",qmake_QT=['widgets','printsupport'])", string=True
                 )
-
-                # Fix build errors
-                # "QAbstractScrollArea: No such file or directory"
-                # "qprinter.h: No such file or directory"
-                # ".../Qsci.so: undefined symbol: _ZTI10Qsci...."
-                qscipro = FileFilter("Qsci/Qsci.pro")
-                if "^qt@4" in self.spec:
-                    qtx = "qt4"
-                elif "^qt@5" in self.spec:
-                    qtx = "qt5"
-
-                link_qscilibs = "LIBS += -L" + self.prefix.lib + " -lqscintilla2_" + qtx
-                qscipro.filter(
-                    "TEMPLATE = lib",
-                    "TEMPLATE = lib\nQT += widgets" + "\nQT += printsupport\n" + link_qscilibs,
-                )
-
-                make()
-
-                # Fix installation prefixes
-                makefile = FileFilter("Makefile")
-                makefile.filter(r"\$\(INSTALL_ROOT\)", "")
-                makefile = FileFilter("Qsci/Makefile")
-                makefile.filter(r"\$\(INSTALL_ROOT\)", "")
-
-                if "@2.11:" in self.spec:
-                    make("install", parallel=False)
-                else:
-                    make("install")
-
-    @run_after("install")
-    def extend_path_setup(self):
-        if self.spec["py-sip"].satisfies("@:4"):
-            # See github issue #14121 and PR #15297
-            module = self.spec["py-sip"].variants["module"].value
-            if module != "sip":
-                module = module.split(".")[0]
-                with working_dir(python_platlib):
-                    with open(os.path.join(module, "__init__.py"), "w") as f:
-                        f.write("from pkgutil import extend_path\n")
-                        f.write("__path__ = extend_path(__path__, __name__)\n")
+            sip_build = Executable(self.spec["py-sip"].prefix.bin.join("sip-build"))
+            sip_build(
+                "--target-dir=" + python_platlib,
+                "--qsci-include-dir=" + self.spec.prefix.include,
+                "--qsci-library-dir=" + self.spec.prefix.lib,
+                "--api-dir=" + self.prefix.share.qsci,
+                "--verbose",
+            )
+
+            makefile = FileFilter(join_path("build", "Qsci", "Makefile"))
+            makefile.filter("$(INSTALL_ROOT)", "", string=True)
+            make("install", "-C", join_path("build", "Qsci"))
+
+            makefile = FileFilter(join_path("build", "Makefile"))
+            makefile.filter("$(INSTALL_ROOT)", "", string=True)
+            make("install", "-C", "build/")
diff --git a/var/spack/repos/builtin/packages/qt-base/package.py b/var/spack/repos/builtin/packages/qt-base/package.py
index ebc87629928ef4..4345e6b34d7508 100644
--- a/var/spack/repos/builtin/packages/qt-base/package.py
+++ b/var/spack/repos/builtin/packages/qt-base/package.py
@@ -33,10 +33,12 @@ def get_list_url(qualname):
 
     maintainers("wdconinc", "sethrj")
 
+    provides("qmake")
+
     # Default dependencies for all qt-* components
     generator("ninja")
     depends_on("cmake@3.16:", type="build")
-    depends_on("pkgconfig", type="build")
+    depends_on("pkgconfig", type="build", when="platform=linux")
     depends_on("python", type="build")
 
     # List of unnecessary directories in src/3rdparty
@@ -89,6 +91,9 @@ class QtBase(QtPackage):
     url = QtPackage.get_url(__qualname__)
     list_url = QtPackage.get_list_url(__qualname__)
 
+    version("6.6.0", sha256="882f39ea3a40a0894cd64e515ce51711a4fab79b8c47bc0fe0279e99493a62cf")
+    version("6.5.3", sha256="174021c4a630df2e7e912c2e523844ad3cb5f90967614628fd8aa15ddbab8bc5")
+    version("6.5.2", sha256="221cafd400c0a992a42746b43ea879d23869232e56d9afe72cb191363267c674")
     version("6.5.1", sha256="fdde60cdc5c899ab7165f1c3f7b93bc727c2484c348f367d155604f5d901bfb6")
     version("6.5.0", sha256="7b0de20e177335927c55c58a3e1a7e269e32b044936e97e9a82564f0f3e69f99")
     version("6.4.3", sha256="e156692029a5503bad5f681bda856dd9df9dec17baa0ca7ee36b10178503ed40")
@@ -126,12 +131,11 @@ class QtBase(QtPackage):
     depends_on("icu4c")
     depends_on("libxml2")
     depends_on("pcre2+multibyte")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("zstd")
     with when("platform=linux"):
         depends_on("libdrm")
-
-    depends_on("at-spi2-core", when="+accessibility")
+        depends_on("at-spi2-core", when="+accessibility")
     depends_on("dbus", when="+dbus")
     depends_on("gl", when="+opengl")
     depends_on("sqlite", when="+sql")
@@ -148,11 +152,27 @@ class QtBase(QtPackage):
             depends_on("libxrender")
 
     with when("+network"):
-        depends_on("libproxy")
         depends_on("openssl")
+        with when("platform=linux"):
+            depends_on("libproxy")
 
     # Qt6 requires newer compilers: see https://github.com/spack/spack/issues/34418
     conflicts("%gcc@:7")
+    # The oldest compiler for Qt 6.5 is GCC 9: https://doc.qt.io/qt-6.5/supported-platforms.html
+    with when("@6.5:"):
+        conflicts("%gcc@:8")
+
+    # ensure that Qt links against GSS framework on macOS: https://bugreports.qt.io/browse/QTBUG-114537
+    with when("@6.3.2:6.5.1"):
+        patch(
+            "https://github.com/qt/qtbase/commit/c3d3e7312499189dde2ff9c0cb14bd608d6fd1cd.patch?full_index=1",
+            sha256="85c16db15406b0094831bb57016dab7e0c0fd0978b082a1dc103c87334db7915",
+        )
+    with when("@6.3.2:6.5.2"):
+        patch(
+            "https://github.com/qt/qtbase/commit/1bf144ba78ff10d712b4de55d2797b9256948a1d.patch?full_index=1",
+            sha256="e4d9f1aee0566558e77eef5609b63c1fde3f3986bea1b9d5d7930b297f916a5e",
+        )
 
     @property
     def archive_files(self):
@@ -211,10 +231,15 @@ def define_feature(key, variant=None):
         if "+dbus" in spec:
             features.append("dbus_linked")
         if "+network" in spec:
-            features += ["openssl_linked", "openssl", "libproxy"]
+            features.extend(["openssl_linked", "openssl"])
+            if sys.platform == "linux":
+                features.append("libproxy")
         for k in features:
             define("FEATURE_" + k, True)
 
+        if "~opengl" in spec:
+            args.append(self.define("INPUT_opengl", "no"))
+
         # INPUT_* arguments: undefined/no/qt/system
         sys_inputs = ["doubleconversion"]
         if "+sql" in spec:
diff --git a/var/spack/repos/builtin/packages/qt-declarative/package.py b/var/spack/repos/builtin/packages/qt-declarative/package.py
index 5a1b669cc37cd9..b93141c4e438d7 100644
--- a/var/spack/repos/builtin/packages/qt-declarative/package.py
+++ b/var/spack/repos/builtin/packages/qt-declarative/package.py
@@ -14,6 +14,9 @@ class QtDeclarative(QtPackage):
     url = QtPackage.get_url(__qualname__)
     list_url = QtPackage.get_list_url(__qualname__)
 
+    version("6.6.0", sha256="2e52ef00736a9954426adf454cfb365fabdffb5703c814c188bc866cbf9f4dad")
+    version("6.5.3", sha256="563924e58ac517492acb1952af0fb950cd54045ef6d61b98de06fac728239811")
+    version("6.5.2", sha256="8b9eed849c90fb301d5399c545c2c926c18dc889d724df2b284253152a2ee139")
     version("6.5.1", sha256="b6f81ee73e8dbc30601c022b30ceb592fd2f8a5a79e7bc48fcd7feef80e3cc7a")
     version("6.5.0", sha256="38281cdfc60b8820ac2943eebabe968138f90629edc8c6c5e88a72a7ec05e303")
     version("6.4.3", sha256="9549668f8ec28199ba19d73fb535855dc5bea690097f43c2f91954bc27ee0fa3")
diff --git a/var/spack/repos/builtin/packages/qt-quick3d/package.py b/var/spack/repos/builtin/packages/qt-quick3d/package.py
index 28db712fd9efd6..b2d4fb0456b23f 100644
--- a/var/spack/repos/builtin/packages/qt-quick3d/package.py
+++ b/var/spack/repos/builtin/packages/qt-quick3d/package.py
@@ -14,6 +14,9 @@ class QtQuick3d(QtPackage):
     url = QtPackage.get_url(__qualname__)
     list_url = QtPackage.get_list_url(__qualname__)
 
+    version("6.6.0", sha256="2cda12649cfb6c23261c48e626714ca7eb01fa4b20e0bed02031f9c488c820ad")
+    version("6.5.3", sha256="5df7494824c44fc73c03348b218166db5c4d8d42bd7d221f15e58c962cf657e5")
+    version("6.5.2", sha256="7b40e578fc1ee2a5f5c413873fdb0552bb97829b70296ba3c6844da062608a7e")
     version("6.5.1", sha256="2b4f65f6c616302b38656f287e9acdf5a9f0e220ef79eaa2e80946780898fa51")
     version("6.5.0", sha256="eaf41f06450b2be50f16b39ec06c06d10dd337b7516aba1d95695b326fd9ef40")
     version("6.4.3", sha256="609f37efb97c9ef893630b5d29f1ad98a51a47700e260c604931285ac8521479")
diff --git a/var/spack/repos/builtin/packages/qt-quicktimeline/package.py b/var/spack/repos/builtin/packages/qt-quicktimeline/package.py
index 3b203ffe950a4e..42fc1a93e2dd18 100644
--- a/var/spack/repos/builtin/packages/qt-quicktimeline/package.py
+++ b/var/spack/repos/builtin/packages/qt-quicktimeline/package.py
@@ -14,6 +14,9 @@ class QtQuicktimeline(QtPackage):
     url = QtPackage.get_url(__qualname__)
     list_url = QtPackage.get_list_url(__qualname__)
 
+    version("6.6.0", sha256="06b94443da3f81153f04dca0cce781481462310d51f97d5550f81322a7a88cd0")
+    version("6.5.3", sha256="fddd90cdb15af093673c6da924e18e22ebd364b9ab215356e1b40db28ac66640")
+    version("6.5.2", sha256="96389af740fde3b2a655bf994001b94fd6e151ef84958ff9982e2ae799f1c3a2")
     version("6.5.1", sha256="d7d845f877f9b990e63ab14c9152f18e290611e760719a9c22f7740b91bd2ed1")
     version("6.5.0", sha256="ff862aad1aa4327c39c071ad1ca6eea6c64d4937521f9ed5d022a70cb3df92a7")
     version("6.4.3", sha256="e0f8f616a9c7d150dc73eccf7546ab4db041a05b85eafcb44b999cb41549dbed")
diff --git a/var/spack/repos/builtin/packages/qt-shadertools/package.py b/var/spack/repos/builtin/packages/qt-shadertools/package.py
index 182a407d995671..5ac23d8626c861 100644
--- a/var/spack/repos/builtin/packages/qt-shadertools/package.py
+++ b/var/spack/repos/builtin/packages/qt-shadertools/package.py
@@ -16,6 +16,9 @@ class QtShadertools(QtPackage):
     url = QtPackage.get_url(__qualname__)
     list_url = QtPackage.get_list_url(__qualname__)
 
+    version("6.6.0", sha256="8b34908f8bbc7fb00a00babede91dbbeec9826f5138d390041f239d483e1162a")
+    version("6.5.3", sha256="e6c627763db8c60799218947443efb90fb3511342f2212f5e99cd98f6942ed08")
+    version("6.5.2", sha256="2b14cf982753f19cf48a4780bc7d96d8fc0ad3ed1049ae5d3292fc5fc1fd6aef")
     version("6.5.1", sha256="642bf97498d54b4471bf4cc227709c6b676dbd520765f82b0749a2b4ef833d25")
     version("6.5.0", sha256="ef2c71fac111a837914b7dc2b46c26579ea50b05fbd60022d430da88bdb211cb")
     version("6.4.3", sha256="a018883e889540ab828db0b0b49a3c5d810dd7283bb84abf43d1622daf474f7d")
diff --git a/var/spack/repos/builtin/packages/qt-svg/package.py b/var/spack/repos/builtin/packages/qt-svg/package.py
new file mode 100644
index 00000000000000..dfd063bda82e20
--- /dev/null
+++ b/var/spack/repos/builtin/packages/qt-svg/package.py
@@ -0,0 +1,45 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+from spack.pkg.builtin.qt_base import QtBase, QtPackage
+
+
+class QtSvg(QtPackage):
+    """Scalable Vector Graphics (SVG) is an XML-based language for describing
+    two-dimensional vector graphics. Qt provides classes for rendering and
+    displaying SVG drawings in widgets and on other paint devices."""
+
+    url = QtPackage.get_url(__qualname__)
+    list_url = QtPackage.get_list_url(__qualname__)
+
+    version("6.6.0", sha256="4fd6b4d9307c3cd8fd207e60334823fed07a9acb32f7d53cd9c9be9b6a2f8a30")
+    version("6.5.3", sha256="fb8e5574c2480aab78062fad2d0a521633b4591ada600130b918b703c2ddb09a")
+    version("6.5.2", sha256="2d0c8780f164472ad968bb4eff325a86b2826f101efedbeca5662acdc0b294ba")
+    version("6.5.1", sha256="1b262f860c51bc5af5034d88e74bb5584ecdc661f4903c9ba27c8edad14fc403")
+    version("6.5.0", sha256="2f96e22858de18de02b05eb6bcc96fadb6d77f4dadd407e1fa4aebcceb6dd154")
+    version("6.4.3", sha256="3cc7479f7787a19e7af8923547dfc35b7b3fd658e3701577e76b2c1e4c1c0c23")
+    version("6.4.2", sha256="2f5fa08dbe6f3aea0c1c77acb74b6164dc069e15010103377186902b018fb623")
+    version("6.4.1", sha256="be6300292a6f38d85c13bb750890af268bd979fb18ab754f88d5332935215e47")
+    version("6.4.0", sha256="375eb69f320121e42d5dc107f9455008980c149646931b8ace19e6bc235dcd80")
+    version("6.3.2", sha256="781055bca458be46ef69f2fff147a00226e41f3a23d02c91238b0328a7156518")
+
+    variant("widgets", default=False, description="Build SVG widgets.")
+
+    depends_on("qt-base +gui")
+    depends_on("qt-base +widgets", when="+widgets")
+
+    for _v in QtBase.versions:
+        v = str(_v)
+        depends_on("qt-base@" + v, when="@" + v)
+
+    def cmake_args(self):
+        args = super().cmake_args() + []
+        return args
+
+    def setup_run_environment(self, env):
+        # to make plugins from SVG module to base, for e.g. icon loading
+        env.prepend_path("QT_PLUGIN_PATH", self.prefix.plugins)
diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py
index 5f6f5576b3d868..1fe61b3870fced 100644
--- a/var/spack/repos/builtin/packages/qt/package.py
+++ b/var/spack/repos/builtin/packages/qt/package.py
@@ -31,6 +31,7 @@ class Qt(Package):
 
     phases = ["configure", "build", "install"]
 
+    version("5.15.11", sha256="7426b1eaab52ed169ce53804bdd05dfe364f761468f888a0f15a308dc1dc2951")
     version("5.15.10", sha256="b545cb83c60934adc9a6bbd27e2af79e5013de77d46f5b9f5bb2a3c762bf55ca")
     version("5.15.9", sha256="26d5f36134db03abe4a6db794c7570d729c92a3fc1b0bf9b1c8f86d0573cd02f")
     version("5.15.8", sha256="776a9302c336671f9406a53bd30b8e36f825742b2ec44a57c08217bff0fa86b9")
@@ -61,6 +62,7 @@ class Qt(Package):
     variant("gtk", default=False, description="Build with gtkplus.")
     variant("gui", default=True, description="Build the Qt GUI module and dependencies")
     variant("opengl", default=False, description="Build with OpenGL support.")
+    variant("location", default=False, when="+opengl", description="Build the Qt Location module.")
     variant("phonon", default=False, description="Build with phonon support.")
     variant("shared", default=True, description="Build shared libraries.")
     variant("sql", default=True, description="Build with SQL support.")
@@ -68,6 +70,8 @@ class Qt(Package):
     variant("tools", default=True, description="Build tools, including Qt Designer.")
     variant("webkit", default=False, description="Build the Webkit extension")
 
+    provides("qmake")
+
     # Patches for qt@3
     patch("qt3-accept.patch", when="@3")
     patch("qt3-headers.patch", when="@3")
@@ -135,6 +139,15 @@ class Qt(Package):
         working_dir="qtwebsockets",
         when="@5.14: %gcc@11:",
     )
+    # patch that adds missing `#include ` in several files
+    # required for gcc 13 (even though the original patch was developed for gcc 10)
+    # (see https://gcc.gnu.org/gcc-13/porting_to.html)
+    patch(
+        "https://src.fedoraproject.org/rpms/qt5-qtlocation/raw/b6d99579de9ce5802c592b512a9f644a5e4690b9/f/qtlocation-gcc10.patch",
+        sha256="78c70fbd0c74031c5f0f1f5990e0b4214fc04c5073c67ce1f23863373932ec86",
+        working_dir="qtlocation",
+        when="@5.15.10: %gcc@10:",
+    )
     # https://github.com/microsoft/vcpkg/issues/21055
     patch("qt5-macos12.patch", working_dir="qtbase", when="@5.14: %apple-clang@13:")
 
@@ -157,7 +170,7 @@ class Qt(Package):
     depends_on("libmng")
     depends_on("libtiff")
     depends_on("libxml2")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("freetype", when="+gui")
     depends_on("gtkplus", when="+gtk")
     depends_on("sqlite+column_metadata", when="+sql", type=("build", "run"))
@@ -261,8 +274,6 @@ def determine_version(cls, exe):
             msg="Apple Silicon requires a very new version of qt",
         )
 
-    use_xcode = True
-
     # Mapping for compilers/systems in the QT 'mkspecs'
     compiler_mapping = {
         "intel": ("icc",),
@@ -332,9 +343,10 @@ def setup_build_environment(self, env):
                 llvm_path = "/spack-disable-llvm"
             env.set("LLVM_INSTALL_DIR", llvm_path)
 
-        if self.spec.satisfies("+ssl"):
-            if self.spec["openssl"].satisfies("~shared"):
-                env.set("OPENSSL_LIBS", "-lssl -lcrypto -lz")
+        if self.spec.satisfies("+ssl ^openssl~shared"):
+            pc = which("pkg-config")
+            ssl_flags = pc("openssl", "--libs-only-l", "--static", output=str).strip()
+            env.set("OPENSSL_LIBS", ssl_flags)
 
     def setup_run_environment(self, env):
         env.set("QTDIR", self.prefix)
@@ -597,7 +609,7 @@ def common_config_args(self):
             use_spack_dep("libtiff", "tiff")
             use_spack_dep("libpng")
             use_spack_dep("jpeg", "libjpeg")
-            use_spack_dep("zlib")
+            use_spack_dep("zlib-api", "zlib")
 
         if "@:5.5" in spec:
             config_args.extend(
@@ -702,9 +714,12 @@ def configure(self, spec, prefix):
                 # Errors on bluetooth even when bluetooth is disabled...
                 # at least on apple-clang%12
                 config_args.extend(["-skip", "connectivity"])
-        elif version < Version("5.15") and "+gui" in spec:
+        elif "+gui" in spec:
             # Linux-only QT5 dependencies
-            config_args.append("-system-xcb")
+            if version < Version("5.9.9"):
+                config_args.append("-system-xcb")
+            else:
+                config_args.append("-xcb")
             if "+opengl" in spec:
                 config_args.append("-I{0}/include".format(spec["libx11"].prefix))
                 config_args.append("-I{0}/include".format(spec["xproto"].prefix))
@@ -735,6 +750,10 @@ def configure(self, spec, prefix):
             # https://wiki.qt.io/QtWayland
             config_args.extend(["-skip", "wayland"])
 
+        if "~location" in spec:
+            if version >= Version("5.15"):
+                config_args.extend(["-skip", "qtlocation"])
+
         if "~opengl" in spec:
             config_args.extend(["-skip", "multimedia"])
             config_args.extend(["-skip", "qt3d"])
@@ -745,9 +764,6 @@ def configure(self, spec, prefix):
             if version >= Version("5.14"):
                 config_args.extend(["-skip", "qtquick3d"])
 
-            if version >= Version("5.15"):
-                config_args.extend(["-skip", "qtlocation"])
-
         else:
             # v5.0: qt3d uses internal-only libassimp
             # v5.5: external-only libassimp
diff --git a/var/spack/repos/builtin/packages/quantum-espresso/package.py b/var/spack/repos/builtin/packages/quantum-espresso/package.py
index b0a5ee37ff9ff5..40c036320d7a7a 100644
--- a/var/spack/repos/builtin/packages/quantum-espresso/package.py
+++ b/var/spack/repos/builtin/packages/quantum-espresso/package.py
@@ -242,6 +242,11 @@ class QuantumEspresso(CMakePackage, Package):
     depends_on("git@2.13:", type="build")
     depends_on("m4", type="build")
 
+    # If the Intel suite is used for Lapack, it must be used for fftw and vice-versa
+    for _intel_pkg in INTEL_MATH_LIBRARIES:
+        requires(f"^[virtuals=fftw-api] {_intel_pkg}", when=f"^[virtuals=lapack]   {_intel_pkg}")
+        requires(f"^[virtuals=lapack]   {_intel_pkg}", when=f"^[virtuals=fftw-api] {_intel_pkg}")
+
     # CONFLICTS SECTION
     # Omitted for now due to concretizer bug
     # MKL with 64-bit integers not supported.
@@ -489,7 +494,8 @@ def install(self, pkg, spec, prefix):
         # you need to pass it in the FFTW_INCLUDE and FFT_LIBS directory.
         # QE supports an internal FFTW2, but only an external FFTW3 interface.
 
-        if "^mkl" in spec:
+        is_using_intel_libraries = spec["lapack"].name in INTEL_MATH_LIBRARIES
+        if is_using_intel_libraries:
             # A seperate FFT library is not needed when linking against MKL
             options.append("FFTW_INCLUDE={0}".format(join_path(env["MKLROOT"], "include/fftw")))
         if "^fftw@3:" in spec:
@@ -531,11 +537,11 @@ def install(self, pkg, spec, prefix):
         if spec.satisfies("@:6.4"):  # set even if MKL is selected
             options.append("BLAS_LIBS={0}".format(lapack_blas.ld_flags))
         else:  # behavior changed at 6.5 and later
-            if not spec.satisfies("^mkl"):
+            if not is_using_intel_libraries:
                 options.append("BLAS_LIBS={0}".format(lapack_blas.ld_flags))
 
         if "+scalapack" in spec:
-            if "^mkl" in spec:
+            if is_using_intel_libraries:
                 if "^openmpi" in spec:
                     scalapack_option = "yes"
                 else:  # mpich, intel-mpi
@@ -602,8 +608,8 @@ def install(self, pkg, spec, prefix):
         if spec.variants["hdf5"].value != "none":
             if spec.satisfies("@6.1.0:6.4.0") or (spec.satisfies("@6.4.1") and "+qmcpack" in spec):
                 make_inc = join_path(self.pkg.stage.source_path, "make.inc")
-                zlib_libs = spec["zlib"].prefix.lib + " -lz"
-                filter_file(zlib_libs, format(spec["zlib"].libs.ld_flags), make_inc)
+                zlib_libs = spec["zlib-api"].prefix.lib + " -lz"
+                filter_file(zlib_libs, format(spec["zlib-api"].libs.ld_flags), make_inc)
 
         # QE 6.8 and later has parallel builds fixed
         if spec.satisfies("@:6.7"):
diff --git a/var/spack/repos/builtin/packages/quickjs/package.py b/var/spack/repos/builtin/packages/quickjs/package.py
new file mode 100644
index 00000000000000..6f77137b75da58
--- /dev/null
+++ b/var/spack/repos/builtin/packages/quickjs/package.py
@@ -0,0 +1,40 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class Quickjs(MakefilePackage):
+    """QuickJS is a small and embeddable Javascript engine."""
+
+    homepage = "https://bellard.org/quickjs/"
+    git = "https://github.com/bellard/quickjs.git"
+    url = "https://bellard.org/quickjs/quickjs-2021-03-27.tar.xz"
+
+    version("master", branch="master")
+    version(
+        "2021-03-27", sha256="a45bface4c3379538dea8533878d694e289330488ea7028b105f72572fe7fe1a"
+    )
+    version(
+        "2020-11-08", sha256="2e9d63dab390a95ed365238f21d8e9069187f7ed195782027f0ab311bb64187b"
+    )
+    version(
+        "2020-09-06", sha256="0021a3e8cdc6b61e225411d05e2841d2437e1ccf4b4cabb9a5f7685ebfb57717"
+    )
+
+    variant("lto", default=True, when="%gcc", description="Enable link-time optimization")
+
+    def edit(self, spec, prefix):
+        makefile = FileFilter("Makefile")
+        makefile.filter("prefix=/usr/local", "prefix={}".format(prefix))
+        makefile.filter("lib/quickjs", "lib")
+        makefile.filter("CFLAGS=", "CFLAGS+=-fPIC ")
+        if "+lto" not in spec:
+            makefile.filter("CONFIG_LTO=y", "")
+        cc = self.compiler.cc
+        makefile.filter("^ *CC=.*", "  CC={}".format(cc))
+        makefile.filter("^ *HOST_CC=.*", "  HOST_CC={}".format(cc))
+        makefile.filter("gcc-ar", "{}-ar".format(cc))
diff --git a/var/spack/repos/builtin/packages/quo-vadis/package.py b/var/spack/repos/builtin/packages/quo-vadis/package.py
new file mode 100644
index 00000000000000..029b4926c9128c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/quo-vadis/package.py
@@ -0,0 +1,55 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class QuoVadis(CMakePackage):
+    """A cross-stack coordination layer to dynamically
+    map runtime components to hardware resources"""
+
+    homepage = "https://github.com/hpc/quo-vadis"
+    git = "https://github.com/hpc/quo-vadis.git"
+
+    maintainers("samuelkgutierrez")
+
+    version("master", branch="master")
+
+    variant("fortran", default=True, description="Build with Fortran bindings")
+    variant("mpi", default=True, description="Build with MPI support")
+    variant("mpipat", default=False, description="Affirm MPI processes are threads")
+    variant("omp", default=True, description="Build with OpenMP support")
+
+    variant(
+        "gpu",
+        values=("nvidia", "amd", "none"),
+        default="none",
+        multi=True,
+        description="Build with GPU support",
+    )
+
+    depends_on("libzmq")
+
+    with when("+mpi"):
+        depends_on("mpi")
+
+    with when("gpu=nvidia"):
+        depends_on("libpciaccess")
+        depends_on("cuda")
+
+    with when("gpu=amd"):
+        depends_on("libpciaccess")
+        depends_on("rocm-smi-lib")
+
+    def cmake_args(self):
+        spec = self.spec
+
+        return [
+            self.define_from_variant("QV_FORTRAN_SUPPORT", "fortran"),
+            self.define_from_variant("QV_MPI_SUPPORT", "mpi"),
+            self.define_from_variant("QV_MPI_PROCESSES_ARE_THREADS", "mpipat"),
+            self.define_from_variant("QV_OMP_SUPPORT", "omp"),
+            self.define("QV_GPU_SUPPORT", not spec.satisfies("gpu=none")),
+        ]
diff --git a/var/spack/repos/builtin/packages/r-affyio/package.py b/var/spack/repos/builtin/packages/r-affyio/package.py
index d196223dd1884e..06dd560743064d 100644
--- a/var/spack/repos/builtin/packages/r-affyio/package.py
+++ b/var/spack/repos/builtin/packages/r-affyio/package.py
@@ -28,4 +28,4 @@ class RAffyio(RPackage):
 
     depends_on("r@2.6.0:", type=("build", "run"))
     depends_on("r-zlibbioc", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-affyplm/package.py b/var/spack/repos/builtin/packages/r-affyplm/package.py
index 4badcd5a8a4194..508d94b05d7fa6 100644
--- a/var/spack/repos/builtin/packages/r-affyplm/package.py
+++ b/var/spack/repos/builtin/packages/r-affyplm/package.py
@@ -35,4 +35,4 @@ class RAffyplm(RPackage):
     depends_on("r-gcrma", type=("build", "run"))
     depends_on("r-preprocesscore@1.5.1:", type=("build", "run"))
     depends_on("r-zlibbioc", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-bio3d/package.py b/var/spack/repos/builtin/packages/r-bio3d/package.py
index b0559d67e8255e..accb4018505fac 100644
--- a/var/spack/repos/builtin/packages/r-bio3d/package.py
+++ b/var/spack/repos/builtin/packages/r-bio3d/package.py
@@ -32,4 +32,4 @@ class RBio3d(RPackage):
 
     depends_on("r@3.1.0:", type=("build", "run"))
     depends_on("r-rcpp", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-biocgenerics/package.py b/var/spack/repos/builtin/packages/r-biocgenerics/package.py
index 2ce882d7220758..b510d8385bfa8c 100644
--- a/var/spack/repos/builtin/packages/r-biocgenerics/package.py
+++ b/var/spack/repos/builtin/packages/r-biocgenerics/package.py
@@ -18,7 +18,7 @@ class RBiocgenerics(RPackage):
     version("0.42.0", commit="3582d47b836387afc08157f3d6a5013fd64334ed")
     version("0.40.0", commit="0bc1e0ed4d20c7101cd782a14f6373e27478acfc")
     version("0.36.0", commit="0d5d169d7d64d648a22f9043837c93bc784e71ed")
-    version("0.34.0", commit="f7c2020")
+    version("0.34.0", commit="f7c2020b6fb06ade6d70b4de17672735edfa45df")
     version("0.30.0", commit="fc7c3af4a5635a30988a062ed09332c13ca1d1a8")
     version("0.28.0", commit="041fc496504f2ab1d4d863fffb23372db214394b")
     version("0.26.0", commit="5b2a6df639e48c3cd53789e0b174aec9dda6b67d")
diff --git a/var/spack/repos/builtin/packages/r-bsseq/package.py b/var/spack/repos/builtin/packages/r-bsseq/package.py
index 7e83045ea3de57..7b96c3370b99e5 100644
--- a/var/spack/repos/builtin/packages/r-bsseq/package.py
+++ b/var/spack/repos/builtin/packages/r-bsseq/package.py
@@ -19,9 +19,9 @@ class RBsseq(RPackage):
     version("1.32.0", commit="a0c1eacbc479c57bd836e885e162c28bbe08e115")
     version("1.30.0", commit="7eb5223e9ee02fd08a52be56eaa9316a67c0d66b")
     version("1.26.0", commit="fae32292687625012a2938a48c93df55ad4257b5")
-    version("1.24.4", commit="8fe7a03")
-    version("1.22.0", commit="d4f7301")
-    version("1.20.0", commit="07e398b")
+    version("1.24.4", commit="8fe7a035802055cf14783d2ab92af70c5d5800ed")
+    version("1.22.0", commit="d4f7301dcd4a03431b0833302b5a79c6f1b186cc")
+    version("1.20.0", commit="07e398bc38ba903881df9a5d0577cca15788e0cd")
 
     depends_on("r@3.5:", type=("build", "run"))
     depends_on("r@4.0:", type=("build", "run"), when="@1.26.0:")
diff --git a/var/spack/repos/builtin/packages/r-cner/package.py b/var/spack/repos/builtin/packages/r-cner/package.py
index bad0fad593dd58..fbb94c6375f379 100644
--- a/var/spack/repos/builtin/packages/r-cner/package.py
+++ b/var/spack/repos/builtin/packages/r-cner/package.py
@@ -47,4 +47,4 @@ class RCner(RPackage):
     depends_on("r-go-db@3.3.0:", type=("build", "run"))
     depends_on("r-r-utils@2.3.0:", type=("build", "run"))
     depends_on("r-keggrest@1.14.0:", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-data-table/package.py b/var/spack/repos/builtin/packages/r-data-table/package.py
index 8573dd4d4ef2b0..a967de5f6661b1 100644
--- a/var/spack/repos/builtin/packages/r-data-table/package.py
+++ b/var/spack/repos/builtin/packages/r-data-table/package.py
@@ -38,4 +38,4 @@ class RDataTable(RPackage):
     version("1.9.6", sha256="6f74c349c1731823aef6899edcf18418454167d04eba983e3a6fe17ee9fd236e")
 
     depends_on("r@3.1.0:", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-dss/package.py b/var/spack/repos/builtin/packages/r-dss/package.py
index 60c72ed030127c..8ec607c0dbb8e1 100644
--- a/var/spack/repos/builtin/packages/r-dss/package.py
+++ b/var/spack/repos/builtin/packages/r-dss/package.py
@@ -23,9 +23,9 @@ class RDss(RPackage):
     version("2.44.0", commit="b9f44106f139c93564dfb4afab50555d24a657ba")
     version("2.42.0", commit="33e87450fbb64bb3e321688ff613e83cd40efe48")
     version("2.38.0", commit="82e65b92e6e227f1f99620362db8b03059e07e98")
-    version("2.36.0", commit="841c7ed")
-    version("2.34.0", commit="f9819c7")
-    version("2.32.0", commit="ffb502d")
+    version("2.36.0", commit="841c7ed72a138a458ee2df52336050dbe581e727")
+    version("2.34.0", commit="f9819c7d7927c8e6d9963632cdeab36c8c22caa8")
+    version("2.32.0", commit="ffb502d20810a873c2376199d44adedf7c83912d")
 
     depends_on("r@3.3:", type=("build", "run"))
     depends_on("r@3.5.0:", type=("build", "run"), when="@2.44.0:")
diff --git a/var/spack/repos/builtin/packages/r-functional/package.py b/var/spack/repos/builtin/packages/r-functional/package.py
new file mode 100644
index 00000000000000..2123ffc7e225a6
--- /dev/null
+++ b/var/spack/repos/builtin/packages/r-functional/package.py
@@ -0,0 +1,21 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class RFunctional(RPackage):
+    """Curry, Compose, and other higher-order functions
+
+    Curry, Compose, and other higher-order functions"""
+
+    cran = "functional"
+
+    maintainers("jgaeb")
+
+    version("0.6", sha256="19b78e27c27b1081245222c42da4dd1cb65c5643e6da9d6c1a6e997755c21888")
+    version("0.4", sha256="05d1a50de6a574d938471c9a615c840871df9f879b2cbbcabc6b25b5809a70a8")
+    version("0.2", sha256="1b11d039153a214e89e4903163522d8e15c1fcf84495023d9e463487bde1e8d8")
+    version("0.1", sha256="148301d066f9c7e450d809a130d31b0763424f65f177704856d76143ded3db7e")
diff --git a/var/spack/repos/builtin/packages/r-furrr/package.py b/var/spack/repos/builtin/packages/r-furrr/package.py
new file mode 100644
index 00000000000000..690cde5cc2cd88
--- /dev/null
+++ b/var/spack/repos/builtin/packages/r-furrr/package.py
@@ -0,0 +1,48 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class RFurrr(RPackage):
+    """Apply Mapping Functions in Parallel using Futures
+
+    Implementations of the family of map() functions from 'purrr' that can be
+    resolved using any 'future'-supported backend, e.g. parallel on the local
+    machine or distributed on a compute cluster."""
+
+    homepage = "https://furrr.futureverse.org"
+    cran = "furrr"
+
+    maintainers("jgaeb")
+
+    version("0.3.1", sha256="0d91735e2e9be759b1ab148d115c2c7429b79740514778828e5dab631dc0e48b")
+    version("0.3.0", sha256="3fe91cc1614f9404c708ea3a15b6a40289fa57f40f3ece54452093408d91fd84")
+    version("0.2.3", sha256="0a213422dc0a2e84173f2d3e6c7900dcb677f980c255d6b6ccf666fba1173700")
+    version("0.2.2", sha256="e5c10353dc47416eda870d16cf810c576f11bdc9e4c7277f7755581f3824cd4d")
+    version("0.2.1", sha256="07b3c98324aeb6a7e77a3d48c54fb90696a6e14efeee391cfc5e05f8dcd3469b")
+    version("0.2.0", sha256="9d6483656fdb5b90e998e2c2f1494c721185079a1412316c6d391e1eade89e1b")
+    version("0.1.0", sha256="dd2937f7cad1bc69e7a512b2a777f82d6cb7e40fe99afa2049ca360f9352a9d1")
+
+    depends_on("r@3.2.0:", type=("build", "run"), when="@0.1.0:")
+    depends_on("r@3.4.0:", type=("build", "run"), when="@0.3.0:")
+    depends_on("r-future@1.6.2:1.19.1", type=("build", "run"), when="@0.1.0")
+    depends_on("r-future@1.19.1:1.22.1", type=("build", "run"), when="@0.2.0:0.2.3")
+    depends_on("r-future@1.25.0:", type=("build", "run"), when="@0.3.0:")
+    depends_on("r-globals@0.10.3:", type=("build", "run"), when="@0.1.0:")
+    depends_on("r-globals@0.13.1:", type=("build", "run"), when="@0.2.0:")
+    depends_on("r-globals@0.14.0:", type=("build", "run"), when="@0.3.0:")
+    depends_on("r-rlang@0.2.0:", type=("build", "run"), when="@0.1.0:")
+    depends_on("r-rlang@0.3.0:", type=("build", "run"), when="@0.2.0:")
+    depends_on("r-rlang@1.0.2:", type=("build", "run"), when="@0.3.0:")
+    depends_on("r-purrr@0.2.4:", type=("build", "run"), when="@0.1.0:")
+    depends_on("r-purrr@0.3.0:", type=("build", "run"), when="@0.2.0:")
+    depends_on("r-purrr@0.3.4:", type=("build", "run"), when="@0.3.0:")
+    depends_on("r-ellipsis", type=("build", "run"), when="@0.2.0:0.3.0")
+    depends_on("r-lifecycle@0.2.0:", type=("build", "run"), when="@0.2.0:")
+    depends_on("r-lifecycle@1.0.0:", type=("build", "run"), when="@0.2.3:")
+    depends_on("r-lifecycle@1.0.1:", type=("build", "run"), when="@0.3.0:")
+    depends_on("r-vctrs@0.3.2:", type=("build", "run"), when="@0.2.0:")
+    depends_on("r-vctrs@0.4.1:", type=("build", "run"), when="@0.3.0:")
diff --git a/var/spack/repos/builtin/packages/r-git2r/package.py b/var/spack/repos/builtin/packages/r-git2r/package.py
index 15f165f9f4bfbe..61b448242bbd74 100644
--- a/var/spack/repos/builtin/packages/r-git2r/package.py
+++ b/var/spack/repos/builtin/packages/r-git2r/package.py
@@ -29,6 +29,6 @@ class RGit2r(RPackage):
     depends_on("r@3.1:", type=("build", "run"))
     depends_on("r@3.4:", type=("build", "run"), when="@0.31.0:")
     depends_on("libgit2")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("openssl")
     depends_on("libssh2")
diff --git a/var/spack/repos/builtin/packages/r-haven/package.py b/var/spack/repos/builtin/packages/r-haven/package.py
index 213c3f2c2db4ac..d7edaed6bf81a3 100644
--- a/var/spack/repos/builtin/packages/r-haven/package.py
+++ b/var/spack/repos/builtin/packages/r-haven/package.py
@@ -37,6 +37,6 @@ class RHaven(RPackage):
     depends_on("r-vctrs@0.3.0:", type=("build", "run"), when="@2.3.1:")
     depends_on("r-cpp11", type=("build", "run"), when="@2.4:")
     depends_on("gmake", type="build")
-    depends_on("zlib", when="@2.4:")
+    depends_on("zlib-api", when="@2.4:")
 
     depends_on("r-rcpp@0.11.4:", type=("build", "run"), when="@:2.3")
diff --git a/var/spack/repos/builtin/packages/r-hdf5array/package.py b/var/spack/repos/builtin/packages/r-hdf5array/package.py
index dcb575aa951035..b8e0e5ed1ff81a 100644
--- a/var/spack/repos/builtin/packages/r-hdf5array/package.py
+++ b/var/spack/repos/builtin/packages/r-hdf5array/package.py
@@ -53,4 +53,4 @@ class RHdf5array(RPackage):
     depends_on("r-iranges", type=("build", "run"))
     depends_on("r-rhdf5lib", type=("build", "run"), when="@1.12.3:")
     depends_on("gmake", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-httpuv/package.py b/var/spack/repos/builtin/packages/r-httpuv/package.py
index 9b9825b6c17c73..3a539d278fb6fb 100644
--- a/var/spack/repos/builtin/packages/r-httpuv/package.py
+++ b/var/spack/repos/builtin/packages/r-httpuv/package.py
@@ -35,6 +35,6 @@ class RHttpuv(RPackage):
     depends_on("r-later@0.8.0:", type=("build", "run"), when="@1.5.0:")
     depends_on("gmake", type="build")
     depends_on("zip")
-    depends_on("zlib", when="@1.6.4:")
+    depends_on("zlib-api", when="@1.6.4:")
 
     depends_on("r-bh", type=("build", "run"), when="@1.5.5")
diff --git a/var/spack/repos/builtin/packages/r-humaniformat/package.py b/var/spack/repos/builtin/packages/r-humaniformat/package.py
new file mode 100644
index 00000000000000..643574468a1eda
--- /dev/null
+++ b/var/spack/repos/builtin/packages/r-humaniformat/package.py
@@ -0,0 +1,26 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class RHumaniformat(RPackage):
+    """A Parser for Human Names
+
+    Human names are complicated and nonstandard things. Humaniformat, which is
+    based on Anthony Ettinger's 'humanparser' project
+    (https://github.com/chovy/humanparser) provides functions for parsing human
+    names, making a best- guess attempt to distinguish sub-components such as
+    prefixes, suffixes, middle names and salutations."""
+
+    homepage = "https://github.com/Ironholds/humaniformat"
+    cran = "humaniformat"
+
+    maintainers("jgaeb")
+
+    version("0.6.0", sha256="861232c66bf6d4ff91b073193506104f4d99eca5e9a9488327f39ef2bfb45e6d")
+    version("0.5.0", sha256="02b585e3623a5c5faa7dc3abff92b932d748900be39097c5db8434b8e92709a0")
+
+    depends_on("r-rcpp", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/r-iranges/package.py b/var/spack/repos/builtin/packages/r-iranges/package.py
index 55b509f2a0e52d..e998ffe89d0e1a 100644
--- a/var/spack/repos/builtin/packages/r-iranges/package.py
+++ b/var/spack/repos/builtin/packages/r-iranges/package.py
@@ -24,7 +24,7 @@ class RIranges(RPackage):
     version("2.30.0", commit="9b5f3ca12812fb76c23b1550aa3a794384384d9b")
     version("2.28.0", commit="d85ee908a379e12d1e32599e999c71ab37c25e57")
     version("2.24.1", commit="6c61fddf4c5830f69a0f7f108888c67cd0a12b19")
-    version("2.22.2", commit="8c5e991")
+    version("2.22.2", commit="8c5e99131c419224f92921b9a13255b705a293ad")
     version("2.18.3", commit="c98a7ba074e72f2e5ec98252dffe9d3392711972")
     version("2.16.0", commit="26834c6868d7c279dd8ac1bb9daa16e6fef273c2")
     version("2.14.12", commit="00af02756c14771a23df9efcf379409ab6eb3041")
diff --git a/var/spack/repos/builtin/packages/r-mzr/package.py b/var/spack/repos/builtin/packages/r-mzr/package.py
index b083a4d14b3e90..fc2faa2600019b 100644
--- a/var/spack/repos/builtin/packages/r-mzr/package.py
+++ b/var/spack/repos/builtin/packages/r-mzr/package.py
@@ -40,6 +40,6 @@ class RMzr(RPackage):
     depends_on("r-ncdf4", type=("build", "run"), when="@2.16.2:")
     depends_on("r-rhdf5lib@1.1.4:", type=("build", "run"), when="@2.14.0:")
     depends_on("gmake", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     depends_on("r-zlibbioc", type=("build", "run"), when="@:2.28.0")
diff --git a/var/spack/repos/builtin/packages/r-piggyback/package.py b/var/spack/repos/builtin/packages/r-piggyback/package.py
new file mode 100644
index 00000000000000..76cf04b88c62e5
--- /dev/null
+++ b/var/spack/repos/builtin/packages/r-piggyback/package.py
@@ -0,0 +1,53 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class RPiggyback(RPackage):
+    """Managing Larger Data on a GitHub Repository
+
+    Because larger (> 50 MB) data files cannot easily be committed to git, a
+    different approach is required to manage data associated with an analysis in
+    a GitHub repository. This package provides a simple work-around by allowing
+    larger (up to 2 GB) data files to piggyback on a repository as assets
+    attached to individual GitHub releases. These files are not handled by git
+    in any way, but instead are uploaded, downloaded, or edited directly by
+    calls through the GitHub API. These data files can be versioned manually by
+    creating different releases. This approach works equally well with public or
+    private repositories. Data can be uploaded and downloaded programmatically
+    from scripts. No authentication is required to download data from public
+    repositories."""
+
+    homepage = "https://github.com/ropensci/piggyback"
+    cran = "piggyback"
+
+    maintainers("jgaeb")
+
+    version("0.1.5", sha256="983b5a46792ff5f2895f36ee29afcbd7723d05fe3daefdaefaada12987e36aee")
+    version("0.1.4", sha256="9e9d6d75e13f82559e5322182af557b3c79f9a6e9b0bc8c1e1b193544dcda511")
+    version("0.1.3", sha256="6fccae21a97653f6a1d90d97c4f089bf403126a808f4b4023f4b0c6a496e0b07")
+    version("0.1.2", sha256="39611437a136e98a89c157ec3aa9f5daf3c38e0b761c446d9eaf76c50ee6ef62")
+    version("0.1.1", sha256="03d966bbcbac8dcda4afc1667faf95fe6aa0047e2cd0d063104d9c33274484b4")
+    version("0.1.0", sha256="71c7484c879f9e9644f3d6d6fe021213a8995d998f6b90b7143093ed11f858a0")
+    version("0.0.11", sha256="bac9b0cde1be9b01574a0dffa8ffd08457bdd4edb2bcb1d455ec9547ee9f959d")
+    version("0.0.10", sha256="7f0134c234cf24ded142037f132349574851d22dc97f949784deed7a7897a721")
+    version("0.0.9", sha256="05668540d457d5945512a802a364487d10671556d675f4c287f2b3afbc9156bf")
+    version("0.0.8", sha256="2e38f51d0ab3f55c5ddb49002693164a5f4c630b2055b8ae36d94875b1f427b0")
+    version("0.0.7", sha256="df0692deacca58b893360bdfb2e295ba8bc785bcb99f4a2cd8c6738ed0c4f865")
+
+    depends_on("r-gh", type=("build", "run"), when="@0.0.7:")
+    depends_on("r-httr", type=("build", "run"), when="@0.0.7:")
+    depends_on("r-jsonlite", type=("build", "run"), when="@0.0.7:")
+    depends_on("r-git2r", type=("build", "run"), when="@0.0.7:0.0.11")
+    depends_on("r-fs", type=("build", "run"), when="@0.0.7:")
+    depends_on("r-usethis", type=("build", "run"), when="@0.0.7:0.0.11")
+    depends_on("r-crayon", type=("build", "run"), when="@0.0.7:0.1.1")
+    depends_on("r-clisymbols", type=("build", "run"), when="@0.0.7:0.1.1")
+    depends_on("r-magrittr", type=("build", "run"), when="@0.0.7:0.1.0")
+    depends_on("r-lubridate", type=("build", "run"), when="@0.0.7:")
+    depends_on("r-memoise", type=("build", "run"), when="@0.0.7:")
+    depends_on("r-cli", type=("build", "run"), when="@0.1.2:")
+    depends_on("r-glue", type=("build", "run"), when="@0.1.2:")
diff --git a/var/spack/repos/builtin/packages/r-pl94171/package.py b/var/spack/repos/builtin/packages/r-pl94171/package.py
new file mode 100644
index 00000000000000..6cd51f523e701c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/r-pl94171/package.py
@@ -0,0 +1,37 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class RPl94171(RPackage):
+    """Tabulate P.L. 94-171 Redistricting Data Summary Files
+
+    Tools to process legacy format summary redistricting data files produced by
+    the United States Census Bureau pursuant to P.L. 94-171. These files are
+    generally available earlier but are difficult to work with as-is."""
+
+    homepage = "https://corymccartan.com/PL94171/"
+    cran = "PL94171"
+
+    maintainers("jgaeb")
+
+    version("1.1.2", sha256="53ca90801eb04a0535dda0f98869fe1d7f67e40702e5f77570303bbbb5289c73")
+    version("1.1.1", sha256="a4016b94070c9e811f33ee7f0b662d968d250391848b8afd7f7386c625b6c2fe")
+    version("1.0.2", sha256="3cbe058ab8f99944dd7b034ba93044d03d8351c3036759d8980378f78bef0330")
+    version("1.0.1", sha256="c31ccf045a742719efe94fa08109b52c6a986d86d15815041fb93c877cf5f474")
+    version("0.3.2", sha256="9135e4a1405e90ae7855af568794eb71702a3142d3ab089702061a359e9bba1e")
+    version("0.2.0", sha256="8d4cf1199812dacfc9a62b975056ba3c763a899bfc177e48f059581a3e4550e1")
+
+    depends_on("r@4.0.0:", type=("build", "run"))
+    depends_on("r-stringr", type=("build", "run"))
+    depends_on("r-readr", type=("build", "run"))
+    depends_on("r-dplyr@1.0.0:", type=("build", "run"))
+    depends_on("r-sf", type=("build", "run"))
+    depends_on("r-withr", type=("build", "run"))
+    depends_on("r-httr", type=("build", "run"))
+    depends_on("r-tigris", type=("build", "run"), when="@0.2.0:1.0.2")
+    depends_on("r-tinytiger", type=("build", "run"), when="@1.1.1:")
+    depends_on("r-cli", type=("build", "run"), when="@1.1.1:")
diff --git a/var/spack/repos/builtin/packages/r-popgenome/package.py b/var/spack/repos/builtin/packages/r-popgenome/package.py
index 8e8f7cbda866da..d53b3fcefcb912 100644
--- a/var/spack/repos/builtin/packages/r-popgenome/package.py
+++ b/var/spack/repos/builtin/packages/r-popgenome/package.py
@@ -24,4 +24,4 @@ class RPopgenome(RPackage):
 
     depends_on("r@2.14.2:", type=("build", "run"))
     depends_on("r-ff", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-rcppcnpy/package.py b/var/spack/repos/builtin/packages/r-rcppcnpy/package.py
index 56abc4f7dc35c4..096a5ad025b60c 100644
--- a/var/spack/repos/builtin/packages/r-rcppcnpy/package.py
+++ b/var/spack/repos/builtin/packages/r-rcppcnpy/package.py
@@ -24,4 +24,4 @@ class RRcppcnpy(RPackage):
 
     depends_on("r@3.1.0:", type=("build", "run"))
     depends_on("r-rcpp", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-rgl/package.py b/var/spack/repos/builtin/packages/r-rgl/package.py
index ed0bd9004a003a..6f96a6f712d2ee 100644
--- a/var/spack/repos/builtin/packages/r-rgl/package.py
+++ b/var/spack/repos/builtin/packages/r-rgl/package.py
@@ -45,7 +45,7 @@ class RRgl(RPackage):
     depends_on("libx11")
     depends_on("gl")
     depends_on("glu")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("libpng@1.2.9:", type="link")
     depends_on("freetype", type="link")
     depends_on("pandoc@1.14:", type="build")
diff --git a/var/spack/repos/builtin/packages/r-rhdf5/package.py b/var/spack/repos/builtin/packages/r-rhdf5/package.py
index 1acd7473968676..c09c4cb20a06b5 100644
--- a/var/spack/repos/builtin/packages/r-rhdf5/package.py
+++ b/var/spack/repos/builtin/packages/r-rhdf5/package.py
@@ -38,6 +38,6 @@ class RRhdf5(RPackage):
     depends_on("r-rhdf5lib@1.13.4:", type=("build", "run"), when="@2.38.0:")
     depends_on("r-rhdf5filters", type=("build", "run"), when="@2.34.0:")
     depends_on("gmake", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     depends_on("r-zlibbioc", type=("build", "run"), when="@:2.28.1")
diff --git a/var/spack/repos/builtin/packages/r-rhdf5filters/package.py b/var/spack/repos/builtin/packages/r-rhdf5filters/package.py
index b13de3ad5e21a0..9f29a05f4a54e8 100644
--- a/var/spack/repos/builtin/packages/r-rhdf5filters/package.py
+++ b/var/spack/repos/builtin/packages/r-rhdf5filters/package.py
@@ -21,7 +21,7 @@ class RRhdf5filters(RPackage):
 
     depends_on("r-rhdf5lib", type=("build", "run"))
     depends_on("gmake", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("zstd")
 
     def configure_args(self):
diff --git a/var/spack/repos/builtin/packages/r-rhdf5lib/package.py b/var/spack/repos/builtin/packages/r-rhdf5lib/package.py
index 8005ec16f9284c..396626eae4384b 100644
--- a/var/spack/repos/builtin/packages/r-rhdf5lib/package.py
+++ b/var/spack/repos/builtin/packages/r-rhdf5lib/package.py
@@ -27,4 +27,4 @@ class RRhdf5lib(RPackage):
     depends_on("r@4.0.0:", type="build", when="@1.16.0:")
     depends_on("r@4.2.0:", type=("build", "run"), when="@1.22.0:")
     depends_on("gmake", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-rjava/package.py b/var/spack/repos/builtin/packages/r-rjava/package.py
index 812d4daacb6289..fcf5c95a06a6b6 100644
--- a/var/spack/repos/builtin/packages/r-rjava/package.py
+++ b/var/spack/repos/builtin/packages/r-rjava/package.py
@@ -30,7 +30,7 @@ class RRjava(RPackage):
     depends_on("iconv")
     depends_on("pcre2")
     depends_on("xz")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def setup_build_environment(self, env):
         spec = self.spec
diff --git a/var/spack/repos/builtin/packages/r-rtracklayer/package.py b/var/spack/repos/builtin/packages/r-rtracklayer/package.py
index 81412dc8e295cc..832d129de6e778 100644
--- a/var/spack/repos/builtin/packages/r-rtracklayer/package.py
+++ b/var/spack/repos/builtin/packages/r-rtracklayer/package.py
@@ -58,5 +58,5 @@ class RRtracklayer(RPackage):
     depends_on("r-genomicalignments@1.15.6:", type=("build", "run"), when="@1.40.6:")
     depends_on("r-biocio", type=("build", "run"), when="@1.54.0:")
     depends_on("r-restfulr@0.0.13:", type=("build", "run"), when="@1.54.0:")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("openssl")
diff --git a/var/spack/repos/builtin/packages/r-s4vectors/package.py b/var/spack/repos/builtin/packages/r-s4vectors/package.py
index bb8848ff290ef8..4760e02e88935d 100644
--- a/var/spack/repos/builtin/packages/r-s4vectors/package.py
+++ b/var/spack/repos/builtin/packages/r-s4vectors/package.py
@@ -25,7 +25,7 @@ class RS4vectors(RPackage):
     version("0.34.0", commit="f590de3ec4d896a63351d0c1925d3856c0bd5292")
     version("0.32.3", commit="ad90e78fd3a4059cfcf2846498fb0748b4394e1a")
     version("0.28.1", commit="994cb7ef830e76f8b43169cc72b553869fafb2ed")
-    version("0.26.1", commit="935769c")
+    version("0.26.1", commit="935769c7e2767230feb47f6f8147e0e2908af4f0")
     version("0.22.1", commit="d25e517b48ca4184a4c2ee1f8223c148a55a8b8a")
     version("0.20.1", commit="1878b2909086941e556c5ea953c6fd86aebe9b02")
     version("0.18.3", commit="d6804f94ad3663828440914920ac933b934aeff1")
diff --git a/var/spack/repos/builtin/packages/r-seqinr/package.py b/var/spack/repos/builtin/packages/r-seqinr/package.py
index 4115f90af27662..278c13b498cf45 100644
--- a/var/spack/repos/builtin/packages/r-seqinr/package.py
+++ b/var/spack/repos/builtin/packages/r-seqinr/package.py
@@ -27,4 +27,4 @@ class RSeqinr(RPackage):
     depends_on("r@2.10.0:", type=("build", "run"), when="@4.2-30:")
     depends_on("r-ade4", type=("build", "run"))
     depends_on("r-segmented", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-sets/package.py b/var/spack/repos/builtin/packages/r-sets/package.py
new file mode 100644
index 00000000000000..289b03d111d025
--- /dev/null
+++ b/var/spack/repos/builtin/packages/r-sets/package.py
@@ -0,0 +1,54 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class RSets(RPackage):
+    """Sets, Generalized Sets, Customizable Sets and Intervals
+
+    Data structures and basic operations for ordinary sets, generalizations such
+    as fuzzy sets, multisets, and fuzzy multisets, customizable sets, and
+    intervals."""
+
+    cran = "sets"
+
+    maintainers("jgaeb")
+
+    version("1.0-24", sha256="e75733f5c9418eb09fb950a4a94ccf84ddd88231c61ee80d02b7f0917debcac9")
+    version("1.0-23", sha256="e5b6bc52060421c572d7f2d99b25909a38eacabd5344a47e1cdb2662c62d690b")
+    version("1.0-22", sha256="6fbf9aa6b0113a58e04f803ab35593feabb0fb55d486d54afb59e027008f9ec6")
+    version("1.0-21", sha256="5733f0be59189c058c069583f5c4dc1d772bfad5abbfd16081131414d6002ac0")
+    version("1.0-20", sha256="234b724d40afcabc57eaf42dd34c6cb846e26803796e7fc80c00d26047c475d6")
+    version("1.0-19", sha256="ae93e56bb0b4fd361349faf962772bb5eab965966e9c9bbf8bd4a2426a2e28a0")
+    version("1.0-18", sha256="74d1e057e5b84197edb120665831d7b0565e2945e903be56c6701e724131679b")
+    version("1.0-17", sha256="17817a386d725a458d95368795e4c31ef5dbc00439df24daf9bda996bfe767c5")
+    version("1.0-16", sha256="5d36bc40937283112287d543f86a8fd470ce587420f5690f6a82f9ffa8e5805e")
+    version("1.0-15", sha256="6f65ebfda2a94707b98cecdb1d3dcd0c0d1fd2f6a5c36eb128de7c2d5f7c1f8b")
+    version("1.0-14", sha256="8fe81fc8d296484ffe9d796820dc259c0e6ab69d65d1f18564f89f3b9827cff1")
+    version("1.0-13", sha256="5ffdc1a0e59c2a9e314b652ad72e0af5d138bad3e69190c6b56eca277c3c41fb")
+    version("1.0-12", sha256="b1d1868bfba7c22e4bd726d534b1afbe593bde1f8e209ddb76613d1dd9c9954a")
+    version("1.0-11", sha256="133d36b6fc3cb75097a829edbc15542f4817e2b6edf2a4d4830004a74308449f")
+    version("1.0-10", sha256="5a631056ceb192ca35ecfc1cf10a0cf5a1671a3d5e50f942b0ac2e2098c909d0")
+    version("1.0-9", sha256="748b254fedfe710bd295eb99168799c711f6a563b986b4f98e32f6ecc0c6de54")
+    version("1.0-8", sha256="fa93e8e44b12cba33e9e1ca71e0d5ea84f3beb5656de9031c78e87ebfc2ee799")
+    version("1.0-7", sha256="29f717a4b71fb2e72f3ce04f4cd703cc860c28c0f58a0e3b2adc1bcaa3b742ba")
+    version("1.0", sha256="02b85933d9cd55e281c3a87889d827021394b51ba714e87359a36cbf60b50980")
+    version("0.7", sha256="f450feaa2df5071c2029367edac867d7dbe435d202e5b1475e48827bc10bdf06")
+    version("0.6", sha256="d682e5fe37d7fb2ded11ca702f9af2bf744cc56b3d5b310dba20dda2df6b1dc6")
+    version("0.5", sha256="b4d8298e6f169d70b969ef6a49ed0583e2efecbdaa883c45142156bc97263149")
+    version("0.4", sha256="f013228cbc3e63eb0a5b6ad8e217099482f275316f78f9bad380da6faa0defc6")
+    version("0.3-2", sha256="c971cf712de2503f6605b883e36e1e1639ab8534d7c78ebc5a6a347e09771d37")
+    version("0.3-1", sha256="0e6d4cd4eaf29edafbe4dbf4b71bfa3ada0f4ffb4a3becb144c023ef4478c944")
+    version("0.3", sha256="a10c390d0571ab50a3c6e128d9157a9afc7c87b7befdc61cf86fa4cc9ef9c36d")
+    version("0.2-1", sha256="f59ea29e7d87ba195909ab11102d14bb9b89c8fd76b25d273e79cd9227753aa2")
+    version("0.2", sha256="987ff3ec1597a3d25d39ac62853443e43e77d3fdc450bed43f5ed72a30b142cb")
+    version("0.1-3", sha256="e91d5b70ddd74ab50c944722b5dab2032cccd5cbfa740dd2b5a744ff0a89ce90")
+    version("0.1-2", sha256="87fa3292eca69d358ea615c39240bb2151afc3a64f004b975f1918602ff9c694")
+    version("0.1-1", sha256="4e41480757e33897a26974e5234801ff1c15f1a3952c96071787b43141a130de")
+    version("0.1", sha256="18dda6c9d526a2f41f2b49a472fb27a7f1bb9ce6ea137b8963e8ad6c378825d0")
+
+    depends_on("r@2.6:", type=("build", "run"), when="@0.1:")
+    depends_on("r@2.7:", type=("build", "run"), when="@0.1-2:")
diff --git a/var/spack/repos/builtin/packages/r-shortread/package.py b/var/spack/repos/builtin/packages/r-shortread/package.py
index 15cee95daef7d2..85b766e191d204 100644
--- a/var/spack/repos/builtin/packages/r-shortread/package.py
+++ b/var/spack/repos/builtin/packages/r-shortread/package.py
@@ -53,4 +53,4 @@ class RShortread(RPackage):
     depends_on("r-latticeextra", type=("build", "run"))
     depends_on("r-xvector", type=("build", "run"))
     depends_on("r-rhtslib", type=("build", "run"), when="@1.48.0:")
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-signac/package.py b/var/spack/repos/builtin/packages/r-signac/package.py
index 669775b0962f67..84bb7113ccc254 100644
--- a/var/spack/repos/builtin/packages/r-signac/package.py
+++ b/var/spack/repos/builtin/packages/r-signac/package.py
@@ -47,4 +47,4 @@ class RSignac(RPackage):
     depends_on("r-rcpp", type=("build", "run"))
     depends_on("r-tidyselect", type=("build", "run"))
     depends_on("r-vctrs", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-snpstats/package.py b/var/spack/repos/builtin/packages/r-snpstats/package.py
index cf6460c2f9bff1..b4038acea31c23 100644
--- a/var/spack/repos/builtin/packages/r-snpstats/package.py
+++ b/var/spack/repos/builtin/packages/r-snpstats/package.py
@@ -31,4 +31,4 @@ class RSnpstats(RPackage):
     depends_on("r-matrix", type=("build", "run"))
     depends_on("r-biocgenerics", type=("build", "run"))
     depends_on("r-zlibbioc", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-summarizedexperiment/package.py b/var/spack/repos/builtin/packages/r-summarizedexperiment/package.py
index bb074df20f87dc..a421fef13066d3 100644
--- a/var/spack/repos/builtin/packages/r-summarizedexperiment/package.py
+++ b/var/spack/repos/builtin/packages/r-summarizedexperiment/package.py
@@ -21,7 +21,7 @@ class RSummarizedexperiment(RPackage):
     version("1.26.1", commit="c8cbd3b4f0fa1d686c4d7ce5b8614a24c74b2074")
     version("1.24.0", commit="d37f19383d03c107a8a41c0df2326e28efe46b28")
     version("1.20.0", commit="874aa87a481e4076a0ec3369f55c9c0a1ab8025e")
-    version("1.18.2", commit="e22fafe")
+    version("1.18.2", commit="e22fafe8c314b4e216fb130012d631b4626bc605")
     version("1.14.1", commit="2c68d99e11c7345e5ed388370822ea48395c64a4")
     version("1.12.0", commit="5f8416864636add121ec1d6737ebb89a42227fd7")
     version("1.10.1", commit="7ad2e991c8285bfc4b2e15b29d94cc86d07f8f2b")
diff --git a/var/spack/repos/builtin/packages/r-tinytiger/package.py b/var/spack/repos/builtin/packages/r-tinytiger/package.py
new file mode 100644
index 00000000000000..d0b66c81f9e00d
--- /dev/null
+++ b/var/spack/repos/builtin/packages/r-tinytiger/package.py
@@ -0,0 +1,33 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class RTinytiger(RPackage):
+    """Lightweight Interface to TIGER/Line Shapefiles
+
+    Download geographic shapes from the United States Census Bureau TIGER/Line
+    Shapefiles
+    .
+    Functions support downloading and reading in geographic boundary data. All
+    downloads can be set up with a cache to avoid multiple downloads. Data is
+    available back to 2000 for most geographies."""
+
+    homepage = "https://alarm-redist.org/tinytiger/"
+    cran = "tinytiger"
+
+    maintainers("jgaeb")
+
+    version("0.0.4", sha256="818328b5095d9e8b302f1a04d004cd3ec6e62d945dbd757fe15e9ab768a7459e")
+    version("0.0.3", sha256="841d92dd4185b9bff5eef0d3635805c5a3efb1bc4ff0a1101ef264417e37921c")
+
+    depends_on("r@2.0.0:", type=("build", "run"))
+    depends_on("r@2.10:", type=("build", "run"), when="@0.0.4:")
+    depends_on("r-rlang")
+    depends_on("r-cli")
+    depends_on("r-glue")
+    depends_on("r-curl")
+    depends_on("r-sf")
diff --git a/var/spack/repos/builtin/packages/r-vcfr/package.py b/var/spack/repos/builtin/packages/r-vcfr/package.py
index 9bed9c832300e5..83d2fa0c0d2b91 100644
--- a/var/spack/repos/builtin/packages/r-vcfr/package.py
+++ b/var/spack/repos/builtin/packages/r-vcfr/package.py
@@ -37,4 +37,4 @@ class RVcfr(RPackage):
     depends_on("r-tibble", type=("build", "run"))
     depends_on("r-vegan", type=("build", "run"))
     depends_on("r-viridislite", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r-wru/package.py b/var/spack/repos/builtin/packages/r-wru/package.py
new file mode 100644
index 00000000000000..d513a33f2d5bdb
--- /dev/null
+++ b/var/spack/repos/builtin/packages/r-wru/package.py
@@ -0,0 +1,56 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class RWru(RPackage):
+    """Who are You? Bayesian Prediction of Racial Category Using Surname, First
+    Name, Middle Name, and Geolocation
+
+    Predicts individual race/ethnicity using surname, first name, middle name,
+    geolocation, and other attributes, such as gender and age. The method
+    utilizes Bayes' Rule (with optional measurement error correction) to compute
+    the posterior probability of each racial category for any given individual.
+    The package implements methods described in Imai and Khanna (2016)
+    "Improving Ecological Inference by Predicting Individual Ethnicity from
+    Voter Registration Records" Political Analysis ."""
+
+    homepage = "https://github.com/kosukeimai/wru"
+    cran = "wru"
+
+    maintainers("jgaeb")
+
+    version("1.0.1", sha256="80b3f54cb2de77ea005755a2de3acfb923a1d380c0dbd52bc4d3e3fcb1d6f1fc")
+    version("1.0.0", sha256="4eae65644981d0b99d3610adf40340b3606f40e6cd578e76a745524ba927e417")
+    version("0.1-12", sha256="896ef4718109ab9fee686f050a3269cbab1589ef2aff7a45fc11a67f7bb35a29")
+    version("0.1-11", sha256="a6af1a7e822b4d45759525872df3aa704d2cd431b47e583052d67de89c9bdba3")
+    version("0.1-10", sha256="9bd3946b5005e8a7b1ff3e217690d0c02264b51f854785e6699d2533dd8145eb")
+    version("0.1-9", sha256="105dc4d785b6044fac68696688be9c234acfdcb2b598f97fdbec3d38831b3577")
+    version("0.1-8", sha256="efa7960a389ecf4c3cb6527848e06ae7c1d0d7f0da3a48c66453920f8b8752f0")
+    version("0.1-7", sha256="8962f09e92f4209d8e52c86eba50cd35e7c0265154e608256fd3b80362a16bbb")
+    version("0.1-6", sha256="81b4fa397d1cb56ea7635e76d198ccbfccd4141484f1f28f25c7dac408bfeed2")
+    version("0.1-5", sha256="2a9d656e371c6d55c425f8e31e388ce574b9527fa674a7a0e65844d5b9e983e1")
+    version("0.1-4", sha256="d463553cd835f85042b7679a14cf935efa9406a1fb4a67513ab251df59d6bd58")
+    version("0.1-3", sha256="4e676ddb2084499404932163c858ec7574e2596e3bcb8cfac93d829dbe6b820a")
+    version("0.1-2", sha256="ce9e622c89439f51698fdbb3dbbed7b8dda5aee886de5747e09e70c9982cf881")
+    version("0.1-1", sha256="9b4d665867803f8992f7d2bf809da290396a1c7070fd72bf0c66c6408e1a7180")
+    version("0.0-2", sha256="4ad5a09a5663906ea546fd9169444a559643d87ac18115e88b538a9260d40d4d")
+    version("0.0-1", sha256="604a3e0f04fa3b1c5ee13543c0e3841f5a050f784f561d7ed8576542a27584ae")
+
+    depends_on("r@3.2.0:", type=("build", "run"), when="@0.0-1:")
+    depends_on("r@3.5.0:", type=("build", "run"), when="@0.0-10:")
+    depends_on("r@4.1.0:", type=("build", "run"), when="@1.0.0:")
+    depends_on("r-devtools", type=("build", "run"), when="@0.0-2:0.1-12")
+    depends_on("r-devtools@1.10.0:", type=("build", "run"), when="@0.1-1:0.1-12")
+    depends_on("r-dplyr", type=("build", "run"), when="@1.0.0:")
+    depends_on("r-future", type=("build", "run"), when="@1.0.0:")
+    depends_on("r-furrr", type=("build", "run"), when="@1.0.0:")
+    depends_on("r-purrr", type=("build", "run"), when="@1.0.0:")
+    depends_on("r-rcpp", type=("build", "run"), when="@1.0.0:")
+    depends_on("r-rcpparmadillo", type=("build", "run"), when="@1.0.0:")
+    depends_on("r-piggyback", type=("build", "run"), when="@1.0.0:")
+    depends_on("r-piggyback@0.1.4:", type=("build", "run"), when="@1.0.0:")
+    depends_on("r-pl94171", type=("build", "run"), when="@1.0.0:")
diff --git a/var/spack/repos/builtin/packages/r-xvector/package.py b/var/spack/repos/builtin/packages/r-xvector/package.py
index 7f4d0cc7a6df1f..7bf098cc8c6b36 100644
--- a/var/spack/repos/builtin/packages/r-xvector/package.py
+++ b/var/spack/repos/builtin/packages/r-xvector/package.py
@@ -41,4 +41,4 @@ class RXvector(RPackage):
     depends_on("r-iranges@2.15.12:", type=("build", "run"), when="@0.22.0:")
     depends_on("r-iranges@2.23.9:", type=("build", "run"), when="@0.30.0:")
     depends_on("r-zlibbioc", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/r/package.py b/var/spack/repos/builtin/packages/r/package.py
index c2d0218d561551..7232a7e6c165dc 100644
--- a/var/spack/repos/builtin/packages/r/package.py
+++ b/var/spack/repos/builtin/packages/r/package.py
@@ -70,9 +70,9 @@ class R(AutotoolsPackage):
     depends_on("blas", when="+external-lapack")
     depends_on("lapack", when="+external-lapack")
     depends_on("bzip2")
-    # R didn't anticipate the celebratory
-    # non-breaking major version bump of curl 8.
-    depends_on("curl+libidn2@:7")
+    depends_on("curl+libidn2")
+    # R didn't anticipate the celebratory non-breaking major version bump of curl 8.
+    depends_on("curl@:7", when="@:4.2")
     depends_on("icu4c")
     depends_on("java")
     depends_on("ncurses")
@@ -81,7 +81,8 @@ class R(AutotoolsPackage):
     depends_on("readline")
     depends_on("xz")
     depends_on("which", type=("build", "run"))
-    depends_on("zlib@1.2.5:")
+    depends_on("zlib-api")
+    depends_on("zlib@1.2.5:", when="^zlib")
     depends_on("texinfo", type="build")
     depends_on("cairo+X+gobject+pdf", when="+X")
     depends_on("pango+X", when="+X")
@@ -102,6 +103,8 @@ class R(AutotoolsPackage):
     # temporary fix to lower the optimization level.
     patch("change_optflags_tmp.patch", when="%fj@4.1.0")
 
+    build_directory = "spack-build"
+
     # R custom URL version
     def url_for_version(self, version):
         """Handle R's customed URL versions"""
@@ -134,7 +137,7 @@ def configure_args(self):
         ]
 
         if "+external-lapack" in spec:
-            if "^mkl" in spec and "gfortran" in self.compiler.fc:
+            if spec["lapack"].name in INTEL_MATH_LIBRARIES and "gfortran" in self.compiler.fc:
                 mkl_re = re.compile(r"(mkl_)intel(_i?lp64\b)")
                 config_args.extend(
                     [
diff --git a/var/spack/repos/builtin/packages/raja/package.py b/var/spack/repos/builtin/packages/raja/package.py
index 5483cbdfccc895..99221b9b08c7d3 100644
--- a/var/spack/repos/builtin/packages/raja/package.py
+++ b/var/spack/repos/builtin/packages/raja/package.py
@@ -20,25 +20,90 @@ class Raja(CachedCMakePackage, CudaPackage, ROCmPackage):
 
     version("develop", branch="develop", submodules=False)
     version("main", branch="main", submodules=False)
-    version("2022.10.4", tag="v2022.10.4", submodules=False)
-    version("2022.03.0", tag="v2022.03.0", submodules=False)
-    version("0.14.0", tag="v0.14.0", submodules="True")
-    version("0.13.0", tag="v0.13.0", submodules="True")
-    version("0.12.1", tag="v0.12.1", submodules="True")
-    version("0.12.0", tag="v0.12.0", submodules="True")
-    version("0.11.0", tag="v0.11.0", submodules="True")
-    version("0.10.1", tag="v0.10.1", submodules="True")
-    version("0.10.0", tag="v0.10.0", submodules="True")
-    version("0.9.0", tag="v0.9.0", submodules="True")
-    version("0.8.0", tag="v0.8.0", submodules="True")
-    version("0.7.0", tag="v0.7.0", submodules="True")
-    version("0.6.0", tag="v0.6.0", submodules="True")
-    version("0.5.3", tag="v0.5.3", submodules="True")
-    version("0.5.2", tag="v0.5.2", submodules="True")
-    version("0.5.1", tag="v0.5.1", submodules="True")
-    version("0.5.0", tag="v0.5.0", submodules="True")
-    version("0.4.1", tag="v0.4.1", submodules="True")
-    version("0.4.0", tag="v0.4.0", submodules="True")
+    version(
+        "2022.10.4",
+        tag="v2022.10.4",
+        commit="c2a6b1740759ae3ae7c85b35e20dbffbe235355d",
+        submodules=False,
+    )
+    version(
+        "2022.03.0",
+        tag="v2022.03.0",
+        commit="4351fe6a50bd579511a625b017c9e054885e7fd2",
+        submodules=False,
+    )
+    version(
+        "0.14.0",
+        tag="v0.14.0",
+        commit="357933a42842dd91de5c1034204d937fce0a2a44",
+        submodules="True",
+    )
+    version(
+        "0.13.0",
+        tag="v0.13.0",
+        commit="3047fa720132d19ee143b1fcdacaa72971f5988c",
+        submodules="True",
+    )
+    version(
+        "0.12.1",
+        tag="v0.12.1",
+        commit="9cb6370bb2868e35ebba23cdce927f5f7f9da530",
+        submodules="True",
+    )
+    version(
+        "0.12.0",
+        tag="v0.12.0",
+        commit="32d92e38da41cc8d4db25ec79b9884a73a0cb3a1",
+        submodules="True",
+    )
+    version(
+        "0.11.0",
+        tag="v0.11.0",
+        commit="0502b9b69c4cb60aa0afbdf699b555c76cb18f22",
+        submodules="True",
+    )
+    version(
+        "0.10.1",
+        tag="v0.10.1",
+        commit="be91e040130678b1350dbda56cc352433db758bd",
+        submodules="True",
+    )
+    version(
+        "0.10.0",
+        tag="v0.10.0",
+        commit="53cb89cf788d28bc4ed2b4e6f75483fdd26024aa",
+        submodules="True",
+    )
+    version(
+        "0.9.0", tag="v0.9.0", commit="df7ca1fa892b6ac4147c614d2d739d5022f63fc7", submodules="True"
+    )
+    version(
+        "0.8.0", tag="v0.8.0", commit="8d19a8c2cbac611de6f92ad8852b9f3454b27e63", submodules="True"
+    )
+    version(
+        "0.7.0", tag="v0.7.0", commit="caa33b371b586dfae3d8569caee91c5eddfd7b31", submodules="True"
+    )
+    version(
+        "0.6.0", tag="v0.6.0", commit="cc7a97e8b4e52c3de820c9dfacd358822a147871", submodules="True"
+    )
+    version(
+        "0.5.3", tag="v0.5.3", commit="1ca35c0ed2a43a3fa9c6cd70c5d25f16d88ecd8c", submodules="True"
+    )
+    version(
+        "0.5.2", tag="v0.5.2", commit="4d5c3d5d7f311838855f7010810610349e729f64", submodules="True"
+    )
+    version(
+        "0.5.1", tag="v0.5.1", commit="bf340abe5199d7e051520913c9a7a5de336b5820", submodules="True"
+    )
+    version(
+        "0.5.0", tag="v0.5.0", commit="9b539d84fdad049f65caeba836f41031f5baf4cc", submodules="True"
+    )
+    version(
+        "0.4.1", tag="v0.4.1", commit="3618cfe95d6a442fa50fbe7bfbcf654cf9f800b9", submodules="True"
+    )
+    version(
+        "0.4.0", tag="v0.4.0", commit="31b2a48192542c2da426885baa5af0ed57606b78", submodules="True"
+    )
 
     # export targets when building pre-2.4.0 release with BLT 0.4.0+
     patch(
@@ -49,17 +114,19 @@ class Raja(CachedCMakePackage, CudaPackage, ROCmPackage):
 
     variant("openmp", default=True, description="Build OpenMP backend")
     variant("shared", default=True, description="Build Shared Libs")
+    variant("plugins", default=False, description="Enable runtime plugins")
     variant("examples", default=True, description="Build examples.")
     variant("exercises", default=True, description="Build exercises.")
     # TODO: figure out gtest dependency and then set this default True
     # and remove the +tests conflict below.
     variant("tests", default=False, description="Build tests")
 
-    depends_on("blt")
+    depends_on("blt", type="build")
     depends_on("blt@0.5.0:", type="build", when="@0.14.1:")
     depends_on("blt@0.4.1", type="build", when="@0.14.0")
     depends_on("blt@0.4.0:", type="build", when="@0.13.0")
     depends_on("blt@0.3.6:", type="build", when="@:0.12.0")
+    conflicts("^blt@:0.3.6", when="+rocm")
 
     depends_on("camp@0.2.2:0.2.3", when="@0.14.0")
     depends_on("camp@0.1.0", when="@0.10.0:0.13.0")
@@ -71,7 +138,7 @@ class Raja(CachedCMakePackage, CudaPackage, ROCmPackage):
 
     depends_on("cmake@:3.20", when="@:2022.03+rocm", type="build")
     depends_on("cmake@3.23:", when="@2022.10:+rocm", type="build")
-    depends_on("cmake@3.14:", when="@2022.03.0:")
+    depends_on("cmake@3.14:", when="@2022.03.0:", type="build")
 
     depends_on("llvm-openmp", when="+openmp %apple-clang")
 
@@ -159,6 +226,7 @@ def initconfig_package_entries(self):
         if "camp" in self.spec:
             entries.append(cmake_cache_path("camp_DIR", spec["camp"].prefix))
         entries.append(cmake_cache_option("BUILD_SHARED_LIBS", "+shared" in spec))
+        entries.append(cmake_cache_option("RAJA_ENABLE_RUNTIME_PLUGINS", "+plugins" in spec))
         entries.append(
             cmake_cache_option("{}ENABLE_EXAMPLES".format(option_prefix), "+examples" in spec)
         )
diff --git a/var/spack/repos/builtin/packages/ratel/package.py b/var/spack/repos/builtin/packages/ratel/package.py
index bdea843bd133c6..9eeaf2ec8c8a7d 100644
--- a/var/spack/repos/builtin/packages/ratel/package.py
+++ b/var/spack/repos/builtin/packages/ratel/package.py
@@ -15,13 +15,16 @@ class Ratel(MakefilePackage, CudaPackage, ROCmPackage):
     maintainers("jedbrown", "jeremylt")
 
     version("develop", branch="main")
-    version("0.2.1", tag="v0.2.1")
-    version("0.1.2", tag="v0.1.2")
+    version("0.3.0", tag="v0.3.0", commit="ca2f3357e10b89fb274626fba104aad30c72774b")
+    version("0.2.1", tag="v0.2.1", commit="043b61696a2407205fdfd898681467d1a7ff59e0")
+    version("0.1.2", tag="v0.1.2", commit="94ad630bf897d231af7a94bf08257f6067258aae")
 
     # development version
     depends_on("libceed@develop", when="@develop")
     depends_on("petsc@main", when="@develop")
     # released versions
+    depends_on("libceed@0.12.0:0.12", when="@0.3.0")
+    depends_on("petsc@3.20.0:3.20", when="@0.3.0")
     depends_on("libceed@0.11.0:0.11", when="@0.2.1")
     depends_on("petsc@3.18.3:3.18", when="@0.2.1")
     depends_on("libceed@0.10.1:0.10", when="@0.1.2")
diff --git a/var/spack/repos/builtin/packages/rccl/0003-Fix-numactl-rocm-smi-path-issue.patch b/var/spack/repos/builtin/packages/rccl/0003-Fix-numactl-rocm-smi-path-issue.patch
index 1451e545830604..9e1b26a10f5433 100644
--- a/var/spack/repos/builtin/packages/rccl/0003-Fix-numactl-rocm-smi-path-issue.patch
+++ b/var/spack/repos/builtin/packages/rccl/0003-Fix-numactl-rocm-smi-path-issue.patch
@@ -1,25 +1,27 @@
-From 5d40f88d3af525326aaeae052f0e7345445336d8 Mon Sep 17 00:00:00 2001
+From d929b8a0c4fcaaf2ce37cff335b89455db043780 Mon Sep 17 00:00:00 2001
 From: Renjith Ravindran 
-Date: Wed, 21 Sep 2022 05:42:54 +0000
-Subject: [PATCH] 5.2.3 changes for spack
+Date: Fri, 19 May 2023 22:36:11 +0000
+Subject: [PATCH] Updating with hsa include path in CMakeList
 
 ---
- CMakeLists.txt | 2 ++
- 1 file changed, 2 insertions(+)
+ CMakeLists.txt | 4 ++++
+ 1 file changed, 4 insertions(+)
 
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 79e0a35..21f2a54 100644
+index 9c139ef..f53cedf 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -123,6 +123,8 @@ include_directories(src)
- include_directories(src/include)
+@@ -134,6 +134,10 @@ include_directories(${PROJECT_BINARY_DIR}/src/include) # for hipified header fil
+ include_directories(src)
  include_directories(src/collectives)
  include_directories(src/collectives/device)
 +include_directories(${NUMACTL_DIR}/include)
++include_directories(${HSA_INCLUDE_PATH})
 +link_directories(${NUMACTL_DIR}/lib)
++
  
- # Create git version file
- # _git_version.cpp is a dummy output to force re-run prior to build
+ if (BUILD_ALLREDUCE_ONLY)
+   add_definitions(-DBUILD_ALLREDUCE_ONLY)
 -- 
 2.17.1
 
diff --git a/var/spack/repos/builtin/packages/rccl/package.py b/var/spack/repos/builtin/packages/rccl/package.py
index bdae9578de75fa..677b077b4bdce7 100644
--- a/var/spack/repos/builtin/packages/rccl/package.py
+++ b/var/spack/repos/builtin/packages/rccl/package.py
@@ -16,12 +16,15 @@ class Rccl(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/rccl"
     git = "https://github.com/ROCmSoftwarePlatform/rccl.git"
-    url = "https://github.com/ROCmSoftwarePlatform/rccl/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/rccl/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
     libraries = ["librccl"]
-
+    version("5.6.1", sha256="27ec6b86a1a329684d808f728c1fce134517ac8e6e7047689f95dbf8386c077e")
+    version("5.6.0", sha256="cce13c8a9e233e7ddf91a67b1626b7aaeaf818fefe61af8de6b6b6ff47cb358c")
+    version("5.5.1", sha256="f6b9dc6dafeb49d95c085825876b09317d8252771c746ccf5aa19a9204a404b2")
+    version("5.5.0", sha256="be2964b408741d046bcd606d339a233d1d1deac7b841647ec53d6d62d71452ba")
     version("5.4.3", sha256="a2524f602bd7b3b6afeb8ba9aff660216ee807fa836e46442d068b5ed5f51a4d")
     version("5.4.0", sha256="213f4f3d75389be588673e43f563e5c0d6908798228b0b6a71f27138fd4ed0c7")
     version("5.3.3", sha256="8995a2d010ad0748fc85ac06e8da7e8d110ba996db04d42b77526c9c059c05bb")
@@ -104,12 +107,11 @@ class Rccl(CMakePackage):
 
     amdgpu_targets = ROCmPackage.amdgpu_targets
 
-    variant("amdgpu_target", values=auto_or_any_combination_of(*amdgpu_targets), sticky=True)
     variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
+        "amdgpu_target",
+        description="AMD GPU architecture",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
     )
 
     patch("0001-Fix-numactl-path-issue.patch", when="@3.7.0:4.3.2")
@@ -141,6 +143,10 @@ class Rccl(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocm-cmake@%s:" % ver, type="build", when="@" + ver)
         depends_on("hip@" + ver, when="@" + ver)
@@ -170,6 +176,10 @@ class Rccl(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("numactl@2:", when="@" + ver)
     for ver in [
@@ -186,10 +196,18 @@ class Rccl(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocm-smi-lib@" + ver, when="@" + ver)
         depends_on("chrpath", when="@5.3.0:")
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+    depends_on("googletest@1.11.0:", when="@5.3:")
+
     @classmethod
     def determine_version(cls, lib):
         match = re.search(r"lib\S*\.so\.\d+\.\d+\.(\d)(\d\d)(\d\d)", lib)
diff --git a/var/spack/repos/builtin/packages/rclone/package.py b/var/spack/repos/builtin/packages/rclone/package.py
index 01341dbd004a2f..a1dacabc7708e7 100644
--- a/var/spack/repos/builtin/packages/rclone/package.py
+++ b/var/spack/repos/builtin/packages/rclone/package.py
@@ -15,6 +15,7 @@ class Rclone(Package):
 
     maintainers("alecbcs")
 
+    version("1.63.1", sha256="0d8bf8b7460681f7906096a9d37eedecc5a1d1d3ad17652e68f0c6de104c2412")
     version("1.62.2", sha256="340371f94604e6771cc4a2c91e37d1bf00a524deab520340440fb0968e783f63")
     version("1.61.1", sha256="34b5f52047741c7bbf54572c02cc9998489c4736a753af3c99255296b1af125d")
     version("1.59.1", sha256="db3860e4549af28d87aa83f2035a57c5d081b179e40d4c828db19c3c3545831e")
@@ -27,44 +28,9 @@ class Rclone(Package):
     version("1.55.1", sha256="25da7fc5c9269b3897f27b0d946919df595c6dda1b127085fda0fe32aa59d29d")
     version("1.55.0", sha256="75accdaedad3b82edc185dc8824a19a59c30dc6392de7074b6cd98d1dc2c9040")
 
-    # https://nvd.nist.gov/vuln/detail/CVE-2020-28924
-    version(
-        "1.51.0",
-        sha256="3eb5b7ffce17e56fadb29bf854666723a14c93fedc02046c7f34c792dbd227ee",
-        deprecated=True,
-    )
-    version(
-        "1.50.2",
-        sha256="6dd8998a72514d3820d241ae46dc609c0305b742aee3db6aaf6017b46c996091",
-        deprecated=True,
-    )
-    version(
-        "1.50.1",
-        sha256="48d6c80883427469682b4d97099d7631cf3b67aa85e652c254423bd1422ce216",
-        deprecated=True,
-    )
-    version(
-        "1.50.0",
-        sha256="f901fd1752aae6116d94fd08d010a70d94535257c2d23caa505e631cce1e802a",
-        deprecated=True,
-    )
-    version(
-        "1.49.5",
-        sha256="abd2c83d71c63a4b0a30b1980b942868e707d05e14ae76ad39abf5cc5a5fde63",
-        deprecated=True,
-    )
-    version(
-        "1.49.4",
-        sha256="070afc85e4e9921151d7cb67247db8f0ff2f06fcf2652c43a42fa6e1e35847af",
-        deprecated=True,
-    )
-    version(
-        "1.43",
-        sha256="d30527b00cecb4e5e7188dddb78e5cec62d67cf2422dab82190db58512b5a4e3",
-        deprecated=True,
-    )
-
-    depends_on("go@1.15:", type="build")
+    depends_on("go@1.14:", type="build")
+    depends_on("go@1.17:", type="build", when="@1.58.0:")
+    depends_on("go@1.18:", type="build", when="@1.62.0:")
 
     phases = ["build", "install"]
 
diff --git a/var/spack/repos/builtin/packages/rdc/package.py b/var/spack/repos/builtin/packages/rdc/package.py
index 92b6ae06a2de85..8f88417ebf127c 100644
--- a/var/spack/repos/builtin/packages/rdc/package.py
+++ b/var/spack/repos/builtin/packages/rdc/package.py
@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 
+import re
+
 from spack.package import *
 
 
@@ -11,7 +13,7 @@ class Rdc(CMakePackage):
     """ROCm Data Center Tool"""
 
     homepage = "https://github.com/RadeonOpenCompute/rdc"
-    url = "https://github.com/RadeonOpenCompute/rdc/archive/rocm-5.4.0.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/rdc/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
@@ -24,6 +26,10 @@ def url_for_version(self, version):
         url = "https://github.com/RadeonOpenCompute/rdc/archive/rocm-{0}.tar.gz"
         return url.format(version)
 
+    version("5.6.1", sha256="9e9f57cebbc5ae386a405957ed2c17344cdb42db5e1a71285f2c9bc09eea6519")
+    version("5.6.0", sha256="5213cd89215463862f6a1e9480ebe017944a6bb6b0db1722628afaa34af57991")
+    version("5.5.1", sha256="a58a319ee702cf61cf71a4eba647c231392f68449b35419d941079c6de944844")
+    version("5.5.0", sha256="56e85e77581963fbcfcc43e091a91773de470152347808ae730bcaf92c9f5ee8")
     version("5.4.3", sha256="c44f0b070b5650bc78e2eb968aae57a8ac1e1fd160e897055b79f3026c4fbad3")
     version("5.4.0", sha256="268aab43e31045443b08a21aee8750da4cf04750c6f419ec171ec704d377a4e4")
     version("5.3.3", sha256="1bf1a02f305e3a629801e62584116a34eafbd1b26627837a2a8c10550fcf611b")
@@ -94,19 +100,13 @@ def url_for_version(self, version):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3.15:3.19.7", type="build", when="@:4.3.1")
     depends_on("cmake@3.15:", type="build", when="@4.5.0:")
     depends_on("grpc@1.28.1+shared", type="build", when="@:5.3")
-    depends_on("grpc@1.44.0+shared", when="@5.4.0:")
-    depends_on("protobuf", type=("build", "link"))
-    depends_on("libcap", type=("build", "link"))
+    depends_on("grpc@1.44.0+shared", when="@5.4.0:5.4")
+    depends_on("grpc@1.55.0+shared", when="@5.5.0:")
+    depends_on("protobuf")
+    depends_on("libcap")
 
     for ver in [
         "3.8.0",
@@ -130,6 +130,10 @@ def url_for_version(self, version):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocm-smi-lib@" + ver, type=("build", "link"), when="@" + ver)
 
@@ -145,9 +149,16 @@ def url_for_version(self, version):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hsa-rocr-dev@" + ver, when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     def patch(self):
         filter_file(r"\${ROCM_DIR}/rocm_smi", "${ROCM_SMI_DIR}", "CMakeLists.txt")
         filter_file(
diff --git a/var/spack/repos/builtin/packages/rdkit/package.py b/var/spack/repos/builtin/packages/rdkit/package.py
index eed3b8d8ae585d..15a68ab8b1ff0b 100644
--- a/var/spack/repos/builtin/packages/rdkit/package.py
+++ b/var/spack/repos/builtin/packages/rdkit/package.py
@@ -14,7 +14,7 @@ class Rdkit(CMakePackage):
     homepage = "https://www.rdkit.org"
     url = "https://github.com/rdkit/rdkit/archive/refs/tags/Release_2021_03_2.tar.gz"
 
-    maintainers("bvanessen")
+    maintainers("bvanessen", "RMeli")
 
     version("2023_03_1", sha256="db346afbd0ba52c843926a2a62f8a38c7b774ffab37eaf382d789a824f21996c")
     version("2022_09_5", sha256="2efe7ce3b527df529ed3e355e2aaaf14623e51876be460fa4ad2b7f7ad54c9b1")
@@ -46,17 +46,57 @@ class Rdkit(CMakePackage):
 
     variant("freetype", default=True, description="Build freetype support")
 
-    depends_on("python@3:")
-    depends_on("boost@1.53.0: +python +serialization +iostreams +system")
+    with when("@2022_09_5:"):
+        variant(
+            "python",
+            default=True,
+            when="@2022_09_5:",
+            description="Build standard Python wrappers",
+        )
+        variant("contrib", default=False, description="Build Contrib directory")
+        variant("freesasa", default=False, description="Build freesasa wrapper")
+        variant("coordgen", default=True, description="Build coordgen wrapper")
+        variant("maeparser", default=True, description="Build MAE parser wrapper")
+        variant("yaehmop", default=True, description="Build YAeHMOP wrapper")
+        variant("xyz2mol", default=False, description="Build support for RDKit xyz2mol")
+        variant("descriptors3d", default=True, description="Build 3D descriptors calculators")
 
-    depends_on("py-numpy")
-    depends_on("sqlite")
+        depends_on("freesasa", when="+freesasa")
+        depends_on("coordgen", when="+coordgen")
+        depends_on("maeparser", when="+maeparser")
+        depends_on("eigen@3:", when="+descriptors3d")
+        depends_on("python@3:", when="+python")
+        depends_on("py-numpy", when="+python")
+
+        extends("python", when="+python")
+
+        conflicts("+xyz2mol", when="~yaehmop", msg="XY2MOL requires YAeHMOP")
 
+    depends_on("boost@1.53.0: +python +serialization +iostreams +system")
+    depends_on("sqlite")
     depends_on("freetype", when="@2020_09_1: +freetype")
 
-    extends("python")
+    with when("@:2021_09_5"):
+        depends_on("python@3:")
+        depends_on("py-numpy")
+        extends("python")
 
     def cmake_args(self):
-        args = ["-DCMAKE_CXX_STANDARD=14", "-DRDK_INSTALL_INTREE=OFF"]
-        args.append(self.define_from_variant("RDK_BUILD_FREETYPE_SUPPORT", "freetype"))
+        args = [
+            "-DRDK_INSTALL_INTREE=OFF",
+            self.define_from_variant("RDK_BUILD_FREETYPE_SUPPORT", "freetype"),
+        ]
+        if "@2022_09_5:" in self.spec:
+            args.extend(
+                [
+                    self.define_from_variant("RDK_BUILD_PYTHON_WRAPPERS", "python"),
+                    self.define_from_variant("RDK_BUILD_CONTRIB", "contrib"),
+                    self.define_from_variant("RDK_BUILD_FREESASA_SUPPORT", "freesasa"),
+                    self.define_from_variant("RDK_BUILD_COORDGEN_SUPPORT", "coordgen"),
+                    self.define_from_variant("RDK_BUILD_MAEPARSER_SUPPORT", "maeparser"),
+                    self.define_from_variant("RDK_BUILD_XYZ2MOL_SUPPORT", "xyz2mol"),
+                    self.define_from_variant("RDK_BUILD_DESCRIPTORS3D", "descriptors3d"),
+                ]
+            )
+
         return args
diff --git a/var/spack/repos/builtin/packages/re2/package.py b/var/spack/repos/builtin/packages/re2/package.py
index 761005949b60e2..3c62d3da76217c 100644
--- a/var/spack/repos/builtin/packages/re2/package.py
+++ b/var/spack/repos/builtin/packages/re2/package.py
@@ -13,6 +13,9 @@ class Re2(CMakePackage):
     homepage = "https://github.com/google/re2"
     url = "https://github.com/google/re2/archive/2020-08-01.tar.gz"
 
+    version(
+        "2023-09-01", sha256="5bb6875ae1cd1e9fedde98018c346db7260655f86fdb8837e3075103acd3649b"
+    )
     version(
         "2021-06-01", sha256="26155e050b10b5969e986dab35654247a3b1b295e0532880b5a9c13c0a700ceb"
     )
@@ -26,6 +29,8 @@ class Re2(CMakePackage):
     variant("shared", default=False, description="Build shared instead of static libraries")
     variant("pic", default=True, description="Enable position independent code")
 
+    depends_on("abseil-cpp", when="@2023-09-01:")
+
     # shared libs must have position-independent code
     conflicts("+shared ~pic")
 
diff --git a/var/spack/repos/builtin/packages/recola/package.py b/var/spack/repos/builtin/packages/recola/package.py
index 80d11f2433444d..c4cb8d3c5f48b1 100644
--- a/var/spack/repos/builtin/packages/recola/package.py
+++ b/var/spack/repos/builtin/packages/recola/package.py
@@ -15,20 +15,27 @@ class Recola(CMakePackage):
 
     tags = ["hep"]
 
-    homepage = "https://recola.hepforge.org"
-    url = "https://recola.hepforge.org/downloads/?f=recola2-2.2.3.tar.gz"
+    homepage = "https://recola.gitlab.io/recola2/"
+    url = "https://gitlab.com/recola/recola2/-/archive/2.2.4/recola2-2.2.4.tar.gz"
 
     maintainers("vvolkl")
 
     variant("python", default=True, description="Build py-recola python bindings.")
 
-    version("2.2.4", sha256="16bdefb633d51842b4d32c39a43118d7052302cd63be456a473557e9b7e0316e")
-    version("2.2.3", sha256="db0f5e448ed603ac4073d4bbf36fd74f401a22876ad390c0d02c815a78106c5f")
+    version("2.2.4", sha256="212ae6141bc5de38c50be3e0c6947a3b0752aeb463cf850c22cfed5e61b1a64b")
+    version("2.2.3", sha256="8dc25798960c272434fcde93817ed92aad82b2a7cf07438bb4deb5688d301086")
+    version("2.2.2", sha256="a64cf2b4aa213289dfab6e2255a77264f281cd0ac85f5e9770c82b815272c5c9")
+    version("2.2.0", sha256="a64cf2b4aa213289dfab6e2255a77264f281cd0ac85f5e9770c82b815272c5c9")
     version(
         "1.4.3",
         url="https://recola.hepforge.org/downloads/?f=recola-1.4.3.tar.gz",
         sha256="f6a7dce6e1f09821ba919524f786557984f216c001ab63e7793e8aa9a8560ceb",
     )
+    version(
+        "1.4.0",
+        url="https://recola.hepforge.org/downloads/?f=recola-1.4.0.tar.gz",
+        sha256="dc7db5ac9456dda2e6c03a63ad642066b0b5e4ceb8cae1f2a13ab33b35caaba8",
+    )
 
     depends_on("collier")
     depends_on("recola-sm")
diff --git a/var/spack/repos/builtin/packages/redis-ai/package.py b/var/spack/repos/builtin/packages/redis-ai/package.py
new file mode 100644
index 00000000000000..05a3d1a0e51187
--- /dev/null
+++ b/var/spack/repos/builtin/packages/redis-ai/package.py
@@ -0,0 +1,126 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+
+
+class RedisAi(MakefilePackage):
+    """A Redis module for serving tensors and executing deep learning graphs"""
+
+    homepage = "https://oss.redis.com/redisai/"
+    git = "https://github.com/RedisAI/RedisAI.git"
+
+    maintainers("MattToast")
+
+    version(
+        "1.2.7", tag="v1.2.7", commit="1bf38d86233ba06e1350ca9de794df2b07cdb274", submodules=True
+    )
+
+    variant("torch", default=True, description="Build with the pytorch backend")
+    variant("cuda", default=False, description="Use CUDA")
+    variant("rocm", default=False, description="Use ROCm")
+
+    conflicts("+cuda+rocm")
+
+    # Required dependencies
+    depends_on("git", type=("build", "link"))
+    depends_on("git-lfs", type=("build", "link"))
+    depends_on("python@3:", type=("build", "link"))
+    depends_on("py-pip", type=("build", "link"))
+    depends_on("cmake@3.0:", type=("build", "link"))
+    depends_on("gmake", type=("build", "link"))
+
+    # GPU deps
+    depends_on("cuda@11.2:", type=("build", "link", "run"), when="+cuda")
+    depends_on("cudnn@8.1:", type=("build", "link", "run"), when="+cuda")
+
+    with when("+rocm"):
+        depends_on("hsa-rocr-dev")
+        depends_on("hip")
+        depends_on("rocprim")
+        depends_on("hipcub")
+        depends_on("rocthrust")
+        depends_on("roctracer-dev")
+        depends_on("rocrand")
+        depends_on("hipsparse")
+        depends_on("hipfft")
+        depends_on("rocfft")
+        depends_on("rocblas")
+        depends_on("miopen-hip")
+        depends_on("rocminfo")
+
+    # Optional Deps
+    with when("+torch"):
+        depends_on("py-torch@1.11.0:~cuda~rocm", type=("build", "link"), when="~cuda~rocm")
+        depends_on("py-torch@1.11.0:+cuda+cudnn~rocm", type=("build", "link"), when="+cuda")
+        depends_on("py-torch@1.11.0:~cuda+rocm", type=("build", "link"), when="+rocm")
+
+    build_directory = "opt"
+    parallel = False
+
+    @property
+    def use_gpu(self):
+        return self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm")
+
+    @property
+    def with_torch(self):
+        return self.spec.satisfies("+torch")
+
+    @property
+    def torch_dir(self):
+        return (
+            join_path(self.spec["py-torch"].package.cmake_prefix_paths[0], "Torch")
+            if self.with_torch
+            else None
+        )
+
+    @property
+    def build_env(self):
+        build_env = {
+            "WITH_TF": "0",
+            "WITH_TFLITE": "0",
+            "WITH_PT": "0",
+            "WITH_ORT": "0",
+            "WITH_UNIT_TESTS": "0",
+            "GPU": "1" if self.use_gpu else "0",
+        }
+        if self.with_torch:
+            build_env["WITH_PT"] = "1"
+            build_env["Torch_DIR"] = self.torch_dir
+        return build_env
+
+    def edit(self, spec, prefix):
+        # resolve deps not provided through spack
+        Executable(join_path(".", "get_deps.sh"))(
+            extra_env={
+                "VERBOSE": "1",
+                # Need to grab the RAI specific version of dlpack
+                "WITH_DLPACK": "1",
+                # Do not get ml backends, they should be retrieved through spack
+                "WITH_TF": "0",
+                "WITH_TFLITE": "0",
+                "WITH_PT": "0",
+                "WITH_ORT": "0",
+                # Decide if we want GPU
+                "GPU": "1" if self.use_gpu else "0",
+            }
+        )
+        env.update(self.build_env)
+
+    def install(self, spec, prefix):
+        super(RedisAi, self).install(spec, prefix)
+        install_tree("install-*", prefix)
+
+    @run_after("install")
+    @on_package_attributes(with_torch=True)
+    def copy_libtorch(self):
+        torch_site_dir = os.path.dirname(os.path.dirname(os.path.dirname(self.torch_dir)))
+        torch_lib_dir = join_path(torch_site_dir, "lib")
+        install_tree(torch_lib_dir, self.prefix.backends.redisai_torch.lib)
+
+    def setup_run_environment(self, env):
+        env.set("REDIS_AI", self.prefix.join("redisai.so"))
diff --git a/var/spack/repos/builtin/packages/redset/package.py b/var/spack/repos/builtin/packages/redset/package.py
index c0689bf4169595..f89b1a6a63e025 100644
--- a/var/spack/repos/builtin/packages/redset/package.py
+++ b/var/spack/repos/builtin/packages/redset/package.py
@@ -27,7 +27,7 @@ class Redset(CMakePackage):
     depends_on("mpi")
     depends_on("kvtree+mpi")
     depends_on("rankstr")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     depends_on("kvtree@:1.3.0", when="@:0.2.0")
     depends_on("kvtree@1.4.0:", when="@0.3.0:")
diff --git a/var/spack/repos/builtin/packages/regale/package.py b/var/spack/repos/builtin/packages/regale/package.py
new file mode 100644
index 00000000000000..705533fabd86b1
--- /dev/null
+++ b/var/spack/repos/builtin/packages/regale/package.py
@@ -0,0 +1,29 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Regale(CMakePackage):
+    """REGALE library is a C/C++ library developed in the European Project
+    REGALE used to allow standard communication among HPC power stack tools."""
+
+    homepage = "https://regale-project.eu/"
+    url = "https://gricad-gitlab.univ-grenoble-alpes.fr/regale/tools/regale/-/archive/v1.0/regale-v1.0.tar.gz"
+
+    version("1.0", sha256="894b0927372467e765049e79b855a9a277def65638013f68a1f2b6e837e35663")
+
+    depends_on("eprosima-fastdds")
+
+    variant("examples", default=False, description="Build examples")
+    variant("shared", default=True, description="Build shared libraries")
+
+    def cmake_args(self):
+        args = [
+            self.define_from_variant("REGALE_EXAMPLES", "examples"),
+            self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
+        ]
+
+        return args
diff --git a/var/spack/repos/builtin/packages/relion/package.py b/var/spack/repos/builtin/packages/relion/package.py
index 404f50d9e2944c..a35a092f99e98f 100644
--- a/var/spack/repos/builtin/packages/relion/package.py
+++ b/var/spack/repos/builtin/packages/relion/package.py
@@ -17,6 +17,7 @@ class Relion(CMakePackage, CudaPackage):
     url = "https://github.com/3dem/relion/archive/4.0.0.zip"
     maintainers("dacolombo")
 
+    version("4.0.1", sha256="7e0d56fd4068c99f943dc309ae533131d33870392b53a7c7aae7f65774f667be")
     version("4.0.0", sha256="0987e684e9d2dfd630f1ad26a6847493fe9fcd829ec251d8bc471d11701d51dd")
 
     # 3.1.4 latest release in 3.1 branch
@@ -75,6 +76,7 @@ class Relion(CMakePackage, CudaPackage):
 
     depends_on("mpi")
     depends_on("cmake@3:", type="build")
+    depends_on("binutils@2.32:", type="build")
     depends_on("fftw precision=float,double", when="~mklfft")
 
     # use the +xft variant so the interface is not so horrible looking
diff --git a/var/spack/repos/builtin/packages/rempi/package.py b/var/spack/repos/builtin/packages/rempi/package.py
index ee0a86eba39251..63461f40e3cf6b 100644
--- a/var/spack/repos/builtin/packages/rempi/package.py
+++ b/var/spack/repos/builtin/packages/rempi/package.py
@@ -17,7 +17,7 @@ class Rempi(AutotoolsPackage):
     version("1.0.0", sha256="1cb21f457cf8a04632150156a2ba699dd0c3f81d47e8881a9b943b9bf575fa01")
 
     depends_on("mpi")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
     depends_on("libtool", type="build")
diff --git a/var/spack/repos/builtin/packages/restic/package.py b/var/spack/repos/builtin/packages/restic/package.py
index c7eb94de479df9..493e0c098331f2 100644
--- a/var/spack/repos/builtin/packages/restic/package.py
+++ b/var/spack/repos/builtin/packages/restic/package.py
@@ -14,6 +14,7 @@ class Restic(Package):
 
     maintainers("alecbcs")
 
+    version("0.16.0", sha256="b91f5ef6203a5c50a72943c21aaef336e1344f19a3afd35406c00f065db8a8b9")
     version("0.15.2", sha256="52aca841486eaf4fe6422b059aa05bbf20db94b957de1d3fca019ed2af8192b7")
     version("0.15.1", sha256="fce382fdcdac0158a35daa640766d5e8a6e7b342ae2b0b84f2aacdff13990c52")
     version("0.15.0", sha256="85a6408cfb0798dab52335bcb00ac32066376c32daaa75461d43081499bc7de8")
diff --git a/var/spack/repos/builtin/packages/riscv-gnu-toolchain/package.py b/var/spack/repos/builtin/packages/riscv-gnu-toolchain/package.py
index 37f2ab81970dd7..71ff595e64247e 100644
--- a/var/spack/repos/builtin/packages/riscv-gnu-toolchain/package.py
+++ b/var/spack/repos/builtin/packages/riscv-gnu-toolchain/package.py
@@ -18,7 +18,12 @@ class RiscvGnuToolchain(AutotoolsPackage):
     maintainers("wanlinwang")
 
     version("develop", branch="master", submodules=True)
-    version("2022.08.08", tag="2022.08.08", submodules=True)
+    version(
+        "2022.08.08",
+        tag="2022.08.08",
+        commit="cb25bb862a3bf56d1577d7930bc41f259632ae24",
+        submodules=True,
+    )
 
     # Dependencies:
     depends_on("pkgconfig", type="build")
@@ -32,7 +37,7 @@ class RiscvGnuToolchain(AutotoolsPackage):
     depends_on("mpc", type="build")
     depends_on("gmp", type="build")
     depends_on("mpfr", type="build")
-    depends_on("zlib", type=("build", "link"))
+    depends_on("zlib-api", type=("build", "link"))
     depends_on("expat", type=("build", "link"))
     depends_on("bzip2", type="build")
     depends_on("gmake@4.3:", type="build")
diff --git a/var/spack/repos/builtin/packages/rivet/package.py b/var/spack/repos/builtin/packages/rivet/package.py
index fe23f40caccfda..dd20cd7273ee40 100644
--- a/var/spack/repos/builtin/packages/rivet/package.py
+++ b/var/spack/repos/builtin/packages/rivet/package.py
@@ -6,7 +6,6 @@
 import os
 
 from spack.package import *
-from spack.pkg.builtin.boost import Boost
 
 
 class Rivet(AutotoolsPackage):
@@ -18,6 +17,7 @@ class Rivet(AutotoolsPackage):
 
     tags = ["hep"]
 
+    version("3.1.8", sha256="75b3f3d419ca6388d1fd2ec0eda7e1f90f324b996ccf0591f48a5d2e28dccc13")
     version("3.1.7", sha256="27c7dbbcb5fd7ee81caf136daf4e960bca0ec255d9fa1abe602f4d430861b27a")
     version("3.1.6", sha256="1cf6ebb6a79d181c441d1d0c7c6d623c423817c61093f36f21adaae23e679090")
     version("3.1.4", sha256="37edc80a2968ce1031589e43ba6b492877ca7901bea38f8bb7536a5c8cf8100d")
@@ -28,206 +28,9 @@ class Rivet(AutotoolsPackage):
     version("3.0.2", sha256="9624d6cdcad77eafde40312cf6a1c97f4263f22faf9244b198c140b2c256d2f3")
     version("3.0.1", sha256="e7551168b86a05c9c029c319c313a0aa142a476195e7ff986c896c1b868f89dd")
     version("3.0.0", sha256="3944434d3791dccb54f7b2257589df6252cc7c065ce9deb57fbef466ff9e62b1")
-    version(
-        "2.7.2b",
-        sha256="e9f0a709f8226cde54f9406d36efab1e1b8ed0c6574fbcb1d72a186b09188106",
-        deprecated=True,
-    )
-    version(
-        "2.7.2",
-        sha256="a6634537c005660e56514b70ab9efb9d466c50685d6fb45ed03e9e1988479f02",
-        deprecated=True,
-    )
-    version(
-        "2.7.1",
-        sha256="b4145d8369b8a9fa0ada7ba2e5a2e9992d8e4a12ca4874d835246d2e708cbdef",
-        deprecated=True,
-    )
-    version(
-        "2.7.0",
-        sha256="34ad6a0b47dc4736feac8580a275e8b3a46df8fbeefd91e813add0a1525aacaf",
-        deprecated=True,
-    )
-    version(
-        "2.6.2",
-        sha256="9dde49d5c02038a295f03d2972f85be8746205bdb5ca1eab868b2c9129ade37a",
-        deprecated=True,
-    )
-    version(
-        "2.6.1",
-        sha256="e490d1f35aafa3e175690ae92f862c07a5fe2c51f693c88c87789f0441c89022",
-        deprecated=True,
-    )
-    version(
-        "2.6.0",
-        sha256="fb3229dccd31ea40b0af09974253073f6ad0a3a97e9a0cf44b53748ea8e2f900",
-        deprecated=True,
-    )
-    version(
-        "2.5.4",
-        sha256="772252193698d994fd111f790e72a4024df7572d492e3d5a9e840a074c5527e2",
-        deprecated=True,
-    )
-    version(
-        "2.5.3",
-        sha256="99e10330564ac479c6637d317c08882889a272db8ee204ad45a6ee1dcb291de4",
-        deprecated=True,
-    )
-    version(
-        "2.5.2",
-        sha256="70aa27764a14159c94c0b753a0c3d3600ac669def398cb2d8a6c63ae17704f05",
-        deprecated=True,
-    )
-    version(
-        "2.5.1",
-        sha256="14ee5828de217e96a30e666192515a2083afee030d81d36fc6bea948e9f7810a",
-        deprecated=True,
-    )
-    version(
-        "2.5.0",
-        sha256="c59ff35715be0caf65d6ba808b3badad0f6f7e7758f2049fb6ba43ed579bd4af",
-        deprecated=True,
-    )
-    version(
-        "2.4.3",
-        sha256="18aafecab6c3baeac871a9743946433c2dc01825d8fe291b157719a15c342682",
-        deprecated=True,
-    )
-    version(
-        "2.4.2",
-        sha256="accb146f3082719781a51eec879427c31401577c44f60b27ec8450102fe68aff",
-        deprecated=True,
-    )
-    version(
-        "2.4.1",
-        sha256="c14f0f58d1792d84d62c62b44ebb94db004776feba83fd8186bba898d55123cf",
-        deprecated=True,
-    )
-    version(
-        "2.4.0",
-        sha256="5ee2f34a277ed058b8aef750d946b40d4cf781023b9adab03ca95e803a39fb06",
-        deprecated=True,
-    )
-    version(
-        "2.3.0",
-        sha256="dd07702981d586e4b97b0fa56ae08cd08a631a952322a9b52e7622a46a7741ab",
-        deprecated=True,
-    )
-    version(
-        "2.2.1",
-        sha256="9e64ba19d567bdf4d0cc42b435491c4363b5fec90170d034445a99a1e752b691",
-        deprecated=True,
-    )
-    version(
-        "2.2.0",
-        sha256="3bdafe2007ff54c4734e0c8bc6ba9dc97028d4c41d538201b9582a869af8ae1a",
-        deprecated=True,
-    )
-    version(
-        "2.1.2",
-        sha256="40a20c1ee186326e5bfd906e0bc88f16dc740551be9cc274e9a826873d9c1eed",
-        deprecated=True,
-    )
-    version(
-        "2.1.1",
-        sha256="eefa936de6f6c34a6bab39899841f3189d7621c8ba227032f0f32e6e20dfcf85",
-        deprecated=True,
-    )
-    version(
-        "2.1.0",
-        sha256="58a1ca7b5a47719933782c050e67d0eb3823a7384cfc3c434fece41724c307e6",
-        deprecated=True,
-    )
-    version(
-        "2.0.0",
-        sha256="038f81f92fbba001ed23b56c1229a4f3b41e0c32e00bc92ea58d042909e3855a",
-        deprecated=True,
-    )
-    version(
-        "1.9.0",
-        sha256="55ef552b351328c287194aa99fa2b797e6632dc3fa88dfccd58264602012e044",
-        deprecated=True,
-    )
-    version(
-        "1.8.3",
-        sha256="aa82742fd4d7c68bfbef1367c4c605e06f9fed479a753db96aa6659407fcc4fd",
-        deprecated=True,
-    )
-    version(
-        "1.8.2",
-        sha256="56be98d31693253543f3e657c8f8edc7979c89fdb0ede1bdddfb3a9f5d4cfc3a",
-        deprecated=True,
-    )
-    version(
-        "1.8.1",
-        sha256="7e06d22350bec30220186e796caa93e9bfebd8d771a7efd35673897248437c61",
-        deprecated=True,
-    )
-    version(
-        "1.8.0",
-        sha256="7b28f9163f74583b1542b87c48f28a3ad1338da6136d8e3ca0aeba21095f5fe0",
-        deprecated=True,
-    )
-    version(
-        "1.7.0",
-        sha256="180741f590f210474b686d60241ad59e008221751ead21f0950c59aff93e54fd",
-        deprecated=True,
-    )
-    version(
-        "1.6.0",
-        sha256="1affd9e779f48477402e4150f315b3179204cbbc920db2d0129cd9c38bd18b26",
-        deprecated=True,
-    )
-    version(
-        "1.5.1",
-        sha256="9f24e9824286d5b0302c7e440f4803a8e3b8da50e1260e78c3b3c2eb587b317a",
-        deprecated=True,
-    )
-    version(
-        "1.5.0",
-        sha256="b7fe63e8caacc5c038ab567fe505d275288eedaa1aed6c379057629eef126006",
-        deprecated=True,
-    )
-    version(
-        "1.4.0",
-        sha256="067c94659bb7859904e20e72a676f94f103e6e012b7dba8071f51e8a6e624dbb",
-        deprecated=True,
-    )
-    version(
-        "1.3.0",
-        sha256="5ce41c8492c2fcf809a7135bf8335a01a98ea85fb556b3d00bd4260151efd12f",
-        deprecated=True,
-    )
-    version(
-        "1.2.1",
-        sha256="2d0380b819f778d8d9c2a462af90bd6a6188121e4edcc6202d936130b59bab17",
-        deprecated=True,
-    )
-    version(
-        "1.2.0",
-        sha256="ff5869f6dc9465f429e54686e12c39becac57a83273542179a59bac7561b6404",
-        deprecated=True,
-    )
-    version(
-        "1.1.3",
-        sha256="4be3cd9e6f808cdc5511991be2756f5fa838b6ecd01806fdbe7aec0aa382f946",
-        deprecated=True,
-    )
-    version(
-        "1.1.2",
-        sha256="a15b5d3339481446dec1b719d7d531a87a2e9d11c9fe8044e270ea69611b07c8",
-        deprecated=True,
-    )
-    version(
-        "1.1.1",
-        sha256="bd87fefee6bb8368216755342dc80ab3f8f3c813732dd03c6f94135d45f7036b",
-        deprecated=True,
-    )
 
     variant("hepmc", default="2", values=("2", "3"), description="HepMC version to link against")
 
-    conflicts("hepmc=3", when="@:2", msg="HepMC support was added in 3.0")
-
     # According to A. Buckley (main Rivet developer):
     # "typically a given Rivet version will work with
     # all YODA releases of that middle-digit version,
@@ -235,25 +38,6 @@ class Rivet(AutotoolsPackage):
     # to be using the latest versions of both.". The versions below
     # are taken from LCG stack which, in most cases, is the definition
     # of "latest" at the moment of release.
-    depends_on("yoda@1.0.4", when="@2.0.0")
-    depends_on("yoda@1.0.5", when="@2.1.0")
-    depends_on("yoda@1.0.6", when="@2.1.1")
-    depends_on("yoda@1.1.0", when="@2.1.2")
-    depends_on("yoda@1.3.0", when="@2.2.0")
-    depends_on("yoda@1.3.1", when="@2.2.1")
-    depends_on("yoda@1.4.0", when="@2.3.0")
-    depends_on("yoda@1.5.5", when="@2.4.0")
-    depends_on("yoda@1.5.9", when="@2.4.2")
-    depends_on("yoda@1.6.1", when="@2.4.3")
-    depends_on("yoda@1.6.2", when="@2.5.0")
-    depends_on("yoda@1.6.3", when="@2.5.1")
-    depends_on("yoda@1.6.5", when="@2.5.2")
-    depends_on("yoda@1.6.6", when="@2.5.3")
-    depends_on("yoda@1.6.7", when="@2.5.4")
-    depends_on("yoda@1.7.1", when="@2.6.1")
-    depends_on("yoda@1.7.4", when="@2.6.2")
-    depends_on("yoda@1.7.5", when="@2.7.2")
-    depends_on("yoda@1.7.5", when="@2.7.2b")
     depends_on("yoda@1.7.7", when="@3.0.1")
     depends_on("yoda@1.8.0", when="@3.1.0")
     depends_on("yoda@1.8.2", when="@3.1.1")
@@ -264,24 +48,16 @@ class Rivet(AutotoolsPackage):
 
     # The following versions were not a part of LCG stack
     # and thus the exact version of YODA is unknown
-    depends_on("yoda@1.7.0:1.7", when="@2.6.0,2.7.0,2.7.1,3.0.0,3.0.2")
-    depends_on("yoda@1.5.0:1.5", when="@2.4.1")
+    depends_on("yoda@1.7.0:1.7", when="@3.0.0,3.0.2")
 
     depends_on("hepmc", when="hepmc=2")
     depends_on("hepmc3", when="hepmc=3")
-    depends_on("boost", when="@:2.5.0")
-    # TODO: replace this with an explicit list of components of Boost,
-    # for instance depends_on('boost +filesystem')
-    # See https://github.com/spack/spack/pull/22303 for reference
-    depends_on(Boost.with_default_variants, when="@:2.5.0")
     depends_on("fastjet")
     depends_on("fastjet@3.4.0:", when="@3.1.7:")
-    depends_on("fjcontrib", when="@3.0.0:")
-    depends_on("gsl", when="@:2.6.0,2.6.2:2")
+    depends_on("fjcontrib")
     depends_on("python", type=("build", "run"))
     depends_on("py-cython@0.24.0:", type="build")
     depends_on("swig", type="build")
-    depends_on("yaml-cpp", when="@2.0.0:2.1.2")
 
     depends_on("autoconf", type="build")
     depends_on("autoconf@2.71:", when="@3.1.7", type="build")
@@ -294,21 +70,6 @@ class Rivet(AutotoolsPackage):
 
     filter_compiler_wrappers("rivet-build", relative_root="bin")
 
-    patch("rivet-1.8.2.patch", when="@1.8.2", level=0)
-    patch("rivet-1.9.0.patch", when="@1.9.0", level=0)
-    patch("rivet-2.2.0.patch", when="@2.2.0", level=0)
-    patch("rivet-2.2.1.patch", when="@2.2.1", level=0)
-    patch("rivet-2.4.0.patch", when="@2.4.0", level=0)
-    patch("rivet-2.4.2.patch", when="@2.4.2", level=0)
-    patch("rivet-2.4.3.patch", when="@2.4.3", level=0)
-    patch("rivet-2.5.1.patch", when="@2.5.1", level=0)
-    patch("rivet-2.5.2.patch", when="@2.5.2", level=0)
-    patch("rivet-2.5.3.patch", when="@2.5.3", level=0)
-    patch("rivet-2.5.4.patch", when="@2.5.4", level=0)
-    patch("rivet-2.6.0.patch", when="@2.6.0", level=0)
-    patch("rivet-2.6.1.patch", when="@2.6.1", level=0)
-    patch("rivet-2.6.2.patch", when="@2.6.2", level=0)
-    patch("rivet-2.7.0.patch", when="@2.7.0", level=0)
     patch("rivet-3.0.0.patch", when="@3.0.0", level=0)
     patch("rivet-3.0.1.patch", when="@3.0.1", level=0)
     patch("rivet-3.1.0.patch", when="@3.1.0", level=0)
@@ -316,12 +77,11 @@ class Rivet(AutotoolsPackage):
 
     @run_before("configure")
     def copy_gsl_m4(self):
-        if self.spec.satisfies("@2.6.2:"):
-            copy(join_path(os.path.dirname(__file__), "gsl.m4"), "m4/gsl.m4")
+        copy(join_path(os.path.dirname(__file__), "gsl.m4"), "m4/gsl.m4")
 
     @property
     def force_autoreconf(self):
-        return self.version >= Version("2.6.2")
+        return True
 
     def setup_build_environment(self, env):
         # this avoids an "import site" error in the build
@@ -340,27 +100,10 @@ def configure_args(self):
         else:
             args += ["--with-hepmc3=" + self.spec["hepmc3"].prefix]
 
-        if self.spec.satisfies("@:1"):
-            args += ["--with-boost-incpath=" + self.spec["boost"].includes]
-        else:
-            if self.spec.satisfies("@:2.5.0"):
-                args += ["--with-boost=" + self.spec["boost"].prefix]
-
         args += ["--with-fastjet=" + self.spec["fastjet"].prefix]
-        if self.spec.satisfies("@2:"):
-            args += ["--with-yoda=" + self.spec["yoda"].prefix]
-
-        if self.spec.satisfies("@:2.6.0,2.6.2:2"):
-            args += ["--with-gsl=" + self.spec["gsl"].prefix]
-
-        if self.spec.satisfies("@3.0.0:"):
-            args += ["--with-fjcontrib=" + self.spec["fjcontrib"].prefix]
-
-        if self.spec.satisfies("@:2.5.1"):
-            args += ["--enable-unvalidated"]
+        args += ["--with-yoda=" + self.spec["yoda"].prefix]
 
-        if self.spec.satisfies("@2:2.4"):
-            args += ["--enable-stdcxx11"]
+        args += ["--with-fjcontrib=" + self.spec["fjcontrib"].prefix]
 
         args += ["--disable-pdfmanual"]
 
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-1.8.2.patch b/var/spack/repos/builtin/packages/rivet/rivet-1.8.2.patch
deleted file mode 100644
index 95cbe774a0e5de..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-1.8.2.patch
+++ /dev/null
@@ -1,44 +0,0 @@
---- src/Tools/Makefile.in.orig	2013-05-15 14:02:25.000000000 +0200
-+++ src/Tools/Makefile.in	2013-05-15 14:03:14.000000000 +0200
-@@ -79,7 +79,7 @@
- 	libRivetTools_la-ParticleIdUtils.lo \
- 	libRivetTools_la-tinyxml.lo libRivetTools_la-tinyxmlerror.lo \
- 	libRivetTools_la-tinyxmlparser.lo \
--	libRivetTools_la-BinnedHistogram.lo \
-+#	libRivetTools_la-BinnedHistogram.lo \
- 	libRivetTools_la-mt2_bisect.lo libRivetTools_la-RivetMT2.lo \
- 	libRivetTools_la-BinnedHistogram.lo
- libRivetTools_la_OBJECTS = $(am_libRivetTools_la_OBJECTS)
-@@ -371,7 +371,7 @@
-   TinyXML/tinyxml.cpp \
-   TinyXML/tinyxmlerror.cpp \
-   TinyXML/tinyxmlparser.cpp \
--  BinnedHistogram.cc \
-+#  BinnedHistogram.cc \
-   mt2_bisect.cc \
-   RivetMT2.cc \
-   BinnedHistogram.cc
---- include/LWH/Axis.h.orig	2013-05-14 18:30:02.000000000 +0200
-+++ include/LWH/Axis.h	2013-05-14 18:31:35.000000000 +0200
-
-@@ -115,7 +115,7 @@
-    *
-    */
-   int coordToIndex(double coord) const {
--    assert( ! isnan(coord) );
-+    assert( ! std::isnan(coord) );
-     if ( coord >= upper ) return OVERFLOW_BIN;
-     else if ( coord < lower ) return UNDERFLOW_BIN;
-     else return int((coord - lower)/binWidth(0));
-
---- src/Tools/Makefile.am.orig  2013-05-15 13:14:11.000000000 +0200
-+++ src/Tools/Makefile.am       2013-05-15 13:16:53.000000000 +0200
-@@ -14,7 +14,7 @@
-   TinyXML/tinyxml.cpp \
-   TinyXML/tinyxmlerror.cpp \
-   TinyXML/tinyxmlparser.cpp \
--  BinnedHistogram.cc \
-+#  BinnedHistogram.cc \
-   mt2_bisect.cc \
-   RivetMT2.cc \
-   BinnedHistogram.cc
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-1.9.0.patch b/var/spack/repos/builtin/packages/rivet/rivet-1.9.0.patch
deleted file mode 100644
index 9d274550b7eab0..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-1.9.0.patch
+++ /dev/null
@@ -1,30 +0,0 @@
---- bin/rivet-buildplugin.in.orig	2014-09-02 11:22:32.000000000 +0200
-+++ bin/rivet-buildplugin.in	2014-09-02 11:26:31.000000000 +0200
-@@ -62,6 +62,7 @@
- mycppflags=""
- prefix="@prefix@"
- irivet="@includedir@"
-+exec_prefix=@exec_prefix@
- test -n "$irivet" && mycppflags="$mycppflags -I${irivet}"
- ihepmc="@HEPMCINCPATH@"
- test -n "$ihepmc" && mycppflags="$mycppflags -I${ihepmc}"
-@@ -74,7 +75,7 @@
- 
- ## Get Rivet system linker flags (duplicating that in rivet-config.in)
- myldflags=""
--lrivet="@libdir@"
-+lrivet="${exec_prefix}/lib"
- test -n "$lrivet" && myldflags="$myldflags -L${lrivet}"
- lhepmc="@HEPMCLIBPATH@"
- test -n "$lhepmc" && myldflags="$myldflags -L${lhepmc}"
---- bin/rivet-findid	2015-06-23 09:15:28.000000001 +0200
-+++ bin/rivet-findid	2015-06-23 09:15:38.000000001 +0200
-@@ -170,7 +170,7 @@
-                     if k=='code':
-                         entries[v] = c.text
-             if entries.get('9') == 'SPIRESTeX':
--                result['bibtex'] = entries['z']
-+                result['bibtex'] = entries['a']
- 
-         if i.get('tag') == '037':
-             entries = {}
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.2.0.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.2.0.patch
deleted file mode 100644
index ac9268a25220ee..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.2.0.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- bin/rivet-findid	2015-06-23 09:39:49.000000001 +0200
-+++ bin/rivet-findid	2015-06-23 09:40:09.000000001 +0200
-@@ -160,7 +160,7 @@
-                     if k=='code':
-                         entries[v] = c.text
-             if entries.get('9') == 'SPIRESTeX':
--                result['bibtex'] = entries['z']
-+                result['bibtex'] = entries['a']
- 
-         if i.get('tag') == '037':
-             entries = {}
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.2.1.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.2.1.patch
deleted file mode 100644
index a9fceb92c0fe3e..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.2.1.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- bin/rivet-findid	2015-06-23 09:40:38.000000001 +0200
-+++ bin/rivet-findid	2015-06-23 09:40:45.000000001 +0200
-@@ -160,7 +160,7 @@
-                     if k=='code':
-                         entries[v] = c.text
-             if entries.get('9') == 'SPIRESTeX':
--                result['bibtex'] = entries['z']
-+                result['bibtex'] = entries['a']
- 
-         if i.get('tag') == '037':
-             entries = {}
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.4.0.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.4.0.patch
deleted file mode 100644
index 7ce4f1dd55d208..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.4.0.patch
+++ /dev/null
@@ -1,12 +0,0 @@
---- configure	2015-10-07 12:35:57.000000000 +0300
-+++ configure	2015-10-09 16:18:02.432562522 +0300
-@@ -18728,7 +18728,7 @@
- ## Boost utility library
- echo "$as_me: this is boost.m4 serial 24" >&5
- boost_save_IFS=$IFS
--boost_version_req=1.55.0
-+boost_version_req=1.53.0
- IFS=.
- set x $boost_version_req 0 0 0
- IFS=$boost_save_IFS
-
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.4.2.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.4.2.patch
deleted file mode 100644
index 98436c5acfb50e..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.4.2.patch
+++ /dev/null
@@ -1,39 +0,0 @@
---- configure.orig      2016-09-30 14:12:42.317369715 +0200
-+++ configure   2016-09-30 14:17:21.690229397 +0200
-@@ -19644,11 +19644,15 @@
-         cat >conftest.py <<_ACEOF
-
- import sys, string
-+# Python 2 and 3 compatible
-+from future.builtins import map
-+# Python 2 and 3: forward-compatible
-+from future.builtins import range
- # split strings by '.' and convert to numeric.  Append some zeros
- # because we need at least 4 digits for the hex conversion.
--minver = map(int, string.split('2.5', '.')) + [0, 0, 0]
-+minver = list(map(int, '2.5'.split('.'))) + [0, 0, 0]
- minverhex = 0
--for i in xrange(0, 4): minverhex = (minverhex << 8) + minver[i]
-+for i in range(0, 4): minverhex = (minverhex << 8) + minver[i]
- if sys.hexversion >= minverhex:
-     sys.exit( 0 )
- else:
-@@ -19676,15 +19680,15 @@
-     fi
-
-
--  PYTHON_VERSION=`$PYTHON -c "import sys; print '.'.join(map(str, sys.version_info[:2]));"`
-+  PYTHON_VERSION=`$PYTHON -c "from __future__ import print_function; import sys; print('.'.join(map(str, sys.version_info[:2])));"`
-
--  RIVET_PYTHONPATH=`$PYTHON -c "import distutils.sysconfig; print distutils.sysconfig.get_python_lib(prefix='$prefix', plat_specific=True);"`
-+  RIVET_PYTHONPATH=`$PYTHON -c "from __future__ import print_function; import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(prefix='$prefix', plat_specific=True));"`
-
-   ## Test for Python header
-   if test -x "$PYTHON"; then
-     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Python include path" >&5
- $as_echo_n "checking for Python include path... " >&6; }
--    python_incpath=`$PYTHON -c "import distutils.sysconfig; print distutils.sysconfig.get_python_inc();"`
-+    python_incpath=`$PYTHON -c "from __future__ import print_function; import distutils.sysconfig; print(distutils.sysconfig.get_python_inc());"`
-     { $as_echo "$as_me:${as_lineno-$LINENO}: result: $python_incpath" >&5
- $as_echo "$python_incpath" >&6; }
-     python_header="$python_incpath/Python.h"
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.4.3.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.4.3.patch
deleted file mode 100644
index 9d006df8c0cec8..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.4.3.patch
+++ /dev/null
@@ -1,39 +0,0 @@
---- configure.orig	2016-06-28 23:57:35.000000000 +0200
-+++ configure	2016-09-19 12:44:23.411192406 +0200
-@@ -19643,11 +19643,15 @@
-         cat >conftest.py <<_ACEOF
-
- import sys, string
-+# Python 2 and 3 compatible
-+from future.builtins import map
-+# Python 2 and 3: forward-compatible
-+from future.builtins import range
- # split strings by '.' and convert to numeric.  Append some zeros
- # because we need at least 4 digits for the hex conversion.
--minver = map(int, string.split('2.5', '.')) + [0, 0, 0]
-+minver = list(map(int, '2.5'.split('.'))) + [0, 0, 0]
- minverhex = 0
--for i in xrange(0, 4): minverhex = (minverhex << 8) + minver[i]
-+for i in range(0, 4): minverhex = (minverhex << 8) + minver[i]
- if sys.hexversion >= minverhex:
-     sys.exit( 0 )
- else:
-@@ -19675,15 +19679,15 @@
-     fi
-
-
--  PYTHON_VERSION=`$PYTHON -c "import sys; print '.'.join(map(str, sys.version_info[:2]));"`
-+    PYTHON_VERSION=`$PYTHON -c "from __future__ import print_function; import sys; print('.'.join(map(str, sys.version_info[:2])));"`
-
--  RIVET_PYTHONPATH=`$PYTHON -c "import distutils.sysconfig; print distutils.sysconfig.get_python_lib(prefix='$prefix', plat_specific=True);"`
-+    RIVET_PYTHONPATH=`$PYTHON -c "from __future__ import print_function; import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(prefix='$prefix', plat_specific=True));"`
-
-   ## Test for Python header
-   if test -x "$PYTHON"; then
-     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Python include path" >&5
- $as_echo_n "checking for Python include path... " >&6; }
--    python_incpath=`$PYTHON -c "import distutils.sysconfig; print distutils.sysconfig.get_python_inc();"`
-+python_incpath=`$PYTHON -c "from __future__ import print_function; import distutils.sysconfig; print(distutils.sysconfig.get_python_inc());"`
-     { $as_echo "$as_me:${as_lineno-$LINENO}: result: $python_incpath" >&5
- $as_echo "$python_incpath" >&6; }
-     python_header="$python_incpath/Python.h"
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.5.1.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.5.1.patch
deleted file mode 100644
index dc5f484203a30f..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.5.1.patch
+++ /dev/null
@@ -1,39 +0,0 @@
---- configure.orig      2016-09-30 09:25:10.000000000 +0200
-+++ configure   2016-09-30 09:27:01.000000000 +0200
-@@ -19681,11 +19681,15 @@
-         cat >conftest.py <<_ACEOF
-
- import sys, string
-+# Python 2 and 3 compatible
-+from future.builtins import map
-+# Python 2 and 3: forward-compatible
-+from future.builtins import range
- # split strings by '.' and convert to numeric.  Append some zeros
- # because we need at least 4 digits for the hex conversion.
--minver = map(int, string.split('2.5', '.')) + [0, 0, 0]
-+minver = list(map(int, '2.5'.split('.'))) + [0, 0, 0]
- minverhex = 0
--for i in xrange(0, 4): minverhex = (minverhex << 8) + minver[i]
-+for i in range(0, 4): minverhex = (minverhex << 8) + minver[i]
- if sys.hexversion >= minverhex:
-     sys.exit( 0 )
- else:
-@@ -19713,15 +19717,15 @@
-     fi
-
-
--  PYTHON_VERSION=`$PYTHON -c "import sys; print '.'.join(map(str, sys.version_info[:2]));"`
-+    PYTHON_VERSION=`$PYTHON -c "from __future__ import print_function; import sys; print('.'.join(map(str, sys.version_info[:2])));"`
-
--  RIVET_PYTHONPATH=`$PYTHON -c "import distutils.sysconfig; print distutils.sysconfig.get_python_lib(prefix='$prefix', plat_specific=True);"`
-+    RIVET_PYTHONPATH=`$PYTHON -c "from __future__ import print_function; import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(prefix='$prefix', plat_specific=True));"`
-
-   ## Test for Python header
-   if test -x "$PYTHON"; then
-     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Python include path" >&5
- $as_echo_n "checking for Python include path... " >&6; }
--    python_incpath=`$PYTHON -c "import distutils.sysconfig; print distutils.sysconfig.get_python_inc();"`
-+python_incpath=`$PYTHON -c "from __future__ import print_function; import distutils.sysconfig; print(distutils.sysconfig.get_python_inc());"`
-     { $as_echo "$as_me:${as_lineno-$LINENO}: result: $python_incpath" >&5
- $as_echo "$python_incpath" >&6; }
-     python_header="$python_incpath/Python.h"
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.5.2.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.5.2.patch
deleted file mode 100644
index 9f536d37511631..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.5.2.patch
+++ /dev/null
@@ -1,10 +0,0 @@
---- ./include/Rivet/Tools/RivetSTL.hh.orig	2017-08-03 11:49:33.005431033 +0200
-+++ ./include/Rivet/Tools/RivetSTL.hh	2017-08-03 11:50:13.817841608 +0200
-@@ -19,6 +19,7 @@
- #include 
- #include 
- #include 
-+#include 
- 
- 
- #ifndef foreach
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.5.3.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.5.3.patch
deleted file mode 100644
index bebf5c73ab01fd..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.5.3.patch
+++ /dev/null
@@ -1,43 +0,0 @@
---- include/Rivet/Config/RivetCommon.hh.orig	2017-05-31 12:32:04.547062840 +0200
-+++ include/Rivet/Config/RivetCommon.hh	2017-05-31 12:32:12.379054886 +0200
-@@ -2,6 +2,7 @@
- #define RIVET_RivetCommon_HH
- 
- // Convenience build-setup header for Rivet internal use
-+#include 
- 
- // Automatic build info from autoconf
- #include "Rivet/Config/RivetConfig.hh"
---- include/Rivet/Run.hh.orig	2017-05-31 12:36:00.225823486 +0200
-+++ include/Rivet/Run.hh	2017-05-31 12:36:18.653804770 +0200
-@@ -4,6 +4,7 @@
- 
- #include "Rivet/Tools/RivetSTL.hh"
- #include "Rivet/Tools/RivetHepMC.hh"
-+#include 
- 
- namespace Rivet {
- 
---- include/Rivet/Particle.hh.orig	2017-05-31 12:39:14.273626411 +0200
-+++ include/Rivet/Particle.hh	2017-05-31 14:00:02.387272379 +0200
-@@ -11,6 +11,8 @@
- // NOTE: Rivet/Tools/ParticleUtils.hh included at the end
- #include "fastjet/PseudoJet.hh"
- 
-+#include 
-+
- namespace Rivet {
- 
- 
---- include/Rivet/Tools/RivetSTL.hh.orig	2017-05-31 14:05:28.368975178 +0200
-+++ include/Rivet/Tools/RivetSTL.hh	2017-05-31 14:05:44.432960512 +0200
-@@ -1,6 +1,8 @@
- #ifndef RIVET_RivetSTL_HH
- #define RIVET_RivetSTL_HH
- 
-+#include 
-+
- #include 
- #include 
- #include 
-
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.5.4.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.5.4.patch
deleted file mode 100644
index af338da6d5f421..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.5.4.patch
+++ /dev/null
@@ -1,42 +0,0 @@
-patch written by GENSER to avoid system libraries of GLS coming from fastjet-config
---- include/Rivet/Tools/JetUtils.hh
-+++ include/Rivet/Tools/JetUtils.hh
-@@ -47,4 +47,5 @@
-   struct BoolJetFunctor {
-     virtual bool operator()(const Jet& p) const = 0;
-+    virtual ~BoolJetFunctor() {}
-   };
-
---- include/Rivet/Tools/ParticleBaseUtils.hh
-+++ include/Rivet/Tools/ParticleBaseUtils.hh
-@@ -23,4 +23,5 @@
-   struct BoolParticleBaseFunctor {
-     virtual bool operator()(const ParticleBase& p) const = 0;
-+    virtual ~BoolParticleBaseFunctor() {}
-   };
- 
-@@ -397,4 +398,5 @@
-   struct DoubleParticleBaseFunctor {
-     virtual double operator()(const ParticleBase& p) const = 0;
-+    virtual ~DoubleParticleBaseFunctor() {}
-   };
-
---- include/Rivet/Tools/ParticleUtils.hh
-+++ include/Rivet/Tools/ParticleUtils.hh
-@@ -486,4 +486,5 @@
-   struct BoolParticleFunctor {
-     virtual bool operator()(const Particle& p) const = 0;
-+    virtual ~BoolParticleFunctor() {}
-   };
-
---- src/Makefile.in.orig	2018-01-30 13:52:58.641670247 +0100
-+++ src/Makefile.in	2018-01-30 13:53:19.377586118 +0100
-@@ -449,7 +449,7 @@
- libRivet_la_SOURCES = 
- libRivet_la_LDFLAGS = -export-dynamic -avoid-version -L$(YODALIBPATH) -L$(HEPMCLIBPATH)
- libRivet_la_LIBADD = Core/libRivetCore.la Projections/libRivetProjections.la Tools/libRivetTools.la $(ANA_LIBADD) \
--  -lYODA -lHepMC -ldl -lm  $(FASTJETCONFIGLIBADD) $(GSL_LDFLAGS)
-+  -lYODA -lHepMC -ldl -lm  $(GSL_LDFLAGS) $(FASTJETCONFIGLIBADD)
- 
- all: all-recursive
- 
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.6.0.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.6.0.patch
deleted file mode 100644
index 2cb7cab14b5e83..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.6.0.patch
+++ /dev/null
@@ -1,90 +0,0 @@
---- include/Rivet/Tools/JetUtils.hh
-+++ include/Rivet/Tools/JetUtils.hh
-@@ -47,4 +47,5 @@
-   struct BoolJetFunctor {
-     virtual bool operator()(const Jet& p) const = 0;
-+    virtual ~BoolJetFunctor() {}
-   };
-
---- include/Rivet/Tools/ParticleBaseUtils.hh
-+++ include/Rivet/Tools/ParticleBaseUtils.hh
-@@ -23,4 +23,5 @@
-   struct BoolParticleBaseFunctor {
-     virtual bool operator()(const ParticleBase& p) const = 0;
-+    virtual ~BoolParticleBaseFunctor() {}
-   };
- 
-@@ -397,4 +398,5 @@
-   struct DoubleParticleBaseFunctor {
-     virtual double operator()(const ParticleBase& p) const = 0;
-+    virtual ~DoubleParticleBaseFunctor() {}
-   };
-
---- include/Rivet/Tools/ParticleUtils.hh
-+++ include/Rivet/Tools/ParticleUtils.hh
-@@ -486,4 +486,5 @@
-   struct BoolParticleFunctor {
-     virtual bool operator()(const Particle& p) const = 0;
-+    virtual ~BoolParticleFunctor() {}
-   };
-
---- analyses/Makefile.in.orig	2018-06-28 09:22:46.722022129 +0200
-+++ analyses/Makefile.in	2018-06-28 10:11:59.772373529 +0200
-@@ -523,21 +523,21 @@
- @ENABLE_ANALYSES_TRUE@%.so:
- @ENABLE_ANALYSES_TRUE@	@+echo && RIVET_BUILDPLUGIN_BEFORE_INSTALL=1 bash $(top_builddir)/bin/rivet-buildplugin -j2 $@ $^ -I$(top_builddir)/include
- 
--@ENABLE_ANALYSES_TRUE@RivetALICEAnalyses.so: $(shell ls $(srcdir)/pluginALICE/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetATLASAnalyses.so: $(shell ls $(srcdir)/pluginATLAS/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetCDFAnalyses.so  : $(shell ls $(srcdir)/pluginCDF/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetCMSAnalyses.so  : $(shell ls $(srcdir)/pluginCMS/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetD0Analyses.so   : $(shell ls $(srcdir)/pluginD0/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetHERAAnalyses.so : $(shell ls $(srcdir)/pluginHERA/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetLEPAnalyses.so  : $(shell ls $(srcdir)/pluginLEP/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetLHCbAnalyses.so : $(shell ls $(srcdir)/pluginLHCb/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetLHCfAnalyses.so : $(shell ls $(srcdir)/pluginLHCf/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetMCAnalyses.so   : $(shell ls $(srcdir)/pluginMC/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetMiscAnalyses.so : $(shell ls $(srcdir)/pluginMisc/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetPetraAnalyses.so: $(shell ls $(srcdir)/pluginPetra/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetRHICAnalyses.so : $(shell ls $(srcdir)/pluginRHIC/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetSPSAnalyses.so  : $(shell ls $(srcdir)/pluginSPS/*.cc)
--@ENABLE_ANALYSES_TRUE@RivetTOTEMAnalyses.so: $(shell ls $(srcdir)/pluginTOTEM/*.cc)
-+@ENABLE_ANALYSES_TRUE@RivetALICEAnalyses.so: $(shell find $(srcdir)/pluginALICE/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetATLASAnalyses.so: $(shell find $(srcdir)/pluginATLAS/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetCDFAnalyses.so  : $(shell find $(srcdir)/pluginCDF/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetCMSAnalyses.so  : $(shell find $(srcdir)/pluginCMS/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetD0Analyses.so   : $(shell find $(srcdir)/pluginD0/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetHERAAnalyses.so : $(shell find $(srcdir)/pluginHERA/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetLEPAnalyses.so  : $(shell find $(srcdir)/pluginLEP/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetLHCbAnalyses.so : $(shell find $(srcdir)/pluginLHCb/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetLHCfAnalyses.so : $(shell find $(srcdir)/pluginLHCf/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetMCAnalyses.so   : $(shell find $(srcdir)/pluginMC/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetMiscAnalyses.so : $(shell find $(srcdir)/pluginMisc/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetPetraAnalyses.so: $(shell find $(srcdir)/pluginPetra/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetRHICAnalyses.so : $(shell find $(srcdir)/pluginRHIC/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetSPSAnalyses.so  : $(shell find $(srcdir)/pluginSPS/ -name '*.cc' -not -name 'tmp*')
-+@ENABLE_ANALYSES_TRUE@RivetTOTEMAnalyses.so: $(shell find $(srcdir)/pluginTOTEM/ -name '*.cc' -not -name 'tmp*')
- 
- @ENABLE_ANALYSES_TRUE@all-local: $(PLUGIN_LIBS) $(PLUGIN_DATAFILES)
- @ENABLE_ANALYSES_TRUE@	mkdir -p $(builddir)/data
---- analyses/pluginATLAS/ATLAS_2016_CONF_2016_037.cc.orig	2018-07-02 10:43:08.656094368 +0200
-+++ analyses/pluginATLAS/ATLAS_2016_CONF_2016_037.cc	2018-07-02 10:44:04.920361160 +0200
-@@ -2,6 +2,7 @@
- #include "Rivet/Analysis.hh"
- #include "Rivet/Projections/FinalState.hh"
- #include "Rivet/Projections/PromptFinalState.hh"
-+#include "Rivet/Projections/ChargedFinalState.hh"
- #include "Rivet/Projections/FastJets.hh"
- #include "Rivet/Projections/Sphericity.hh"
- #include "Rivet/Projections/SmearedParticles.hh"
---- include/Rivet/AnalysisHandler.hh.orig	2018-07-10 10:39:21.719532209 +0200
-+++ include/Rivet/AnalysisHandler.hh	2018-07-10 10:39:29.225608530 +0200
-@@ -17,7 +17,7 @@
- 
-   // Needed to make smart pointers compare equivalent in the STL set
-   struct CmpAnaHandle {
--    bool operator() (const AnaHandle& a, const AnaHandle& b) {
-+    bool operator() (const AnaHandle& a, const AnaHandle& b) const {
-       return a.get() < b.get();
-     }
-   };
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.6.1.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.6.1.patch
deleted file mode 100644
index 878e72d7393d1a..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.6.1.patch
+++ /dev/null
@@ -1,24 +0,0 @@
---- doc/Makefile.in.orig	2018-10-17 18:29:15.568630034 +0200
-+++ doc/Makefile.in	2018-10-17 18:31:45.829375147 +0200
-@@ -594,15 +594,15 @@
- .PRECIOUS: Makefile
- 
- @ENABLE_PYEXT_TRUE@analyses.html: $(top_srcdir)/analyses $(srcdir)/mk-analysis-html
--@ENABLE_PYEXT_TRUE@	LD_LIBRARY_PATH=$(top_builddir)/src/.libs:$(LD_LIBRARY_PATH) \
--@ENABLE_PYEXT_TRUE@    DYLD_LIBRARY_PATH=$(top_builddir)/src/.libs:$(DYLD_LIBRARY_PATH) \
-+@ENABLE_PYEXT_TRUE@	LD_LIBRARY_PATH=$(top_builddir)/src/.libs:$(HEPMCLIBPATH):$(FASTJETLIBPATH):$(YODALIBPATH):$(RIVETLIBPATH):$(LD_LIBRARY_PATH) \
-+@ENABLE_PYEXT_TRUE@    DYLD_LIBRARY_PATH=$(top_builddir)/src/.libs:$(HEPMCLIBPATH):$(FASTJETLIBPATH):$(YODALIBPATH):$(RIVETLIBPATH):$(DYLD_LIBRARY_PATH) \
- @ENABLE_PYEXT_TRUE@    PYTHONPATH=$(YODA_PYTHONPATH):$(RIVET_PYTHONPATH):$(PYTHONPATH) \
--@ENABLE_PYEXT_TRUE@    $(srcdir)/mk-analysis-html
-+@ENABLE_PYEXT_TRUE@    $(PYTHON) $(srcdir)/mk-analysis-html
- @ENABLE_PYEXT_TRUE@analyses.json: $(top_srcdir)/analyses $(srcdir)/mk-analysis-json
--@ENABLE_PYEXT_TRUE@	LD_LIBRARY_PATH=$(top_builddir)/src/.libs:$(LD_LIBRARY_PATH) \
--@ENABLE_PYEXT_TRUE@    DYLD_LIBRARY_PATH=$(top_builddir)/src/.libs:$(DYLD_LIBRARY_PATH) \
-+@ENABLE_PYEXT_TRUE@	LD_LIBRARY_PATH=$(top_builddir)/src/.libs:$(HEPMCLIBPATH):$(FASTJETLIBPATH):$(YODALIBPATH):$(RIVETLIBPATH):$(LD_LIBRARY_PATH) \
-+@ENABLE_PYEXT_TRUE@    DYLD_LIBRARY_PATH=$(top_builddir)/src/.libs:$(HEPMCLIBPATH):$(FASTJETLIBPATH):$(YODALIBPATH):$(RIVETLIBPATH):$(DYLD_LIBRARY_PATH) \
- @ENABLE_PYEXT_TRUE@    PYTHONPATH=$(YODA_PYTHONPATH):$(RIVET_PYTHONPATH):$(PYTHONPATH) \
--@ENABLE_PYEXT_TRUE@    $(srcdir)/mk-analysis-json
-+@ENABLE_PYEXT_TRUE@    $(PYTHON) $(srcdir)/mk-analysis-json
- 
- @WITH_ASCIIDOC_TRUE@compare-histos.html: compare-histos.txt
- @WITH_ASCIIDOC_TRUE@	asciidoc -a toc compare-histos.txt
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.6.2.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.6.2.patch
deleted file mode 100644
index a327f481d49a61..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.6.2.patch
+++ /dev/null
@@ -1,97 +0,0 @@
---- configure.ac.orig	2019-01-23 11:02:06.206570992 +0100
-+++ configure.ac	2019-01-23 11:02:26.704761899 +0100
-@@ -151,12 +151,12 @@
- 
- 
- # ## GNU Scientific Library
--# AC_SEARCH_GSL
--# AC_CEDAR_HEADERS([gsl], , , [AC_MSG_ERROR([GSL (GNU Scientific Library) is required])])
--# oldCPPFLAGS=$CPPFLAGS
--# CPPFLAGS="$CPPFLAGS -I$GSLINCPATH"
--# AC_CHECK_HEADER([gsl/gsl_vector.h], [], [AC_MSG_ERROR([GSL vectors not found.])])
--# CPPFLAGS=$oldCPPFLAGS
-+AC_SEARCH_GSL
-+AC_CEDAR_HEADERS([gsl], , , [AC_MSG_ERROR([GSL (GNU Scientific Library) is required])])
-+oldCPPFLAGS=$CPPFLAGS
-+CPPFLAGS="$CPPFLAGS -I$GSLINCPATH"
-+AC_CHECK_HEADER([gsl/gsl_vector.h], [], [AC_MSG_ERROR([GSL vectors not found.])])
-+CPPFLAGS=$oldCPPFLAGS
- 
- 
- ## Disable build/install of standard analyses
-@@ -256,7 +256,7 @@
- ## Set default build flags
- AM_CPPFLAGS="-I\$(top_srcdir)/include -I\$(top_builddir)/include"
- #AM_CPPFLAGS="$AM_CPPFLAGS -I\$(top_srcdir)/include/eigen3"
--#AM_CPPFLAGS="$AM_CPPFLAGS \$(GSL_CPPFLAGS)"
-+AM_CPPFLAGS="$AM_CPPFLAGS \$(GSL_CPPFLAGS)"
- dnl AM_CPPFLAGS="$AM_CPPFLAGS \$(BOOST_CPPFLAGS)"
- AM_CPPFLAGS="$AM_CPPFLAGS -I\$(YODAINCPATH)"
- AM_CPPFLAGS="$AM_CPPFLAGS -I\$(HEPMCINCPATH)"
---- bin/rivet-config.in.orig	2019-01-23 11:08:04.608907832 +0100
-+++ bin/rivet-config.in	2019-01-23 11:08:25.058098155 +0100
-@@ -82,8 +82,8 @@
-     test -n "$iyoda" && OUT="$OUT -I${iyoda}"
-     ifastjet="@FASTJETINCPATH@"
-     test -n "$ifastjet" && OUT="$OUT -I${ifastjet}"
--    # igsl="@GSLINCPATH@"
--    # test -n "$igsl" && OUT="$OUT -I${igsl}"
-+    igsl="@GSLINCPATH@"
-+    test -n "$igsl" && OUT="$OUT -I${igsl}"
-     # iboost="@BOOST_CPPFLAGS@"
-     # test -n "$iboost" && OUT="$OUT ${iboost}"
- fi
-@@ -98,8 +98,8 @@
-     test -n "$lyoda" && OUT="$OUT -L${lyoda} -lYODA"
-     lfastjet="@FASTJETCONFIGLIBADD@"
-     test -n "$lfastjet" && OUT="$OUT ${lfastjet}"
--    # lgsl="@GSLLIBPATH@"
--    # test -n "$lgsl" && OUT="$OUT -L${lgsl}"
-+    lgsl="@GSLLIBPATH@"
-+    test -n "$lgsl" && OUT="$OUT -L${lgsl}"
- fi
- 
- tmp=$( echo "$*" | egrep -- '--\|--\')
---- bin/rivet-buildplugin.in.orig	2019-01-23 11:10:07.804054317 +0100
-+++ bin/rivet-buildplugin.in	2019-01-23 11:10:34.370301517 +0100
-@@ -169,8 +169,8 @@
- test -n "$iyoda" && mycppflags="$mycppflags -I${iyoda}"
- ifastjet="@FASTJETINCPATH@"
- test -n "$ifastjet" && mycppflags="$mycppflags -I${ifastjet}"
--# igsl="@GSLINCPATH@"
--# test -n "$igsl" && mycppflags="$mycppflags -I${igsl}"
-+igsl="@GSLINCPATH@"
-+test -n "$igsl" && mycppflags="$mycppflags -I${igsl}"
- # iboost="@BOOST_CPPFLAGS@"
- # test -n "$iboost" && mycppflags="$mycppflags ${iboost}"
- 
---- pyext/setup.py.in.orig	2019-01-23 11:12:04.694141901 +0100
-+++ pyext/setup.py.in	2019-01-23 11:12:44.531512512 +0100
-@@ -29,11 +29,11 @@
- 
- 
- ## Be careful with extracting the GSL path from the flags string
--# import re
--# re_libdirflag = re.compile(r".*-L\s*(\S+).*")
--# re_match = re_libdirflag.search("@GSL_LDFLAGS@")
--# if re_match:
--    # lookupdirs.append( re_match.group(1) )
-+import re
-+re_libdirflag = re.compile(r".*-L\s*(\S+).*")
-+re_match = re_libdirflag.search("@GSL_LDFLAGS@")
-+if re_match:
-+    lookupdirs.append( re_match.group(1) )
- 
- ## A helper function
- def ext(name, depends=[], statics=[]):
-@@ -47,8 +47,8 @@
-         language="c++",
- #        depends=depends,
-         include_dirs=[incdir1, incdir2, incdir3, incdir4],
--        # extra_compile_args="-I@prefix@/include @PYEXT_CXXFLAGS@ @HEPMCCPPFLAGS@ @FASTJETCPPFLAGS@ @YODACPPFLAGS@ @GSLCPPFLAGS@".split(),
--        extra_compile_args="-I@prefix@/include @PYEXT_CXXFLAGS@ @HEPMCCPPFLAGS@ @FASTJETCPPFLAGS@ @YODACPPFLAGS@".split(),
-+        extra_compile_args="-I@prefix@/include @PYEXT_CXXFLAGS@ @HEPMCCPPFLAGS@ @FASTJETCPPFLAGS@ @YODACPPFLAGS@ @GSLCPPFLAGS@".split(),
-+        # extra_compile_args="-I@prefix@/include @PYEXT_CXXFLAGS@ @HEPMCCPPFLAGS@ @FASTJETCPPFLAGS@ @YODACPPFLAGS@".split(),
-         extra_link_args=BASE_LINK_ARGS,
-         library_dirs=lookupdirs,
-         runtime_library_dirs=lookupdirs[1:],
diff --git a/var/spack/repos/builtin/packages/rivet/rivet-2.7.0.patch b/var/spack/repos/builtin/packages/rivet/rivet-2.7.0.patch
deleted file mode 100644
index a327f481d49a61..00000000000000
--- a/var/spack/repos/builtin/packages/rivet/rivet-2.7.0.patch
+++ /dev/null
@@ -1,97 +0,0 @@
---- configure.ac.orig	2019-01-23 11:02:06.206570992 +0100
-+++ configure.ac	2019-01-23 11:02:26.704761899 +0100
-@@ -151,12 +151,12 @@
- 
- 
- # ## GNU Scientific Library
--# AC_SEARCH_GSL
--# AC_CEDAR_HEADERS([gsl], , , [AC_MSG_ERROR([GSL (GNU Scientific Library) is required])])
--# oldCPPFLAGS=$CPPFLAGS
--# CPPFLAGS="$CPPFLAGS -I$GSLINCPATH"
--# AC_CHECK_HEADER([gsl/gsl_vector.h], [], [AC_MSG_ERROR([GSL vectors not found.])])
--# CPPFLAGS=$oldCPPFLAGS
-+AC_SEARCH_GSL
-+AC_CEDAR_HEADERS([gsl], , , [AC_MSG_ERROR([GSL (GNU Scientific Library) is required])])
-+oldCPPFLAGS=$CPPFLAGS
-+CPPFLAGS="$CPPFLAGS -I$GSLINCPATH"
-+AC_CHECK_HEADER([gsl/gsl_vector.h], [], [AC_MSG_ERROR([GSL vectors not found.])])
-+CPPFLAGS=$oldCPPFLAGS
- 
- 
- ## Disable build/install of standard analyses
-@@ -256,7 +256,7 @@
- ## Set default build flags
- AM_CPPFLAGS="-I\$(top_srcdir)/include -I\$(top_builddir)/include"
- #AM_CPPFLAGS="$AM_CPPFLAGS -I\$(top_srcdir)/include/eigen3"
--#AM_CPPFLAGS="$AM_CPPFLAGS \$(GSL_CPPFLAGS)"
-+AM_CPPFLAGS="$AM_CPPFLAGS \$(GSL_CPPFLAGS)"
- dnl AM_CPPFLAGS="$AM_CPPFLAGS \$(BOOST_CPPFLAGS)"
- AM_CPPFLAGS="$AM_CPPFLAGS -I\$(YODAINCPATH)"
- AM_CPPFLAGS="$AM_CPPFLAGS -I\$(HEPMCINCPATH)"
---- bin/rivet-config.in.orig	2019-01-23 11:08:04.608907832 +0100
-+++ bin/rivet-config.in	2019-01-23 11:08:25.058098155 +0100
-@@ -82,8 +82,8 @@
-     test -n "$iyoda" && OUT="$OUT -I${iyoda}"
-     ifastjet="@FASTJETINCPATH@"
-     test -n "$ifastjet" && OUT="$OUT -I${ifastjet}"
--    # igsl="@GSLINCPATH@"
--    # test -n "$igsl" && OUT="$OUT -I${igsl}"
-+    igsl="@GSLINCPATH@"
-+    test -n "$igsl" && OUT="$OUT -I${igsl}"
-     # iboost="@BOOST_CPPFLAGS@"
-     # test -n "$iboost" && OUT="$OUT ${iboost}"
- fi
-@@ -98,8 +98,8 @@
-     test -n "$lyoda" && OUT="$OUT -L${lyoda} -lYODA"
-     lfastjet="@FASTJETCONFIGLIBADD@"
-     test -n "$lfastjet" && OUT="$OUT ${lfastjet}"
--    # lgsl="@GSLLIBPATH@"
--    # test -n "$lgsl" && OUT="$OUT -L${lgsl}"
-+    lgsl="@GSLLIBPATH@"
-+    test -n "$lgsl" && OUT="$OUT -L${lgsl}"
- fi
- 
- tmp=$( echo "$*" | egrep -- '--\|--\')
---- bin/rivet-buildplugin.in.orig	2019-01-23 11:10:07.804054317 +0100
-+++ bin/rivet-buildplugin.in	2019-01-23 11:10:34.370301517 +0100
-@@ -169,8 +169,8 @@
- test -n "$iyoda" && mycppflags="$mycppflags -I${iyoda}"
- ifastjet="@FASTJETINCPATH@"
- test -n "$ifastjet" && mycppflags="$mycppflags -I${ifastjet}"
--# igsl="@GSLINCPATH@"
--# test -n "$igsl" && mycppflags="$mycppflags -I${igsl}"
-+igsl="@GSLINCPATH@"
-+test -n "$igsl" && mycppflags="$mycppflags -I${igsl}"
- # iboost="@BOOST_CPPFLAGS@"
- # test -n "$iboost" && mycppflags="$mycppflags ${iboost}"
- 
---- pyext/setup.py.in.orig	2019-01-23 11:12:04.694141901 +0100
-+++ pyext/setup.py.in	2019-01-23 11:12:44.531512512 +0100
-@@ -29,11 +29,11 @@
- 
- 
- ## Be careful with extracting the GSL path from the flags string
--# import re
--# re_libdirflag = re.compile(r".*-L\s*(\S+).*")
--# re_match = re_libdirflag.search("@GSL_LDFLAGS@")
--# if re_match:
--    # lookupdirs.append( re_match.group(1) )
-+import re
-+re_libdirflag = re.compile(r".*-L\s*(\S+).*")
-+re_match = re_libdirflag.search("@GSL_LDFLAGS@")
-+if re_match:
-+    lookupdirs.append( re_match.group(1) )
- 
- ## A helper function
- def ext(name, depends=[], statics=[]):
-@@ -47,8 +47,8 @@
-         language="c++",
- #        depends=depends,
-         include_dirs=[incdir1, incdir2, incdir3, incdir4],
--        # extra_compile_args="-I@prefix@/include @PYEXT_CXXFLAGS@ @HEPMCCPPFLAGS@ @FASTJETCPPFLAGS@ @YODACPPFLAGS@ @GSLCPPFLAGS@".split(),
--        extra_compile_args="-I@prefix@/include @PYEXT_CXXFLAGS@ @HEPMCCPPFLAGS@ @FASTJETCPPFLAGS@ @YODACPPFLAGS@".split(),
-+        extra_compile_args="-I@prefix@/include @PYEXT_CXXFLAGS@ @HEPMCCPPFLAGS@ @FASTJETCPPFLAGS@ @YODACPPFLAGS@ @GSLCPPFLAGS@".split(),
-+        # extra_compile_args="-I@prefix@/include @PYEXT_CXXFLAGS@ @HEPMCCPPFLAGS@ @FASTJETCPPFLAGS@ @YODACPPFLAGS@".split(),
-         extra_link_args=BASE_LINK_ARGS,
-         library_dirs=lookupdirs,
-         runtime_library_dirs=lookupdirs[1:],
diff --git a/var/spack/repos/builtin/packages/rkcommon/package.py b/var/spack/repos/builtin/packages/rkcommon/package.py
index 50409f10a4e0de..4f0a07559ef9c6 100644
--- a/var/spack/repos/builtin/packages/rkcommon/package.py
+++ b/var/spack/repos/builtin/packages/rkcommon/package.py
@@ -16,6 +16,7 @@ class Rkcommon(CMakePackage):
 
     # maintainers("github_user1",o"github_user2")
 
+    version("1.11.0", sha256="9cfeedaccdefbdcf23c465cb1e6c02057100c4a1a573672dc6cfea5348cedfdd")
     version("1.10.0", sha256="57a33ce499a7fc5a5aaffa39ec7597115cf69ed4ff773546b5b71ff475ee4730")
     version("1.9.0", sha256="b68aa02ef44c9e35c168f826a14802bb5cc6a9d769ba4b64b2c54f347a14aa53")
     version("1.8.0", sha256="f037c15f7049610ef8bca37500b2ab00775af60ebbb9d491ba5fc2e5c04a7794")
diff --git a/var/spack/repos/builtin/packages/rkt-racket-lib/package.py b/var/spack/repos/builtin/packages/rkt-racket-lib/package.py
new file mode 100644
index 00000000000000..a64e9527638f64
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rkt-racket-lib/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class RktRacketLib(RacketPackage):
+    """Stub package for packages which are currently part of core
+    racket installation (but which may change in the future)."""
+
+    git = "ssh://git@github.com/racket/racket.git"
+
+    maintainers = ["elfprince13"]
+
+    version("8.3", commit="cab83438422bfea0e4bd74bc3e8305e6517cf25f")  # tag="v8.3"
+    depends_on("racket@8.3", type=("build", "run"), when="@8.3")
+
+    racket_name = "racket-lib"
diff --git a/var/spack/repos/builtin/packages/rkt-scheme-lib/package.py b/var/spack/repos/builtin/packages/rkt-scheme-lib/package.py
index e366a23b7951f2..d29ae350b7779a 100644
--- a/var/spack/repos/builtin/packages/rkt-scheme-lib/package.py
+++ b/var/spack/repos/builtin/packages/rkt-scheme-lib/package.py
@@ -17,3 +17,4 @@ class RktSchemeLib(RacketPackage):
     depends_on("rkt-base@8.3", type=("build", "run"), when="@8.3")
 
     racket_name = "scheme-lib"
+    subdirectory = None
diff --git a/var/spack/repos/builtin/packages/rlwrap/package.py b/var/spack/repos/builtin/packages/rlwrap/package.py
index 2df72bfa2b3f0b..8203856915270c 100644
--- a/var/spack/repos/builtin/packages/rlwrap/package.py
+++ b/var/spack/repos/builtin/packages/rlwrap/package.py
@@ -13,6 +13,17 @@ class Rlwrap(AutotoolsPackage):
     homepage = "https://github.com/hanslub42/rlwrap"
     url = "https://github.com/hanslub42/rlwrap/releases/download/v0.43/rlwrap-0.43.tar.gz"
 
+    version("0.46.1", sha256="2711986a1248f6ac59e2aecf5586205835970040d300a42b4bf8014397e73e37")
+    version("0.46", sha256="b4bd79fda824426dae65236e338ba7daf3f0d0acad7c1561d4d5e6dadcfd539d")
+    version("0.45.2", sha256="9f8870deb46e473d21b5db89d709b6497f4ef9fa06d44eebc5f821daa00c8eca")
+    version("0.44", sha256="cd7ff50cde66e443cbea0049b4abf1cca64a74948371fa4f1b5d9a5bbce1e13c")
     version("0.43", sha256="8e86d0b7882d9b8a73d229897a90edc207b1ae7fa0899dca8ee01c31a93feb2f")
 
     depends_on("readline@4.2:")
+
+    def url_for_version(self, version):
+        if version < Version("0.46.1"):
+            return super().url_for_version(version)
+        # The latest release (0.46.1) removed the "v" prefix.
+        url_fmt = "https://github.com/hanslub42/rlwrap/releases/download/{0}/rlwrap-{0}.tar.gz"
+        return url_fmt.format(version)
diff --git a/var/spack/repos/builtin/packages/rmgdft/package.py b/var/spack/repos/builtin/packages/rmgdft/package.py
index 7f7af92286fde1..23cec275799fe7 100644
--- a/var/spack/repos/builtin/packages/rmgdft/package.py
+++ b/var/spack/repos/builtin/packages/rmgdft/package.py
@@ -7,7 +7,7 @@
 from spack.package import *
 
 
-class Rmgdft(CMakePackage):
+class Rmgdft(CMakePackage, CudaPackage):
     """RMGDFT is a high performance real-space density functional code
     designed for large scale electronic structure calculations."""
 
@@ -16,14 +16,12 @@ class Rmgdft(CMakePackage):
     maintainers("elbriggs")
     tags = ["ecp", "ecp-apps"]
     version("master", branch="master")
-    version("5.2.0", tag="v5.2.0")
-    version("5.0.5", tag="v5.0.5")
-    version("5.0.4", tag="v5.0.4")
-    version("5.0.1", tag="v5.0.1")
-    version("4.3.1", tag="v4.3.1")
-    version("4.3.0", tag="v4.3.0")
-    version("4.2.2", tag="v4.2.2")
-    version("4.2.1", tag="v4.2.1")
+    version("5.4.0", tag="v5.4.0", commit="471251b191abb5f6ffdca4333c1fcb2add3c52f2")
+    version("5.3.1", tag="v5.3.1", commit="dd6217ed82a8fe335acd0c030023b539d1be920a")
+    version("5.2.0", tag="v5.2.0", commit="e95a84a258f84a3c33f36eb34ebb9daba691b649")
+    version("5.0.5", tag="v5.0.5", commit="f67a5d80e4bb418d31f35586a19b21c9b52e7832")
+    version("5.0.4", tag="v5.0.4", commit="30faadeff7dc896169d011910831263fb19eb965")
+    version("5.0.1", tag="v5.0.1", commit="60b3ad64b09a4fccdd2b84052350e7947e3e8ad0")
 
     variant(
         "build_type",
@@ -36,6 +34,8 @@ class Rmgdft(CMakePackage):
 
     variant("local_orbitals", default=True, description="Build O(N) variant.")
 
+    variant("rocm", default=False, description="Build rocm enabled variant.")
+
     # Normally we want this but some compilers (e.g. IBM) are
     # very slow when this is on so provide the option to disable
     variant(
@@ -63,6 +63,11 @@ class Rmgdft(CMakePackage):
     depends_on("fftw-api@3")
     depends_on("mpi")
     depends_on("hdf5")
+    depends_on("cuda", when="+cuda")
+    with when("+rocm"):
+        depends_on("hipblas")
+        depends_on("rocfft")
+        depends_on("rocsolver")
 
     # RMG is a hybrid MPI/threads code and performance is
     # highly dependent on the threading model of the blas
@@ -75,9 +80,18 @@ class Rmgdft(CMakePackage):
     @property
     def build_targets(self):
         spec = self.spec
-        targets = ["rmg-cpu"]
-        if "+local_orbitals" in spec:
-            targets.append("rmg-on-cpu")
+        if "+cuda" in spec:
+            targets = ["rmg-gpu"]
+            cuda_arch_list = spec.variants["cuda_arch"].value
+            cuda_arch = cuda_arch_list[0]
+            if cuda_arch != "none":
+                args.append("-DCUDA_FLAGS=-arch=sm_{0}".format(cuda_arch))
+            if "+local_orbitals" in spec:
+                targets.append("rmg-on-gpu")
+        else:
+            targets = ["rmg-cpu"]
+            if "+local_orbitals" in spec:
+                targets.append("rmg-on-cpu")
         return targets
 
     def cmake_args(self):
@@ -91,6 +105,8 @@ def cmake_args(self):
             args.append("-DUSE_INTERNAL_PSEUDOPOTENTIALS=1")
         else:
             args.append("-DUSE_INTERNAL_PSEUDOPOTENTIALS=0")
+        if "+cuda" in spec:
+            args.append("-DRMG_CUDA_ENABLED=1")
         return args
 
     def install(self, spec, prefix):
@@ -99,9 +115,14 @@ def install(self, spec, prefix):
         mkdirp(prefix.share.tests.RMG)
 
         with working_dir(self.build_directory):
-            install("rmg-cpu", prefix.bin)
-            if "+local_orbitals" in spec:
-                install("rmg-on-cpu", prefix.bin)
+            if "+cuda" in spec:
+                install("rmg-gpu", prefix.bin)
+                if "+local_orbitals" in spec:
+                    install("rmg-on-gpu", prefix.bin)
+            else:
+                install("rmg-cpu", prefix.bin)
+                if "+local_orbitals" in spec:
+                    install("rmg-on-cpu", prefix.bin)
 
         # install tests
         with working_dir(self.build_directory):
diff --git a/var/spack/repos/builtin/packages/rocalution/package.py b/var/spack/repos/builtin/packages/rocalution/package.py
index c089cc1da8f52a..b0ba2021ba6804 100644
--- a/var/spack/repos/builtin/packages/rocalution/package.py
+++ b/var/spack/repos/builtin/packages/rocalution/package.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import itertools
+import re
 
 from spack.package import *
 
@@ -18,12 +19,15 @@ class Rocalution(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/rocALUTION"
     git = "https://github.com/ROCmSoftwarePlatform/rocALUTION.git"
-    url = "https://github.com/ROCmSoftwarePlatform/rocALUTION/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/rocALUTION/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath")
     libraries = ["librocalution_hip"]
-
+    version("5.6.1", sha256="7197b3617a0c91e90adaa32003c04d247a5f585d216e77493d20984ba215addb")
+    version("5.6.0", sha256="7397a2039e9615c0cf6776c33c4083c00b185b5d5c4149c89fea25a8976a3097")
+    version("5.5.1", sha256="4612e30a0290b1732c8862eea655122abc2d22ce4345b8498fe4127697e880b4")
+    version("5.5.0", sha256="626e966b67b83a1ef79f9bf27aba998c49cf65c4208092516aa1e32a6cbd8c36")
     version("5.4.3", sha256="39d00951a9b3cbdc4205a7e3ce75c026d9428c71c784815288c445f84a7f8a0e")
     version("5.4.0", sha256="dccf004434e0fee6d0c7bedd46827f5a2af0392bc4807a08403b130e461f55eb")
     version("5.3.3", sha256="3af022250bc25bebdee12bfb8fdbab4b60513b537b9fe15dfa82ded8850c5066")
@@ -106,12 +110,11 @@ class Rocalution(CMakePackage):
 
     amdgpu_targets = ROCmPackage.amdgpu_targets
 
-    variant("amdgpu_target", values=auto_or_any_combination_of(*amdgpu_targets), sticky=True)
     variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
+        "amdgpu_target",
+        description="AMD GPU architecture",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
     )
 
     depends_on("cmake@3.5:", type="build")
@@ -151,6 +154,10 @@ class Rocalution(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("rocprim@" + ver, when="@" + ver)
diff --git a/var/spack/repos/builtin/packages/rocblas/package.py b/var/spack/repos/builtin/packages/rocblas/package.py
index ba92d43aec24eb..0b59eadd7e559b 100644
--- a/var/spack/repos/builtin/packages/rocblas/package.py
+++ b/var/spack/repos/builtin/packages/rocblas/package.py
@@ -13,7 +13,7 @@ class Rocblas(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/rocBLAS/"
     git = "https://github.com/ROCmSoftwarePlatform/rocBLAS.git"
-    url = "https://github.com/ROCmSoftwarePlatform/rocBLAS/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/rocBLAS/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath", "haampie")
@@ -21,7 +21,10 @@ class Rocblas(CMakePackage):
 
     version("develop", branch="develop")
     version("master", branch="master")
-
+    version("5.6.1", sha256="73896ebd445162a69af97f9fd462684609b4e0cf617eab450cd4558b4a23941e")
+    version("5.6.0", sha256="6a70b27eede02c45f46095a6ce8421af9a774a565e39f5e1074783ecf00c1ea7")
+    version("5.5.1", sha256="7916a8d238d51cc239949d799f0b61c9d5cd63c6ccaed0e16749489b89ca8ff3")
+    version("5.5.0", sha256="b5260517f199e806ae18f2c4495f163884e0d7a0a7c67af0770f7428ea50f898")
     version("5.4.3", sha256="d82cd334b7a9b40d16ec4f4bb1fb5662382dcbfc86ee5e262413ed63d9e6a701")
     version("5.4.0", sha256="261e05375024a01e68697c5d175210a07f0f5fc63a756234d996ddedffde78a2")
     version("5.3.3", sha256="62a3b5f415bd8e0dcd0d68233d379f1a928ec0349977c32b4eea72ae5004e805")
@@ -104,14 +107,13 @@ class Rocblas(CMakePackage):
 
     amdgpu_targets = ROCmPackage.amdgpu_targets
 
-    variant("amdgpu_target", values=auto_or_any_combination_of(*amdgpu_targets), sticky=True)
-    variant("tensile", default=True, description="Use Tensile as a backend")
     variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
+        "amdgpu_target",
+        description="AMD GPU architecture",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
     )
+    variant("tensile", default=True, description="Use Tensile as a backend")
 
     # gfx906, gfx908,gfx803,gfx900 are valid for @:4.0.0
     # gfx803,gfx900,gfx:xnack-,gfx908:xnack- are valid gpus for @4.1.0:4.2.0
@@ -172,6 +174,10 @@ def check(self):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("llvm-amdgpu@" + ver, type="build", when="@" + ver)
@@ -189,6 +195,8 @@ def check(self):
         depends_on("py-wheel", type="build")
         depends_on("py-msgpack", type="build")
         depends_on("py-pip", type="build")
+        depends_on("py-joblib", type="build", when="@5.6:")
+        depends_on("procps", type="build", when="@5.6:")
 
     for t_version, t_commit in [
         ("@3.5.0", "f842a1a4427624eff6cbddb2405c36dec9a210cd"),
@@ -214,6 +222,10 @@ def check(self):
         ("@5.3.3", "006a5d653ce0d82fecb05d5e215d053749b57c04"),
         ("@5.4.0", "5aec08937473b27865fa969bb38a83bcf9463c2b"),
         ("@5.4.3", "5aec08937473b27865fa969bb38a83bcf9463c2b"),
+        ("@5.5.0", "38d444a9f2b6cddfeaeedcb39a5688150fa27093"),
+        ("@5.5.1", "38d444a9f2b6cddfeaeedcb39a5688150fa27093"),
+        ("@5.6.0", "7d0a9d040c3bbae893df7ecef6a19d9cd1c304aa"),
+        ("@5.6.1", "7d0a9d040c3bbae893df7ecef6a19d9cd1c304aa"),
     ]:
         resource(
             name="Tensile",
@@ -236,8 +248,8 @@ def check(self):
     patch("0002-Fix-rocblas-clients-blas.patch", when="@4.2.0:4.3.1")
     patch("0003-Fix-rocblas-gentest.patch", when="@4.2.0:5.1")
     # Finding Python package and set command python as python3
-    patch("0004-Find-python.patch", when="@5.2.0:")
-    patch("0006-Guard-use-of-OpenMP-to-make-it-optional-5.4.patch", when="@5.4:")
+    patch("0004-Find-python.patch", when="@5.2.0:5.4")
+    patch("0006-Guard-use-of-OpenMP-to-make-it-optional-5.4.patch", when="@5.4")
 
     def setup_build_environment(self, env):
         env.set("CXX", self.spec["hip"].hipcc)
@@ -271,7 +283,6 @@ def cmake_args(self):
                 self.define("Tensile_TEST_LOCAL_PATH", tensile_path),
                 self.define("Tensile_COMPILER", "hipcc"),
                 self.define("Tensile_LOGIC", "asm_full"),
-                self.define("Tensile_CODE_OBJECT_VERSION", "V3"),
                 self.define("BUILD_WITH_TENSILE_HOST", "@3.7.0:" in self.spec),
             ]
             if self.spec.satisfies("@3.7.0:"):
@@ -296,5 +307,9 @@ def cmake_args(self):
             args.append(self.define("BUILD_FILE_REORG_BACKWARD_COMPATIBILITY", True))
         if self.spec.satisfies("@5.3.0:"):
             args.append("-DCMAKE_INSTALL_LIBDIR=lib")
+        if self.spec.satisfies("@:5.4"):
+            args.append(self.define("Tensile_CODE_OBJECT_VERSION", "V3"))
+        else:
+            args.append(self.define("Tensile_CODE_OBJECT_VERSION", "default"))
 
         return args
diff --git a/var/spack/repos/builtin/packages/rocfft/package.py b/var/spack/repos/builtin/packages/rocfft/package.py
index 63aca056165f2d..63c0548ce3a6b3 100644
--- a/var/spack/repos/builtin/packages/rocfft/package.py
+++ b/var/spack/repos/builtin/packages/rocfft/package.py
@@ -13,12 +13,15 @@ class Rocfft(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/rocFFT/"
     git = "https://github.com/ROCmSoftwarePlatform/rocFFT.git"
-    url = "https://github.com/ROCmSoftwarePlatform/rocfft/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/rocfft/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath", "haampie")
     libraries = ["librocfft"]
-
+    version("5.6.1", sha256="a65861e453587c3e6393da75b0b1976508c61f968aecda77fbec920fea48489e")
+    version("5.6.0", sha256="e3d4a6c1bdac78f9a22033f57011af783d560308103f73542f9e0e4dd133d38a")
+    version("5.5.1", sha256="57423a64f5cdb1c37ff0891b6c17b59f73198d46be42db4ae23781ef2c0cd49d")
+    version("5.5.0", sha256="9288152e66504b06082e4eed8cdb791b4f9ae2836b3defbeb4d2b54901b96485")
     version("5.4.3", sha256="ed9664adc9825c237327497bc4b23f020d50be7645647f14a45f4d943dd506e7")
     version("5.4.0", sha256="d35a67332f4425fba1824eed78cf98d5c9a17a422614ff3f4cba2461df952336")
     version("5.3.3", sha256="678c18710578c1fb36a0009311bb79de7607c3468f9102cfba56a866ebb7ff78")
@@ -102,14 +105,16 @@ class Rocfft(CMakePackage):
     amdgpu_targets = ROCmPackage.amdgpu_targets
 
     variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
+        "amdgpu_target",
+        description="AMD GPU architecture",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
     )
-    variant("amdgpu_target", values=auto_or_any_combination_of(*amdgpu_targets), sticky=True)
     variant(
-        "amdgpu_target_sram_ecc", values=auto_or_any_combination_of(*amdgpu_targets), sticky=True
+        "amdgpu_target_sram_ecc",
+        description="AMD GPU architecture",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
     )
 
     depends_on("cmake@3.16:", type="build", when="@4.5.0:")
@@ -150,6 +155,10 @@ def check(self):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("rocm-cmake@%s:" % ver, type="build", when="@" + ver)
@@ -159,7 +168,7 @@ def check(self):
     patch("0002-Fix-clients-fftw3-include-dirs-rocm-4.2.patch", when="@4.2.0:4.3.1")
     patch("0003-Fix-clients-fftw3-include-dirs-rocm-4.5.patch", when="@4.5.0:5.1")
     # Patch to add install prefix header location for sqlite for 5.4
-    patch("0004-fix-missing-sqlite-include-paths.patch", when="@5.4.0:5.4")
+    patch("0004-fix-missing-sqlite-include-paths.patch", when="@5.4.0:5.5")
 
     def setup_build_environment(self, env):
         env.set("CXX", self.spec["hip"].hipcc)
diff --git a/var/spack/repos/builtin/packages/rocksdb/package.py b/var/spack/repos/builtin/packages/rocksdb/package.py
index b14675e4fb6d88..80f4b88b822ea9 100644
--- a/var/spack/repos/builtin/packages/rocksdb/package.py
+++ b/var/spack/repos/builtin/packages/rocksdb/package.py
@@ -14,6 +14,7 @@ class Rocksdb(MakefilePackage):
     git = "https://github.com/facebook/rocksdb.git"
 
     version("master", git=git, branch="master", submodules=True)
+    version("8.6.7", sha256="cdb2fc3c6a556f20591f564cb8e023e56828469aa3f76e1d9535c443ba1f0c1a")
     version("8.1.1", sha256="9102704e169cfb53e7724a30750eeeb3e71307663852f01fa08d5a320e6155a8")
     version("7.7.3", sha256="b8ac9784a342b2e314c821f6d701148912215666ac5e9bdbccd93cf3767cb611")
     version("7.2.2", sha256="c4ea6bd2e3ffe3f0f8921c699234d59108c9122d61b0ba2aa78358642a7b614e")
@@ -42,7 +43,7 @@ class Rocksdb(MakefilePackage):
     depends_on("gflags")
     depends_on("lz4", when="+lz4")
     depends_on("snappy", when="+snappy")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("zstd", when="+zstd")
     depends_on("tbb", when="+tbb")
 
@@ -67,8 +68,8 @@ def install(self, spec, prefix):
             cflags.append("-Wno-error=redundant-move")
 
         if "+zlib" in self.spec:
-            cflags.append("-I" + self.spec["zlib"].prefix.include)
-            ldflags.append(self.spec["zlib"].libs.ld_flags)
+            cflags.append("-I" + self.spec["zlib-api"].prefix.include)
+            ldflags.append(self.spec["zlib-api"].libs.ld_flags)
         else:
             env["ROCKSDB_DISABLE_ZLIB"] = "YES"
 
diff --git a/var/spack/repos/builtin/packages/rocm-bandwidth-test/package.py b/var/spack/repos/builtin/packages/rocm-bandwidth-test/package.py
index 866bb5dd270061..af829cf7ade83a 100644
--- a/var/spack/repos/builtin/packages/rocm-bandwidth-test/package.py
+++ b/var/spack/repos/builtin/packages/rocm-bandwidth-test/package.py
@@ -12,13 +12,16 @@ class RocmBandwidthTest(CMakePackage):
 
     homepage = "https://github.com/RadeonOpenCompute/rocm_bandwidth_test"
     git = "https://github.com/RadeonOpenCompute/rocm_bandwidth_test.git"
-    url = "https://github.com/RadeonOpenCompute/rocm_bandwidth_test/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/rocm_bandwidth_test/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
 
     version("master", branch="master")
-
+    version("5.6.1", sha256="849af715d08dfd89e7aa5e4453b624151db1cafaa567ab5fa36a77948b90bf0d")
+    version("5.6.0", sha256="ae2f7263a21a3a650068f43e3112b2b765eea80a5af2297572f850c77f83c85e")
+    version("5.5.1", sha256="768b3da49fe7d4bb4e6536a8ee15be9f5e865d961e813ed4a407f32402685e1f")
+    version("5.5.0", sha256="1070ce14d45f34c2c6b2fb003184f3ae735ccfd640e9df1c228988b2a5a82949")
     version("5.4.3", sha256="a2f5a75bf47db1e39a4626a9f5cd2d120bcafe56b1baf2455d794f7a4734993e")
     version("5.4.0", sha256="47a1ef92e565d5ce7a167cc1ebe3d4198cc04d598b259426245b8c11eb795677")
     version("5.3.3", sha256="2bc079297e639d45d57c8017f6f47bc44d4ed34613ec76c80574bb703d79b498")
@@ -99,13 +102,6 @@ class RocmBandwidthTest(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3:", type="build")
 
     for ver in [
@@ -132,9 +128,16 @@ class RocmBandwidthTest(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
     ]:
         depends_on("hsa-rocr-dev@" + ver, when="@" + ver)
         depends_on("hsakmt-roct@" + ver, when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     build_targets = ["package"]
diff --git a/var/spack/repos/builtin/packages/rocm-clang-ocl/package.py b/var/spack/repos/builtin/packages/rocm-clang-ocl/package.py
index d797cb8289cca0..941b1900f07802 100644
--- a/var/spack/repos/builtin/packages/rocm-clang-ocl/package.py
+++ b/var/spack/repos/builtin/packages/rocm-clang-ocl/package.py
@@ -11,12 +11,15 @@ class RocmClangOcl(CMakePackage):
 
     homepage = "https://github.com/RadeonOpenCompute/clang-ocl"
     git = "https://github.com/RadeonOpenCompute/clang-ocl.git"
-    url = "https://github.com/RadeonOpenCompute/clang-ocl/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/clang-ocl/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
     version("master", branch="master")
-
+    version("5.6.1", sha256="c41deb1b564d939fc897b2bbdb13570b2234fa4c052a39783f5ad2dd1052f901")
+    version("5.6.0", sha256="1afc47dee02d73c10de422f254067f4ef3ff921c4a1204d54ecc40e61fc63497")
+    version("5.5.1", sha256="bfa62ad14830e2bd5afbc346685216c69f8cbef0eb449954f793178e10b19a38")
+    version("5.5.0", sha256="43a5459165693301ba2ebcc41b2b0705df9a3a47571d43bdc2cc49cfdd0833a7")
     version("5.4.3", sha256="689e0354ea685bd488116de8eb902b902492e9ace184c3109b97b9a43f8b2d59")
     version("5.4.0", sha256="602f8fb1f36587543cc0ee95fd1938f8eeb03de79119101e128150332cc8d89c")
     version("5.3.3", sha256="549d5bf37507f67c5277abdeed4ec40b5d0edbfbb72907c685444c26b9ce6f8a")
@@ -97,13 +100,6 @@ class RocmClangOcl(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3.5:", type="build")
 
     for ver in [
@@ -130,6 +126,10 @@ class RocmClangOcl(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
     ]:
         depends_on("rocm-cmake@%s:" % ver, type="build", when="@" + ver)
@@ -139,6 +139,8 @@ class RocmClangOcl(CMakePackage):
         depends_on(
             "rocm-device-libs@" + ver, when="@{0} ^llvm-amdgpu ~rocm-device-libs".format(ver)
         )
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
 
     test_src_dir = "test"
 
diff --git a/var/spack/repos/builtin/packages/rocm-cmake/package.py b/var/spack/repos/builtin/packages/rocm-cmake/package.py
index b0fd7526782f4d..c833db675557c3 100644
--- a/var/spack/repos/builtin/packages/rocm-cmake/package.py
+++ b/var/spack/repos/builtin/packages/rocm-cmake/package.py
@@ -13,12 +13,16 @@ class RocmCmake(CMakePackage):
 
     homepage = "https://github.com/RadeonOpenCompute/rocm-cmake"
     git = "https://github.com/RadeonOpenCompute/rocm-cmake.git"
-    url = "https://github.com/RadeonOpenCompute/rocm-cmake/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/rocm-cmake/archive/rocm-5.6.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
 
     version("master", branch="master")
+    version("5.6.1", sha256="98bf5fe2e6e12f55d122807d0060f1bb19c80d63d2c2f6fee579c40bfd244fa6")
+    version("5.6.0", sha256="a118ca937856a4d0039955a8aef2466ef1fd1f08f7f7221cda53e1b5d02e476a")
+    version("5.5.1", sha256="60113412b35d94e20e8100ed3db688c35801991b4b8fa282fdc6fd6fd413fb6e")
+    version("5.5.0", sha256="b7884c346737eba70ae11044e41598b2482a92e21f3e0719b1ca11619f02a20b")
     version("5.4.3", sha256="c185b3a10d191d73b76770ca0f9d6bdc355ee91fe0c9016a3779c9cfe042ba0f")
     version("5.4.0", sha256="617faa9a1e51db3c7a59bd0393e054ab67e57be357d59cb0cd9b677f47824946")
     version("5.3.3", sha256="3e527f99db52e301ab4f1b994029585951e2ae685f0cdfb7b8529c72f4b77af4")
@@ -99,16 +103,12 @@ class RocmCmake(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3:", type="build")
     depends_on("cmake@3.6:", type="build", when="@4.1.0:")
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     test_src_dir = "test"
 
     @run_after("install")
diff --git a/var/spack/repos/builtin/packages/rocm-core/package.py b/var/spack/repos/builtin/packages/rocm-core/package.py
new file mode 100644
index 00000000000000..45d947ce0e7f2a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rocm-core/package.py
@@ -0,0 +1,29 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class RocmCore(CMakePackage):
+    """rocm-core is a utility which can be used to get ROCm release version.
+    It also provides the Lmod modules files for the ROCm release.
+    getROCmVersion function provides the ROCm version."""
+
+    homepage = "https://github.com/RadeonOpenCompute/rocm-core"
+    url = "https://github.com/RadeonOpenCompute/rocm-core/archive/refs/tags/rocm-5.5.0.tar.gz"
+    tags = ["rocm"]
+
+    maintainers("srekolam", "renjithravindrankannath")
+    libraries = ["librocm-core"]
+
+    version("5.6.1", sha256="eeef75e16e05380ccbc8df17a02dc141a66dddaadb444a97f7278f78067c498c")
+    version("5.6.0", sha256="3c3d47c8b774968d768d42810a3fed42d058b7d6da248d5295df2a7ffb262568")
+    version("5.5.1", sha256="bc73060432ffdc2e210394835d383890b9652476074ef4708d447473f273ce76")
+    version("5.5.0", sha256="684d3312bb14f05dc280cf136f5eddff38ba340cd85c383d6a217d8e27d3d57d")
+
+    def cmake_args(self):
+        args = [self.define("ROCM_VERSION", self.spec.version)]
+        return args
diff --git a/var/spack/repos/builtin/packages/rocm-dbgapi/package.py b/var/spack/repos/builtin/packages/rocm-dbgapi/package.py
index da26d7f704233c..6b28adb40c121d 100644
--- a/var/spack/repos/builtin/packages/rocm-dbgapi/package.py
+++ b/var/spack/repos/builtin/packages/rocm-dbgapi/package.py
@@ -16,14 +16,17 @@ class RocmDbgapi(CMakePackage):
 
     homepage = "https://github.com/ROCm-Developer-Tools/ROCdbgapi"
     git = "https://github.com/ROCm-Developer-Tools/ROCdbgapi.git"
-    url = "https://github.com/ROCm-Developer-Tools/ROCdbgapi/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCm-Developer-Tools/ROCdbgapi/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
     libraries = ["librocm-dbgapi"]
 
     version("master", branch="amd-master")
-
+    version("5.6.1", sha256="c7241bf94bdb97a4cf1befbf25b8c35720797710da6f6b5b9d6a4094c1bc9c8b")
+    version("5.6.0", sha256="9b66e47f4eccb3c8bbc324aade92aac6139539dda449427b7823d0c45341afc8")
+    version("5.5.1", sha256="c41dfc62591bcf42003fe744d8bd03a51311d54e4b012f946ca0ede0c14dd977")
+    version("5.5.0", sha256="ce572340a3fe99e4f1538eb614933153456003f8dfe9306a5735cdd25b451e25")
     version("5.4.3", sha256="d647c9121a50f2c54367c567d8f39a145cb135e1ceed931581659f57f49f61e5")
     version("5.4.0", sha256="895eb7056864daada40c3f9cd37645b0bdf4b6dc408b5f8cc974fc4cd9ab7ccb")
     version("5.3.3", sha256="3c81cb23fe671d391557a63c13b6a13d4dc367db5cb5de55592a6758284d8a3f")
@@ -104,14 +107,8 @@ class RocmDbgapi(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3:", type="build")
+    depends_on("hwdata", when="@5.5.0:")
 
     for ver in [
         "3.5.0",
@@ -137,11 +134,18 @@ class RocmDbgapi(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
     ]:
         depends_on("hsa-rocr-dev@" + ver, type="build", when="@" + ver)
         depends_on("comgr@" + ver, type=("build", "link"), when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     @classmethod
     def determine_version(cls, lib):
         match = re.search(r"lib\S*\.so\.\d+\.\d+\.(\d)(\d\d)(\d\d)", lib)
diff --git a/var/spack/repos/builtin/packages/rocm-debug-agent/package.py b/var/spack/repos/builtin/packages/rocm-debug-agent/package.py
index dbca1e5c351714..4b5850f0d0b8fe 100644
--- a/var/spack/repos/builtin/packages/rocm-debug-agent/package.py
+++ b/var/spack/repos/builtin/packages/rocm-debug-agent/package.py
@@ -13,12 +13,15 @@ class RocmDebugAgent(CMakePackage):
 
     homepage = "https://github.com/ROCm-Developer-Tools/rocr_debug_agent"
     git = "https://github.com/ROCm-Developer-Tools/rocr_debug_agent.git"
-    url = "https://github.com/ROCm-Developer-Tools/rocr_debug_agent/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCm-Developer-Tools/rocr_debug_agent/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
     libraries = ["librocm-debug-agent"]
-
+    version("5.6.1", sha256="d3b1d5d757489ed3cc66d351cec56b7b850aaa7ecf6a55b0350b89c3dee3153a")
+    version("5.6.0", sha256="0bed788f07906afeb9092d0bec184a7963233ac9d8ccd20b4afeb624a1d20698")
+    version("5.5.1", sha256="1bb66734f11bb57df6efa507f0217651446653bf28b3ca36acfcf94511a7c2bc")
+    version("5.5.0", sha256="4f2431a395a77a06dc417ed1e9188731b031a0c680e62c6eee19d60965317f5a")
     version("5.4.3", sha256="b2c9ac198ea3cbf35e7e80f57c5d81c461de78b821d07b637ea4037a65cdf49f")
     version("5.4.0", sha256="94bef73ea0a6d385dab2292ee591ca1dc268a5585cf9f1b5092a1530949f575e")
     version("5.3.3", sha256="7170312d08e91334ee03586aa1f23d67f33d9ec0df25a5556cbfa3f210b15b06")
@@ -108,13 +111,6 @@ def url_for_version(self, version):
 
         return url
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3:", type="build")
     depends_on("elfutils@:0.168", type="link")
 
@@ -142,6 +138,10 @@ def url_for_version(self, version):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hsa-rocr-dev@" + ver, when="@" + ver)
         depends_on("hsakmt-roct@" + ver, when="@" + ver)
@@ -169,10 +169,17 @@ def url_for_version(self, version):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocm-dbgapi@" + ver, when="@" + ver)
         depends_on("hip@" + ver, when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     # https://github.com/ROCm-Developer-Tools/rocr_debug_agent/pull/4
     patch("0001-Drop-overly-strict-Werror-flag.patch", when="@3.7.0:")
     patch("0002-add-hip-architecture.patch", when="@3.9.0:")
diff --git a/var/spack/repos/builtin/packages/rocm-device-libs/package.py b/var/spack/repos/builtin/packages/rocm-device-libs/package.py
index fff38c1e626251..cb784a050f195c 100644
--- a/var/spack/repos/builtin/packages/rocm-device-libs/package.py
+++ b/var/spack/repos/builtin/packages/rocm-device-libs/package.py
@@ -12,13 +12,16 @@ class RocmDeviceLibs(CMakePackage):
 
     homepage = "https://github.com/RadeonOpenCompute/ROCm-Device-Libs"
     git = "https://github.com/RadeonOpenCompute/ROCm-Device-Libs.git"
-    url = "https://github.com/RadeonOpenCompute/ROCm-Device-Libs/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/ROCm-Device-Libs/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath", "haampie")
 
     version("master", branch="amd-stg-open")
-
+    version("5.6.1", sha256="f0dfab272ff936225bfa1e9dabeb3c5d12ce08b812bf53ffbddd2ddfac49761c")
+    version("5.6.0", sha256="efb5dcdca9b3a9fbe408d494fb4a23e0b78417eb5fa8eebd4a5d226088f28921")
+    version("5.5.1", sha256="3b5f6dd85f0e3371f6078da7b59bf77d5b210e30f1cc66ef1e2de6bbcb775833")
+    version("5.5.0", sha256="5ab95aeb9c8bed0514f96f7847e21e165ed901ed826cdc9382c14d199cbadbd3")
     version("5.4.3", sha256="f4f7281f2cea6d268fcc3662b37410957d4f0bc23e0df9f60b12eb0fcdf9e26e")
     version("5.4.0", sha256="d68813ded47179c39914c8d1b76af3dad8c714b10229d1e2246af67609473951")
     version("5.3.3", sha256="963c9a0561111788b55a8c3b492e2a5737047914752376226c97a28122a4d768")
@@ -99,17 +102,10 @@ class RocmDeviceLibs(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3.13.4:", type="build", when="@3.9.0:")
     depends_on("cmake@3.4.3:", type="build")
 
-    depends_on("zlib", type="link", when="@3.9.0:")
+    depends_on("zlib-api", type="link", when="@3.9.0:")
     depends_on("texinfo", type="link", when="@3.9.0:")
 
     depends_on("rocm-cmake@3.5.0:", type="build")
@@ -142,10 +138,17 @@ class RocmDeviceLibs(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
     ]:
         depends_on("llvm-amdgpu@" + ver, when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     def cmake_args(self):
         spec = self.spec
         return [
diff --git a/var/spack/repos/builtin/packages/rocm-gdb/package.py b/var/spack/repos/builtin/packages/rocm-gdb/package.py
index b1b67ed8388417..a752f0c4d79056 100644
--- a/var/spack/repos/builtin/packages/rocm-gdb/package.py
+++ b/var/spack/repos/builtin/packages/rocm-gdb/package.py
@@ -12,11 +12,14 @@ class RocmGdb(AutotoolsPackage):
     based on GDB, the GNU source-level debugger."""
 
     homepage = "https://github.com/ROCm-Developer-Tools/ROCgdb/"
-    url = "https://github.com/ROCm-Developer-Tools/ROCgdb/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCm-Developer-Tools/ROCgdb/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
-
+    version("5.6.1", sha256="d2b40d4c5aa41a6ce2a84307627b30d16a458672e03e13f9d27c12f2dc3f21d6")
+    version("5.6.0", sha256="997ef1883aac2769552bc7082c70b837f4e98b57d24c133cea52b9c92fb0dee1")
+    version("5.5.1", sha256="359258548bc7e6abff16bb13c301339fb96560b2b961433c9e0712e4aaf2d9e1")
+    version("5.5.0", sha256="d3b100e332facd9635e328f5efd9f0565250edbe05be986baa2e0470a19bcd79")
     version("5.4.3", sha256="28c1ce39fb1fabe61f86f6e3c6940c10f9a8b8de77f7bb4fdd73b04e172f85f6")
     version("5.4.0", sha256="7ee984d99818da04733030b140c1f0929639bc719a5e418d53cc2c2a8cbc9a79")
     version("5.3.3", sha256="9fc3ccd9378ad40f2f0c9577bc400cc9a202d0ae4656378813b67653b9023c46")
@@ -104,7 +107,7 @@ class RocmGdb(AutotoolsPackage):
     depends_on("libunwind", type="build")
     depends_on("expat", type=("build", "link"))
     depends_on("python", type=("build", "link"))
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("babeltrace@1.2.4", type="link")
     depends_on("gmp", type=("build", "link"), when="@4.5.0:")
 
@@ -132,10 +135,17 @@ class RocmGdb(AutotoolsPackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocm-dbgapi@" + ver, type="link", when="@" + ver)
         depends_on("comgr@" + ver, type="link", when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     build_directory = "spack-build"
 
     def configure_args(self):
diff --git a/var/spack/repos/builtin/packages/rocm-opencl/package.py b/var/spack/repos/builtin/packages/rocm-opencl/package.py
index 38c8aa9df8589c..f79496e91dc2c2 100644
--- a/var/spack/repos/builtin/packages/rocm-opencl/package.py
+++ b/var/spack/repos/builtin/packages/rocm-opencl/package.py
@@ -29,7 +29,10 @@ def url_for_version(self, version):
         return url.format(version)
 
     version("master", branch="main")
-
+    version("5.6.1", sha256="ec26049f7d93c95050c27ba65472736665ec7a40f25920a868616b2970f6b845")
+    version("5.6.0", sha256="52ab260d00d279c2a86c353901ffd88ee61b934ad89e9eb480f210656705f04e")
+    version("5.5.1", sha256="a8a62a7c6fc5398406d2203b8cb75621a24944688e545d917033d87de2724498")
+    version("5.5.0", sha256="0df9fa0b8aa0c8e6711d34eec0fdf1ed356adcd9625bc8f1ce9b3e72090f3e4f")
     version("5.4.3", sha256="b0f8339c844a2e62773bd85cd1e7c5ecddfe71d7c8e8d604e1a1d60900c30873")
     version("5.4.0", sha256="a294639478e76c75dac0e094b418f9bd309309b07faf6af126cdfad9aab3c5c7")
     version("5.3.3", sha256="cab394e6ef16c35bab8de29a66b96a7dc0e7d1297aaacba3718fa1d369233c9f")
@@ -110,18 +113,15 @@ def url_for_version(self, version):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3:", type="build")
     depends_on("gl@4.5:", type="link")
     depends_on("numactl", type="link", when="@3.7.0:")
 
     for d_version, d_shasum in [
+        ("5.6.1", "cc9a99c7e4de3d9360c0a471b27d626e84a39c9e60e0aff1e8e1500d82391819"),
+        ("5.6.0", "864f87323e793e60b16905284fba381a7182b960dd4a37fb67420c174442c03c"),
+        ("5.5.1", "1375fc7723cfaa0ae22a78682186d4804188b0a54990bfd9c0b8eb421b85e37e"),
+        ("5.5.0", "efbae9a1ef2ab3de5ca44091e9bb78522e76759c43524c1349114f9596cc61d1"),
         ("5.4.3", "71d9668619ab57ec8a4564d11860438c5aad5bd161a3e58fbc49555fbd59182d"),
         ("5.4.0", "46a1579310b3ab9dc8948d0fb5bed4c6b312f158ca76967af7ab69e328d43138"),
         ("5.3.3", "f8133a5934f9c53b253d324876d74f08a19e2f5b073bc94a62fe64b0d2183a18"),
@@ -188,11 +188,18 @@ def url_for_version(self, version):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
     ]:
         depends_on("comgr@" + ver, type="build", when="@" + ver)
         depends_on("hsa-rocr-dev@" + ver, type="link", when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     @classmethod
     def determine_version(cls, lib):
         match = re.search(r"lib\S*\.so\.\d+\.\d+\.(\d)(\d\d)(\d\d)", lib)
diff --git a/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py b/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py
index 3792e96b889c1b..dedba382c56714 100644
--- a/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py
+++ b/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py
@@ -35,6 +35,10 @@
     "c86141fcde879fc78d06a41ba6a26ff528da539c6a1be8b714f635182c66e3f4",
     "bbca540897848fa95fd0f14fc05ab6deda31299a061424972d5e2bc09c7543dc",
     "7f90634fb621169b21bcbd920c2e299acc88ba0eeb1a33fd40ae26e13201b652",
+    "23cc7d1c82e35c74f48285a0a1c27e7b3cae1767568bb7b9367ea21f53dd6598",
+    "9ec03a69cc462ada43e1fd4ca905a765b08c10e0911fb7a202c893cc577855e6",
+    "0673820a81986c9e2f28f15bbb45ad18934bca56a9d08aae6c49ec3895b38487",
+    "6c051bf7625f682ba3d2ea80b46a38ca2cbcd20f5d89ae3433602d3e7ef0403a",
 ]
 
 devlib = [
@@ -58,6 +62,10 @@
     "963c9a0561111788b55a8c3b492e2a5737047914752376226c97a28122a4d768",
     "d68813ded47179c39914c8d1b76af3dad8c714b10229d1e2246af67609473951",
     "f4f7281f2cea6d268fcc3662b37410957d4f0bc23e0df9f60b12eb0fcdf9e26e",
+    "5ab95aeb9c8bed0514f96f7847e21e165ed901ed826cdc9382c14d199cbadbd3",
+    "3b5f6dd85f0e3371f6078da7b59bf77d5b210e30f1cc66ef1e2de6bbcb775833",
+    "efb5dcdca9b3a9fbe408d494fb4a23e0b78417eb5fa8eebd4a5d226088f28921",
+    "f0dfab272ff936225bfa1e9dabeb3c5d12ce08b812bf53ffbddd2ddfac49761c",
 ]
 
 llvm = [
@@ -81,6 +89,10 @@
     "5296d5e474811c7d1e456cb6d5011db248b79b8d0512155e8a6c2aa5b5f12d38",
     "ff54f45a17723892cd775c1eaff9e5860527fcfd33d98759223c70e3362335bf",
     "a844d3cc01613f6284a75d44db67c495ac1e9b600eacbb1eb13d2649f5d5404d",
+    "5dc6c99f612b69ff73145bee17524e3712990100e16445b71634106acf7927cf",
+    "7d7181f20f89cb0715191aa32914186c67a34258c13457055570d47e15296553",
+    "e922bd492b54d99e56ed88c81e2009ed6472059a180b10cc56ce1f9bd2d7b6ed",
+    "045e43c0c4a3f4f2f1db9fb603a4f1ea3d56e128147e19ba17909eb57d7f08e5",
 ]
 
 flang = [
@@ -104,6 +116,10 @@
     "ddccd866d0c01086087fe21b5711668f85bcf9cbd9f62853f8bda32eaedb5339",
     "fae8195a5e1b3778e31dbc6cbeedeae9998ea4b5a54215534af41e91fdcb8ba0",
     "b283d76244d19ab16c9d087ee7de0d340036e9c842007aa9d288aa4e6bf3749f",
+    "a18522588686672150c7862f2b23048a429baa4a66010c4196e969cc77bd152c",
+    "7c3b4eb3e95b9e2f91234f202a76034628d230a92e57b7c5ee9dcca1097bec46",
+    "fcefebddca0b373da81ff84f0f5469a1ef77a05430a5195d0f2e6399d3af31c3",
+    "5ebcbca2e03bd0686e677f44ea551e97bd9395c6b119f832fa784818733aa652",
 ]
 
 extras = [
@@ -127,6 +143,10 @@
     "b26b9f4b11a9ccfab53d0dd55aada7e5b98f7ab51981cb033b376321dd44bf87",
     "2546becd4b182d1e366f47660c731c8ff7366b6306782f04706b6a7bf4e2094c",
     "d393f27a85c9229433b50daee8154e11517160beb1049c1de9c55fc31dd11fac",
+    "8f49026a80eb8685cbfb6d3d3b9898dd083df4d71893984ae5330d4804c685fb",
+    "8955aa9d039fd6c1ff2e26d7298f0bf09bbcf03f09c6df92c91a9ab2510df9da",
+    "017bfed52fbe08185d8dbde79377918454215683562519a9e47acf403d9a1c29",
+    "437e2017cfe2ab73b15ada0fc1ea88f794f0b108cc5410f457268ae7e4e8985a",
 ]
 
 versions = [
@@ -150,6 +170,10 @@
     "5.3.3",
     "5.4.0",
     "5.4.3",
+    "5.5.0",
+    "5.5.1",
+    "5.6.0",
+    "5.6.1",
 ]
 versions_dict = dict()  # type: Dict[str,Dict[str,str]]
 components = ["aomp", "devlib", "llvm", "flang", "extras"]
@@ -167,10 +191,14 @@ class RocmOpenmpExtras(Package):
     """OpenMP support for ROCm LLVM."""
 
     homepage = tools_url + "/aomp"
-    url = tools_url + "/aomp/archive/rocm-5.4.3.tar.gz"
+    url = tools_url + "/aomp/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath", "estewart08")
+    version("5.6.1", sha256=versions_dict["5.6.1"]["aomp"])
+    version("5.6.0", sha256=versions_dict["5.6.0"]["aomp"])
+    version("5.5.1", sha256=versions_dict["5.5.1"]["aomp"])
+    version("5.5.0", sha256=versions_dict["5.5.0"]["aomp"])
     version("5.4.3", sha256=versions_dict["5.4.3"]["aomp"])
     version("5.4.0", sha256=versions_dict["5.4.0"]["aomp"])
     version("5.3.3", sha256=versions_dict["5.3.3"]["aomp"])
@@ -221,12 +249,19 @@ class RocmOpenmpExtras(Package):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hsakmt-roct@" + ver, when="@" + ver)
         depends_on("comgr@" + ver, when="@" + ver)
         depends_on("hsa-rocr-dev@" + ver, when="@" + ver)
         depends_on("llvm-amdgpu@{0} ~openmp".format(ver), when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
         # tag changed to 'rocm-' in 4.0.0
         if ver == "3.9.0" or ver == "3.10.0":
             tag = "rocm-uc-"
diff --git a/var/spack/repos/builtin/packages/rocm-smi-lib/package.py b/var/spack/repos/builtin/packages/rocm-smi-lib/package.py
index 485b46e878bb2b..11ad3aa2ab257e 100644
--- a/var/spack/repos/builtin/packages/rocm-smi-lib/package.py
+++ b/var/spack/repos/builtin/packages/rocm-smi-lib/package.py
@@ -18,14 +18,17 @@ class RocmSmiLib(CMakePackage):
 
     homepage = "https://github.com/RadeonOpenCompute/rocm_smi_lib"
     git = "https://github.com/RadeonOpenCompute/rocm_smi_lib.git"
-    url = "https://github.com/RadeonOpenCompute/rocm_smi_lib/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/rocm_smi_lib/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
     libraries = ["librocm_smi64"]
 
     version("master", branch="master")
-
+    version("5.6.1", sha256="9e94f9a941202c3d7ce917fd1cd78c4e0f06f48d6c929f3aa916378ccef1e02c")
+    version("5.6.0", sha256="88be875948a29454b8aacced8bb8ad967502a7a074ecbc579ed673c1650a2f7e")
+    version("5.5.1", sha256="37f32350bfaf6c697312628696d1b1d5fd9165f183882759bc6cb9a5d65b9430")
+    version("5.5.0", sha256="0703f49b1c2924cc1d3f613258eabdff1925cb5bcf7cf22bb6b955dd065e4ce8")
     version("5.4.3", sha256="34d550272e420684230ceb7845aefcef79b155e51cf9ec55e31fdba2a4ed177b")
     version("5.4.0", sha256="4b110c9ec104ec39fc458b1b6f693662ab75395b75ed402b671d8e58c7ae63fe")
     version("5.3.3", sha256="c2c2a377c2e84f0c40297a97b6060dddc49183c2771b833ebe91ed98a98e4119")
@@ -106,23 +109,19 @@ class RocmSmiLib(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
     variant("shared", default=True, description="Build shared or static library")
 
     depends_on("cmake@3:", type="build")
     depends_on("python@3:", type=("build", "run"), when="@3.9.0:")
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
     patch("disable_pdf_generation_with_doxygen_and_latex.patch", when="@4.5.2:")
 
     def cmake_args(self):
         args = [
             self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
-            self.define("CMAKE_INSTALL_LIBDIR", self.prefix.lib),
+            self.define("CMAKE_INSTALL_LIBDIR", "lib"),
         ]
         return args
 
diff --git a/var/spack/repos/builtin/packages/rocm-tensile/package.py b/var/spack/repos/builtin/packages/rocm-tensile/package.py
index 6ab928332cfba0..29a6e82e092e3c 100644
--- a/var/spack/repos/builtin/packages/rocm-tensile/package.py
+++ b/var/spack/repos/builtin/packages/rocm-tensile/package.py
@@ -13,11 +13,15 @@ class RocmTensile(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/Tensile/"
     git = "https://github.com/ROCmSoftwarePlatform/Tensile.git"
-    url = "https://github.com/ROCmSoftwarePlatform/Tensile/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/Tensile/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath", "haampie")
 
+    version("5.6.1", sha256="3e78c933563fade8781a1dca2079bff135af2f5d2c6eb0147797d2c1f24d006c")
+    version("5.6.0", sha256="383728ecf49def59ab9a7f8a1d1e2eaf8b528e36b461e27030a2aab1a1ed80cb")
+    version("5.5.1", sha256="b65cb7335abe51ba33be9d46a5ede992b4e5932fa33797397899a6bf33a770e9")
+    version("5.5.0", sha256="70fd736d40bb4c3461f07c77ad3ae6c485e3e842671ce9b223d023d836884ae2")
     version("5.4.3", sha256="a4c5e62edd33ea6b8528eb3f017a14c28eaa67c540f5c9023f6a245340198b0f")
     version("5.4.0", sha256="2da9c1df3c6d9b44afdad621ef59a03389fb1a38a61a8b8bad9c9991b97157eb")
     version("5.3.3", sha256="ecb99243edf1cd2bb5e953915a7dae7867c3cdb0cd8ed15b8618aaaeb2bd7b29")
@@ -114,12 +118,12 @@ class RocmTensile(CMakePackage):
     )
 
     variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
+        "tensile_architecture",
+        default="all",
+        description="AMD GPU architecture",
+        values=tensile_architecture,
+        multi=True,
     )
-    variant("tensile_architecture", default="all", values=tensile_architecture, multi=True)
     variant("openmp", default=True, description="Enable OpenMP")
     conflicts("tensile_architecture=gfx906", when="@4.0.1:")
     conflicts("tensile_architecture=gfx908", when="@4.0.1:")
@@ -153,13 +157,30 @@ class RocmTensile(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocm-cmake@" + ver, type="build", when="@" + ver)
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("comgr@" + ver, when="@" + ver)
         depends_on("rocminfo@" + ver, type="build", when="@" + ver)
-
-    for ver in ["5.1.0", "5.1.3", "5.2.0", "5.2.1", "5.2.3", "5.3.0", "5.3.3", "5.4.0", "5.4.3"]:
+    for ver in [
+        "5.1.0",
+        "5.1.3",
+        "5.2.0",
+        "5.2.1",
+        "5.2.3",
+        "5.3.0",
+        "5.3.3",
+        "5.4.0",
+        "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
+    ]:
         depends_on("rocm-openmp-extras@" + ver, when="@" + ver)
 
     for ver in ["3.5.0", "3.7.0", "3.8.0", "3.9.0"]:
@@ -184,6 +205,10 @@ class RocmTensile(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("rocm-smi-lib@" + ver, type="build", when="@" + ver)
 
@@ -196,6 +221,7 @@ class RocmTensile(CMakePackage):
 
     def setup_build_environment(self, env):
         env.set("CXX", self.spec["hip"].hipcc)
+        env.append_flags("LDFLAGS", "-pthread")
 
     def get_gpulist_for_tensile_support(self):
         arch = self.spec.variants["tensile_architecture"].value
diff --git a/var/spack/repos/builtin/packages/rocm-validation-suite/007-cleanup-path-reference-donot-download-googletest-yaml-library-path_5.6.patch b/var/spack/repos/builtin/packages/rocm-validation-suite/007-cleanup-path-reference-donot-download-googletest-yaml-library-path_5.6.patch
new file mode 100644
index 00000000000000..7acd9606141590
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rocm-validation-suite/007-cleanup-path-reference-donot-download-googletest-yaml-library-path_5.6.patch
@@ -0,0 +1,532 @@
+From 795e7474acf23eb2f7815fd54ffdd3fd41ff8c35 Mon Sep 17 00:00:00 2001
+From: Renjith Ravindran 
+Date: Tue, 12 Sep 2023 07:00:31 +0000
+Subject: [PATCH] 5.6 Patch to add rocm-smi library and include path
+
+---
+ CMakeLists.txt                 | 105 ++++-----------------------------
+ babel.so/CMakeLists.txt        |  16 ++---
+ cmake_modules/tests_unit.cmake |   2 +-
+ edp.so/CMakeLists.txt          |   3 +-
+ gm.so/CMakeLists.txt           |   4 +-
+ gpup.so/CMakeLists.txt         |   2 +-
+ gst.so/CMakeLists.txt          |   4 +-
+ iet.so/CMakeLists.txt          |   6 +-
+ mem.so/CMakeLists.txt          |   4 +-
+ pbqt.so/CMakeLists.txt         |   2 +-
+ pebb.so/CMakeLists.txt         |   2 +-
+ peqt.so/CMakeLists.txt         |   4 +-
+ perf.so/CMakeLists.txt         |   4 +-
+ pesm.so/CMakeLists.txt         |   2 +-
+ rcqt.so/CMakeLists.txt         |   2 +-
+ rvs/CMakeLists.txt             |   2 +-
+ rvs/tests.cmake                |   2 +-
+ rvslib/CMakeLists.txt          |   2 +-
+ smqt.so/CMakeLists.txt         |   2 +-
+ testif.so/CMakeLists.txt       |   2 +-
+ 20 files changed, 45 insertions(+), 127 deletions(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index a12eb41..900657a 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -68,13 +68,12 @@ endif(rocblas_FOUND)
+ # variables since we will pass them as cmake params appropriately, and 
+ # all find_packages relevant to this build will be in ROCM path hence appending it to CMAKE_PREFIX_PATH 
+ set(ROCM_PATH "/opt/rocm" CACHE PATH "ROCM install path")
+-set(CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE PATH "CMAKE installation directory")
+-set(CMAKE_PACKAGING_INSTALL_PREFIX "/opt/rocm" CACHE PATH "Prefix used in built packages")
++set (CMAKE_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" )
+ list(APPEND CMAKE_PREFIX_PATH "${ROCM_PATH}")
+-set(ROCR_INC_DIR "${ROCM_PATH}/include" CACHE PATH "Contains header files exported by ROC Runtime" FORCE)
+-set(ROCR_LIB_DIR "${ROCM_PATH}/lib" CACHE PATH "Contains library files exported by ROC Runtime" FORCE)
+-set(HIP_INC_DIR "${ROCM_PATH}" CACHE PATH "Contains header files exported by ROC Runtime")
+-set(ROCT_INC_DIR "${ROCM_PATH}/include" CACHE PATH "Contains header files exported by ROC Trunk" FORCE)
++set(ROCR_INC_DIR "${HSA_PATH}/include" CACHE PATH "Contains header files exported by ROC Runtime")
++set(ROCR_LIB_DIR "${HSA_PATH}/lib" CACHE PATH "Contains library files exported by ROC Runtime")
++set(HIP_INC_DIR "${HIP_PATH}" CACHE PATH "Contains header files exported by ROC Runtime")
++set(ROCT_INC_DIR "${ROCM_PATH}/include" CACHE PATH "Contains header files exported by ROC Trunk")
+ 
+ 
+ #
+@@ -193,8 +192,6 @@ set(RVS_ROCBLAS "0" CACHE STRING "1 = use local rocBLAS")
+ set(RVS_ROCMSMI "0" CACHE STRING "1 = use local rocm_smi_lib")
+ 
+ set(RVS_LIB_DIR "${CMAKE_BINARY_DIR}/rvslib" CACHE PATH "Contains RVS library")
+-set(YAML_INC_DIR "${CMAKE_BINARY_DIR}/yaml-src/include" CACHE PATH "Contains header files exported by yaml-cpp")
+-set(YAML_LIB_DIR "${CMAKE_BINARY_DIR}/yaml-build" CACHE PATH "Contains library files exported by yaml-cpp")
+ 
+ if (${RVS_OS_TYPE} STREQUAL "centos")
+   set(ROCT_LIB_DIR "${ROCM_PATH}/lib64" CACHE PATH "Contains library files exported by ROC Trunk")
+@@ -238,86 +235,6 @@ if (NOT DEFINED CPACK_GENERATOR )
+ endif()
+ message (STATUS "CPACK_GENERATOR ${CPACK_GENERATOR}" )
+ 
+-
+-################################################################################
+-# Download and unpack yaml-cpp at configure time
+-configure_file(CMakeYamlDownload.cmake yaml-download/CMakeLists.txt)
+-execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
+-  RESULT_VARIABLE result
+-  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/yaml-download )
+-if(result)
+-  message(FATAL_ERROR "CMake step for yaml-download failed: ${result}")
+-endif()
+-execute_process(COMMAND ${CMAKE_COMMAND} --build .
+-  RESULT_VARIABLE result
+-  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/yaml-download )
+-if(result)
+-  message(FATAL_ERROR "Build step for yaml-download failed: ${result}")
+-endif()
+-execute_process(COMMAND ${CMAKE_COMMAND} ${CMAKE_BINARY_DIR}/yaml-src -B${CMAKE_BINARY_DIR}/yaml-build
+-  RESULT_VARIABLE result
+-  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/yaml-src )
+-if(result)
+-  message(FATAL_ERROR "Config step for yaml-src failed: ${result}")
+-endif()
+-
+-add_custom_target(rvs_yaml_target
+-  DEPENDS ${CMAKE_BINARY_DIR}/yaml-build/libyaml-cpp.a
+-)
+-
+-add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/yaml-build/libyaml-cpp.a
+-  COMMAND make -C ${CMAKE_BINARY_DIR}/yaml-build
+-  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/yaml-src
+-  COMMENT "Generating yaml-cpp targets"
+-  VERBATIM)
+-
+-################################################################################
+-## GOOGLE TEST
+-if(RVS_BUILD_TESTS)
+-  # Download and unpack googletest at configure time
+-  configure_file(CMakeGtestDownload.cmake googletest-download/CMakeLists.txt)
+-  execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
+-    RESULT_VARIABLE result
+-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/googletest-download )
+-  if(result)
+-    message(FATAL_ERROR "CMake step for googletest failed: ${result}")
+-  endif()
+-  execute_process(COMMAND ${CMAKE_COMMAND} --build .
+-    RESULT_VARIABLE result
+-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/googletest-download )
+-  if(result)
+-    message(FATAL_ERROR "Build step for googletest failed: ${result}")
+-  endif()
+-  execute_process(COMMAND ${CMAKE_COMMAND} ${CMAKE_BINARY_DIR}/googletest-src -B${CMAKE_BINARY_DIR}/googletest-build
+-    RESULT_VARIABLE result
+-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/googletest-src )
+-  if(result)
+-    message(FATAL_ERROR "Config step for googletest-src failed: ${result}")
+-  endif()
+-
+-  add_custom_target(rvs_gtest_target
+-    DEPENDS ${CMAKE_BINARY_DIR}/googletest-build/lib/libgtest_main.a
+-  )
+-
+-  add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/googletest-build/lib/libgtest_main.a
+-    COMMAND make -C ${CMAKE_BINARY_DIR}/googletest-build
+-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/googletest-src
+-    COMMENT "Generating googletest targets"
+-    VERBATIM)
+-
+-  ## Set default unit test framework include path
+-  if (NOT DEFINED UT_INC)
+-      set (UT_INC "${CMAKE_BINARY_DIR}/googletest-src/googletest/include")
+-      message ("UT_INC ${UT_INC}")
+-  endif ()
+-
+-  ## Set default unit test framework include path
+-  if (NOT DEFINED UT_LIB)
+-      set (UT_LIB "${CMAKE_BINARY_DIR}/googletest-build/lib")
+-      message ("UT_LIB ${UT_LIB}")
+-  endif()
+-
+-endif()
+ ################################################################################
+ ## rocBLAS
+ 
+@@ -441,8 +358,8 @@ if (RVS_ROCBLAS EQUAL 1)
+   set(ROCBLAS_INC_DIR "${CMAKE_BINARY_DIR}/rvs_rblas-src/build/release/rocblas-install")
+   set(ROCBLAS_LIB_DIR "${CMAKE_BINARY_DIR}/rvs_rblas-src/build/release/rocblas-install/lib/")
+ else()
+-  set(ROCBLAS_INC_DIR "${ROCM_PATH}/include")
+-  set(ROCBLAS_LIB_DIR "${ROCM_PATH}/lib")
++  set(ROCBLAS_INC_DIR "${ROCBLAS_DIR}/include")
++  set(ROCBLAS_LIB_DIR "${ROCBLAS_DIR}/lib")
+ endif()
+ 
+ if (RVS_ROCMSMI EQUAL 1)
+@@ -457,8 +374,8 @@ else()
+     set(ROCM_SMI_LIB_DIR "${ROCM_PATH}/rocm_smi/lib")
+   else()
+     message( STATUS "ROCBLAS REORG Enabled Version: ${RVS_ROCBLAS_VERSION_FLAT}" )
+-    set(ROCM_SMI_INC_DIR "${ROCM_PATH}/include")
+-    set(ROCM_SMI_LIB_DIR "${ROCM_PATH}/lib")
++    set(ROCM_SMI_INC_DIR "${ROCM_SMI_DIR}/include")
++    set(ROCM_SMI_LIB_DIR "${ROCM_SMI_DIR}/lib")
+   endif()
+ endif()
+ set(ROCM_SMI_LIB "rocm_smi64" CACHE STRING "rocm_smi library name")
+@@ -502,7 +419,7 @@ if (RVS_BUILD_TESTS)
+   add_subdirectory(testif.so)
+ endif()
+ 
+-add_dependencies(rvshelper rvs_bin_folder rvs_doc rvs_yaml_target)
++add_dependencies(rvshelper rvs_bin_folder rvs_doc)
+ 
+ 
+ add_dependencies(pesm rvslib rvslibrt)
+@@ -537,7 +454,7 @@ if (RVS_BUILD_TESTS)
+   WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+   COMMENT "Create the bintest directory"
+   VERBATIM)
+-  add_dependencies(rvshelper rvs_bintest_folder rvs_gtest_target)
++  add_dependencies(rvshelper rvs_bintest_folder)
+ endif()
+ 
+ add_custom_target(rvs_doc ALL
+diff --git a/babel.so/CMakeLists.txt b/babel.so/CMakeLists.txt
+index 7290cef..ebd55ad 100644
+--- a/babel.so/CMakeLists.txt
++++ b/babel.so/CMakeLists.txt
+@@ -107,13 +107,13 @@ set(HIP_HCC_LIB "amdhip64")
+ add_compile_options(-DRVS_ROCBLAS_VERSION_FLAT=${RVS_ROCBLAS_VERSION_FLAT})
+ 
+ # Determine Roc Runtime header files are accessible
+-if(NOT EXISTS ${HIP_INC_DIR}/include/hip/hip_runtime.h)
+-  message("ERROR: ROC Runtime headers can't be found under specified path. Please set HIP_INC_DIR path. Current value is : " ${HIP_INC_DIR})
++if(NOT EXISTS ${HIP_PATH}/include/hip/hip_runtime.h)
++	message("ERROR: ROC Runtime headers can't be found under specified path. Please set HIP_PATH path. Current value is : " ${HIP_PATH})
+   RETURN()
+ endif()
+ 
+-if(NOT EXISTS ${HIP_INC_DIR}/include/hip/hip_runtime_api.h)
+-  message("ERROR: ROC Runtime headers can't be found under specified path. Please set HIP_INC_DIR path. Current value is : " ${HIP_INC_DIR})
++if(NOT EXISTS ${HIP_PATH}/include/hip/hip_runtime_api.h)
++	message("ERROR: ROC Runtime headers can't be found under specified path. Please set HIP_PATH path. Current value is : " ${HIP_PATH})
+   RETURN()
+ endif()
+ 
+@@ -133,16 +133,16 @@ if(DEFINED RVS_ROCMSMI)
+ endif()
+ 
+ 
+-if(NOT EXISTS "${ROCR_LIB_DIR}/lib${HIP_HCC_LIB}.so")
+-  message("ERROR: ROC Runtime libraries can't be found under specified path. Please set ROCR_LIB_DIR path. Current value is : " ${ROCR_LIB_DIR})
++if(NOT EXISTS "${HIP_PATH}/lib/lib${HIP_HCC_LIB}.so")
++	message("ERROR: ROC Runtime libraries can't be found under specified path. Please set HIP_PATH path. Current value is : " ${HIP_PATH})
+   RETURN()
+ endif()
+ 
+ ## define include directories
+-include_directories(./ ../ ${ROCR_INC_DIR} ${HIP_INC_DIR})
++include_directories(./ ../ ${HIP_PATH})
+ 
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${HIP_PATH}/lib/ ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS rvslibrt rvslib libpthread.so libpci.so libm.so)
+ 
+diff --git a/cmake_modules/tests_unit.cmake b/cmake_modules/tests_unit.cmake
+index 586f453..c8b6560 100644
+--- a/cmake_modules/tests_unit.cmake
++++ b/cmake_modules/tests_unit.cmake
+@@ -27,7 +27,7 @@
+ ## define additional unit testing include directories
+ include_directories(${UT_INC})
+ ## define additional unit testing lib directories
+-link_directories(${UT_LIB} ${RVS_LIB_DIR})
++link_directories(${UT_LIB} ${RVS_LIB_DIR} ${ROCM_SMI_LIB_DIR})
+ 
+ file(GLOB TESTSOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} test/test*.cpp )
+ #message ( "TESTSOURCES: ${TESTSOURCES}" )
+diff --git a/edp.so/CMakeLists.txt b/edp.so/CMakeLists.txt
+index a933061..d117e03 100644
+--- a/edp.so/CMakeLists.txt
++++ b/edp.so/CMakeLists.txt
+@@ -129,6 +129,7 @@ endif()
+ 
+ 
+ if(NOT EXISTS "${ROCR_LIB_DIR}/lib${HIP_HCC_LIB}.so")
++  message("${ROCR_LIB_DIR}/lib${HIP_HCC_LIB}.so not found")
+   message("ERROR: ROC Runtime libraries can't be found under specified path. Please set ROCR_LIB_DIR path. Current value is : " ${ROCR_LIB_DIR})
+   RETURN()
+ endif()
+@@ -136,7 +137,7 @@ endif()
+ ## define include directories
+ include_directories(./ ../ ${ROCR_INC_DIR} ${ROCBLAS_INC_DIR} ${HIP_INC_DIR})
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCBLAS_LIB_DIR})
++link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCBLAS_LIB_DIR} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS rvslibrt rvslib libpthread.so libpciaccess.so libpci.so libm.so)
+ 
+diff --git a/gm.so/CMakeLists.txt b/gm.so/CMakeLists.txt
+index afaafcb..7c0cd79 100644
+--- a/gm.so/CMakeLists.txt
++++ b/gm.so/CMakeLists.txt
+@@ -122,7 +122,7 @@ include_directories(./ ../ ${ROCM_SMI_INC_DIR})
+ # Add directories to look for library files to link
+ link_directories(${RVS_LIB_DIR} ${ROCM_SMI_LIB_DIR} ${ASAN_LIB_PATH})
+ ## additional libraries
+-set (PROJECT_LINK_LIBS rvslibrt rvslib libpthread.so libpci.so libm.so)
++set (PROJECT_LINK_LIBS rvslibrt rvslib libpthread.so libpci.so libm.so librocm_smi64.so)
+ 
+ ## define source files
+ set(SOURCES  src/rvs_module.cpp src/action.cpp src/worker.cpp)
+@@ -133,7 +133,7 @@ add_library( ${RVS_TARGET} SHARED ${SOURCES})
+ set_target_properties(${RVS_TARGET} PROPERTIES
+         SUFFIX .so.${LIB_VERSION_STRING}
+         LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
+-target_link_libraries(${RVS_TARGET} ${PROJECT_LINK_LIBS} ${ROCM_SMI_LIB})
++target_link_libraries(${RVS_TARGET} ${PROJECT_LINK_LIBS})
+ add_dependencies(${RVS_TARGET} rvslibrt rvslib)
+ 
+ add_custom_command(TARGET ${RVS_TARGET} POST_BUILD
+diff --git a/gpup.so/CMakeLists.txt b/gpup.so/CMakeLists.txt
+index ca1674b..a9e4d16 100644
+--- a/gpup.so/CMakeLists.txt
++++ b/gpup.so/CMakeLists.txt
+@@ -111,7 +111,7 @@ endif()
+ ## define include directories
+ include_directories(./ ../ include ../include)
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS rvslibrt rvslib libpci.so libm.so)
+ 
+diff --git a/gst.so/CMakeLists.txt b/gst.so/CMakeLists.txt
+index d85eadb..ca7fff4 100644
+--- a/gst.so/CMakeLists.txt
++++ b/gst.so/CMakeLists.txt
+@@ -137,7 +137,7 @@ if(DEFINED RVS_ROCMSMI)
+ endif()
+ 
+ 
+-if(NOT EXISTS "${ROCR_LIB_DIR}/lib${HIP_HCC_LIB}.so")
++if(NOT EXISTS "${HIP_INC_DIR}/lib/lib${HIP_HCC_LIB}.so")
+   message("ERROR: ROC Runtime libraries can't be found under specified path. Please set ROCR_LIB_DIR path. Current value is : " ${ROCR_LIB_DIR})
+   RETURN()
+ endif()
+@@ -145,7 +145,7 @@ endif()
+ ## define include directories
+ include_directories(./ ../ ${ROCR_INC_DIR} ${ROCBLAS_INC_DIR} ${HIP_INC_DIR})
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${HIP_INC_DIR}/lib/ ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS rvslibrt rvslib libpthread.so libpci.so libm.so)
+ 
+diff --git a/iet.so/CMakeLists.txt b/iet.so/CMakeLists.txt
+index 3263d12..62f4318 100644
+--- a/iet.so/CMakeLists.txt
++++ b/iet.so/CMakeLists.txt
+@@ -140,7 +140,7 @@ if(DEFINED RVS_ROCMSMI)
+   endif()
+ endif()
+ 
+-if(NOT EXISTS "${ROCR_LIB_DIR}/lib${HIP_HCC_LIB}.so")
++if(NOT EXISTS "${HIP_INC_DIR}/lib/lib${HIP_HCC_LIB}.so")
+   message("ERROR: ROC Runtime libraries can't be found under specified path. Please set ROCR_LIB_DIR path. Current value is : " ${ROCR_LIB_DIR})
+   RETURN()
+ endif()
+@@ -159,7 +159,7 @@ include_directories(./ ../ ${ROCM_SMI_INC_DIR} ${ROCBLAS_INC_DIR} ${ROCR_INC_DIR
+ # Add directories to look for library files to link
+ link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCBLAS_LIB_DIR} ${ROCM_SMI_LIB_DIR} ${ASAN_LIB_PATH})
+ ## additional libraries
+-set (PROJECT_LINK_LIBS rvslibrt rvslib libpthread.so libpci.so libm.so)
++set (PROJECT_LINK_LIBS rvslibrt rvslib libpthread.so libpci.so libm.so librocm_smi64.so)
+ 
+ set(SOURCES src/rvs_module.cpp src/action.cpp src/iet_worker.cpp )
+ 
+@@ -168,7 +168,7 @@ add_library( ${RVS_TARGET} SHARED ${SOURCES})
+ set_target_properties(${RVS_TARGET} PROPERTIES
+         SUFFIX .so.${LIB_VERSION_STRING}
+         LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
+-target_link_libraries(${RVS_TARGET} ${PROJECT_LINK_LIBS} ${HIP_HCC_LIB} ${ROCBLAS_LIB} ${ROCM_SMI_LIB})
++target_link_libraries(${RVS_TARGET} ${PROJECT_LINK_LIBS} ${HIP_INC_DIR}/lib/ ${HIP_HCC_LIB} ${ROCBLAS_LIB} ${ROCM_SMI_LIB_DIR})
+ add_dependencies(${RVS_TARGET} rvslibrt rvslib)
+ 
+ add_custom_command(TARGET ${RVS_TARGET} POST_BUILD
+diff --git a/mem.so/CMakeLists.txt b/mem.so/CMakeLists.txt
+index 5a0f401..3fc4f51 100644
+--- a/mem.so/CMakeLists.txt
++++ b/mem.so/CMakeLists.txt
+@@ -134,7 +134,7 @@ if(DEFINED RVS_ROCMSMI)
+ endif()
+ 
+ 
+-if(NOT EXISTS "${ROCR_LIB_DIR}/lib${HIP_HCC_LIB}.so")
++if(NOT EXISTS "${HIP_INC_DIR}/lib/lib${HIP_HCC_LIB}.so")
+   message("ERROR: ROC Runtime libraries can't be found under specified path. Please set ROCR_LIB_DIR path. Current value is : " ${ROCR_LIB_DIR})
+   RETURN()
+ endif()
+@@ -143,7 +143,7 @@ endif()
+ include_directories(./ ../ ${ROCR_INC_DIR} ${HIP_INC_DIR})
+ 
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${HIP_INC_DIR}/lib ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS rvslibrt rvslib libpthread.so libpci.so libm.so)
+ 
+diff --git a/pbqt.so/CMakeLists.txt b/pbqt.so/CMakeLists.txt
+index d75211d..80abe22 100644
+--- a/pbqt.so/CMakeLists.txt
++++ b/pbqt.so/CMakeLists.txt
+@@ -138,7 +138,7 @@ endif()
+ ## define include directories
+ include_directories(./ ../ pci ${ROCR_INC_DIR})
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCT_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${HSAKMT_LIB_DIR} ${ROCT_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS rvslibrt rvslib libpthread.so libpci.so libm.so)
+ 
+diff --git a/pebb.so/CMakeLists.txt b/pebb.so/CMakeLists.txt
+index 7ba031c..e64be8e 100644
+--- a/pebb.so/CMakeLists.txt
++++ b/pebb.so/CMakeLists.txt
+@@ -139,7 +139,7 @@ endif()
+ ## define include directories
+ include_directories(./ ../ pci ${ROCR_INC_DIR})
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCT_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${HSAKMT_LIB_DIR} ${ROCT_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR} )
+ ## additional libraries
+ set (PROJECT_LINK_LIBS rvslibrt rvslib libpthread.so libpci.so libm.so)
+ 
+diff --git a/peqt.so/CMakeLists.txt b/peqt.so/CMakeLists.txt
+index 2248d91..7f5912d 100644
+--- a/peqt.so/CMakeLists.txt
++++ b/peqt.so/CMakeLists.txt
+@@ -107,9 +107,9 @@ else()
+ endif()
+ 
+ ## define include directories
+-include_directories(./ ../)
++include_directories(./ ../ ${HSA_PATH})
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${HSA_PATH}/lib/ ${HSAKMT_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS rvslibrt rvslib libpci.so libm.so)
+ 
+diff --git a/perf.so/CMakeLists.txt b/perf.so/CMakeLists.txt
+index b319396..b9abe15 100644
+--- a/perf.so/CMakeLists.txt
++++ b/perf.so/CMakeLists.txt
+@@ -137,7 +137,7 @@ if(DEFINED RVS_ROCMSMI)
+ endif()
+ 
+ 
+-if(NOT EXISTS "${ROCR_LIB_DIR}/lib${HIP_HCC_LIB}.so")
++if(NOT EXISTS "${HIP_INC_DIR}/lib/lib${HIP_HCC_LIB}.so")
+   message("ERROR: ROC Runtime libraries can't be found under specified path. Please set ROCR_LIB_DIR path. Current value is : " ${ROCR_LIB_DIR})
+   RETURN()
+ endif()
+@@ -145,7 +145,7 @@ endif()
+ ## define include directories
+ include_directories(./ ../ ${ROCR_INC_DIR} ${ROCBLAS_INC_DIR} ${HIP_INC_DIR})
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${HIP_INC_DIR}/lib ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS rvslibrt rvslib libpthread.so libpci.so libm.so)
+ 
+diff --git a/pesm.so/CMakeLists.txt b/pesm.so/CMakeLists.txt
+index ff60729..e7a2402 100644
+--- a/pesm.so/CMakeLists.txt
++++ b/pesm.so/CMakeLists.txt
+@@ -109,7 +109,7 @@ endif()
+ ## define include directories
+ include_directories(./ ../ pci)
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS libpthread.so libpci.so libm.so)
+ 
+diff --git a/rcqt.so/CMakeLists.txt b/rcqt.so/CMakeLists.txt
+index 32e1004..ac826ea 100644
+--- a/rcqt.so/CMakeLists.txt
++++ b/rcqt.so/CMakeLists.txt
+@@ -110,7 +110,7 @@ endif()
+ ## define include directories
+ include_directories(./ ../)
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${ASAN_LIB_PATH} ${ASAN_LIB_PATH} ${HSAKMT_LIB_DIR} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS rvslibrt rvslib)
+ 
+diff --git a/rvs/CMakeLists.txt b/rvs/CMakeLists.txt
+index b350429..c855a32 100644
+--- a/rvs/CMakeLists.txt
++++ b/rvs/CMakeLists.txt
+@@ -115,7 +115,7 @@ endif()
+ ## define include directories
+ include_directories(./ ../ ${YAML_INC_DIR} ${YAML_LIB_DIR}/include)
+ ## define lib directories
+-link_directories(${CMAKE_CURRENT_BINARY_DIR} ${RVS_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${CMAKE_CURRENT_BINARY_DIR} ${RVS_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS libdl.so "${YAML_LIB_DIR}/libyaml-cpp.a" libpthread.so)
+ 
+diff --git a/rvs/tests.cmake b/rvs/tests.cmake
+index 32301c8..a058749 100644
+--- a/rvs/tests.cmake
++++ b/rvs/tests.cmake
+@@ -179,7 +179,7 @@ add_test(NAME unit.ttf.rvs.config.noconfig
+ ## define include directories
+ include_directories(${UT_INC})
+ ## define lib directories
+-link_directories(${UT_LIB})
++link_directories(${UT_LIB} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries for unit tests
+ set (PROJECT_TEST_LINK_LIBS ${PROJECT_LINK_LIBS} libpci.so)
+ 
+diff --git a/rvslib/CMakeLists.txt b/rvslib/CMakeLists.txt
+index 31e6143..4ffed0f 100644
+--- a/rvslib/CMakeLists.txt
++++ b/rvslib/CMakeLists.txt
+@@ -115,7 +115,7 @@ endif()
+ 
+ ## define include directories
+ include_directories(./ ../
+-  ${ROCM_SMI_INC_DIR} ${ROCR_INC_DIR} ${ROCBLAS_INC_DIR} ${HIP_INC_DIR}
++  ${ROCM_SMI_INC_DIR} ${HIP_PATH} ${ROCBLAS_INC_DIR}
+ )
+ link_directories(${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ 
+diff --git a/smqt.so/CMakeLists.txt b/smqt.so/CMakeLists.txt
+index e6b8ec4..722f329 100644
+--- a/smqt.so/CMakeLists.txt
++++ b/smqt.so/CMakeLists.txt
+@@ -108,7 +108,7 @@ endif()
+ ## define include directories
+ include_directories(./ ../ pci)
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS rvslibrt rvslib libpci.so libm.so)
+ 
+diff --git a/testif.so/CMakeLists.txt b/testif.so/CMakeLists.txt
+index ed7d3d3..f09951e 100644
+--- a/testif.so/CMakeLists.txt
++++ b/testif.so/CMakeLists.txt
+@@ -110,7 +110,7 @@ endif()
+ ## define include directories
+ include_directories(./ ../ pci)
+ # Add directories to look for library files to link
+-link_directories(${RVS_LIB_DIR} ${ROCR_LIB_DIR} ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH})
++link_directories(${RVS_LIB_DIR} ${ROCBLAS_LIB_DIR} ${ASAN_LIB_PATH} ${ROCM_SMI_LIB_DIR})
+ ## additional libraries
+ set (PROJECT_LINK_LIBS libpthread.so libpci.so libm.so)
+ 
+-- 
+2.39.3
+
diff --git a/var/spack/repos/builtin/packages/rocm-validation-suite/package.py b/var/spack/repos/builtin/packages/rocm-validation-suite/package.py
index d99cb1c7e7bca2..adad90b646e628 100644
--- a/var/spack/repos/builtin/packages/rocm-validation-suite/package.py
+++ b/var/spack/repos/builtin/packages/rocm-validation-suite/package.py
@@ -16,11 +16,15 @@ class RocmValidationSuite(CMakePackage):
     compatible platform."""
 
     homepage = "https://github.com/ROCm-Developer-Tools/ROCmValidationSuite"
-    url = "https://github.com/ROCm-Developer-Tools/ROCmValidationSuite/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCm-Developer-Tools/ROCmValidationSuite/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
 
+    version("5.6.1", sha256="d5e4100e2d07311dfa101563c15d026a8130442cdee8af9ef861832cd7866c0d")
+    version("5.6.0", sha256="54cc5167055870570c97ee7114f48d24d5415f984e0c9d7b58b83467e0cf18fb")
+    version("5.5.1", sha256="0fbfaa9f68642b590ef04f9778013925bbf3f17bdcd35d4c85a8ffd091169a6e")
+    version("5.5.0", sha256="296add772171db67ab8838d2db1ea56df21e895c0348c038768e40146e4fe86a")
     version("5.4.3", sha256="1f0888e559104a4b8c2f5322f7463e425f2baaf12aeb1a8982a5974516e7b667")
     version("5.4.0", sha256="ca2abfa739c2853f71453e65787e318ab879be8a6a362c4cb4d27baa90f3cd5f")
     version("5.3.3", sha256="9acbc8de9b2e18659f51bd49f6e92ab6c93742e2ed0046322025f017fc12497f")
@@ -101,13 +105,6 @@ class RocmValidationSuite(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     patch("001-fixes-for-rocblas-rocm-smi-install-prefix-path.patch", when="@4.1.0:4.3.2")
     patch("002-remove-force-setting-hip-inc-path.patch", when="@4.1.0:4.3.2")
     patch("003-cmake-change-to-remove-installs-and-sudo.patch", when="@4.1.0:4.3.2")
@@ -116,11 +113,15 @@ class RocmValidationSuite(CMakePackage):
     patch("006-library-path.patch", when="@4.5.0:5.2")
     patch(
         "007-cleanup-path-reference-donot-download-googletest-yaml-library-path_5.3.patch",
-        when="@5.3.0:",
+        when="@5.3.0:5.5",
+    )
+    patch(
+        "007-cleanup-path-reference-donot-download-googletest-yaml-library-path_5.6.patch",
+        when="@5.6:",
     )
 
     depends_on("cmake@3.5:", type="build")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("yaml-cpp~shared")
     depends_on("googletest", when="@4.5.0:")
     depends_on("doxygen", type="build", when="@4.5.0:")
@@ -153,6 +154,10 @@ def setup_build_environment(self, build_env):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("rocminfo@" + ver, when="@" + ver)
@@ -174,14 +179,18 @@ def setup_build_environment(self, build_env):
         depends_on("hip-rocclr@" + ver, when="@" + ver)
 
     def patch(self):
-        if "@4.5.0:5.1" in self.spec:
+        if self.spec.satisfies("@4.5:5.1"):
             filter_file(
                 "@ROCM_PATH@/rvs", self.spec.prefix.rvs, "rvs/conf/deviceid.sh.in", string=True
             )
-        elif "@5.2.0:" in self.spec:
+        elif self.spec.satisfies("@5.2:5.4"):
             filter_file(
                 "@ROCM_PATH@/bin", self.spec.prefix.bin, "rvs/conf/deviceid.sh.in", string=True
             )
+        elif self.spec.satisfies("@5.5:"):
+            filter_file(
+                "@ROCM_PATH@/rvs", self.spec.prefix.rvs, "rvs/conf/deviceid.sh.in", string=True
+            )
 
     def cmake_args(self):
         args = [
diff --git a/var/spack/repos/builtin/packages/rocminfo/package.py b/var/spack/repos/builtin/packages/rocminfo/package.py
index dc731e501e601d..92fcd8c826cc3c 100644
--- a/var/spack/repos/builtin/packages/rocminfo/package.py
+++ b/var/spack/repos/builtin/packages/rocminfo/package.py
@@ -12,13 +12,16 @@ class Rocminfo(CMakePackage):
 
     homepage = "https://github.com/RadeonOpenCompute/rocminfo"
     git = "https://github.com/RadeonOpenCompute/rocminfo.git"
-    url = "https://github.com/RadeonOpenCompute/rocminfo/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/RadeonOpenCompute/rocminfo/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath", "haampie")
 
     version("master", branch="master")
-
+    version("5.6.1", sha256="780b186ac7410a503eca1060f4bbc35db1b7b4d1d714d15c7534cd26d8af7b54")
+    version("5.6.0", sha256="87d98a736e4f7510d1475d35717842068d826096a0af7c15a395bcf9d36d7fa0")
+    version("5.5.1", sha256="bcab27bb3595d5a4c981e2416458d169e85c27e603c22e743d9240473bfbe98a")
+    version("5.5.0", sha256="b6107d362b70e20a10911741eb44247139b4eb43489f7fa648daff880b6de37f")
     version("5.4.3", sha256="72159eed31f8deee0df9228b9e306a18fe9efdd4d6c0eead871cad4617874170")
     version("5.4.0", sha256="79123b92992cce75ae679caf9a6bf57b16d24e96e54b36eb002511f3800e29c6")
     version("5.3.3", sha256="77e6adc81da6c1d153517e1d28db774205531a2ec188e6518f998328ef7897c6")
@@ -99,13 +102,6 @@ class Rocminfo(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3:", type="build")
 
     for ver in [
@@ -132,10 +128,17 @@ class Rocminfo(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
         "master",
     ]:
         depends_on("hsakmt-roct@" + ver, when="@" + ver)
         depends_on("hsa-rocr-dev@" + ver, when="@" + ver)
 
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
     def cmake_args(self):
         return [self.define("ROCM_DIR", self.spec["hsa-rocr-dev"].prefix)]
diff --git a/var/spack/repos/builtin/packages/rocmlir/package.py b/var/spack/repos/builtin/packages/rocmlir/package.py
index a01392a0bebdc2..4df599d253f24e 100644
--- a/var/spack/repos/builtin/packages/rocmlir/package.py
+++ b/var/spack/repos/builtin/packages/rocmlir/package.py
@@ -14,10 +14,11 @@ class Rocmlir(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/rocMLIR"
     git = "https://github.com/ROCmSoftwarePlatform/rocMLIR.git"
-    url = "https://github.com/ROCmSoftwarePlatform/rocMLIR/archive/refs/tags/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/rocMLIR/archive/refs/tags/rocm-5.5.0.tar.gz"
 
     maintainers("srekolam")
-
+    version("5.5.1", commit="8c29325e7e68e3248e863172bf0e7f97055d45ee")
+    version("5.5.0", sha256="a5f62769d28a73e60bc8d61022820f050e97c977c8f6f6275488db31512e1f42")
     version("5.4.3", sha256="c0ba0f565e1c6614c9e6091a24cbef67b734a29e4a4ed7a8a57dc43f58ed8d53")
     version("5.4.0", sha256="3823f455ee392118c3281e27d45fa0e5381f3c4070eb4e06ba13bc6b34a90a60")
     version("5.3.0", sha256="e8471a13cb39d33adff34730d3162adaa5d20f9544d61a6a94b39b9b5762ad6d")
@@ -39,14 +40,14 @@ def patch(self):
 
     depends_on("python", type="build")
     depends_on("z3", type="link")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("ncurses+termlib", type="link")
     depends_on("bzip2")
     depends_on("sqlite")
     depends_on("half")
     depends_on("pkgconfig", type="build")
 
-    for ver in ["5.3.0", "5.4.0", "5.4.3"]:
+    for ver in ["5.3.0", "5.4.0", "5.4.3", "5.5.0", "5.5.1"]:
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("llvm-amdgpu@" + ver, when="@" + ver)
         depends_on("hsa-rocr-dev@" + ver, when="@" + ver)
diff --git a/var/spack/repos/builtin/packages/rocprim/package.py b/var/spack/repos/builtin/packages/rocprim/package.py
index 16a99d5ff708b5..5394f73958f428 100644
--- a/var/spack/repos/builtin/packages/rocprim/package.py
+++ b/var/spack/repos/builtin/packages/rocprim/package.py
@@ -11,11 +11,15 @@ class Rocprim(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/rocPRIM"
     git = "https://github.com/ROCmSoftwarePlatform/rocPRIM.git"
-    url = "https://github.com/ROCmSoftwarePlatform/rocPRIM/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/rocPRIM/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath")
 
+    version("5.6.1", sha256="e9ec1b0039c07cf3096653a04224fe5fe755afc6ba000f6838b3a8bc84df27de")
+    version("5.6.0", sha256="360d6ece3c4a3c289dd88043432026fb989e982ae4d05230d8cdc858bcd50466")
+    version("5.5.1", sha256="63cdc682afb39efd18f097faf695ce64c851c4a550a8ad96fa89d694451b6a42")
+    version("5.5.0", sha256="968d9059f93d3f0f8a602f7b989e54e36cff2f9136486b6869e4534a5bf8c7d9")
     version("5.4.3", sha256="7be6314a46195912d3203e7e59cb8880a46ed7c1fd221e92fadedd20532e0e48")
     version("5.4.0", sha256="1740dca11c70ed350995331c292f7e3cb86273614e4a5ce9f0ea64dea5364318")
     version("5.3.3", sha256="21a6b352ad3f5b2b7d05a5ed55e612feb3c5c19d34fdb8f80260b6d25af18b2d")
@@ -98,12 +102,11 @@ class Rocprim(CMakePackage):
 
     amdgpu_targets = ROCmPackage.amdgpu_targets
 
-    variant("amdgpu_target", values=auto_or_any_combination_of(*amdgpu_targets), sticky=True)
     variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
+        "amdgpu_target",
+        description="AMD GPU architecture",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
     )
 
     depends_on("cmake@3.10.2:", type="build", when="@4.2.0:")
@@ -135,6 +138,10 @@ class Rocprim(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("comgr@" + ver, when="@" + ver)
diff --git a/var/spack/repos/builtin/packages/rocprofiler-dev/package.py b/var/spack/repos/builtin/packages/rocprofiler-dev/package.py
index c2bb3854cc7a9d..7da0ef96de2469 100644
--- a/var/spack/repos/builtin/packages/rocprofiler-dev/package.py
+++ b/var/spack/repos/builtin/packages/rocprofiler-dev/package.py
@@ -18,7 +18,6 @@ class RocprofilerDev(CMakePackage):
 
     maintainers("srekolam", "renjithravindrankannath")
     libraries = ["librocprofiler64"]
-
     version("5.4.3", sha256="86c3f43ee6cb9808796a21409c853cc8fd496578b9eef4de67ca77830229cac1")
     version("5.4.0", sha256="0322cbe5d1d3182e616f472da31f0707ad6040833c38c28f2b39381a85210f43")
     version("5.3.3", sha256="07ee28f3420a07fc9d45910e78ad7961b388109cfc0e74cfdf2666789e6af171")
@@ -99,13 +98,6 @@ class RocprofilerDev(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3:", type="build")
     for ver in [
         "3.5.0",
@@ -138,7 +130,6 @@ class RocprofilerDev(CMakePackage):
         depends_on("roctracer-dev-api@" + ver, when="@" + ver)
 
     depends_on("numactl", type="link", when="@4.3.1")
-
     # See https://github.com/ROCm-Developer-Tools/rocprofiler/pull/50
     patch("fix-includes.patch")
     patch("0001-Continue-build-in-absence-of-aql-profile-lib.patch", when="@5.3:")
diff --git a/var/spack/repos/builtin/packages/rocrand/package.py b/var/spack/repos/builtin/packages/rocrand/package.py
index 3d1f0d16aa4001..eb6496d3386b3d 100644
--- a/var/spack/repos/builtin/packages/rocrand/package.py
+++ b/var/spack/repos/builtin/packages/rocrand/package.py
@@ -16,7 +16,7 @@ class Rocrand(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/rocRAND"
     git = "https://github.com/ROCmSoftwarePlatform/rocRAND.git"
-    url = "https://github.com/ROCmSoftwarePlatform/rocRAND/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/rocRAND/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath")
@@ -24,6 +24,11 @@ class Rocrand(CMakePackage):
 
     version("develop", branch="develop")
     version("master", branch="master")
+
+    version("5.6.1", sha256="6bf71e687ffa0fcc1b00e3567dd43da4147a82390f1b2db5e6f1f594dee6066d")
+    version("5.6.0", sha256="cc894d2f1af55e16b62c179062063946609c656043556189c656a115fd7d6f5f")
+    version("5.5.1", sha256="e8bed3741b19e296bd698fc55b43686206f42f4deea6ace71513e0c48258cc6e")
+    version("5.5.0", sha256="0481e7ef74c181026487a532d1c17e62dd468e508106edde0279ca1adeee6f9a")
     version("5.4.3", sha256="463aa760e9f74e45b326765040bb8a8a4fa27aaeaa5e5df16f8289125f88a619")
     version("5.4.0", sha256="0f6a0279b8b5a6dfbe32b45e1598218fe804fee36170d5c1f7b161c600544ef2")
     version("5.3.3", sha256="b0aae79dce7f6f9ef76ad2594745fe1f589a7b675b22f35b4d2369e7d5e1985a")
@@ -106,12 +111,11 @@ class Rocrand(CMakePackage):
 
     amdgpu_targets = ROCmPackage.amdgpu_targets
 
-    variant("amdgpu_target", values=auto_or_any_combination_of(*amdgpu_targets), sticky=True)
     variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
+        "amdgpu_target",
+        description="AMD GPU architecture",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
     )
     variant("hiprand", default=True, when="@5.1.0:", description="Build the hiprand library")
 
@@ -124,7 +128,9 @@ class Rocrand(CMakePackage):
     # own directory first thanks to the $ORIGIN RPATH setting. Otherwise,
     # libhiprand.so cannot find dependency librocrand.so despite being in the
     # same directory.
-    patch("hiprand_prefer_samedir_rocrand.patch", working_dir="hiprand", when="@5.2.0: +hiprand")
+    patch(
+        "hiprand_prefer_samedir_rocrand.patch", working_dir="hiprand", when="@5.2.0:5.4 +hiprand"
+    )
 
     # Add hiprand sources thru the below
     for d_version, d_commit in [
@@ -187,12 +193,16 @@ class Rocrand(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("rocm-cmake@%s:" % ver, type="build", when="@" + ver)
 
     def patch(self):
-        if self.spec.satisfies("@5.1.0: +hiprand"):
+        if self.spec.satisfies("@5.1.0:5.4 +hiprand"):
             os.rmdir("hipRAND")
             os.rename("hiprand", "hipRAND")
 
@@ -270,7 +280,9 @@ def cmake_args(self):
         if self.spec.satisfies("^cmake@3.21.0:3.21.2"):
             args.append(self.define("__skip_rocmclang", "ON"))
 
-        if self.spec.satisfies("@5.1.0:"):
+        if self.spec.satisfies("@5.1.0:5.4"):
             args.append(self.define_from_variant("BUILD_HIPRAND", "hiprand"))
+        else:
+            args.append(self.define("BUILD_HIPRAND", "OFF"))
 
         return args
diff --git a/var/spack/repos/builtin/packages/rocsolver/package.py b/var/spack/repos/builtin/packages/rocsolver/package.py
index b9becd57f5e87f..3b1cfcb51173dd 100644
--- a/var/spack/repos/builtin/packages/rocsolver/package.py
+++ b/var/spack/repos/builtin/packages/rocsolver/package.py
@@ -15,7 +15,7 @@ class Rocsolver(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/rocSOLVER"
     git = "https://github.com/ROCmSoftwarePlatform/rocSOLVER.git"
-    url = "https://github.com/ROCmSoftwarePlatform/rocSOLVER/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/rocSOLVER/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath", "haampie")
@@ -23,7 +23,12 @@ class Rocsolver(CMakePackage):
 
     amdgpu_targets = ROCmPackage.amdgpu_targets
 
-    variant("amdgpu_target", values=auto_or_any_combination_of(*amdgpu_targets), sticky=True)
+    variant(
+        "amdgpu_target",
+        description="AMD GPU architecture",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
+    )
     variant(
         "optimal",
         default=True,
@@ -34,7 +39,10 @@ class Rocsolver(CMakePackage):
 
     version("develop", branch="develop")
     version("master", branch="master")
-
+    version("5.6.1", sha256="6a8f366218aee599a0e56755030f94ee690b34f30e6d602748632226c5dc21bb")
+    version("5.6.0", sha256="54baa7f35f3c53da9005054e6f7aeecece5526dafcb277af32cbcb3996b0cbbc")
+    version("5.5.1", sha256="8bf843e42d2e89203ea5fdb6e6082cea90da8d02920ab4c09bcc2b6f69909760")
+    version("5.5.0", sha256="6775aa5b96731208c12c5b450cf218d4c262a80b7ea20c2c3034c448bb2ca4d2")
     version("5.4.3", sha256="5308b68ea72f465239a4bb2ed1a0507f0df7c98d3df3fd1f392e6d9ed7975232")
     version("5.4.0", sha256="69690839cb649dee43353b739d3e6b2312f3d965dfe66705c0ea910e57c6a8cb")
     version("5.3.3", sha256="d2248b5e2e0b20e08dd1ee5408e38deb02ecd28096dc7c7f2539351df6cb6ad5")
@@ -115,13 +123,6 @@ class Rocsolver(CMakePackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3.8:", type="build", when="@4.1.0:")
     depends_on("cmake@3.5:", type="build")
     depends_on("fmt@7:", type="build", when="@4.5.0:")
@@ -133,7 +134,7 @@ class Rocsolver(CMakePackage):
     # Backport https://github.com/ROCmSoftwarePlatform/rocSOLVER/commit/2bbfb8976f6e4d667499c77e41a6433850063e88
     patch("fmt-8.1-compatibility.patch", when="@4.5.0:5.1.3")
     # Maximize compatibility with other libraries that are using fmt.
-    patch("fmt-9-compatibility.patch", when="@5.2.0:")
+    patch("fmt-9-compatibility.patch", when="@5.2.0:5.5")
 
     def check(self):
         exe = join_path(self.build_directory, "clients", "staging", "rocsolver-test")
@@ -172,9 +173,15 @@ def check(self):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("rocblas@" + ver, when="@" + ver)
+    for ver in ["5.6.0", "5.6.1"]:
+        depends_on("rocsparse@5.2:", when="@5.6:")
 
     for tgt in itertools.chain(["auto"], amdgpu_targets):
         depends_on("rocblas amdgpu_target={0}".format(tgt), when="amdgpu_target={0}".format(tgt))
diff --git a/var/spack/repos/builtin/packages/rocsparse/package.py b/var/spack/repos/builtin/packages/rocsparse/package.py
index ec41e97d27ed4c..4fb8fb1646b4fc 100644
--- a/var/spack/repos/builtin/packages/rocsparse/package.py
+++ b/var/spack/repos/builtin/packages/rocsparse/package.py
@@ -17,7 +17,7 @@ class Rocsparse(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/rocSPARSE"
     git = "https://github.com/ROCmSoftwarePlatform/rocSPARSE.git"
-    url = "https://github.com/ROCmSoftwarePlatform/rocSPARSE/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/rocSPARSE/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath")
@@ -25,15 +25,18 @@ class Rocsparse(CMakePackage):
 
     amdgpu_targets = ROCmPackage.amdgpu_targets
 
-    variant("amdgpu_target", values=auto_or_any_combination_of(*amdgpu_targets), sticky=True)
     variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
+        "amdgpu_target",
+        description="AMD GPU architecture",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
     )
     variant("test", default=False, description="Build rocsparse-test client")
 
+    version("5.6.1", sha256="6a50a64354507f1374e1a86aa7f5c07d1aaa96ac193ac292c279153087bb5d54")
+    version("5.6.0", sha256="5797db3deb4a532e691447e3e8c923b93bd9fe4c468f3a88f00cecd80bebcae4")
+    version("5.5.1", sha256="1dd2d18898dfebdf898e8fe7d1c1198e8f8451fd70ff12a1990ec1419cf359e1")
+    version("5.5.0", sha256="cbee79b637691bc710c1c83fbaa91db7498d38d4df873be23e28ed5617acde72")
     version("5.4.3", sha256="9fb633f235eb0567cc54fae6bdc779f16bf0bb4e6f5bdddb40312c6d11ca8478")
     version("5.4.0", sha256="c8f0e920a8ec15b9ae40564c68191363356cc4d793c16247bb6e11ef5293ed11")
     version("5.3.3", sha256="4204035e952e20ada4526a94989e8e5c76c04574176fe63a021522862461c800")
@@ -140,6 +143,10 @@ class Rocsparse(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("rocprim@" + ver, when="@" + ver)
diff --git a/var/spack/repos/builtin/packages/rocthrust/package.py b/var/spack/repos/builtin/packages/rocthrust/package.py
index 522d7a2ac89e1b..196bd7eaa15408 100644
--- a/var/spack/repos/builtin/packages/rocthrust/package.py
+++ b/var/spack/repos/builtin/packages/rocthrust/package.py
@@ -14,11 +14,15 @@ class Rocthrust(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/rocThrust"
     git = "https://github.com/ROCmSoftwarePlatform/rocThrust.git"
-    url = "https://github.com/ROCmSoftwarePlatform/rocThrust/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/rocThrust/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("cgmb", "srekolam", "renjithravindrankannath")
 
+    version("5.6.1", sha256="63df61d5ab46d4cfda6066d748274bacecc77151692e372e6f7df5e91852bdc2")
+    version("5.6.0", sha256="e52a27bcb4add38a5f0f3a5c7e409c230bf4ba9afae19bd2e06c2be00d39db59")
+    version("5.5.1", sha256="66f126e5ea46ca761533411f81e83402773f95d3184cb7645ca73df227413023")
+    version("5.5.0", sha256="c031f71cd4b6eaf98664fd2ad50fc18f7ccbfa67be415dca425169d2d1c81e9e")
     version("5.4.3", sha256="d133e14ea6d27d358d1bd4d31b79fb1562d1aea7c400e5a2d28d0f159cb6c8a8")
     version("5.4.0", sha256="a4799fb1086da3f70c9b95effb1f5f9033c861685e960a8759278463cc55a971")
     version("5.3.3", sha256="0c2fc8d437efaf5c4c859d97adb049d4025025d0be0e0908f59a8112508234e5")
@@ -103,12 +107,11 @@ class Rocthrust(CMakePackage):
 
     # the rocthrust library itself is header-only, but the build_type and amdgpu_target
     # are relevant to the test client
-    variant("amdgpu_target", values=auto_or_any_combination_of(*amdgpu_targets), sticky=True)
     variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
+        "amdgpu_target",
+        description="AMD GPU architecture",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
     )
     depends_on("cmake@3.10.2:", type="build", when="@4.2.0:")
     depends_on("cmake@3.5.1:", type="build")
@@ -139,6 +142,10 @@ class Rocthrust(CMakePackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("rocprim@" + ver, when="@" + ver)
diff --git a/var/spack/repos/builtin/packages/roctracer-dev-api/package.py b/var/spack/repos/builtin/packages/roctracer-dev-api/package.py
index f396982a527bc1..c7a80816c4ed0e 100644
--- a/var/spack/repos/builtin/packages/roctracer-dev-api/package.py
+++ b/var/spack/repos/builtin/packages/roctracer-dev-api/package.py
@@ -13,11 +13,13 @@ class RoctracerDevApi(Package):
 
     homepage = "https://github.com/ROCm-Developer-Tools/roctracer"
     git = "https://github.com/ROCm-Developer-Tools/roctracer.git"
-    url = "https://github.com/ROCm-Developer-Tools/roctracer/archive/refs/tags/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCm-Developer-Tools/roctracer/archive/refs/tags/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
-
+    version("5.6.1", sha256="007c498be25b067ad9a7631a2b0892f9129150ee9714e471a921225875d45e69")
+    version("5.6.0", sha256="cbcfe4fa2e8b627006b320a93992fb3078696d8ef2ef049b4b880b6b7d57e13e")
+    version("5.5.1", sha256="3afc31ebfdb14b0365185ca6b9326a83b1503a94a51d910f5ce7ced192d8c133")
     version("5.5.0", sha256="fe9ad95628fa96639db6fc33f78d334c814c7161b4a754598f5a4a7852625777")
     version("5.4.3", sha256="6b5111be5efd4d7fd6935ca99b06fab19b43d97a58d26fc1fe6e783c4de9a926")
     version("5.4.0", sha256="04c1e955267a3e8440833a177bb976f57697aba0b90c325d07fc0c6bd4065aea")
diff --git a/var/spack/repos/builtin/packages/roctracer-dev/package.py b/var/spack/repos/builtin/packages/roctracer-dev/package.py
index 2823b81436ab57..328aa0844bfa4f 100644
--- a/var/spack/repos/builtin/packages/roctracer-dev/package.py
+++ b/var/spack/repos/builtin/packages/roctracer-dev/package.py
@@ -15,12 +15,15 @@ class RoctracerDev(CMakePackage, ROCmPackage):
 
     homepage = "https://github.com/ROCm-Developer-Tools/roctracer"
     git = "https://github.com/ROCm-Developer-Tools/roctracer.git"
-    url = "https://github.com/ROCm-Developer-Tools/roctracer/archive/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCm-Developer-Tools/roctracer/archive/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
     libraries = ["libroctracer64"]
-
+    version("5.6.1", sha256="007c498be25b067ad9a7631a2b0892f9129150ee9714e471a921225875d45e69")
+    version("5.6.0", sha256="cbcfe4fa2e8b627006b320a93992fb3078696d8ef2ef049b4b880b6b7d57e13e")
+    version("5.5.1", sha256="3afc31ebfdb14b0365185ca6b9326a83b1503a94a51d910f5ce7ced192d8c133")
+    version("5.5.0", sha256="fe9ad95628fa96639db6fc33f78d334c814c7161b4a754598f5a4a7852625777")
     version("5.4.3", sha256="6b5111be5efd4d7fd6935ca99b06fab19b43d97a58d26fc1fe6e783c4de9a926")
     version("5.4.0", sha256="04c1e955267a3e8440833a177bb976f57697aba0b90c325d07fc0c6bd4065aea")
     version("5.3.3", sha256="f2cb1e6bb69ea1a628c04f984741f781ae1d8498dc58e15795bb03015f924d13")
@@ -51,13 +54,6 @@ class RoctracerDev(CMakePackage, ROCmPackage):
         deprecated=True,
     )
 
-    variant(
-        "build_type",
-        default="Release",
-        values=("Release", "Debug", "RelWithDebInfo"),
-        description="CMake build type",
-    )
-
     depends_on("cmake@3:", type="build")
     depends_on("python@3:", type="build")
     depends_on("py-cppheaderparser", type="build")
@@ -76,14 +72,36 @@ class RoctracerDev(CMakePackage, ROCmPackage):
         "5.3.3",
         "5.4.0",
         "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
     ]:
         depends_on("hsakmt-roct@" + ver, when="@" + ver)
         depends_on("hsa-rocr-dev@" + ver, when="@" + ver)
         depends_on("rocminfo@" + ver, when="@" + ver)
         depends_on("hip@" + ver, when="@" + ver)
+    for ver in [
+        "4.5.0",
+        "4.5.2",
+        "5.0.0",
+        "5.0.2",
+        "5.1.0",
+        "5.1.3",
+        "5.2.0",
+        "5.2.1",
+        "5.2.3",
+        "5.3.0",
+        "5.3.3",
+        "5.4.0",
+        "5.4.3",
+    ]:
         depends_on("rocprofiler-dev@" + ver, when="@" + ver)
 
-    patch("0001-include-rocprofiler-dev-path.patch", when="@5.3:")
+    for ver in ["5.5.0", "5.5.1", "5.6.0", "5.6.1"]:
+        depends_on("rocm-core@" + ver, when="@" + ver)
+
+    patch("0001-include-rocprofiler-dev-path.patch", when="@5.3:5.4")
 
     @classmethod
     def determine_version(cls, lib):
@@ -116,7 +134,8 @@ def cmake_args(self):
             "-DHIP_VDI=1",
             "-DCMAKE_MODULE_PATH={0}/cmake_modules".format(self.stage.source_path),
             "-DHSA_RUNTIME_HSA_INC_PATH={0}/include".format(self.spec["hsa-rocr-dev"].prefix),
-            "-DROCPROFILER_PATH={0}".format(self.spec["rocprofiler-dev"].prefix),
             "-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON",
         ]
+        if self.spec.satisfies("@:5.4.0"):
+            "-DROCPROFILER_PATH={0}".format(self.spec["rocprofiler-dev"].prefix)
         return args
diff --git a/var/spack/repos/builtin/packages/rocwmma/0001-add-rocm-smi-lib-path-for-building-tests.patch b/var/spack/repos/builtin/packages/rocwmma/0001-add-rocm-smi-lib-path-for-building-tests.patch
new file mode 100644
index 00000000000000..cfa3cb4180c722
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rocwmma/0001-add-rocm-smi-lib-path-for-building-tests.patch
@@ -0,0 +1,31 @@
+From 099ac638f41d9224f649fe23a64783bb408a2b09 Mon Sep 17 00:00:00 2001
+From: Sreenivasa Murthy Kolam 
+Date: Wed, 30 Aug 2023 09:41:15 +0000
+Subject: [PATCH] add rocm-smi-lib path for building tests
+
+---
+ test/CMakeLists.txt | 5 +++--
+ 1 file changed, 3 insertions(+), 2 deletions(-)
+
+diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
+index 85f98d0..269f517 100644
+--- a/test/CMakeLists.txt
++++ b/test/CMakeLists.txt
+@@ -69,11 +69,12 @@ function(add_rocwmma_test TEST_TARGET TEST_SOURCE)
+ 
+   list(APPEND TEST_SOURCE ${ARGN})
+   add_executable(${TEST_TARGET} ${TEST_SOURCE})
+-  target_link_libraries(${TEST_TARGET} rocwmma gtest)
++  target_link_libraries(${TEST_TARGET} rocwmma gtest ${ROCM_SMI_DIR}/lib)
+   target_link_libraries(${TEST_TARGET} OpenMP::OpenMP_CXX "-L${HIP_CLANG_ROOT}/lib" "-Wl,-rpath=${HIP_CLANG_ROOT}/lib")
+   target_include_directories(${TEST_TARGET} PRIVATE
+                              ${CMAKE_CURRENT_SOURCE_DIR}
+-                             ${ROCWMMA_TEST_INCLUDE_DIRS})
++                             ${ROCWMMA_TEST_INCLUDE_DIRS}
++			     ${ROCM_SMI_DIR}/include)
+ 
+   # Add support to include extended test coverage
+   if(ROCWMMA_BUILD_EXTENDED_TESTS)
+-- 
+2.39.3
+
diff --git a/var/spack/repos/builtin/packages/rocwmma/package.py b/var/spack/repos/builtin/packages/rocwmma/package.py
index 229f6a4add436a..96978f7862ba70 100644
--- a/var/spack/repos/builtin/packages/rocwmma/package.py
+++ b/var/spack/repos/builtin/packages/rocwmma/package.py
@@ -21,11 +21,14 @@ class Rocwmma(CMakePackage):
 
     homepage = "https://github.com/ROCmSoftwarePlatform/rocWMMA"
     git = "https://github.com/ROCmSoftwarePlatform/rocWMMA.git"
-    url = "https://github.com/ROCmSoftwarePlatform/rocWMMA/archive/refs/tags/rocm-5.4.3.tar.gz"
+    url = "https://github.com/ROCmSoftwarePlatform/rocWMMA/archive/refs/tags/rocm-5.5.0.tar.gz"
     tags = ["rocm"]
 
     maintainers("srekolam", "renjithravindrankannath")
-
+    version("5.6.1", sha256="41a5159ee1ad5fc411fe6220f37bd754e26d3883c24c0f2378f50ef628bc1b8f")
+    version("5.6.0", sha256="78b6ab10fce71d10a9d762b2eaab3390eb13b05c764f47a3b0a303ec3d37acf8")
+    version("5.5.1", sha256="ada30d5e52df5da0d3f4e212a25efb492dbedc129628f4db4ef4ed77667da228")
+    version("5.5.0", sha256="b9e1938cba111eeea295414c42de34d54a878f0d41a26e433809d60c12d31dbf")
     version("5.4.3", sha256="0968366c83b78a9d058d483be536aba03e79b300ccb6890d3da43298be54c288")
     version("5.4.0", sha256="a18724c3b45d171e54ef9f85c269124ce8d29b6a2f9dbd76a4806bda2933f7a7")
     version("5.3.3", sha256="cd9bc09f98fb78e53ba4bde1dcfe1817c34c2822234a82b1128d36d92b97ae79")
@@ -39,7 +42,12 @@ class Rocwmma(CMakePackage):
     # releases
 
     amdgpu_targets = ("gfx908:xnack-", "gfx90a", "gfx90a:xnack-", "gfx90a:xnack+")
-    variant("amdgpu_target", values=auto_or_any_combination_of(*amdgpu_targets), sticky=True)
+    variant(
+        "amdgpu_target",
+        description="AMD GPU architecture",
+        values=auto_or_any_combination_of(*amdgpu_targets),
+        sticky=True,
+    )
     variant(
         "build_type",
         default="Release",
@@ -52,16 +60,33 @@ class Rocwmma(CMakePackage):
 
     depends_on("googletest@1.10.0:", type="test")
 
-    for ver in ["5.2.0", "5.2.1", "5.2.3", "5.3.0", "5.3.3", "5.4.0", "5.4.3"]:
+    for ver in [
+        "5.2.0",
+        "5.2.1",
+        "5.2.3",
+        "5.3.0",
+        "5.3.3",
+        "5.4.0",
+        "5.4.3",
+        "5.5.0",
+        "5.5.1",
+        "5.6.0",
+        "5.6.1",
+    ]:
         depends_on("rocm-cmake@%s:" % ver, type="build", when="@" + ver)
         depends_on("llvm-amdgpu@" + ver, type="build", when="@" + ver)
         depends_on("hip@" + ver, when="@" + ver)
         depends_on("rocblas@" + ver, type="build", when="@" + ver)
         depends_on("rocm-openmp-extras@" + ver, type="build", when="@" + ver)
 
+    for ver in ["5.6.0", "5.6.1"]:
+        depends_on("rocm-smi-lib@" + ver, when="@" + ver)
+
     for tgt in itertools.chain(["auto"], amdgpu_targets):
         depends_on("rocblas amdgpu_target={0}".format(tgt), when="amdgpu_target={0}".format(tgt))
 
+    patch("0001-add-rocm-smi-lib-path-for-building-tests.patch", when="@5.6:")
+
     def setup_build_environment(self, env):
         env.set("CXX", self.spec["hip"].hipcc)
 
@@ -86,5 +111,7 @@ def cmake_args(self):
         tgt = self.spec.variants["amdgpu_target"]
         if "auto" not in tgt:
             args.append(self.define_from_variant("AMDGPU_TARGETS", "amdgpu_target"))
+        if self.spec.satisfies("@5.6.0:"):
+            args.append(self.define("ROCM_SMI_DIR", self.spec["rocm-smi-lib"].prefix))
 
         return args
diff --git a/var/spack/repos/builtin/packages/roms/package.py b/var/spack/repos/builtin/packages/roms/package.py
index cd3cd7bd81570b..5590c22c036c9a 100644
--- a/var/spack/repos/builtin/packages/roms/package.py
+++ b/var/spack/repos/builtin/packages/roms/package.py
@@ -44,7 +44,7 @@ class Roms(MakefilePackage):
     depends_on("netcdf-fortran")
     depends_on("netcdf-c")
     depends_on("hdf5+fortran")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("curl")
     depends_on("amdlibm", when="%aocc")
 
diff --git a/var/spack/repos/builtin/packages/root/package.py b/var/spack/repos/builtin/packages/root/package.py
index 347eb46d0fa7a4..6a6484b4b2e25c 100644
--- a/var/spack/repos/builtin/packages/root/package.py
+++ b/var/spack/repos/builtin/packages/root/package.py
@@ -34,6 +34,7 @@ class Root(CMakePackage):
     # Development version (when more recent than production).
 
     # Production version
+    version("6.28.06", sha256="af3b673b9aca393a5c9ae1bf86eab2672aaf1841b658c5c6e7a30ab93c586533")
     version("6.28.04", sha256="70f7f86a0cd5e3f2a0befdc59942dd50140d990ab264e8e56c7f17f6bfe9c965")
     version("6.28.02", sha256="6643c07710e68972b00227c68b20b1016fec16f3fba5f44a571fa1ce5bb42faa")
     version("6.28.00", sha256="afa1c5c06d0915411cb9492e474ea9ab12b09961a358e7e559013ed63b5d8084")
@@ -157,7 +158,7 @@ class Root(CMakePackage):
         default=False,
         description="Enable support for TMultilayerPerceptron " "classes' federation",
     )
-    variant("mysql", default=False)
+    variant("mysql", default=False, description="Enable support for MySQL databases")
     variant("opengl", default=True, description="Enable OpenGL support")
     variant("oracle", default=False, description="Enable support for Oracle databases")
     variant("postgres", default=False, description="Enable postgres support")
@@ -223,7 +224,7 @@ class Root(CMakePackage):
     depends_on("pcre")
     depends_on("xxhash", when="@6.13.02:")  # See cmake_args, below.
     depends_on("xz")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("zstd", when="@6.20:")
 
     # X-Graphics
@@ -277,11 +278,13 @@ class Root(CMakePackage):
     depends_on("sqlite", when="+sqlite")
     depends_on("tbb", when="+tbb")
     # See: https://github.com/root-project/root/issues/6933
-    conflicts("^intel-tbb@2021.1:", when="@:6.22", msg="Please use an older intel-tbb version")
+    conflicts(
+        "^intel-tbb@2021.1:", when="@:6.22", msg="Please use an older intel-tbb version for ROOT"
+    )
     conflicts(
         "^intel-oneapi-tbb@2021.1:",
         when="@:6.22",
-        msg="Please use an older intel-tbb/intel-oneapi-tbb version",
+        msg="Please use an older intel-tbb/intel-oneapi-tbb version for ROOT",
     )
     # depends_on('intel-tbb@:2021.0', when='@:6.22 ^intel-tbb')
     depends_on("unuran", when="+unuran")
@@ -311,20 +314,22 @@ class Root(CMakePackage):
 
     # Incompatible variants
     if sys.platform != "darwin":
-        conflicts("+opengl", when="~x", msg="OpenGL requires X")
-    conflicts("+math", when="~gsl", msg="Math requires GSL")
-    conflicts("+tmva", when="~gsl", msg="TVMA requires GSL")
-    conflicts("+tmva", when="~mlp", msg="TVMA requires MLP")
+        conflicts("+opengl", when="~x", msg="root+opengl requires X")
+    conflicts("+math", when="~gsl", msg="root+math requires GSL")
+    conflicts("+tmva", when="~gsl", msg="root+tmva requires GSL")
+    conflicts("+tmva", when="~mlp", msg="root+tmva requires MLP")
     conflicts("cxxstd=11", when="+root7", msg="root7 requires at least C++14")
     conflicts("cxxstd=11", when="@6.25.02:", msg="This version of root requires at least C++14")
-    conflicts("cxxstd=20", when="@:6.28.02", msg="C++20 support requires at least version 6.28.04")
+    conflicts(
+        "cxxstd=20", when="@:6.28.02", msg="C++20 support requires root version at least 6.28.04"
+    )
 
     # See https://github.com/root-project/root/issues/11128
-    conflicts("%clang@16:", when="@:6.26.07", msg="clang 16+ support was added in 6.26.08")
+    conflicts("%clang@16:", when="@:6.26.07", msg="clang 16+ support was added in root 6.26.08")
 
     # See https://github.com/root-project/root/issues/11714
     if sys.platform == "darwin" and macos_version() >= Version("13"):
-        conflicts("@:6.26.09", msg="macOS 13 support was added in 6.26.10")
+        conflicts("@:6.26.09", msg="macOS 13 support was added in root 6.26.10")
 
     # ROOT <6.14 is incompatible with Python >=3.7, which is the minimum supported by spack
     conflicts("+python", when="@:6.13", msg="Spack wants python >=3.7, too new for ROOT <6.14")
@@ -629,7 +634,7 @@ def add_include_path(dep_name):
 
         # With that done, let's go fixing those deps
         if spec.satisfies("@:6.12"):
-            add_include_path("zlib")
+            add_include_path("zlib-api")
         if "+x" in spec:
             if spec.satisfies("@:6.08") or spec.satisfies("@6.22:"):
                 add_include_path("xextproto")
diff --git a/var/spack/repos/builtin/packages/routino/package.py b/var/spack/repos/builtin/packages/routino/package.py
index 31f14e3c6d4b64..00326fdc94ef2e 100644
--- a/var/spack/repos/builtin/packages/routino/package.py
+++ b/var/spack/repos/builtin/packages/routino/package.py
@@ -17,7 +17,7 @@ class Routino(MakefilePackage):
     version("3.3.3", sha256="abd82b77c314048f45030f7219887ca241b46d40641db6ccb462202b97a047f5")
     version("3.2", sha256="e2a431eaffbafab630835966d342e4ae25d5edb94c8ed419200e1ffb50bc7552")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2")
 
     def edit(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/rpm/package.py b/var/spack/repos/builtin/packages/rpm/package.py
index 67fa90cd5e44ca..66bce60f177a6c 100644
--- a/var/spack/repos/builtin/packages/rpm/package.py
+++ b/var/spack/repos/builtin/packages/rpm/package.py
@@ -86,7 +86,7 @@ class Rpm(AutotoolsPackage):
     # compression support -- there is no configure option for many of these
     # and they autodetect the libraries, so it's better to just make them
     # hard requirements to avoid linking against system libraries.
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2")
     depends_on("gzip")
     depends_on("xz")
diff --git a/var/spack/repos/builtin/packages/rpp/0001-include-half-openmp-through-spack-package.patch b/var/spack/repos/builtin/packages/rpp/0001-include-half-openmp-through-spack-package.patch
new file mode 100644
index 00000000000000..5ef77a80932d96
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rpp/0001-include-half-openmp-through-spack-package.patch
@@ -0,0 +1,80 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index ae50d00..c2deefc 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -129,7 +129,9 @@ include_directories(${Boost_INCLUDE_DIRS})
+ set(LINK_LIBRARY_LIST ${LINK_LIBRARY_LIST} ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY})
+
+ # OpenMP
+-find_package(OpenMP REQUIRED)
++find_path(HALF_INCLUDE_DIR half.hpp)
++message(STATUS "HALF_INCLUDE_DIR: ${HALF_INCLUDE_DIR}")
++
+ if(APPLE)
+     if(CMAKE_C_COMPILER_ID MATCHES "Clang")
+         set(OpenMP_C "${CMAKE_C_COMPILER}")
+@@ -151,7 +153,7 @@ else()
+     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+ endif()
+-set(LINK_LIBRARY_LIST ${LINK_LIBRARY_LIST} OpenMP::OpenMP_CXX)
++ set(LINK_LIBRARY_LIST ${LINK_LIBRARY_LIST} ${ROCM_OPENMP_EXTRAS_DIR}/lib/libomp.so)
+
+ # Threads
+ set(THREADS_PREFER_PTHREAD_FLAG ON)
+@@ -269,6 +271,8 @@ target_include_directories(${PROJECT_NAME}
+     PUBLIC
+         ${CMAKE_CURRENT_SOURCE_DIR}/include
+         ${ROCM_PATH}/include
++        ${HALF_INCLUDE_DIR}
++        ${ROCM_OPENMP_EXTRAS_DIR}/include
+     PRIVATE
+         ${CMAKE_CURRENT_SOURCE_DIR}/src/include/cpu
+         ${CMAKE_CURRENT_SOURCE_DIR}/src/include/common
+diff --git a/src/modules/CMakeLists.txt b/src/modules/CMakeLists.txt
+index f133128..65cb6d5 100644
+--- a/src/modules/CMakeLists.txt
++++ b/src/modules/CMakeLists.txt
+@@ -81,6 +81,8 @@ if("${TIME_INFO}" STREQUAL "1")
+ endif()
+
+ # Backend specific settings
++find_path(HALF_INCLUDE_DIR half.hpp)
++message(STATUS "HALF_INCLUDE_DIR: ${HALF_INCLUDE_DIR}")
+
+ if( "${BACKEND}" STREQUAL "HIP")
+     # Add HIP kernels
+@@ -99,7 +101,7 @@ if( "${BACKEND}" STREQUAL "HIP")
+     # Add HIP specific includes
+     set(ROCM_INC ${ROCM_PATH}/include/)
+     list(APPEND HIP_LOCAL_INCLUDE_DIRS ${CMAKE_SOURCE_DIR}/src/include/hip/ ${CMAKE_SOURCE_DIR}/src/include/common/)
+-    set(INCLUDE_LIST ${ROCM_INC} ${HIP_LOCAL_INCLUDE_DIRS} ${INCLUDE_LIST})
++    set(INCLUDE_LIST ${ROCM_INC} ${HIP_LOCAL_INCLUDE_DIRS} ${INCLUDE_LIST} ${HALF_INCLUDE_DIR} ${ROCM_OPENMP_EXTRAS_DIR}/include)
+ elseif( "${BACKEND}" STREQUAL "OCL")
+     # Add OpenCL kernels
+     file(GLOB MOD_CL_CPP "cl/*.cpp" )
+@@ -114,7 +116,7 @@ elseif( "${BACKEND}" STREQUAL "OCL")
+     # Add OpenCL specific includes
+     set(ROCM_INC ${ROCM_PATH}/include/)
+     list(APPEND OCL_LOCAL_INCLUDE_LIST ${CMAKE_SOURCE_DIR}/src/include/cl/ ${CMAKE_SOURCE_DIR}/src/include/common/)
+-    set(INCLUDE_LIST ${ROCM_INC} ${OCL_LOCAL_INCLUDE_LIST} ${INCLUDE_LIST})
++    set(INCLUDE_LIST ${ROCM_INC} ${OCL_LOCAL_INCLUDE_LIST} ${INCLUDE_LIST} ${HALF_INCLUDE_DIR} ${ROCM_OPENMP_EXTRAS_DIR}/include)
+ elseif( "${BACKEND}" STREQUAL "CPU")
+     # Add CPU specific includes
+     set(INCLUDE_LIST ${CMAKE_SOURCE_DIR}/src/include/common/)
+@@ -134,6 +136,8 @@ target_include_directories( ${PROJECT_NAME}
+     PUBLIC
+         ${CMAKE_SOURCE_DIR}/include
+         ${ROCM_INC}
++        ${HALF_INCLUDE_DIR}
++        ${ROCM_OPENMP_EXTRAS_DIR}/include
+     PRIVATE
+         ${CMAKE_SOURCE_DIR}/src/include/cpu
+         ${CMAKE_SOURCE_DIR}/src/include/common
+@@ -152,4 +156,4 @@ elseif( "${BACKEND}" STREQUAL "OCL")
+     PRIVATE
+         ${CMAKE_SOURCE_DIR}/src/include/cl
+     )
+-endif()
+\ No newline at end of file
++endif()
diff --git a/var/spack/repos/builtin/packages/rpp/0002-declare-handle-in-header.patch b/var/spack/repos/builtin/packages/rpp/0002-declare-handle-in-header.patch
new file mode 100644
index 00000000000000..fb409a9951f8ea
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rpp/0002-declare-handle-in-header.patch
@@ -0,0 +1,12 @@
+diff --git a/src/include/common/rpp/handle.hpp b/src/include/common/rpp/handle.hpp
+index 7dc29e4..6e5ccea 100644
+--- a/src/include/common/rpp/handle.hpp
++++ b/src/include/common/rpp/handle.hpp
+@@ -99,6 +99,7 @@ struct Handle : rppHandle
+     // Device handle related
+     Handle(rppAcceleratorQueue_t stream);
+     Handle(rppAcceleratorQueue_t stream, size_t nBatchSize);
++    Handle(size_t nBatchSize);
+     void rpp_destroy_object_gpu();
+     rppAcceleratorQueue_t GetStream() const;
+     void SetStream(rppAcceleratorQueue_t streamID) const;
diff --git a/var/spack/repos/builtin/packages/rpp/package.py b/var/spack/repos/builtin/packages/rpp/package.py
new file mode 100644
index 00000000000000..bb30bca21b6ade
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rpp/package.py
@@ -0,0 +1,95 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+from spack.pkg.builtin.boost import Boost
+
+
+class Rpp(CMakePackage):
+    """Radeon Performance Primitives (RPP) library is a comprehensive high-
+    performance computer vision library for AMD (CPU and GPU) with HIP
+    and OPENCL back-ends"""
+
+    homepage = "https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp"
+    git = "https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git"
+    url = "https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp/archive/0.97.tar.gz"
+
+    maintainers = ["srekolam", "afzpatel"]
+    tags = ["rocm"]
+
+    version("1.1.0", sha256="9b1b9e721df27ee577819710b261071c68b2dccba96d9daf5d0535ee5f0e045f")
+    version("1.0.0", sha256="040601e356b0a06c4ffb2043320ae822ab0da78af867392002c7b68dbd85989c")
+    version("0.99", sha256="f1d7ec65d0148ddb7b3ce836a7e058727036df940d72d1683dee590a913fd44a")
+    version("0.98", sha256="191b5d89bf990ae22b5ef73675b89ed4371c3ce342ab9cc65383fa12ef13086e")
+    version("0.97", sha256="8ce1a869ff67a29579d87d399d8b0bd97bf12ae1b6b1ca1f161cb8a262fb9939")
+    variant(
+        "build_type",
+        default="Release",
+        values=("Release", "Debug", "RelWithDebInfo"),
+        description="CMake build type",
+    )
+    # Adding 3 variants OPENCL ,HIP , CPU with HIP as default.
+
+    variant("opencl", default=False, description="Use OPENCL as the backend")
+    variant("hip", default=True, description="Use HIP as backend")
+    variant("cpu", default=False, description="Use CPU as backend")
+
+    patch("0001-include-half-openmp-through-spack-package.patch")
+    patch("0002-declare-handle-in-header.patch")
+
+    def patch(self):
+        if self.spec.satisfies("+hip"):
+            filter_file(
+                "${ROCM_PATH}/llvm", self.spec["llvm-amdgpu"].prefix, "CMakeLists.txt", string=True
+            )
+        if self.spec.satisfies("+opencl"):
+            filter_file(
+                "${ROCM_PATH}",
+                self.spec["rocm-opencl"].prefix,
+                "cmake/FindOpenCL.cmake",
+                string=True,
+            )
+
+    depends_on("cmake@3.5:", type="build")
+    depends_on("pkgconfig", type="build")
+    depends_on(Boost.with_default_variants)
+    depends_on("boost@1.72.0:1.80.0")
+    depends_on("bzip2")
+    depends_on("half")
+    depends_on("hwloc")
+    depends_on(
+        "opencv@4.5:"
+        "+calib3d+features2d+highgui+imgcodecs+imgproc"
+        "+video+videoio+flann+photo+objdetect",
+        type="build",
+        when="@1.0:",
+    )
+    depends_on("libjpeg-turbo", type="build")
+    depends_on("rocm-openmp-extras")
+    conflicts("+opencl+hip")
+
+    with when("+hip"):
+        depends_on("hip@5:")
+    with when("~hip"):
+        depends_on("rocm-opencl@5:")
+
+    def cmake_args(self):
+        spec = self.spec
+        args = []
+        args.append(self.define("ROCM_OPENMP_EXTRAS_DIR", spec["rocm-openmp-extras"].prefix))
+        if self.spec.satisfies("+opencl"):
+            args.append(self.define("BACKEND", "OPENCL"))
+        if self.spec.satisfies("+cpu"):
+            args.append(self.define("BACKEND", "CPU"))
+        if self.spec.satisfies("+hip"):
+            args.append(self.define("BACKEND", "HIP"))
+            args.append(self.define("HIP_PATH", spec["hip"].prefix))
+            args.append(
+                self.define(
+                    "COMPILER_FOR_HIP", "{0}/bin/clang++".format(spec["llvm-amdgpu"].prefix)
+                )
+            )
+        return args
diff --git a/var/spack/repos/builtin/packages/rr/package.py b/var/spack/repos/builtin/packages/rr/package.py
index d0f3025f91d9a7..a4fd6c6f8979d5 100644
--- a/var/spack/repos/builtin/packages/rr/package.py
+++ b/var/spack/repos/builtin/packages/rr/package.py
@@ -18,7 +18,7 @@ class Rr(CMakePackage):
 
     depends_on("gdb")
     depends_on("git")
-    depends_on("zlib")
+    depends_on("zlib-api")
     # depends_on('capnproto', when='@4.6:')  # not yet in spack
     # depends_on('libcapnp')    # needed for future releases
     depends_on("pkgconfig", type="build")
diff --git a/var/spack/repos/builtin/packages/rsl/package.py b/var/spack/repos/builtin/packages/rsl/package.py
index 451a56354fd97f..f2339360c76da5 100644
--- a/var/spack/repos/builtin/packages/rsl/package.py
+++ b/var/spack/repos/builtin/packages/rsl/package.py
@@ -17,7 +17,7 @@ class Rsl(AutotoolsPackage):
 
     depends_on("bzip2")
     depends_on("jpeg")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("rpc")
 
     def configure_args(self):
diff --git a/var/spack/repos/builtin/packages/rsync/package.py b/var/spack/repos/builtin/packages/rsync/package.py
index 212e36571c55b0..5b5f7ceeb12386 100644
--- a/var/spack/repos/builtin/packages/rsync/package.py
+++ b/var/spack/repos/builtin/packages/rsync/package.py
@@ -24,7 +24,7 @@ class Rsync(AutotoolsPackage):
     version("3.1.2", sha256="ecfa62a7fa3c4c18b9eccd8c16eaddee4bd308a76ea50b5c02a5840f09c0a1c2")
     version("3.1.1", sha256="7de4364fcf5fe42f3bdb514417f1c40d10bbca896abe7e7f2c581c6ea08a2621")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("popt")
     depends_on("openssl", when="@3.2:")
     depends_on("xxhash", when="@3.2:")
diff --git a/var/spack/repos/builtin/packages/rsyslog/package.py b/var/spack/repos/builtin/packages/rsyslog/package.py
index b5f1f80af254df..73ec7dc8ca48ef 100644
--- a/var/spack/repos/builtin/packages/rsyslog/package.py
+++ b/var/spack/repos/builtin/packages/rsyslog/package.py
@@ -22,7 +22,7 @@ class Rsyslog(AutotoolsPackage):
     depends_on("m4", type="build")
     depends_on("libestr")
     depends_on("libfastjson")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("uuid")
     depends_on("libgcrypt")
     depends_on("curl")
diff --git a/var/spack/repos/builtin/packages/rtags/package.py b/var/spack/repos/builtin/packages/rtags/package.py
index 5aa38a362a9fe1..88e3cc8e6f1f98 100644
--- a/var/spack/repos/builtin/packages/rtags/package.py
+++ b/var/spack/repos/builtin/packages/rtags/package.py
@@ -18,7 +18,7 @@ class Rtags(CMakePackage):
     version("2.17", sha256="288fa49fedf647fb15e2ef10f0ebcd9de1a4ef1bbae3a3940870e136d32a3a60")
 
     depends_on("llvm@3.3: +clang")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("openssl")
     depends_on("lua@5.3:")
     depends_on("bash-completion")
diff --git a/var/spack/repos/builtin/packages/rtmpdump/missing-include.patch b/var/spack/repos/builtin/packages/rtmpdump/missing-include.patch
new file mode 100644
index 00000000000000..4325ed07381f54
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/missing-include.patch
@@ -0,0 +1,23 @@
+https://bugs.gentoo.org/828082
+--- a/librtmp/rtmp.c
++++ b/librtmp/rtmp.c
+@@ -28,6 +28,7 @@
+ #include 
+ #include 
+ #include 
++#include 
+ 
+ #include "rtmp_sys.h"
+ #include "log.h"
+diff --git a/librtmp/hashswf.c b/librtmp/hashswf.c
+index 32b2eed..e3669e3 100644
+--- a/librtmp/hashswf.c
++++ b/librtmp/hashswf.c
+@@ -25,6 +25,7 @@
+ #include 
+ #include 
+ #include 
++#include 
+ 
+ #include "rtmp_sys.h"
+ #include "log.h"
diff --git a/var/spack/repos/builtin/packages/rtmpdump/package.py b/var/spack/repos/builtin/packages/rtmpdump/package.py
new file mode 100644
index 00000000000000..a868e6e3d0d8c9
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/package.py
@@ -0,0 +1,38 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Rtmpdump(MakefilePackage):
+    """rtmpdump is a toolkit for RTMP streams."""
+
+    homepage = "https://rtmpdump.mplayerhq.hu/"
+    git = "https://git.ffmpeg.org/rtmpdump.git"
+
+    maintainers("tobbez")
+
+    license("GPL-2.0-or-later")
+
+    version("2021-02-19", commit="f1b83c10d8beb43fcc70a6e88cf4325499f25857")
+
+    variant("tls", default="openssl", description="TLS backend", values=("gnutls", "openssl"))
+
+    depends_on("openssl@:3", when="tls=openssl")
+    depends_on("gnutls", when="tls=gnutls")
+    depends_on("zlib-api")
+
+    patch("missing-include.patch")
+    patch("rtmpdump-fix-chunk-size.patch")
+    patch("rtmpdump-openssl-1.1-v2.patch")
+    patch("rtmpdump-swf_vertification_type_2.patch")
+    patch("rtmpdump-swf_vertification_type_2_part_2.patch")
+
+    @property
+    def build_targets(self):
+        return [f"CRYPTO={self.spec.variants['tls'].value.upper()}"]
+
+    def install(self, spec, prefix):
+        make("install", f"prefix={prefix}", "sbindir=$(bindir)")
diff --git a/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-fix-chunk-size.patch b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-fix-chunk-size.patch
new file mode 100644
index 00000000000000..1c6cfdc6261075
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-fix-chunk-size.patch
@@ -0,0 +1,48 @@
+https://git.alpinelinux.org/aports/commit/main/rtmpdump/fix-chunk-size.patch?id=bf39fb1177ee77eee6c214a7393cc0054958ce08
+https://git.alpinelinux.org/aports/commit/main/rtmpdump/fix-chunk-size.patch?id=69bc162319b12e9b6c6d3ea345dbf7c218753594
+diff --git a/librtmp/rtmp.c b/librtmp/rtmp.c
+index a2863b0..ac1b3be 100644
+--- a/librtmp/rtmp.c
++++ b/librtmp/rtmp.c
+@@ -2077,6 +2077,29 @@ RTMP_SendClientBW(RTMP *r)
+ }
+ 
+ static int
++SendClientChunkSize(RTMP *r, int chunkSize)
++{
++  RTMPPacket packet;
++  char pbuf[256], *pend = pbuf + sizeof(pbuf);
++  int ret;
++
++  packet.m_nChannel = 0x02;	/* control channel (invoke) */
++  packet.m_headerType = RTMP_PACKET_SIZE_LARGE;
++  packet.m_packetType = RTMP_PACKET_TYPE_CHUNK_SIZE;
++  packet.m_nTimeStamp = 0;
++  packet.m_nInfoField2 = 0;
++  packet.m_hasAbsTimestamp = 0;
++  packet.m_body = pbuf + RTMP_MAX_HEADER_SIZE;
++
++  packet.m_nBodySize = 4;
++
++  AMF_EncodeInt32(packet.m_body, pend, chunkSize);
++  ret = RTMP_SendPacket(r, &packet, FALSE);
++  r->m_outChunkSize = chunkSize;
++  return ret;
++}
++
++static int
+ SendBytesReceived(RTMP *r)
+ {
+   RTMPPacket packet;
+@@ -3349,6 +3372,11 @@ HandleChangeChunkSize(RTMP *r, const RTMPPacket *packet)
+       r->m_inChunkSize = AMF_DecodeInt32(packet->m_body);
+       RTMP_Log(RTMP_LOGDEBUG, "%s, received: chunk size change to %d", __FUNCTION__,
+ 	  r->m_inChunkSize);
++      if (r->Link.protocol & RTMP_FEATURE_WRITE)
++        {
++          RTMP_Log(RTMP_LOGDEBUG, "%s, updating outChunkSize too", __FUNCTION__);
++          SendClientChunkSize(r, r->m_inChunkSize);
++        }
+     }
+ }
+ 
diff --git a/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-openssl-1.1-v2.patch b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-openssl-1.1-v2.patch
new file mode 100644
index 00000000000000..146243bd111188
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-openssl-1.1-v2.patch
@@ -0,0 +1,248 @@
+https://raw.githubusercontent.com/xbmc/inputstream.rtmp/master/depends/common/librtmp/0003-openssl-1.1.patch
+See also https://github.com/xbmc/inputstream.rtmp/pull/46
+--- a/librtmp/dh.h
++++ b/librtmp/dh.h
+@@ -253,20 +253,42 @@
+   if (!dh)
+     goto failed;
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_new(dh->g);
+ 
+   if (!dh->g)
+     goto failed;
++#else
++  BIGNUM *g = NULL;
++  MP_new(g);
++  if (!g)
++    goto failed;
++#endif
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_gethex(dh->p, P1024, res);	/* prime P1024, see dhgroups.h */
++#else
++  BIGNUM* p = NULL;
++  DH_get0_pqg(dh, (BIGNUM const**)&p, NULL, NULL);
++  MP_gethex(p, P1024, res); /* prime P1024, see dhgroups.h */
++#endif
+   if (!res)
+     {
+       goto failed;
+     }
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_set_w(dh->g, 2);	/* base 2 */
++#else
++  MP_set_w(g, 2);   /* base 2 */
++  DH_set0_pqg(dh, p, NULL, g);
++#endif
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   dh->length = nKeyBits;
++#else
++  DH_set_length(dh, nKeyBits);
++#endif
+   return dh;
+ 
+ failed:
+@@ -293,12 +315,24 @@
+       MP_gethex(q1, Q1024, res);
+       assert(res);
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+       res = isValidPublicKey(dh->pub_key, dh->p, q1);
++#else
++      BIGNUM const* pub_key = NULL;
++      BIGNUM const* p = NULL;
++      DH_get0_key(dh, &pub_key, NULL);
++      DH_get0_pqg(dh, &p, NULL, NULL);
++      res = isValidPublicKey((BIGNUM*)pub_key, (BIGNUM*)p, q1);
++#endif
+       if (!res)
+ 	{
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+ 	  MP_free(dh->pub_key);
+ 	  MP_free(dh->priv_key);
+ 	  dh->pub_key = dh->priv_key = 0;
++#else
++          DH_free(dh);
++#endif
+ 	}
+ 
+       MP_free(q1);
+@@ -314,15 +348,29 @@
+ DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen)
+ {
+   int len;
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   if (!dh || !dh->pub_key)
++#else
++  BIGNUM const* pub_key = NULL;
++  DH_get0_key(dh, &pub_key, NULL);
++  if (!dh || !pub_key)
++#endif
+     return 0;
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   len = MP_bytes(dh->pub_key);
++#else
++  len = MP_bytes(pub_key);
++#endif
+   if (len <= 0 || len > (int) nPubkeyLen)
+     return 0;
+ 
+   memset(pubkey, 0, nPubkeyLen);
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_setbin(dh->pub_key, pubkey + (nPubkeyLen - len), len);
++#else
++  MP_setbin(pub_key, pubkey + (nPubkeyLen - len), len);
++#endif
+   return 1;
+ }
+ 
+@@ -364,7 +412,13 @@
+   MP_gethex(q1, Q1024, len);
+   assert(len);
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   if (isValidPublicKey(pubkeyBn, dh->p, q1))
++#else
++  BIGNUM const* p = NULL;
++  DH_get0_pqg(dh, &p, NULL, NULL);
++  if (isValidPublicKey(pubkeyBn, (BIGNUM*)p, q1))
++#endif
+     res = MDH_compute_key(secret, nPubkeyLen, pubkeyBn, dh);
+   else
+     res = -1;
+--- a/librtmp/handshake.h
++++ b/librtmp/handshake.h
+@@ -31,9 +31,9 @@
+ #define SHA256_DIGEST_LENGTH	32
+ #endif
+ #define HMAC_CTX	sha2_context
+-#define HMAC_setup(ctx, key, len)	sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0)
+-#define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(&ctx, buf, len)
+-#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig)
++#define HMAC_setup(ctx, key, len)	sha2_hmac_starts(ctx, (unsigned char *)key, len, 0)
++#define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(ctx, buf, len)
++#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(ctx, dig)
+ 
+ typedef arc4_context *	RC4_handle;
+ #define RC4_alloc(h)	*h = malloc(sizeof(arc4_context))
+@@ -50,9 +50,9 @@
+ #endif
+ #undef HMAC_CTX
+ #define HMAC_CTX	struct hmac_sha256_ctx
+-#define HMAC_setup(ctx, key, len)	hmac_sha256_set_key(&ctx, len, key)
+-#define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(&ctx, len, buf)
+-#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
++#define HMAC_setup(ctx, key, len)	hmac_sha256_set_key(ctx, len, key)
++#define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(ctx, len, buf)
++#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(ctx, SHA256_DIGEST_LENGTH, dig)
+ #define HMAC_close(ctx)
+ 
+ typedef struct arcfour_ctx*	RC4_handle;
+@@ -64,14 +64,23 @@
+ 
+ #else	/* USE_OPENSSL */
+ #include 
++#include 
+ #include 
+ #include 
+ #if OPENSSL_VERSION_NUMBER < 0x0090800 || !defined(SHA256_DIGEST_LENGTH)
+ #error Your OpenSSL is too old, need 0.9.8 or newer with SHA256
+ #endif
+-#define HMAC_setup(ctx, key, len)	HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, key, len, EVP_sha256(), 0)
+-#define HMAC_crunch(ctx, buf, len)	HMAC_Update(&ctx, buf, len)
+-#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(&ctx, dig, &dlen); HMAC_CTX_cleanup(&ctx)
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_setup(ctx, key, len)	HMAC_CTX_init(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0)
++#else
++#define HMAC_setup(ctx, key, len)	ctx = HMAC_CTX_new(); HMAC_CTX_reset(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0)
++#endif
++#define HMAC_crunch(ctx, buf, len)	HMAC_Update(ctx, buf, len)
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(ctx, dig, &dlen); HMAC_CTX_cleanup(ctx)
++#else
++#define HMAC_finish(ctx, dig, dlen)     HMAC_Final(ctx, dig, &dlen); HMAC_CTX_free(ctx)
++#endif
+ 
+ typedef RC4_KEY *	RC4_handle;
+ #define RC4_alloc(h)	*h = malloc(sizeof(RC4_KEY))
+@@ -117,7 +126,7 @@
+ {
+   uint8_t digest[SHA256_DIGEST_LENGTH];
+   unsigned int digestLen = 0;
+-  HMAC_CTX ctx;
++  HMAC_CTX* ctx = NULL;
+ 
+   RC4_alloc(rc4keyIn);
+   RC4_alloc(rc4keyOut);
+@@ -266,7 +275,7 @@
+ 	   size_t keylen, uint8_t *digest)
+ {
+   unsigned int digestLen;
+-  HMAC_CTX ctx;
++  HMAC_CTX* ctx = NULL;
+ 
+   HMAC_setup(ctx, key, keylen);
+   HMAC_crunch(ctx, message, messageLen);
+--- a/librtmp/hashswf.c
++++ b/librtmp/hashswf.c
+@@ -37,9 +37,9 @@
+ #define SHA256_DIGEST_LENGTH	32
+ #endif
+ #define HMAC_CTX	sha2_context
+-#define HMAC_setup(ctx, key, len)	sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0)
+-#define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(&ctx, buf, len)
+-#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig)
++#define HMAC_setup(ctx, key, len)	sha2_hmac_starts(ctx, (unsigned char *)key, len, 0)
++#define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(ctx, buf, len)
++#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(ctx, dig)
+ #define HMAC_close(ctx)
+ #elif defined(USE_GNUTLS)
+ #include 
+@@ -48,19 +48,27 @@
+ #endif
+ #undef HMAC_CTX
+ #define HMAC_CTX	struct hmac_sha256_ctx
+-#define HMAC_setup(ctx, key, len)	hmac_sha256_set_key(&ctx, len, key)
+-#define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(&ctx, len, buf)
+-#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
++#define HMAC_setup(ctx, key, len)	hmac_sha256_set_key(ctx, len, key)
++#define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(ctx, len, buf)
++#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(ctx, SHA256_DIGEST_LENGTH, dig)
+ #define HMAC_close(ctx)
+ #else	/* USE_OPENSSL */
+ #include 
+ #include 
+ #include 
+ #include 
+-#define HMAC_setup(ctx, key, len)	HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, (unsigned char *)key, len, EVP_sha256(), 0)
+-#define HMAC_crunch(ctx, buf, len)	HMAC_Update(&ctx, (unsigned char *)buf, len)
+-#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(&ctx, (unsigned char *)dig, &dlen);
+-#define HMAC_close(ctx)	HMAC_CTX_cleanup(&ctx)
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_setup(ctx, key, len)	HMAC_CTX_init(ctx); HMAC_Init_ex(ctx, (unsigned char *)key, len, EVP_sha256(), 0)
++#else
++#define HMAC_setup(ctx, key, len)	ctx = HMAC_CTX_new(); HMAC_CTX_reset(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0)
++#endif
++#define HMAC_crunch(ctx, buf, len)	HMAC_Update(ctx, (unsigned char *)buf, len)
++#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(ctx, (unsigned char *)dig, &dlen);
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_close(ctx)	HMAC_CTX_cleanup(ctx)
++#else
++#define HMAC_close(ctx) HMAC_CTX_reset(ctx); HMAC_CTX_free(ctx)
++#endif
+ #endif
+ 
+ extern void RTMP_TLS_Init();
+@@ -289,7 +297,7 @@
+ struct info
+ {
+   z_stream *zs;
+-  HMAC_CTX ctx;
++  HMAC_CTX *ctx;
+   int first;
+   int zlib;
+   int size;
diff --git a/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2.patch b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2.patch
new file mode 100644
index 00000000000000..cc7637d84943af
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2.patch
@@ -0,0 +1,14 @@
+https://bugs.gentoo.org/669574
+diff --git a/librtmp/rtmp.c b/librtmp/rtmp.c
+index 5311a8a..79fefae 100644
+--- a/librtmp/rtmp.c
++++ b/librtmp/rtmp.c
+@@ -2854,7 +2854,7 @@ HandleCtrl(RTMP *r, const RTMPPacket *packet)
+   if (nType == 0x1A)
+     {
+       RTMP_Log(RTMP_LOGDEBUG, "%s, SWFVerification ping received: ", __FUNCTION__);
+-      if (packet->m_nBodySize > 2 && packet->m_body[2] > 0x01)
++      if (packet->m_nBodySize > 2 && packet->m_body[2] > 0x02)
+ 	{
+ 	  RTMP_Log(RTMP_LOGERROR,
+             "%s: SWFVerification Type %d request not supported! Patches welcome...",
diff --git a/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2_part_2.patch b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2_part_2.patch
new file mode 100644
index 00000000000000..ade0d9baa79a46
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2_part_2.patch
@@ -0,0 +1,22 @@
+https://bugs.gentoo.org/669574
+diff --git a/librtmp/rtmp.c b/librtmp/rtmp.c
+index df2cb27..b72dc64 100644
+--- a/librtmp/rtmp.c
++++ b/librtmp/rtmp.c
+@@ -2857,14 +2857,14 @@ HandleCtrl(RTMP *r, const RTMPPacket *packet)
+       if (packet->m_nBodySize > 2 && packet->m_body[2] > 0x01)
+ 	{
+ 	  RTMP_Log(RTMP_LOGERROR,
+-            "%s: SWFVerification Type %d request not supported! Patches welcome...",
++            "%s: SWFVerification Type %d request not supported, attempting to use SWFVerification Type 1! Patches welcome...",
+ 	    __FUNCTION__, packet->m_body[2]);
+ 	}
+ #ifdef CRYPTO
+       /*RTMP_LogHex(packet.m_body, packet.m_nBodySize); */
+ 
+       /* respond with HMAC SHA256 of decompressed SWF, key is the 30byte player key, also the last 30 bytes of the server handshake are applied */
+-      else if (r->Link.SWFSize)
++      if (r->Link.SWFSize)
+ 	{
+ 	  RTMP_SendCtrl(r, 0x1B, 0, 0);
+ 	}
diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py
index 87a2e130714a99..694c2551e1b372 100644
--- a/var/spack/repos/builtin/packages/ruby/package.py
+++ b/var/spack/repos/builtin/packages/ruby/package.py
@@ -44,7 +44,7 @@ class Ruby(AutotoolsPackage, NMakePackage):
             depends_on("tcl", when="@:2.3")
             depends_on("tk", when="@:2.3")
             depends_on("readline", when="+readline")
-            depends_on("zlib")
+            depends_on("zlib-api")
             with when("+openssl"):
                 depends_on("openssl@:1")
                 depends_on("openssl@:1.0", when="@:2.3")
@@ -83,9 +83,8 @@ def url_for_version(self, version):
         return url.format(version.up_to(2), version)
 
     def setup_dependent_run_environment(self, env, dependent_spec):
-        for d in dependent_spec.traverse(deptype=("run"), root=True):
-            if d.package.extends(self.spec):
-                env.prepend_path("GEM_PATH", d.prefix)
+        if dependent_spec.package.extends(self.spec):
+            env.prepend_path("GEM_PATH", dependent_spec.prefix)
 
     def setup_dependent_package(self, module, dependent_spec):
         """Called before ruby modules' install() methods.  Sets GEM_HOME
diff --git a/var/spack/repos/builtin/packages/rust-bootstrap/package.py b/var/spack/repos/builtin/packages/rust-bootstrap/package.py
index 84100bf2424a14..337618e20e38f2 100644
--- a/var/spack/repos/builtin/packages/rust-bootstrap/package.py
+++ b/var/spack/repos/builtin/packages/rust-bootstrap/package.py
@@ -73,7 +73,7 @@ class RustBootstrap(Package):
 
     # Determine system os and architecture/target.
     os = platform.system().lower()
-    target = rust_targets[platform.machine().lower()]
+    target = rust_targets.get(platform.machine().lower(), platform.machine().lower())
 
     # Pre-release versions of the bootstrap compiler.
     # Note: These versions are unchecksumed since they will change
diff --git a/var/spack/repos/builtin/packages/rust/package.py b/var/spack/repos/builtin/packages/rust/package.py
index abf73a01af5623..b660697df3653e 100644
--- a/var/spack/repos/builtin/packages/rust/package.py
+++ b/var/spack/repos/builtin/packages/rust/package.py
@@ -116,6 +116,9 @@ def configure(self, spec, prefix):
         opts.append(f"build.cargo={spec['rust-bootstrap'].prefix.bin.cargo}")
         opts.append(f"build.rustc={spec['rust-bootstrap'].prefix.bin.rustc}")
 
+        # Disable bootstrap LLVM download.
+        opts.append("llvm.download-ci-llvm=false")
+
         # Convert opts to '--set key=value' format.
         flags = [flag for opt in opts for flag in ("--set", opt)]
 
diff --git a/var/spack/repos/builtin/packages/sabre/package.py b/var/spack/repos/builtin/packages/sabre/package.py
index 6481a63ea122af..fea74532d91a80 100644
--- a/var/spack/repos/builtin/packages/sabre/package.py
+++ b/var/spack/repos/builtin/packages/sabre/package.py
@@ -22,7 +22,7 @@ class Sabre(MakefilePackage):
 
     version("2013-09-27", commit="039a55e500ba07b7e6432ea6ec2ddcfb3471d949")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/salmon/fix_hts.patch b/var/spack/repos/builtin/packages/salmon/fix_hts.patch
new file mode 100644
index 00000000000000..c5eb36482ffe1a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/salmon/fix_hts.patch
@@ -0,0 +1,11 @@
+--- a/cmake/Modules/Findlibstadenio.cmake
++++ /dev/null
+@@ -13,7 +13,7 @@ find_path(STADEN_INCLUDE_DIR io_lib
+ find_library(STADEN_LIBRARY NAMES staden-read libstaden-read 
+   HINTS ${STADEN_ROOT} ENV STADEN_ROOT PATH_SUFFIXES lib lib64)
+ 
+-find_library(HTSCODEC_LIBRARY NAMES htscodecs libhtscodecs
++find_library(HTSCODEC_LIBRARY NAMES htscodecs libhtscodecs hts
+   HINTS ${STADEN_ROOT} ENV STADEN_ROOT PATH_SUFFIXES lib lib64)
+ 
+ if(STADEN_INCLUDE_DIR)
diff --git a/var/spack/repos/builtin/packages/salmon/package.py b/var/spack/repos/builtin/packages/salmon/package.py
index 553a2d17674a8e..332c9885055e23 100644
--- a/var/spack/repos/builtin/packages/salmon/package.py
+++ b/var/spack/repos/builtin/packages/salmon/package.py
@@ -13,6 +13,7 @@ class Salmon(CMakePackage):
     url = "https://github.com/COMBINE-lab/salmon/archive/v0.8.2.tar.gz"
     maintainers("snehring")
 
+    version("1.10.2", sha256="976989182160fef3afb4429ee8b85d8dd39ed6ca212bb14d6a65cde0e985fb98")
     version("1.9.0", sha256="450d953a5c43fe63fd745733f478d3fbaf24d926cb52731fd38ee21c4990d613")
     version("1.4.0", sha256="6d3e25387450710f0aa779a1e9aaa9b4dec842324ff8551d66962d7c7606e71d")
     version("0.14.1", sha256="05289170e69b5f291a8403b40d6b9bff54cc38825e9f721c210192b51a19273e")
@@ -30,8 +31,8 @@ class Salmon(CMakePackage):
     # 1.8.0 relies on tbb provided config, earlier versions make
     # assumptions about the layout of tbb files that are not true in
     # 2021.1 and later
-    conflicts("intel-tbb@2021.1:", when="@:1.7.0")
-    conflicts("intel-oneapi-tbb@2021.1:", when="@:1.7.0")
+    conflicts("^intel-tbb@2021.1:", when="@:1.7.0")
+    conflicts("^intel-oneapi-tbb@2021.1:", when="@:1.7.0")
     depends_on("tbb")
     depends_on(
         "boost@1.66.0:"
@@ -48,17 +49,25 @@ class Salmon(CMakePackage):
     depends_on("cereal")
     depends_on("jemalloc")
     depends_on("xz")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2")
     depends_on("libdivsufsort")
     depends_on("staden-io-lib~curl")
     depends_on("libgff")
     depends_on("pkgconfig")
     depends_on("curl", when="@0.14.1:")
+    depends_on("htslib", when="@1.10.2")
+
+    patch("fix_hts.patch", when="@1.10.2")
 
     conflicts("%gcc@:5.1", when="@0.14.1:")
 
     resources = [
+        (
+            "1.10.2",
+            "pufferfish",
+            "f225b74833f71dcf767a565345224357fb091f90ce79717abc836814d9ccd101",
+        ),
         (
             "1.9.0",
             "pufferfish",
diff --git a/var/spack/repos/builtin/packages/salome-configuration/package.py b/var/spack/repos/builtin/packages/salome-configuration/package.py
index d0fedc71393702..aeb4551012eef1 100644
--- a/var/spack/repos/builtin/packages/salome-configuration/package.py
+++ b/var/spack/repos/builtin/packages/salome-configuration/package.py
@@ -18,11 +18,11 @@ class SalomeConfiguration(Package):
     homepage = "https://www.salome-platform.org"
     git = "https://git.salome-platform.org/gitpub/tools/configuration.git"
 
-    version("9.7.0", tag="V9_7_0")
-    version("9.6.0", tag="V9_6_0")
-    version("9.5.0", tag="V9_5_0")
-    version("9.4.0", tag="V9_4_0")
-    version("9.3.0", tag="V9_3_0")
+    version("9.7.0", tag="V9_7_0", commit="b1430e72bc252867289b45de9a94041841fade06")
+    version("9.6.0", tag="V9_6_0", commit="02e621fc9e24b4eab20f82ef921859013bf024b4")
+    version("9.5.0", tag="V9_5_0", commit="96ecd4927604943dc80ead4aaf732a9d0215b70c")
+    version("9.4.0", tag="V9_4_0", commit="057e00d65a86f058dd4b0f82a866fcc66d81ed63")
+    version("9.3.0", tag="V9_3_0", commit="de7bac0ee58007a9501fffa7c1488de029b19cdc")
 
     patch("SalomeMacros.patch", working_dir="./cmake")
     patch("FindSalomeHDF5.patch", working_dir="./cmake")
diff --git a/var/spack/repos/builtin/packages/salome-medcoupling/package.py b/var/spack/repos/builtin/packages/salome-medcoupling/package.py
index 09701e535ac5a6..dd0f7dee38b45f 100644
--- a/var/spack/repos/builtin/packages/salome-medcoupling/package.py
+++ b/var/spack/repos/builtin/packages/salome-medcoupling/package.py
@@ -18,11 +18,11 @@ class SalomeMedcoupling(CMakePackage):
     homepage = "https://docs.salome-platform.org/latest/dev/MEDCoupling/developer/index.html"
     git = "https://git.salome-platform.org/gitpub/tools/medcoupling.git"
 
-    version("9.7.0", tag="V9_7_0")
-    version("9.6.0", tag="V9_6_0")
-    version("9.5.0", tag="V9_5_0")
-    version("9.4.0", tag="V9_4_0")
-    version("9.3.0", tag="V9_3_0")
+    version("9.7.0", tag="V9_7_0", commit="773434a7f2a5cbacc2f50e93ea6d6a48a157acd9")
+    version("9.6.0", tag="V9_6_0", commit="2c14a65b40252770b3503945405f5bdb2f29f8e2")
+    version("9.5.0", tag="V9_5_0", commit="dd75474d950baf8ff862b03cb1685f2a2d562846")
+    version("9.4.0", tag="V9_4_0", commit="984fe46c4076f08f42ef43e290e3cd1aea5a8182")
+    version("9.3.0", tag="V9_3_0", commit="32521cd6e5c113de5db7953a80149e5ab492120a")
 
     variant("static", default=False, description="Enable static library build")
     variant("mpi", default=False, description="Enable MPI")
diff --git a/var/spack/repos/builtin/packages/sambamba/package.py b/var/spack/repos/builtin/packages/sambamba/package.py
index 2c771f523277ab..7f70d58f17fcc2 100644
--- a/var/spack/repos/builtin/packages/sambamba/package.py
+++ b/var/spack/repos/builtin/packages/sambamba/package.py
@@ -12,7 +12,9 @@ class Sambamba(Package):
     homepage = "https://lomereiter.github.io/sambamba/"
     git = "https://github.com/lomereiter/sambamba.git"
 
-    version("0.6.6", tag="v0.6.6", submodules=True)
+    version(
+        "0.6.6", tag="v0.6.6", commit="63cfd5c7b3053e1f7045dec0b5a569f32ef73d06", submodules=True
+    )
 
     depends_on("ldc~shared", type=("build", "link"))
     depends_on("python", type="build")
diff --git a/var/spack/repos/builtin/packages/samrai/package.py b/var/spack/repos/builtin/packages/samrai/package.py
index b063b16f595f7a..58e0df44ec2747 100644
--- a/var/spack/repos/builtin/packages/samrai/package.py
+++ b/var/spack/repos/builtin/packages/samrai/package.py
@@ -61,7 +61,7 @@ class Samrai(AutotoolsPackage):
     variant("shared", default=False, description="Build shared libraries")
 
     depends_on("mpi")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("hdf5+mpi")
     depends_on("m4", type="build")
     depends_on("boost@:1.64.0", when="@3.0.0:3.11", type="build")
@@ -91,7 +91,7 @@ def configure_args(self):
                 "--with-F77=%s" % self.spec["mpi"].mpifc,
                 "--with-M4=%s" % self.spec["m4"].prefix,
                 "--with-hdf5=%s" % self.spec["hdf5"].prefix,
-                "--with-zlib=%s" % self.spec["zlib"].prefix,
+                "--with-zlib=%s" % self.spec["zlib-api"].prefix,
                 "--without-blas",
                 "--without-lapack",
                 "--with-hypre=no",
diff --git a/var/spack/repos/builtin/packages/samtools/package.py b/var/spack/repos/builtin/packages/samtools/package.py
index 6f3d530c76b1af..d71ab637a954eb 100644
--- a/var/spack/repos/builtin/packages/samtools/package.py
+++ b/var/spack/repos/builtin/packages/samtools/package.py
@@ -14,6 +14,7 @@ class Samtools(Package):
     homepage = "https://www.htslib.org"
     url = "https://github.com/samtools/samtools/releases/download/1.13/samtools-1.13.tar.bz2"
 
+    version("1.17", sha256="3adf390b628219fd6408f14602a4c4aa90e63e18b395dad722ab519438a2a729")
     version("1.16.1", sha256="2fa0a25f78594cf23d07c9d32d5060a14f1c5ee14d7b0af7a8a71abc9fdf1d07")
     version("1.15.1", sha256="708c525ac76b0532b25f14aadea34a4d11df667bc19bf0a74dae617d80526c6e")
     version("1.15", sha256="35d945a5eee9817a764490870474f24e538400b0397b28f94247a5b91447215d")
@@ -36,12 +37,13 @@ class Samtools(Package):
         url="https://github.com/samtools/samtools/archive/0.1.8.tar.gz",
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("ncurses")
     depends_on("perl", type="run")
     depends_on("python", type="run")
 
     # htslib became standalone @1.3.1, must use corresponding version
+    depends_on("htslib@1.17", when="@1.17")
     depends_on("htslib@1.16", when="@1.16.1")
     depends_on("htslib@1.15.1", when="@1.15.1")
     depends_on("htslib@1.15", when="@1.15")
diff --git a/var/spack/repos/builtin/packages/saws/package.py b/var/spack/repos/builtin/packages/saws/package.py
index 863b898e020226..e843afbda42bcc 100644
--- a/var/spack/repos/builtin/packages/saws/package.py
+++ b/var/spack/repos/builtin/packages/saws/package.py
@@ -16,7 +16,7 @@ class Saws(AutotoolsPackage):
     git = "https://bitbucket.org/saws/saws.git"
 
     version("develop", tag="master")
-    version("0.1.1", tag="v0.1.1")
-    version("0.1.0", tag="v0.1.0")
+    version("0.1.1", tag="v0.1.1", commit="c483e262b22bdf83ab026d20c35f0e09465d768d")
+    version("0.1.0", tag="v0.1.0", commit="312ccc1698cf6c489c0d1eff6db46f54bd9031b7")
 
     depends_on("python", type="build")
diff --git a/var/spack/repos/builtin/packages/sbml/package.py b/var/spack/repos/builtin/packages/sbml/package.py
index 5d7ed286dea26a..d5e25c8e9b1ba2 100644
--- a/var/spack/repos/builtin/packages/sbml/package.py
+++ b/var/spack/repos/builtin/packages/sbml/package.py
@@ -57,7 +57,7 @@ def url_for_version(self, version):
 
     depends_on("swig@2:", type="build")
     depends_on("cmake", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2")
     depends_on("libxml2")
 
diff --git a/var/spack/repos/builtin/packages/sbp/package.py b/var/spack/repos/builtin/packages/sbp/package.py
index 5f04622bb744a1..de9d126a9ed31c 100644
--- a/var/spack/repos/builtin/packages/sbp/package.py
+++ b/var/spack/repos/builtin/packages/sbp/package.py
@@ -14,7 +14,9 @@ class Sbp(CMakePackage):
 
     maintainers("jayvdb")
 
-    version("3.4.10", tag="v3.4.10", submodules=True)
+    version(
+        "3.4.10", tag="v3.4.10", commit="d8ec454a3d7f1b2f8b8f515934612c184b8d5fa3", submodules=True
+    )
 
     root_cmakelists_dir = "c"
 
diff --git a/var/spack/repos/builtin/packages/scantailor/package.py b/var/spack/repos/builtin/packages/scantailor/package.py
index ff696710bd259b..3ffb13c790ebc5 100644
--- a/var/spack/repos/builtin/packages/scantailor/package.py
+++ b/var/spack/repos/builtin/packages/scantailor/package.py
@@ -22,7 +22,7 @@ class Scantailor(CMakePackage):
 
     depends_on("qt@5:")
     depends_on("libjpeg")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("libpng")
     depends_on("libtiff")
     depends_on("boost@1.35:")
diff --git a/var/spack/repos/builtin/packages/scine-qcmaquis/package.py b/var/spack/repos/builtin/packages/scine-qcmaquis/package.py
new file mode 100644
index 00000000000000..0f076227abf64e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/scine-qcmaquis/package.py
@@ -0,0 +1,62 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class ScineQcmaquis(CMakePackage):
+    """Scine QCMaquis DMRG Solver"""
+
+    homepage = "https://scine.ethz.ch/download/qcmaquis"
+    git = "https://github.com/qcscine/qcmaquis.git"
+
+    maintainers("adam-grofe")
+
+    version("master", branch="master")
+    version("3.1.3", branch="release-3.1.3")
+    version("3.1.2", branch="release-3.1.2", preferred=True)
+    variant(
+        "blas",
+        values=("openblas", "mkl"),
+        default="openblas",
+        description="Which blas library to use.",
+    )
+    variant(
+        "symmetries",
+        default="SU2U1PG,TwoU1PG",
+        description='Wave functions symmetries to compile (e.g. "SU2U1PG,TwoU1PG")',
+        values=("U1", "TwoU1", "TwoU1PG", "NU1", "Z2", "SU2U1", "SU2U1PG", "U1DG", "NONE"),
+        multi=True,
+    )
+    variant(
+        "build_tests",
+        default=True,
+        description="Whether to build unit tests using gtest and gmock",
+    )
+
+    root_cmakelists_dir = "dmrg"
+
+    depends_on("hdf5~mpi")
+    depends_on("lapack")
+    depends_on("openblas+ilp64 threads=openmp", when="blas=openblas")
+    depends_on("intel-oneapi-mkl", when="blas=mkl")
+    depends_on("gsl")
+    depends_on("boost+program_options+filesystem+system+thread+serialization+chrono")
+    depends_on("googletest+gmock", when="+build_tests")
+
+    def cmake_args(self):
+        args = [
+            self.define_from_variant("BUILD_SYMMETRIES", "symmetries"),
+            self.define_from_variant("QCMAQUIS_TESTS", "build_tests"),
+        ]
+        return args
+
+    def patch(self):
+        if self.version <= Version("3.1.3"):
+            filter_file(
+                "#include ",
+                "#include \n#include ",
+                "dmrg/framework/dmrg/utils/results_collector.h",
+            )
diff --git a/var/spack/repos/builtin/packages/scipoptsuite/package.py b/var/spack/repos/builtin/packages/scipoptsuite/package.py
index d983055b930bbc..bc0f42f024f915 100644
--- a/var/spack/repos/builtin/packages/scipoptsuite/package.py
+++ b/var/spack/repos/builtin/packages/scipoptsuite/package.py
@@ -27,6 +27,6 @@ class Scipoptsuite(CMakePackage):
     version("7.0.1", sha256="971962f2d896b0c8b8fa554c18afd2b5037092685735d9494a05dc16d56ad422")
 
     depends_on("gmp")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("readline")
     depends_on("ncurses")
diff --git a/var/spack/repos/builtin/packages/scitokens-cpp/package.py b/var/spack/repos/builtin/packages/scitokens-cpp/package.py
index 324c8889d4cafd..5bfd4bcc3a0b66 100644
--- a/var/spack/repos/builtin/packages/scitokens-cpp/package.py
+++ b/var/spack/repos/builtin/packages/scitokens-cpp/package.py
@@ -39,7 +39,7 @@ class ScitokensCpp(CMakePackage):
     depends_on("pkgconfig", type="build")
     depends_on("uuid", type="build")
 
-    conflicts("jwt-cpp@0.5:", when="@:0.7")
+    conflicts("^jwt-cpp@0.5:", when="@:0.7")
 
     # https://github.com/scitokens/scitokens-cpp/issues/72
     @when("@0.7.0 ^openssl@3:")
diff --git a/var/spack/repos/builtin/packages/scorep/package.py b/var/spack/repos/builtin/packages/scorep/package.py
index 0921552edafd71..316173a73e297d 100644
--- a/var/spack/repos/builtin/packages/scorep/package.py
+++ b/var/spack/repos/builtin/packages/scorep/package.py
@@ -16,6 +16,9 @@ class Scorep(AutotoolsPackage):
     url = "https://perftools.pages.jsc.fz-juelich.de/cicd/scorep/tags/scorep-7.1/scorep-7.1.tar.gz"
     maintainers("wrwilliams")
 
+    version("8.3", sha256="76c914e6319221c059234597a3bc53da788ed679179ac99c147284dcefb1574a")
+    # version 8.2 was immediately superseded before it hit Spack
+    version("8.1", sha256="3a40b481fce610871ddf6bdfb88a6d06b9e5eb38c6080faac6d5e44990060a37")
     version("8.0", sha256="4c0f34f20999f92ebe6ca1ff706d0846b8ce6cd537ffbedb49dfaef0faa66311")
     version("7.1", sha256="98dea497982001fb82da3429ca55669b2917a0858c71abe2cfe7cd113381f1f7")
     version("7.0", sha256="68f24a68eb6f94eaecf500e17448f566031946deab74f2cba072ee8368af0996")
@@ -92,8 +95,10 @@ def url_for_version(self, version):
     # SCOREP 8
     depends_on("binutils", type="link", when="@8:")
     depends_on("otf2@3:", when="@8:")
-    depends_on("cubew@4.8:", when="@8:")
-    depends_on("cubelib@4.8:", when="@8:")
+    depends_on("cubew@4.8.2:", when="@8.3:")
+    depends_on("cubelib@4.8.2:", when="@8.3:")
+    depends_on("cubew@4.8:", when="@8:8.2")
+    depends_on("cubelib@4.8:", when="@8:8.2")
     # fall through to Score-P 7's OPARI2, no new release
     # SCOREP 7
     depends_on("otf2@2.3:2.3.99", when="@7.0:7")
@@ -139,6 +144,12 @@ def url_for_version(self, version):
     # https://github.com/spack/spack/issues/1609
     conflicts("platform=darwin")
 
+    def find_libpath(self, libname, root):
+        libs = find_libraries(libname, root, shared=True, recursive=True)
+        if len(libs.directories) == 0:
+            return None
+        return libs.directories[0]
+
     def configure_args(self):
         spec = self.spec
 
@@ -168,7 +179,9 @@ def configure_args(self):
         if "+unwind" in spec:
             config_args.append("--with-libunwind=%s" % spec["libunwind"].prefix)
         if "+cuda" in spec:
-            config_args.append("--with-libcuda=%s" % spec["cuda"].prefix)
+            config_args.append("--with-libcudart=%s" % spec["cuda"].prefix)
+            cuda_driver_path = self.find_libpath("libcuda", spec["cuda"].prefix)
+            config_args.append("--with-libcuda-lib=%s" % cuda_driver_path)
         if "+hip" in spec:
             config_args.append("--with-rocm=%s" % spec["hip"].prefix)
 
diff --git a/var/spack/repos/builtin/packages/scorpio/package.py b/var/spack/repos/builtin/packages/scorpio/package.py
index bfbdac00320bc2..7eca9b539077e3 100644
--- a/var/spack/repos/builtin/packages/scorpio/package.py
+++ b/var/spack/repos/builtin/packages/scorpio/package.py
@@ -16,7 +16,7 @@ class Scorpio(CMakePackage):
     homepage = "https://gitlab.com/truchas/tpl-forks/scorpio"
     git = "https://gitlab.com/truchas/tpl-forks/scorpio.git"
 
-    maintainers("pbrady")
+    maintainers("pbrady", "zjibben")
 
     version("develop", branch="truchas")
 
diff --git a/var/spack/repos/builtin/packages/scotch/package.py b/var/spack/repos/builtin/packages/scotch/package.py
index 5007bdf497b212..5906d77ab96a9b 100644
--- a/var/spack/repos/builtin/packages/scotch/package.py
+++ b/var/spack/repos/builtin/packages/scotch/package.py
@@ -64,7 +64,7 @@ class Scotch(CMakePackage, MakefilePackage):
     depends_on("flex@:2.6.1,2.6.4:", type="build")
     depends_on("bison@3.4:", type="build")
     depends_on("mpi", when="+mpi")
-    depends_on("zlib", when="+compression")
+    depends_on("zlib-api", when="+compression")
 
     # Version-specific patches
     patch("nonthreaded-6.0.4.patch", when="@6.0.4")
@@ -79,8 +79,8 @@ class Scotch(CMakePackage, MakefilePackage):
 
     # Vendored dependency of METIS/ParMETIS conflicts with standard
     # installations
-    conflicts("^metis", when="+metis")
-    conflicts("^parmetis", when="+metis")
+    conflicts("metis", when="+metis")
+    conflicts("parmetis", when="+metis")
 
     parallel = False
 
@@ -112,7 +112,7 @@ def libs(self):
 
         scotchlibs = find_libraries(libraries, root=self.prefix, recursive=True, shared=shared)
         if "+compression" in self.spec:
-            zlibs = self.spec["zlib"].libs
+            zlibs = self.spec["zlib-api"].libs
 
         return scotchlibs + zlibs
 
@@ -220,7 +220,7 @@ def edit(self, pkg, spec, prefix):
 
         if "+compression" in self.spec:
             cflags.append("-DCOMMON_FILE_COMPRESS_GZ")
-            ldflags.append(" {0} ".format(self.spec["zlib"].libs.joined()))
+            ldflags.append(" {0} ".format(self.spec["zlib-api"].libs.joined()))
 
         cflags.append("-DCOMMON_PTHREAD")
 
diff --git a/var/spack/repos/builtin/packages/scr/package.py b/var/spack/repos/builtin/packages/scr/package.py
index 1a5d692106796d..2048d1b57d1541 100644
--- a/var/spack/repos/builtin/packages/scr/package.py
+++ b/var/spack/repos/builtin/packages/scr/package.py
@@ -66,7 +66,7 @@ class Scr(CMakePackage):
     )
 
     depends_on("mpi")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # Use latest iteration of dtcmp and  components when installing scr@develop
     cmpnts = ["axl", "dtcmp", "er", "kvtree", "rankstr", "redset", "shuffile", "spath"]
diff --git a/var/spack/repos/builtin/packages/screen/package.py b/var/spack/repos/builtin/packages/screen/package.py
index 60a1f11da17ae4..0f9002df7d5f45 100644
--- a/var/spack/repos/builtin/packages/screen/package.py
+++ b/var/spack/repos/builtin/packages/screen/package.py
@@ -14,6 +14,7 @@ class Screen(AutotoolsPackage, GNUMirrorPackage):
     homepage = "https://www.gnu.org/software/screen/"
     gnu_mirror_path = "screen/screen-4.3.1.tar.gz"
 
+    version("4.9.1", sha256="26cef3e3c42571c0d484ad6faf110c5c15091fbf872b06fa7aa4766c7405ac69")
     version("4.9.0", sha256="f9335281bb4d1538ed078df78a20c2f39d3af9a4e91c57d084271e0289c730f4")
     version("4.8.0", sha256="6e11b13d8489925fde25dfb0935bf6ed71f9eb47eff233a181e078fde5655aa1")
     version("4.6.2", sha256="1b6922520e6a0ce5e28768d620b0f640a6631397f95ccb043b70b91bb503fa3a")
diff --git a/var/spack/repos/builtin/packages/seacas/package.py b/var/spack/repos/builtin/packages/seacas/package.py
index e058db747a5fd8..44b4b6a6034cce 100644
--- a/var/spack/repos/builtin/packages/seacas/package.py
+++ b/var/spack/repos/builtin/packages/seacas/package.py
@@ -31,6 +31,9 @@ class Seacas(CMakePackage):
 
     # ###################### Versions ##########################
     version("master", branch="master")
+    version(
+        "2023-10-24", sha256="f93bf0327329c302ed3feb6adf2e3968f01ec325084a457b2c2dbbf6c4f751a2"
+    )
     version(
         "2023-05-30", sha256="3dd982841854466820a3902163ad1cf1b3fbab65ed7542456d328f2d1a5373c1"
     )
@@ -132,7 +135,8 @@ class Seacas(CMakePackage):
     variant("x11", default=True, description="Compile with X11")
 
     # ###################### Dependencies ##########################
-    depends_on("cmake@3.17:", type="build")
+    depends_on("cmake@3.22:", when="@2023-10-24:", type="build")
+    depends_on("cmake@3.17:", when="@:2023-05-30", type="build")
     depends_on("mpi", when="+mpi")
 
     # Always depends on netcdf-c
@@ -140,9 +144,10 @@ class Seacas(CMakePackage):
     depends_on("netcdf-c@4.8.0:~mpi", when="~mpi")
     depends_on("hdf5+hl~mpi", when="~mpi")
 
-    depends_on("fmt@8.1.0:", when="@2022-03-04:2022-05-16")
-    depends_on("fmt@9.1.0", when="@2022-10-14")
-    depends_on("fmt@9.1.0:", when="@2023-05-30")
+    depends_on("fmt@10.1.0", when="@2023-10-24:")
+    depends_on("fmt@9.1.0", when="@2022-10-14:2023-05-30")
+    depends_on("fmt@8.1.0:9", when="@2022-03-04:2022-05-16")
+
     depends_on("matio", when="+matio")
     depends_on("libx11", when="+x11")
 
@@ -195,6 +200,8 @@ def cmake_args(self):
                 from_variant("BUILD_SHARED_LIBS", "shared"),
                 from_variant("SEACASExodus_ENABLE_THREADSAFE", "thread_safe"),
                 from_variant("SEACASIoss_ENABLE_THREADSAFE", "thread_safe"),
+                # SEACASExodus_ENABLE_THREADSAFE=ON requires TPL_ENABLE_Pthread=ON
+                from_variant("TPL_ENABLE_Pthread", "thread_safe"),
                 from_variant("TPL_ENABLE_X11", "x11"),
                 from_variant(project_name_base + "_ENABLE_Fortran", "fortran"),
             ]
diff --git a/var/spack/repos/builtin/packages/selalib/package.py b/var/spack/repos/builtin/packages/selalib/package.py
new file mode 100644
index 00000000000000..d36a4d20538ca3
--- /dev/null
+++ b/var/spack/repos/builtin/packages/selalib/package.py
@@ -0,0 +1,74 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Selalib(CMakePackage):
+    """SeLaLib is a modular library for the kinetic and gyrokinetic simulation
+    of tokamak plasmas by the semi-lagrangian or particle-in-cell methods"""
+
+    homepage = "https://selalib.github.io/selalib"
+    url = "https://github.com/selalib/selalib"
+    git = "https://github.com/selalib/selalib"
+
+    maintainers("pnavaro", "freifrauvonbleifrei")
+
+    version("main", branch="main")
+
+    variant("fmempool", default=False, description="Use memory pool")
+    variant("mpi", default=True, description="Build with MPI support")
+    variant("openmp", default=True, description="Build with OpenMP support")
+    variant("compression", default=False, description="Add compression by ZFP")
+
+    requires(
+        "%gcc@9.0.0:",
+        "%clang@16.0.0:",
+        "%intel@18.0:",
+        "%oneapi@18.0:",
+        policy="one_of",
+        msg="SeLaLib requires new-enough Fortran compiler",
+    )
+
+    depends_on("cmake@3.6.0:", type=("build"))
+    depends_on("blas")
+    depends_on("fftw")
+    depends_on("fftw+openmp", when="+openmp")
+    depends_on("fgsl")
+    depends_on("git", type=("build", "run", "test"))
+    depends_on("hdf5+fortran+cxx")
+    depends_on("lapack", when="~mpi")
+    with when("+mpi"):
+        depends_on("mpi")
+        depends_on("fftw+mpi")
+        depends_on("hdf5+mpi")
+        depends_on("scalapack")
+    depends_on("python@3.0.0:", type=("build"))
+    # beware: compiling w/ zfp may throw type mismatch errors
+    depends_on("zfp+fortran", when="+compression")
+
+    def cmake_args(self):
+        args = [
+            self.define_from_variant("OPENMP_ENABLED", "openmp"),
+            self.define_from_variant("HDF5_PARALLEL_ENABLED", "mpi"),
+            self.define_from_variant("USE_FMEMPOOL", "fmempool"),
+            self.define("FFTW_ENABLED", "ON"),
+        ]
+        return args
+
+    def setup_build_environment(self, env):
+        env.set("FFTW_INCLUDE", self.spec["fftw"].prefix.include)
+        env.set("FFTW_ROOT", self.spec["fftw"].prefix)
+
+    @run_after("build")
+    @on_package_attributes(run_tests=True)
+    def quick_serial_ctest(self):
+        """quickly run a serial subset of tests for sanity check"""
+        ctest = which("ctest")
+        with working_dir(self.build_directory):
+            ctest("--output-on-failure", "-R", "test_mud2")
+            ctest("--output-on-failure", "-R", "sparse_grid_4d")
+            ctest("--output-on-failure", "-R", "scalar_field_2d")
+            ctest("--output-on-failure", "-R", "maxwell_3d_fem_fft")
diff --git a/var/spack/repos/builtin/packages/semiprof/package.py b/var/spack/repos/builtin/packages/semiprof/package.py
index 0506ecf7a36467..d934da0fbb70d0 100644
--- a/var/spack/repos/builtin/packages/semiprof/package.py
+++ b/var/spack/repos/builtin/packages/semiprof/package.py
@@ -12,10 +12,10 @@ class Semiprof(CMakePackage):
     homepage = "https://github.com/bcumming/semiprof"
     url = "https://github.com/bcumming/semiprof/archive/refs/tags/v0.1.tar.gz"
 
-    maintainers = ["simonpintarelli"]
+    maintainers("simonpintarelli")
 
-    variant("examples", default=False)
-    variant("shared", default=True)
+    variant("examples", default=False, description="Enable examples")
+    variant("shared", default=True, description="Build shared libraries")
 
     version("0.1", sha256="4fb3823c65a4f5dfbe05e8cbe1911dfd25cd7740597f82c7b3a84472de26f0dc")
 
diff --git a/var/spack/repos/builtin/packages/sensei/package.py b/var/spack/repos/builtin/packages/sensei/package.py
index e063c963001f6f..6e4e0bf6741c9e 100644
--- a/var/spack/repos/builtin/packages/sensei/package.py
+++ b/var/spack/repos/builtin/packages/sensei/package.py
@@ -19,6 +19,7 @@ class Sensei(CMakePackage):
     maintainers("sshudler", "kwryankrattiger")
 
     version("develop", branch="develop")
+    version("4.1.0", sha256="e1154240c022069fee454c747d7c60e065d36b4d1dc71852b3cd527c22b531c1")
     version("4.0.0", sha256="fc1538aa1051789dbdefbe18b7f251bc46e7a6ae1db3a940c123552e0318db8b")
     version("3.2.2", sha256="d554b654880e899d97d572f02de87b0202faadaf899420ef871093b5bce320c0")
     version("3.2.1", sha256="7438fb4b148e4d1eb888c619366d0d8639122ecbbf1767e19549d6ca0c8698ca")
diff --git a/var/spack/repos/builtin/packages/sentieon-genomics/package.py b/var/spack/repos/builtin/packages/sentieon-genomics/package.py
index e31b1f1af14dfc..397ab9ffcabcad 100644
--- a/var/spack/repos/builtin/packages/sentieon-genomics/package.py
+++ b/var/spack/repos/builtin/packages/sentieon-genomics/package.py
@@ -26,6 +26,7 @@ class SentieonGenomics(Package):
     url = "https://s3.amazonaws.com/sentieon-release/software/sentieon-genomics-201808.01.tar.gz"
     maintainers("snehring")
 
+    version("202308", sha256="d663067f46e499c23819e344cf548fdc362abbf94d3ef086a2e655c072ebe0d6")
     version("202112.07", sha256="7178769bb5a9619840996356bda4660410fb6f228b2c0b86611bcb1c6bcfc2e1")
     version("202112.06", sha256="18306036f01c3d41dd7ae738b18ae76fd6b666f1172dd4696cd55b4a8465270d")
     version("202112.05", sha256="c97b14b0484a0c0025115ad7b911453af7bdcd09874c26cbc39fd0bc5588a306")
diff --git a/var/spack/repos/builtin/packages/seqan/package.py b/var/spack/repos/builtin/packages/seqan/package.py
index b3d607c02b38b7..b2022f0f9fa0b2 100644
--- a/var/spack/repos/builtin/packages/seqan/package.py
+++ b/var/spack/repos/builtin/packages/seqan/package.py
@@ -23,7 +23,7 @@ class Seqan(CMakePackage):
     depends_on("py-nose", type="build")
     depends_on("py-sphinx", type="build")
     depends_on("boost+exception+math+serialization+container", type=("build", "link"))
-    depends_on("zlib", type=("build", "link"))
+    depends_on("zlib-api", type=("build", "link"))
     depends_on("bzip2", type=("build", "link"))
 
     conflicts("%intel@:16.0.4")
diff --git a/var/spack/repos/builtin/packages/seqprep/package.py b/var/spack/repos/builtin/packages/seqprep/package.py
index b8f8a5fee5bf42..37eb04380e9e9e 100644
--- a/var/spack/repos/builtin/packages/seqprep/package.py
+++ b/var/spack/repos/builtin/packages/seqprep/package.py
@@ -15,7 +15,7 @@ class Seqprep(MakefilePackage):
 
     version("1.3.2", sha256="2b8a462a0e0a3e51f70be7730dc77b1f2bb69e74845dd0fbd2110a921c32265a")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/seqtk/package.py b/var/spack/repos/builtin/packages/seqtk/package.py
index 95d39f09cdc317..337cb8989a0c8b 100644
--- a/var/spack/repos/builtin/packages/seqtk/package.py
+++ b/var/spack/repos/builtin/packages/seqtk/package.py
@@ -17,7 +17,7 @@ class Seqtk(Package):
     version("1.2", sha256="bd53316645ab10f0aaba59e1e72c28442ee4c9c37fddaacce5e24757eff78d7b")
     version("1.1", sha256="f01b9f9af6e443673a0105a7536a01957a4fc371826385a1f3dd1e417aa91d52")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         make()
diff --git a/var/spack/repos/builtin/packages/serf/package.py b/var/spack/repos/builtin/packages/serf/package.py
index fd4b4375ab13a8..fea878d02e2dec 100644
--- a/var/spack/repos/builtin/packages/serf/package.py
+++ b/var/spack/repos/builtin/packages/serf/package.py
@@ -15,6 +15,7 @@ class Serf(SConsPackage):
 
     maintainers("cosmicexplorer")
 
+    version("1.3.10", sha256="be81ef08baa2516ecda76a77adf7def7bc3227eeb578b9a33b45f7b41dc064e6")
     version("1.3.9", sha256="549c2d21c577a8a9c0450facb5cca809f26591f048e466552240947bdf7a87cc")
     version("1.3.8", sha256="e0500be065dbbce490449837bb2ab624e46d64fc0b090474d9acaa87c82b2590")
 
@@ -28,9 +29,9 @@ class Serf(SConsPackage):
     depends_on("python+pythoncmd", type="build")
     depends_on("scons@2.3.0:", type="build")
     depends_on("uuid")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
-    patch("py3syntax.patch")
+    patch("py3syntax.patch", when="@:1.3.9")
     patch("py3-hashbang.patch")
 
     def build_args(self, spec, prefix):
@@ -39,7 +40,7 @@ def build_args(self, spec, prefix):
             "APR": spec["apr"].prefix,
             "APU": spec["apr-util"].prefix,
             "OPENSSL": spec["openssl"].prefix,
-            "ZLIB": spec["zlib"].prefix,
+            "ZLIB": spec["zlib-api"].prefix,
             "DEBUG": "yes" if "+debug" in spec else "no",
         }
 
diff --git a/var/spack/repos/builtin/packages/sfcio/package.py b/var/spack/repos/builtin/packages/sfcio/package.py
index 60ddd75daba452..bd82260984e272 100644
--- a/var/spack/repos/builtin/packages/sfcio/package.py
+++ b/var/spack/repos/builtin/packages/sfcio/package.py
@@ -14,11 +14,21 @@ class Sfcio(CMakePackage):
 
     homepage = "https://noaa-emc.github.io/NCEPLIBS-sfcio"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-sfcio/archive/refs/tags/v1.4.1.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-sfcio"
 
-    maintainers("t-brown", "AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
+    maintainers("AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
 
+    version("develop", branch="develop")
     version("1.4.1", sha256="d9f900cf18ec1a839b4128c069b1336317ffc682086283443354896746b89c59")
 
+    variant("pfunit", default=False, description="Add pfunit dependency to enable testing")
+
+    depends_on("pfunit", when="+pfunit")
+
+    def cmake_args(self):
+        args = [self.define("ENABLE_TESTS", self.run_tests)]
+        return args
+
     def setup_run_environment(self, env):
         lib = find_libraries("libsfcio", root=self.prefix, shared=False, recursive=True)
         # Only one library version, but still need to set _4 to make NCO happy
@@ -31,3 +41,7 @@ def flag_handler(self, name, flags):
             if name == "fflags":
                 flags.append("-Free")
         return (None, None, flags)
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/sga/package.py b/var/spack/repos/builtin/packages/sga/package.py
index 373346a87da300..d220800555c4bd 100644
--- a/var/spack/repos/builtin/packages/sga/package.py
+++ b/var/spack/repos/builtin/packages/sga/package.py
@@ -24,7 +24,7 @@ class Sga(AutotoolsPackage):
     version("0.10.8", sha256="55c5e0e425e14902e83d68cfb8cee4c86ee186459e54113a484b2a1b06d223c8")
     version("0.10.3", sha256="c000823a58428d9db2979b30a571ad89aec78a8cb1af60bae1ce252dd4e8adac")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("sparsehash")
     depends_on("jemalloc")
     depends_on("bamtools")
diff --git a/var/spack/repos/builtin/packages/sgpp/disable_disutils_deprecation_warning.patch b/var/spack/repos/builtin/packages/sgpp/disable_disutils_deprecation_warning.patch
new file mode 100644
index 00000000000000..32fc5bc235d6ba
--- /dev/null
+++ b/var/spack/repos/builtin/packages/sgpp/disable_disutils_deprecation_warning.patch
@@ -0,0 +1,23 @@
+diff --git a/site_scons/SGppConfigure.py b/site_scons/SGppConfigure.py
+index e5f54931f..2b0adc82a 100644
+--- a/site_scons/SGppConfigure.py
++++ b/site_scons/SGppConfigure.py
+@@ -3,7 +3,8 @@
+ # use, please see the copyright notice provided with SG++ or at
+ # sgpp.sparsegrids.org
+ 
+-
++import warnings
++warnings.filterwarnings("ignore", category=DeprecationWarning) 
+ import distutils.sysconfig
+ import errno
+ import os
+@@ -311,6 +312,8 @@ def checkPython(config):
+         raise Exception("Python 3 is required for SGpp python support!")
+       
+     pythonpath = getOutput(["python3", "-c",
++          "import warnings; "
++          "warnings.filterwarnings(\"ignore\", category=DeprecationWarning); " 
+           "import distutils.sysconfig; "
+           "print(distutils.sysconfig.get_python_inc())"])
+     package = "python3-dev"
diff --git a/var/spack/repos/builtin/packages/sgpp/package.py b/var/spack/repos/builtin/packages/sgpp/package.py
index 3b6a18055a429d..4aa479aaf094bf 100644
--- a/var/spack/repos/builtin/packages/sgpp/package.py
+++ b/var/spack/repos/builtin/packages/sgpp/package.py
@@ -22,6 +22,10 @@ class Sgpp(SConsPackage):
     version("3.4.0", sha256="450d4002850b0a48c561abe221b634261ca44eee111ca605c3e80797182f40b3")
     version("3.3.0", sha256="ca4d5b79f315b425ce69b04940c141451a76848bf1bd7b96067217304c68e2d4")
     version("3.2.0", sha256="dab83587fd447f92ed8546eacaac6b8cbe65b8db5e860218c0fa2e42f776962d")
+    # Note: Older versions of SGpp required Python 2 (and offered Python 2 bindings) and have
+    # thus been removed from this list as Spack now requires Python 3.
+    # The last spack release with support for Python 2 is v0.19 - there, the spack package
+    # still supports SGpp versions 3.1.0 and 3.0.0 if required.
 
     # Patches with bugfixes that are necessary to build old SGpp versions
     # with spack. Patches are submitted upstream, but need to applied
@@ -42,8 +46,13 @@ class Sgpp(SConsPackage):
     # Fixes compilation with AVX512 and datadriven
     # Fixed in SGpp in PR https://github.com/SGpp/SGpp/pull/229
     patch("avx512_datadriven_compilation.patch", when="@:3.3.0+datadriven")
+    # Continue despite distutils deprecation warning!
+    # distutils will be removed in future SGpp versions. See
+    # https://github.com/SGpp/SGpp/issues/263 for associated issue!
+    # TODO Once distutils is removed from SGpp, limit patch to @:3.4.0
+    patch("disable_disutils_deprecation_warning.patch", when="^python@3.10:3.11")
 
-    variant("python", default=True, description="Provide Python bindings for SGpp", when="@3.2:")
+    variant("python", default=True, description="Provide Python bindings for SGpp")
     variant("optimization", default=True, description="Builds the optimization module of SGpp")
     variant("pde", default=True, description="Builds the datadriven module of SGpp")
     variant("quadrature", default=True, description="Builds the datadriven module of SGpp")
@@ -56,22 +65,17 @@ class Sgpp(SConsPackage):
     )
     variant("mpi", default=False, description="Enables support for MPI-distributed operations")
 
-    # Java variant deactivated due to spack issue #987
-    # variant('java', default=False,
-    #         description='Provide Java bindings for SGpp')
-    # depends_on('swig@3:', when='+java', type=('build'))
-    # extends('openjdk', when='+java')
-
     # Mandatory dependencies
-    depends_on("scons", type=("build"))
-    depends_on("scons@3:", when="@3.2.0:", type=("build"))
-    depends_on("zlib", type=("link"))
+    depends_on("scons@3:", type=("build"))
+    depends_on("zlib-api", type=("link"))
     # Python dependencies
     extends("python", when="+python")
     depends_on("py-pip", when="+python", type="build")
     depends_on("py-wheel", when="+python", type="build")
-    depends_on("py-setuptools", when="+python", type=("build"))
-    depends_on("python@3.7:", when="+python", type=("build", "run"))
+    # TODO allow newer versions once distutils is removed from SGpp
+    depends_on("py-setuptools@:59", type=("build"))
+    # TODO allow newer versions once distutils is removed from SGpp
+    depends_on("python@3.7:3.11", type=("build", "run"))
     depends_on("swig@3:", when="+python", type=("build"))
     depends_on("py-numpy@1.17:", when="+python", type=("build", "run"))
     depends_on("py-scipy@1.3:", when="+python", type=("build", "run"))
@@ -105,7 +109,6 @@ class Sgpp(SConsPackage):
     conflicts("+misc", when="-solver")
     conflicts("+misc", when="-optimization")
     conflicts("+misc", when="-pde")
-    conflicts("+misc", when="@1.0.0:3.1.0", msg="The misc module was introduced in version 3.2.0")
     # Combigrid module requirements (for 3.2.0 or older)
     # newer combigrids have no dependencies
     conflicts("+combigrid", when="@1.0.0:3.2.0~optimization")
@@ -138,9 +141,7 @@ def build_args(self, spec, prefix):
         # Generate swig bindings?
         self.args.append("SG_PYTHON={0}".format("1" if "+python" in spec else "0"))
 
-        # Java variant deactivated due to spack issue #987
-        # self.args.append('SG_JAVA={0}'.format(
-        #     '1' if '+java' in spec else '0'))
+        # Java bindings are now deprecated within SGpp
         self.args.append("SG_JAVA=0")
 
         # Which modules to build?
@@ -150,10 +151,7 @@ def build_args(self, spec, prefix):
         self.args.append("SG_DATADRIVEN={0}".format("1" if "+datadriven" in spec else "0"))
         self.args.append("SG_COMBIGRID={0}".format("1" if "+combigrid" in spec else "0"))
         self.args.append("SG_SOLVER={0}".format("1" if "+solver" in spec else "0"))
-
-        # Misc flag did not exist in older versions
-        if not spec.satisfies("@1.0.0:3.2.0"):
-            self.args.append("SG_MISC={0}".format("1" if "+misc" in spec else "0"))
+        self.args.append("SG_MISC={0}".format("1" if "+misc" in spec else "0"))
 
         # SIMD scons parameter (pick according to simd spec)
         if "avx512" in self.spec.target:
@@ -181,6 +179,10 @@ def build_args(self, spec, prefix):
 
         return self.args
 
+    def install_args(self, spec, prefix):
+        # SGpp expects the same args for the install and build commands
+        return self.args
+
     @run_after("install")
     def python_install(self):
         if "+python" in self.spec:
diff --git a/var/spack/repos/builtin/packages/shuffile/package.py b/var/spack/repos/builtin/packages/shuffile/package.py
index 309d30b95cbc84..cf05d17b3d2c83 100644
--- a/var/spack/repos/builtin/packages/shuffile/package.py
+++ b/var/spack/repos/builtin/packages/shuffile/package.py
@@ -25,7 +25,7 @@ class Shuffile(CMakePackage):
 
     depends_on("mpi")
     depends_on("kvtree+mpi")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     depends_on("kvtree@:1.3.0", when="@:0.2.0")
     depends_on("kvtree@1.4.0:", when="@0.3.0:")
diff --git a/var/spack/repos/builtin/packages/sickle/package.py b/var/spack/repos/builtin/packages/sickle/package.py
index b9b0975de8ab7c..90c3e25d0b3a8d 100644
--- a/var/spack/repos/builtin/packages/sickle/package.py
+++ b/var/spack/repos/builtin/packages/sickle/package.py
@@ -17,7 +17,7 @@ class Sickle(MakefilePackage):
 
     version("1.33", sha256="eab271d25dc799e2ce67c25626128f8f8ed65e3cd68e799479bba20964624734")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/sigio/package.py b/var/spack/repos/builtin/packages/sigio/package.py
index 241154dde986eb..069ad8accefda3 100644
--- a/var/spack/repos/builtin/packages/sigio/package.py
+++ b/var/spack/repos/builtin/packages/sigio/package.py
@@ -14,11 +14,21 @@ class Sigio(CMakePackage):
 
     homepage = "https://noaa-emc.github.io/NCEPLIBS-sigio"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-sigio/archive/refs/tags/v2.3.2.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-sigio"
 
-    maintainers("t-brown", "AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
+    maintainers("AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
 
+    version("develop", branch="develop")
     version("2.3.2", sha256="333f3cf3a97f97103cbafcafc2ad89b24faa55b1332a98adc1637855e8a5b613")
 
+    variant("pfunit", default=False, description="Use pFunit to enable unit testing")
+
+    depends_on("pfunit", when="+pfunit")
+
+    def cmake_args(self):
+        args = [self.define("ENABLE_TESTS", self.run_tests)]
+        return args
+
     def setup_run_environment(self, env):
         lib = find_libraries("libsigio", root=self.prefix, shared=False, recursive=True)
         # Only one library version, but still need to set _4 to make NCO happy
@@ -31,3 +41,7 @@ def flag_handler(self, name, flags):
             if name == "fflags":
                 flags.append("-Free")
         return (None, None, flags)
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/silo/mkinc-usr-bin-env-perl.patch b/var/spack/repos/builtin/packages/silo/mkinc-usr-bin-env-perl.patch
new file mode 100644
index 00000000000000..13f233010c16e3
--- /dev/null
+++ b/var/spack/repos/builtin/packages/silo/mkinc-usr-bin-env-perl.patch
@@ -0,0 +1,23 @@
+diff --git a/config/mkinc b/config/mkinc
+index e52bb64..6d96176 100755
+--- a/config/mkinc
++++ b/config/mkinc
+@@ -17,7 +17,7 @@ exec perl /tmp/visitperl$$ $0 ${1+"$@"}
+ unlink $0;
+ $0 = shift @ARGV;
+
+-#!/usr/bin/perl
++#!/usr/bin/env perl
+ #########################################################################
+ # Copyright (C) 1994-2016 Lawrence Livermore National Security, LLC.
+ # LLNL-CODE-425250.
+diff --git a/config/mklite b/config/mklite
+index f9394a9..274c867 100755
+--- a/config/mklite
++++ b/config/mklite
+@@ -1,4 +1,4 @@
+-#!/usr/bin/perl
++#!/usr/bin/env perl
+
+ #
+ # Tool to generate lite_pdb.h file from pdb.h and score.h headers.
diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py
index 123ad8418d6eba..2678b0d7c56021 100644
--- a/var/spack/repos/builtin/packages/silo/package.py
+++ b/var/spack/repos/builtin/packages/silo/package.py
@@ -12,8 +12,21 @@ class Silo(AutotoolsPackage):
     data to binary, disk files."""
 
     homepage = "https://wci.llnl.gov/simulation/computer-codes/silo"
+    git = "https://github.com/LLNL/Silo.git"
     url = "https://wci.llnl.gov/sites/wci/files/2021-01/silo-4.10.2.tgz"
+    maintainers("patrickb314")
 
+    version(
+        "4.11.1",
+        preferred=True,
+        sha256="49eddc00304aa4a19074b099559edbdcaa3532c98df32f99aa62b9ec3ea7cee2",
+        url="https://github.com/LLNL/Silo/releases/download/4.11.1/silo-4.11.1.tar.xz",
+    )
+    version(
+        "4.11.1-bsd",
+        sha256="51ccfdf3c09dfc98c7858a0a6f08cc3b2a07ee3c4142ee6482ba7b24e314c2aa",
+        url="https://github.com/LLNL/Silo/releases/download/4.11.1/silo-4.11.1-bsd.tar.xz",
+    )
     version(
         "4.11",
         sha256="ab936c1f4fc158d9fdc4415965f7d9def7f4abeca596fe5a25bd8485654898ac",
@@ -46,6 +59,7 @@ class Silo(AutotoolsPackage):
     variant("hzip", default=True, description="Enable hzip support")
     variant("fpzip", default=True, description="Enable fpzip support")
 
+    depends_on("perl", type="build")
     depends_on("m4", type="build", when="+shared")
     depends_on("autoconf", type="build", when="+shared")
     depends_on("autoconf-archive", type="build", when="+shared")
@@ -59,7 +73,7 @@ class Silo(AutotoolsPackage):
     # Xmu dependency is required on Ubuntu 18-20
     depends_on("libxmu", when="+silex")
     depends_on("readline")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     patch("remove-mpiposix.patch", when="@4.8:4.10.2")
 
@@ -67,27 +81,29 @@ class Silo(AutotoolsPackage):
     patch("H5FD_class_t-terminate.patch", when="@:4.10.2-bsd")
 
     # H5EPR_SEMI_COLON.patch was fixed in current dev
-    # patch("H5EPR_SEMI_COLON.patch", when="@:4.11-bsd")
-    patch("H5EPR_SEMI_COLON.patch")
+    patch("H5EPR_SEMI_COLON.patch", when="@:4.11-bsd")
 
     # Fix missing F77 init, fixed in 4.9
     patch("48-configure-f77.patch", when="@:4.8")
 
     # The previously used AX_CHECK_COMPILER_FLAGS macro was dropped from
     # autoconf-archive in 2011
-    patch("configure-AX_CHECK_COMPILE_FLAG.patch")
+    patch("configure-AX_CHECK_COMPILE_FLAG.patch", when="@:4.11-bsd")
 
     # API changes in hdf5-1.13 cause breakage
     # See https://github.com/LLNL/Silo/pull/260
-    patch("hdf5-113.patch", when="@4.11: +hdf5 ^hdf5@1.13:")
-    conflicts("hdf5@1.13:", when="@:4.10.2-bsd")
+    patch("hdf5-113.patch", when="@4.11:4.11-bsd +hdf5 ^hdf5@1.13:")
+    conflicts("^hdf5@1.13:", when="@:4.10.2-bsd")
 
     # hzip and fpzip are not available in the BSD releases
     conflicts("+hzip", when="@4.10.2-bsd,4.11-bsd")
     conflicts("+fpzip", when="@4.10.2-bsd,4.11-bsd")
 
     # zfp include missing
-    patch("zfp_error.patch", when="@4.11 +hdf5")
+    patch("zfp_error.patch", when="@4.11:4.11-bsd +hdf5")
+
+    # use /usr/bin/env perl for portability
+    patch("mkinc-usr-bin-env-perl.patch", when="@:4.11-bsd")
 
     def flag_handler(self, name, flags):
         spec = self.spec
@@ -95,7 +111,6 @@ def flag_handler(self, name, flags):
             if "+hdf5" in spec:
                 if spec["hdf5"].satisfies("~shared"):
                     flags.append("-ldl")
-            flags.append(spec["readline"].libs.search_flags)
 
         if "+pic" in spec:
             if name == "cflags":
@@ -105,6 +120,9 @@ def flag_handler(self, name, flags):
             elif name == "fcflags":
                 flags.append(self.compiler.fc_pic_flag)
         if name == "cflags" or name == "cxxflags":
+            if spec.satisfies("%oneapi"):
+                flags.append("-Wno-error=int")
+                flags.append("-Wno-error=int-conversion")
             if "+hdf5" in spec:
                 # @:4.10 can use up to the 1.10 API
                 if "@:4.10" in spec:
@@ -187,7 +205,7 @@ def configure_args(self):
 
         # Do not specify the prefix of zlib if it is in a system directory
         # (see https://github.com/spack/spack/pull/21900).
-        zlib_prefix = self.spec["zlib"].prefix
+        zlib_prefix = self.spec["zlib-api"].prefix
         if is_system_path(zlib_prefix):
             config_args.append("--with-zlib=yes")
         else:
diff --git a/var/spack/repos/builtin/packages/singularityce/package.py b/var/spack/repos/builtin/packages/singularityce/package.py
index f6e05aa1436509..4fccdd059c6088 100644
--- a/var/spack/repos/builtin/packages/singularityce/package.py
+++ b/var/spack/repos/builtin/packages/singularityce/package.py
@@ -79,14 +79,14 @@ def config_options(self):
     # Hijack the edit stage to run mconfig.
     def edit(self, spec, prefix):
         with working_dir(self.build_directory):
-            confstring = "./mconfig --prefix=%s" % prefix
-            confstring += " " + " ".join(self.config_options)
+            _config_options = ["--prefix=%s" % prefix]
+            _config_options += self.config_options
             if "~suid" in spec:
-                confstring += " --without-suid"
+                _config_options += ["--without-suid"]
             if "~network" in spec:
-                confstring += " --without-network"
-            configure = Executable(confstring)
-            configure()
+                _config_options += ["--without-network"]
+            configure = Executable("./mconfig")
+            configure(*_config_options)
 
     # Set these for use by MakefilePackage's default build/install methods.
     build_targets = ["-C", "builddir", "parallel=False"]
@@ -190,7 +190,7 @@ class Singularityce(SingularityBase):
     See package definition or `spack-build-out.txt` build log for details,
     e.g.
 
-    tail -15 $(spack location -i singularity)/.spack/spack-build-out.txt
+    tail -15 $(spack location -i singularityce)/.spack/spack-build-out.txt
     """
 
     homepage = "https://sylabs.io/singularity/"
diff --git a/var/spack/repos/builtin/packages/sio/package.py b/var/spack/repos/builtin/packages/sio/package.py
index 2c86448a514e83..351fa4c3d571cf 100644
--- a/var/spack/repos/builtin/packages/sio/package.py
+++ b/var/spack/repos/builtin/packages/sio/package.py
@@ -37,7 +37,7 @@ class Sio(CMakePackage):
         description="Use the specified C++ standard when building.",
     )
 
-    depends_on("zlib", when="~builtin_zlib")
+    depends_on("zlib-api", when="~builtin_zlib")
 
     def cmake_args(self):
         return [
diff --git a/var/spack/repos/builtin/packages/sirius/cmake-fix-shared-library-installation.patch b/var/spack/repos/builtin/packages/sirius/cmake-fix-shared-library-installation.patch
deleted file mode 100644
index bffa67e5da708b..00000000000000
--- a/var/spack/repos/builtin/packages/sirius/cmake-fix-shared-library-installation.patch
+++ /dev/null
@@ -1,53 +0,0 @@
-From 4b51d07369b5972f3917cc8f2425caa814ae0975 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Tiziano=20M=C3=BCller?= 
-Date: Thu, 16 May 2019 10:53:04 +0200
-Subject: [PATCH] cmake: fix shared library installation
-
-fixes the error during `make install`:
-
-    TARGETS given no LIBRARY DESTINATION for shared library target
-
-when building shared libraries.
-
-... and respect the current OS/distro library dir.
----
- src/CMakeLists.txt | 11 +++++++++--
- 1 file changed, 9 insertions(+), 2 deletions(-)
-
-diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
-index 65307dd3..2b7a5279 100644
---- a/src/CMakeLists.txt
-+++ b/src/CMakeLists.txt
-@@ -2,6 +2,8 @@
- # working correctly
- # list(APPEND CUDA_NVCC_FLAGS "-Xcompiler -fPIC")
- 
-+include(GNUInstallDirs)  # required to get a proper LIBDIR variable
-+
- # keep two libraries: libsirius and libsirius_f
- 
- if(USE_CUDA)
-@@ -9,13 +11,18 @@ if(USE_CUDA)
-   file(GLOB_RECURSE CUFILES_KERNELS "Kernels/*.cu")
-   add_library(sirius_cu "${CUFILES_KERNELS};${CUFILES_SDDK}")
-   set_target_properties(sirius_cu PROPERTIES POSITION_INDEPENDENT_CODE ON)
--  INSTALL (TARGETS sirius_cu ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/)
-+  INSTALL (TARGETS sirius_cu
-+    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-+    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-+    )
- endif()
- if(CREATE_FORTRAN_BINDINGS)
-   add_library(sirius_f "sirius_api.cpp;sirius.f90")
-   SIRIUS_SETUP_TARGET(sirius_f)
-   INSTALL (TARGETS sirius_f ARCHIVE DESTINATION
--    ${CMAKE_INSTALL_PREFIX}/lib/)
-+    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-+    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-+    )
-   set_target_properties(sirius_f PROPERTIES POSITION_INDEPENDENT_CODE ON)
-   set_target_properties(sirius_f PROPERTIES Fortran_MODULE_DIRECTORY mod_files)
-   target_link_libraries(sirius_f PUBLIC OpenMP::OpenMP_CXX)
--- 
-2.16.4
-
diff --git a/var/spack/repos/builtin/packages/sirius/link-libraries-fortran.patch b/var/spack/repos/builtin/packages/sirius/link-libraries-fortran.patch
deleted file mode 100644
index 16de3a1710150a..00000000000000
--- a/var/spack/repos/builtin/packages/sirius/link-libraries-fortran.patch
+++ /dev/null
@@ -1,25 +0,0 @@
-From 76f238d0ebcfc186f5210b0807b6a83a5c17bd43 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Tiziano=20M=C3=BCller?= 
-Date: Wed, 24 Apr 2019 15:30:50 +0200
-Subject: [PATCH 2/2] cmake: properly link Fortran lib against used libraries
-
----
- src/CMakeLists.txt | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
-index b927adef..a0d4aeb3 100644
---- a/src/CMakeLists.txt
-+++ b/src/CMakeLists.txt
-@@ -13,7 +13,7 @@ if(USE_CUDA)
- endif()
- if(CREATE_FORTRAN_BINDINGS)
-   add_library(sirius_f "sirius_api.cpp;sirius.f90")
--  add_dependencies(sirius_f generate_version_hpp runtime_options_json_hpp)
-+  SIRIUS_SETUP_TARGET(sirius_f)
-   INSTALL (TARGETS sirius_f ARCHIVE DESTINATION
-     ${CMAKE_INSTALL_PREFIX}/lib/)
-   set_target_properties(sirius_f PROPERTIES POSITION_INDEPENDENT_CODE ON)
--- 
-2.16.4
-
diff --git a/var/spack/repos/builtin/packages/sirius/package.py b/var/spack/repos/builtin/packages/sirius/package.py
index c171e59113349d..20f5a4246d420c 100644
--- a/var/spack/repos/builtin/packages/sirius/package.py
+++ b/var/spack/repos/builtin/packages/sirius/package.py
@@ -26,111 +26,54 @@ class Sirius(CMakePackage, CudaPackage, ROCmPackage):
     version("7.3.2", sha256="a256508de6b344345c295ad8642dbb260c4753cd87cc3dd192605c33542955d7")
     version("7.3.1", sha256="8bf9848b8ebf0b43797fd359adf8c84f00822de4eb677e3049f22baa72735e98")
     version("7.3.0", sha256="69b5cf356adbe181be6c919032859c4e0160901ff42a885d7e7ea0f38cc772e2")
-    version("7.2.7", sha256="929bf7f131a4847624858b9c4295532c24b0c06f6dcef5453c0dfc33fb78eb03")
-    version("7.2.6", sha256="e751fd46cdc7c481ab23b0839d3f27fb00b75dc61dc22a650c92fe8e35336e3a")
-    version("7.2.5", sha256="794e03d4da91025f77542d3d593d87a8c74e980394f658a0210a4fd91c011f22")
-    version("7.2.4", sha256="aeed0e83b80c3a79a9469e7f3fe10d80ad331795e38dbc3c49cb0308e2bd084d")
-    version("7.2.3", sha256="6c10f0e87e50fcc7cdb4d1b2d35e91dba6144de8f111e36c7d08912e5942a906")
-    version("7.2.1", sha256="01bf6c9893ff471473e13351ca7fdc2ed6c1f4b1bb7afa151909ea7cd6fa0de7")
-    version("7.2.0", sha256="537800459db8a7553d7aa251c19f3a31f911930194b068bc5bca2dfb2c9b71db")
     version(
-        "7.0.2",
-        sha256="ee613607ce3be0b2c3f69b560b2415ce1b0e015179002aa90739430dbfaa0389",
-        deprecated=True,
-    )
-    version(
-        "7.0.1",
-        sha256="cca11433f86e7f4921f7956d6589f27bf0fd5539f3e8f96e66a3a6f274888595",
-        deprecated=True,
-    )
-    version(
-        "7.0.0",
-        sha256="da783df11e7b65668e29ba8d55c8a6827e2216ad6d88040f84f42ac20fd1bb99",
-        deprecated=True,
-    )
-    version(
-        "6.5.7",
-        sha256="d886c3066163c43666ebac2ea50351df03907b5686671e514a75f131ba51b43c",
-        deprecated=True,
-    )
-    version(
-        "6.5.6",
-        sha256="c8120100bde4477545eae489ea7f9140d264a3f88696ec92728616d78f214cae",
-        deprecated=True,
-    )
-    version(
-        "6.5.5",
-        sha256="0b23d3a8512682eea67aec57271031c65f465b61853a165015b38f7477651dd1",
+        "7.2.7",
+        sha256="929bf7f131a4847624858b9c4295532c24b0c06f6dcef5453c0dfc33fb78eb03",
         deprecated=True,
     )
     version(
-        "6.5.4",
-        sha256="5f731926b882a567d117afa5e0ed33291f1db887fce52f371ba51f014209b85d",
+        "7.2.6",
+        sha256="e751fd46cdc7c481ab23b0839d3f27fb00b75dc61dc22a650c92fe8e35336e3a",
         deprecated=True,
     )
     version(
-        "6.5.3",
-        sha256="eae0c303f332425a8c792d4455dca62557931b28a5df8b4c242652d5ffddd580",
+        "7.2.5",
+        sha256="794e03d4da91025f77542d3d593d87a8c74e980394f658a0210a4fd91c011f22",
         deprecated=True,
     )
     version(
-        "6.5.2",
-        sha256="c18adc45b069ebae03f94eeeeed031ee99b3d8171fa6ee73c7c6fb1e42397fe7",
+        "7.2.4",
+        sha256="aeed0e83b80c3a79a9469e7f3fe10d80ad331795e38dbc3c49cb0308e2bd084d",
         deprecated=True,
     )
     version(
-        "6.5.1",
-        sha256="599dd0fa25a4e83db2a359257a125e855d4259188cf5b0065b8e7e66378eacf3",
+        "7.2.3",
+        sha256="6c10f0e87e50fcc7cdb4d1b2d35e91dba6144de8f111e36c7d08912e5942a906",
         deprecated=True,
     )
     version(
-        "6.5.0",
-        sha256="5544f3abbb71dcd6aa08d18aceaf53c38373de4cbd0c3af44fbb39c20cfeb7cc",
+        "7.2.1",
+        sha256="01bf6c9893ff471473e13351ca7fdc2ed6c1f4b1bb7afa151909ea7cd6fa0de7",
         deprecated=True,
     )
     version(
-        "6.4.4",
-        sha256="1c5de9565781847658c3cc11edcb404e6e6d1c5a9dfc81e977de7a9a7a162c8a",
+        "7.2.0",
+        sha256="537800459db8a7553d7aa251c19f3a31f911930194b068bc5bca2dfb2c9b71db",
         deprecated=True,
     )
     version(
-        "6.4.3",
-        sha256="4d1effeadb84b3e1efd7d9ac88018ef567aa2e0aa72e1112f0abf2e493e2a189",
-        deprecated=True,
-    )
-    version(
-        "6.4.2",
-        sha256="40b9b66deebb6538fc0f4cd802554d0d763ea6426b9b2f0e8db8dc617e494479",
-        deprecated=True,
-    )
-    version(
-        "6.4.1",
-        sha256="86f25c71517952a63e92e0a9bcf66d27e4afb2b0d67cf84af480f116b8e7f53c",
-        deprecated=True,
-    )
-    version(
-        "6.4.0",
-        sha256="bc61758b71dd2996e2ff515b8c3560b2c69c00931cb2811a163a31bcfea4436e",
-        deprecated=True,
-    )
-    version(
-        "6.3.4",
-        sha256="8839e988b4bb6ef99b6180f7fba03a5537e31fce51bb3e4c2298b513d6a07e0a",
-        deprecated=True,
-    )
-    version(
-        "6.3.3",
-        sha256="7ba30a4e5c9a545433251211454ec0d59b74ba8941346057bc7de11e7f6886f7",
+        "7.0.2",
+        sha256="ee613607ce3be0b2c3f69b560b2415ce1b0e015179002aa90739430dbfaa0389",
         deprecated=True,
     )
     version(
-        "6.3.2",
-        sha256="1723e5ad338dad9a816369a6957101b2cae7214425406b12e8712c82447a7ee5",
+        "7.0.1",
+        sha256="cca11433f86e7f4921f7956d6589f27bf0fd5539f3e8f96e66a3a6f274888595",
         deprecated=True,
     )
     version(
-        "6.1.5",
-        sha256="379f0a2e5208fd6d91c2bd4939c3a5c40002975fb97652946fa1bfe4a3ef97cb",
+        "7.0.0",
+        sha256="da783df11e7b65668e29ba8d55c8a6827e2216ad6d88040f84f42ac20fd1bb99",
         deprecated=True,
     )
 
@@ -165,6 +108,7 @@ class Sirius(CMakePackage, CudaPackage, ROCmPackage):
         "profiler", default=True, description="Use internal profiler to measure execution time"
     )
 
+    depends_on("cmake@3.23:", type="build")
     depends_on("mpi")
     depends_on("gsl")
     depends_on("lapack")
@@ -191,7 +135,6 @@ class Sirius(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("magma", when="+magma")
     depends_on("boost cxxstd=14 +filesystem", when="+boost_filesystem")
 
-    depends_on("spfft@0.9.6: +mpi", when="@6.4.0:")
     depends_on("spfft@0.9.13:", when="@7.0.1:")
     depends_on("spfft+single_precision", when="+single_precision ^spfft")
     depends_on("spfft+cuda", when="+cuda ^spfft")
@@ -217,7 +160,6 @@ class Sirius(CMakePackage, CudaPackage, ROCmPackage):
     # FindHIP cmake script only works for < 4.1
     depends_on("hip@:4.0", when="@:7.2.0 +rocm")
 
-    conflicts("+shared", when="@6.3.0:6.4")
     conflicts("+boost_filesystem", when="~apps")
     conflicts("^libxc@5.0.0")  # known to produce incorrect results
     conflicts("+single_precision", when="@:7.2.4")
@@ -240,57 +182,37 @@ class Sirius(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("costa+shared", when="@7.3.2:")
 
     with when("@7.5: +memory_pool"):
-        depends_on("umpire")
+        depends_on("umpire~cuda~rocm", when="~cuda~rocm")
         depends_on("umpire+cuda~device_alloc", when="+cuda")
         depends_on("umpire+rocm~device_alloc", when="+rocm")
 
-    patch("strip-spglib-include-subfolder.patch", when="@6.1.5")
-    patch("link-libraries-fortran.patch", when="@6.1.5")
-    patch("cmake-fix-shared-library-installation.patch", when="@6.1.5")
     patch("mpi_datatypes.patch", when="@:7.2.6")
 
-    @property
-    def libs(self):
-        libraries = []
-
-        if "@6.3.0:" in self.spec:
-            libraries += ["libsirius"]
-
-            return find_libraries(
-                libraries, root=self.prefix, shared="+shared" in self.spec, recursive=True
-            )
-        else:
-            if "+fortran" in self.spec:
-                libraries += ["libsirius_f"]
-
-            if "+cuda" in self.spec:
-                libraries += ["libsirius_cu"]
-
-            return find_libraries(
-                libraries, root=self.prefix, shared="+shared" in self.spec, recursive=True
-            )
-
     def cmake_args(self):
         spec = self.spec
 
+        cm_label = ""
+        if "@7.5:" in spec:
+            cm_label = "SIRIUS_"
+
         args = [
-            self.define_from_variant("USE_OPENMP", "openmp"),
-            self.define_from_variant("USE_ELPA", "elpa"),
-            self.define_from_variant("USE_MAGMA", "magma"),
-            self.define_from_variant("USE_NLCGLIB", "nlcglib"),
-            self.define_from_variant("USE_VDWXC", "vdwxc"),
-            self.define_from_variant("USE_MEMORY_POOL", "memory_pool"),
-            self.define_from_variant("USE_SCALAPACK", "scalapack"),
-            self.define_from_variant("CREATE_FORTRAN_BINDINGS", "fortran"),
-            self.define_from_variant("CREATE_PYTHON_MODULE", "python"),
-            self.define_from_variant("USE_CUDA", "cuda"),
-            self.define_from_variant("USE_ROCM", "rocm"),
+            self.define_from_variant(cm_label + "USE_OPENMP", "openmp"),
+            self.define_from_variant(cm_label + "USE_ELPA", "elpa"),
+            self.define_from_variant(cm_label + "USE_MAGMA", "magma"),
+            self.define_from_variant(cm_label + "USE_NLCGLIB", "nlcglib"),
+            self.define_from_variant(cm_label + "USE_VDWXC", "vdwxc"),
+            self.define_from_variant(cm_label + "USE_MEMORY_POOL", "memory_pool"),
+            self.define_from_variant(cm_label + "USE_SCALAPACK", "scalapack"),
+            self.define_from_variant(cm_label + "CREATE_FORTRAN_BINDINGS", "fortran"),
+            self.define_from_variant(cm_label + "CREATE_PYTHON_MODULE", "python"),
+            self.define_from_variant(cm_label + "USE_CUDA", "cuda"),
+            self.define_from_variant(cm_label + "USE_ROCM", "rocm"),
+            self.define_from_variant(cm_label + "BUILD_APPS", "apps"),
+            self.define_from_variant(cm_label + "BUILD_SHARED_LIBS", "shared"),
+            self.define_from_variant(cm_label + "USE_FP32", "single_precision"),
+            self.define_from_variant(cm_label + "USE_PROFILER", "profiler"),
+            self.define_from_variant(cm_label + "USE_WANNIER90", "wannier90"),
             self.define_from_variant("BUILD_TESTING", "tests"),
-            self.define_from_variant("BUILD_APPS", "apps"),
-            self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
-            self.define_from_variant("USE_FP32", "single_precision"),
-            self.define_from_variant("USE_PROFILER", "profiler"),
-            self.define_from_variant("USE_WANNIER90", "wannier90"),
         ]
 
         lapack = spec["lapack"]
@@ -308,43 +230,37 @@ def cmake_args(self):
         if "+scalapack" in spec and "^cray-libsci" not in spec:
             args.extend(
                 [
-                    self.define("SCALAPACK_FOUND", "true"),
-                    self.define("SCALAPACK_INCLUDE_DIRS", spec["scalapack"].prefix.include),
-                    self.define("SCALAPACK_LIBRARIES", spec["scalapack"].libs.joined(";")),
+                    self.define(cm_label + "SCALAPACK_FOUND", "true"),
+                    self.define(
+                        cm_label + "SCALAPACK_INCLUDE_DIRS", spec["scalapack"].prefix.include
+                    ),
+                    self.define(
+                        cm_label + "SCALAPACK_LIBRARIES", spec["scalapack"].libs.joined(";")
+                    ),
                 ]
             )
 
         if "^cray-libsci" in spec:
-            args.append(self.define("USE_CRAY_LIBSCI", "ON"))
+            args.append(self.define(cm_label + "USE_CRAY_LIBSCI", "ON"))
 
         if spec["blas"].name in ["intel-mkl", "intel-parallel-studio", "intel-oneapi-mkl"]:
-            args.append(self.define("USE_MKL", "ON"))
+            args.append(self.define(cm_label + "USE_MKL", "ON"))
 
         if "+elpa" in spec:
             elpa_incdir = os.path.join(spec["elpa"].headers.directories[0], "elpa")
-            args.append(self.define("ELPA_INCLUDE_DIR", elpa_incdir))
+            args.append(self.define(cm_label + "ELPA_INCLUDE_DIR", elpa_incdir))
 
         if "+cuda" in spec:
             cuda_arch = spec.variants["cuda_arch"].value
             if cuda_arch[0] != "none":
-                # Specify a single arch directly
-                if "@:6" in spec:
-                    args.append(
-                        self.define("CMAKE_CUDA_FLAGS", "-arch=sm_{0}".format(cuda_arch[0]))
-                    )
-
                 # Make SIRIUS handle it
+                if "@6:7.4.3" in spec:
+                    args.append(self.define("CMAKE_CUDA_ARCH", ";".join(cuda_arch)))
                 else:
-                    args.append(self.define("CUDA_ARCH", ";".join(cuda_arch)))
+                    args.append(self.define("CMAKE_CUDA_ARCHITECTURES", ";".join(cuda_arch)))
 
         if "+rocm" in spec:
             archs = ",".join(self.spec.variants["amdgpu_target"].value)
-            args.extend(
-                [
-                    self.define("HIP_ROOT_DIR", spec["hip"].prefix),
-                    self.define("HIP_HCC_FLAGS", "--amdgpu-target={0}".format(archs)),
-                    self.define("HIP_CXX_COMPILER", self.spec["hip"].hipcc),
-                ]
-            )
+            args.extend([self.define("CMAKE_HIP_ARCHITECTURES", archs)])
 
         return args
diff --git a/var/spack/repos/builtin/packages/sirius/strip-spglib-include-subfolder.patch b/var/spack/repos/builtin/packages/sirius/strip-spglib-include-subfolder.patch
deleted file mode 100644
index 0e775149a37f62..00000000000000
--- a/var/spack/repos/builtin/packages/sirius/strip-spglib-include-subfolder.patch
+++ /dev/null
@@ -1,25 +0,0 @@
-From 1a4ae29387790183b3de38acf8d38623429f3f62 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Tiziano=20M=C3=BCller?= 
-Date: Wed, 24 Apr 2019 11:29:05 +0200
-Subject: [PATCH] spglib does not install to a subfolder by default
-
----
- src/Unit_cell/unit_cell_symmetry.hpp | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/src/Unit_cell/unit_cell_symmetry.hpp b/src/Unit_cell/unit_cell_symmetry.hpp
-index 9852bd4a..2c5a1e04 100644
---- a/src/Unit_cell/unit_cell_symmetry.hpp
-+++ b/src/Unit_cell/unit_cell_symmetry.hpp
-@@ -26,7 +26,7 @@
- #define __UNIT_CELL_SYMMETRY_HPP__
- 
- extern "C" {
--#include 
-+#include 
- }
- 
- #include "geometry3d.hpp"
--- 
-2.16.4
-
diff --git a/var/spack/repos/builtin/packages/sjpeg/package.py b/var/spack/repos/builtin/packages/sjpeg/package.py
index 4d0dfd7eb9bdc7..1479c736154120 100644
--- a/var/spack/repos/builtin/packages/sjpeg/package.py
+++ b/var/spack/repos/builtin/packages/sjpeg/package.py
@@ -16,7 +16,7 @@ class Sjpeg(CMakePackage):
 
     depends_on("cmake@2.8.7:", type="build")
     # TODO: these dependencies seem to be optional?
-    # depends_on("zlib")
+    # depends_on("zlib-api")
     # depends_on("libpng")
     # depends_on("jpeg")
     # depends_on("gl")
diff --git a/var/spack/repos/builtin/packages/slate/package.py b/var/spack/repos/builtin/packages/slate/package.py
index af13a54de4606c..778beda83ae1d4 100644
--- a/var/spack/repos/builtin/packages/slate/package.py
+++ b/var/spack/repos/builtin/packages/slate/package.py
@@ -24,6 +24,9 @@ class Slate(CMakePackage, CudaPackage, ROCmPackage):
     test_requires_compiler = True
 
     version("master", branch="master")
+    version(
+        "2023.08.25", sha256="0894d8669ed88358cc7c4cb7b77d8467336613245a7b843f3504e9224632ce0e"
+    )
     version(
         "2022.07.00", sha256="176db81aef44b1d498a37c67b30aff88d4025770c9200e19ceebd416e4101327"
     )
@@ -48,17 +51,23 @@ class Slate(CMakePackage, CudaPackage, ROCmPackage):
     )
     variant("openmp", default=True, description="Build with OpenMP support.")
     variant("shared", default=True, description="Build shared library")
+    variant("sycl", default=False, description="Build with SYCL backend")
 
     # The runtime dependency on cmake is needed by the stand-alone tests (spack test).
     depends_on("cmake", type="run")
 
     depends_on("mpi", when="+mpi")
+    depends_on("intel-oneapi-mkl threads=openmp", when="+sycl")
     depends_on("blas")
     depends_on("blaspp ~cuda", when="~cuda")
     depends_on("blaspp +cuda", when="+cuda")
+    depends_on("blaspp ~sycl", when="~sycl")
+    depends_on("blaspp +sycl", when="+sycl")
     depends_on("blaspp ~rocm", when="~rocm")
     depends_on("lapackpp ~cuda", when="~cuda")
     depends_on("lapackpp +cuda", when="+cuda")
+    depends_on("lapackpp ~sycl", when="~sycl")
+    depends_on("lapackpp +sycl", when="+sycl")
     depends_on("lapackpp ~rocm", when="~rocm")
     for val in CudaPackage.cuda_arch_values:
         depends_on("blaspp +cuda cuda_arch=%s" % val, when="cuda_arch=%s" % val)
@@ -66,6 +75,7 @@ class Slate(CMakePackage, CudaPackage, ROCmPackage):
     for val in ROCmPackage.amdgpu_targets:
         depends_on("blaspp +rocm amdgpu_target=%s" % val, when="amdgpu_target=%s" % val)
         depends_on("lapackpp +rocm amdgpu_target=%s" % val, when="amdgpu_target=%s" % val)
+    depends_on("lapackpp@2023.08.25:", when="@2023.08.25:")
     depends_on("lapackpp@2022.07.00:", when="@2022.07.00:")
     depends_on("lapackpp@2022.05.00:", when="@2022.05.00:")
     depends_on("lapackpp@2021.04.00:", when="@2021.05.01:")
@@ -74,6 +84,8 @@ class Slate(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("scalapack", type="test")
     depends_on("hipify-clang", when="@:2021.05.02 +rocm ^hip@5:")
 
+    requires("%oneapi", when="+sycl", msg="slate+sycl must be compiled with %oneapi")
+
     cpp_17_msg = "Requires C++17 compiler support"
     conflicts("%gcc@:5", msg=cpp_17_msg)
     conflicts("%xl", msg=cpp_17_msg)
@@ -82,7 +94,11 @@ class Slate(CMakePackage, CudaPackage, ROCmPackage):
     conflicts(
         "+rocm", when="@:2020.10.00", msg="ROCm support requires SLATE 2021.05.01 or greater"
     )
-    conflicts("+rocm", when="+cuda", msg="SLATE only supports one GPU backend at a time")
+    backend_msg = "SLATE supports only one GPU backend at a time"
+    conflicts("+rocm", when="+cuda", msg=backend_msg)
+    conflicts("+rocm", when="+sycl", msg=backend_msg)
+    conflicts("+cuda", when="+sycl", msg=backend_msg)
+    conflicts("+sycl", when="@:2022.07.00", msg="SYCL support requires SLATE version 2023.08.25")
 
     def cmake_args(self):
         spec = self.spec
@@ -93,6 +109,8 @@ def cmake_args(self):
                 backend = "cuda"
             if "+rocm" in spec:
                 backend = "hip"
+            if "+sycl" in spec:
+                backend = "sycl"
             backend_config = "-Dgpu_backend=%s" % backend
 
         config = [
diff --git a/var/spack/repos/builtin/packages/sleef/package.py b/var/spack/repos/builtin/packages/sleef/package.py
index 663ffff3def032..79227766691a76 100644
--- a/var/spack/repos/builtin/packages/sleef/package.py
+++ b/var/spack/repos/builtin/packages/sleef/package.py
@@ -14,9 +14,7 @@ class Sleef(CMakePackage):
     git = "https://github.com/shibatch/sleef.git"
 
     version("master", branch="master")
-    version(
-        "3.5.1_2020-12-22", commit="e0a003ee838b75d11763aa9c3ef17bf71a725bff"
-    )  # py-torch@1.8:1.9
+    version("3.5.1_2020-12-22", commit="e0a003ee838b75d11763aa9c3ef17bf71a725bff")  # py-torch@1.8:
     version(
         "3.5.1",
         sha256="415ee9b1bcc5816989d3d4d92afd0cd3f9ee89cbd5a33eb008e69751e40438ab",
@@ -40,17 +38,25 @@ class Sleef(CMakePackage):
     )  # py-torch@0.4.1:1.0
     version("3.2", sha256="3130c5966e204e6d6a3ace81e543d12b5b21f60897f1c185bfa587c1bd77bee2")
 
-    # Some versions have ICE when building RelWithDebInfo with GCC 7
-    # See https://github.com/shibatch/sleef/issues/234
-    # See https://github.com/pytorch/pytorch/issues/26892
-    # See https://github.com/pytorch/pytorch/pull/26993
+    # https://github.com/shibatch/sleef/issues/474
+    conflicts("%apple-clang@15:")
 
     generator("ninja")
     depends_on("cmake@3.4.3:", type="build")
 
+    # # https://github.com/shibatch/sleef/issues/475
+    # depends_on("fftw-api")
+    # depends_on("mpfr")
+    # depends_on("openssl")
+
+    # # https://github.com/shibatch/sleef/issues/458
+    # conflicts("^mpfr@4.2:")
+
     def cmake_args(self):
+        # Taken from PyTorch's aten/src/ATen/CMakeLists.txt
         return [
-            self.define("DISABLE_FFTW", True),
-            self.define("DISABLE_MPFR", True),
-            self.define("DISABLE_SSL", True),
+            self.define("BUILD_SHARED_LIBS", False),
+            self.define("BUILD_DFT", False),
+            self.define("BUILD_GNUABI_LIBS", False),
+            self.define("BUILD_TESTS", False),
         ]
diff --git a/var/spack/repos/builtin/packages/slepc/package.py b/var/spack/repos/builtin/packages/slepc/package.py
index 597688cf68a808..979a252dd8ae3a 100644
--- a/var/spack/repos/builtin/packages/slepc/package.py
+++ b/var/spack/repos/builtin/packages/slepc/package.py
@@ -22,6 +22,8 @@ class Slepc(Package, CudaPackage, ROCmPackage):
     test_requires_compiler = True
 
     version("main", branch="main")
+    version("3.20.0", sha256="780c50260a9bc9b72776cb920774800c73832370938f1d48c2ea5c66d31b7380")
+    version("3.19.2", sha256="ca7ed906795971fbe35f08ee251a26b86a4442a18609b878cba00835c9d62034")
     version("3.19.1", sha256="280737e9ef762d7f0079ad3ad29913215c799ebf124651c723c1972f71fbc0db")
     version("3.19.0", sha256="724f6610a2e38b1be7586fd494fe350b58f5aee1ca734bd85e783aa9d3daa8de")
     version("3.18.3", sha256="1b02bdf87c083749e81b3735aae7728098eaab78143b262b92c2ab164924c6f5")
@@ -62,15 +64,8 @@ class Slepc(Package, CudaPackage, ROCmPackage):
 
     # Cannot mix release and development versions of SLEPc and PETSc:
     depends_on("petsc@main", when="@main")
-    depends_on("petsc@3.19.0:3.19", when="@3.19.0:3.19")
-    depends_on("petsc@3.18.0:3.18", when="@3.18.0:3.18")
-    depends_on("petsc@3.17.0:3.17", when="@3.17.0:3.17")
-    depends_on("petsc@3.16.0:3.16", when="@3.16.0:3.16")
-    depends_on("petsc@3.15.0:3.15", when="@3.15.0:3.15")
-    depends_on("petsc@3.14.0:3.14", when="@3.14.0:3.14")
-    depends_on("petsc@3.13.0:3.13", when="@3.13.0:3.13")
-    depends_on("petsc@3.12.0:3.12", when="@3.12.0:3.12")
-    depends_on("petsc@3.11.0:3.11", when="@3.11.0:3.11")
+    for ver in ["3.20", "3.19", "3.18", "3.17", "3.16", "3.15", "3.14", "3.13", "3.12", "3.11"]:
+        depends_on(f"petsc@{ver}", when=f"@{ver}")
     depends_on("petsc+cuda", when="+cuda")
     depends_on("arpack-ng~mpi", when="+arpack^petsc~mpi~int64")
     depends_on("arpack-ng+mpi", when="+arpack^petsc+mpi~int64")
diff --git a/var/spack/repos/builtin/packages/slurm/package.py b/var/spack/repos/builtin/packages/slurm/package.py
index 63585143ae87e9..aa4f126018bf39 100644
--- a/var/spack/repos/builtin/packages/slurm/package.py
+++ b/var/spack/repos/builtin/packages/slurm/package.py
@@ -26,6 +26,8 @@ class Slurm(AutotoolsPackage):
     homepage = "https://slurm.schedmd.com"
     url = "https://github.com/SchedMD/slurm/archive/slurm-21-08-8-2.tar.gz"
 
+    version("23-02-4-1", sha256="7290143a71ce2797d0df3423f08396fd5c0ae4504749ff372d6860b2d6a3a1b0")
+    version("23-02-3-1", sha256="c41747e4484011cf376d6d4bc73b6c4696cdc0f7db4f64174f111bb9f53fb603")
     version("23-02-2-1", sha256="71edcf187a7d68176cca06143adf98e8f332d42cdf000cb534b03b13834ad537")
     version("23-02-1-1", sha256="d827553496ee9158bbf6a862b563cfd48566e6d815ad2f8349950fe6f04934da")
     version("22-05-9-1", sha256="c9aaa2362b5bf7a4745c8bf90e8dd2ca50802f1241dd1f5220aec8448c09b514")
@@ -127,6 +129,10 @@ class Slurm(AutotoolsPackage):
         description="Set system configuration path (possibly /etc/slurm)",
     )
     variant("restd", default=False, description="Enable the slurmrestd server")
+    variant("nvml", default=False, description="Enable NVML autodetection")
+    variant("cgroup", default=False, description="Enable cgroup plugin")
+    variant("pam", default=False, description="Enable PAM support")
+    variant("rsmi", default=False, description="Enable ROCm SMI support")
 
     # TODO: add variant for BG/Q and Cray support
 
@@ -142,7 +148,7 @@ class Slurm(AutotoolsPackage):
     depends_on("openssl")
     depends_on("pkgconfig", type="build")
     depends_on("readline", when="+readline")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     depends_on("gtkplus", when="+gtk")
     depends_on("hdf5", when="+hdf5")
@@ -154,6 +160,11 @@ class Slurm(AutotoolsPackage):
     depends_on("libyaml", when="+restd")
     depends_on("libjwt", when="+restd")
 
+    depends_on("cuda", when="+nvml")
+    depends_on("dbus", when="+cgroup")
+    depends_on("linux-pam", when="+pam")
+    depends_on("rocm-smi-lib", when="+rsmi")
+
     executables = ["^srun$", "^salloc$"]
 
     @classmethod
@@ -180,7 +191,7 @@ def configure_args(self):
             "--with-lz4={0}".format(spec["lz4"].prefix),
             "--with-munge={0}".format(spec["munge"].prefix),
             "--with-ssl={0}".format(spec["openssl"].prefix),
-            "--with-zlib={0}".format(spec["zlib"].prefix),
+            "--with-zlib={0}".format(spec["zlib-api"].prefix),
         ]
 
         if "~gtk" in spec:
@@ -211,6 +222,15 @@ def configure_args(self):
         else:
             args.append("--without-pmix")
 
+        if spec.satisfies("+nvml"):
+            args.append(f"--with-nvml={spec['cuda'].prefix}")
+
+        if spec.satisfies("+pam"):
+            args.append(f"--with-pam_dir={spec['linux-pam'].prefix}")
+
+        if spec.satisfies("+rsmi"):
+            args.append(f"--with-rsmi={spec['rocm-smi-lib'].prefix}")
+
         sysconfdir = spec.variants["sysconfdir"].value
         if sysconfdir != "PREFIX/etc":
             args.append("--sysconfdir={0}".format(sysconfdir))
diff --git a/var/spack/repos/builtin/packages/snap-berkeley/package.py b/var/spack/repos/builtin/packages/snap-berkeley/package.py
index a2347cfcbd2ed0..ab9e3f6e0d1e0e 100644
--- a/var/spack/repos/builtin/packages/snap-berkeley/package.py
+++ b/var/spack/repos/builtin/packages/snap-berkeley/package.py
@@ -27,7 +27,7 @@ class SnapBerkeley(MakefilePackage):
         preferred=True,
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     conflicts("%gcc@6:", when="@:1.0beta.18")
 
diff --git a/var/spack/repos/builtin/packages/snappy/package.py b/var/spack/repos/builtin/packages/snappy/package.py
index e1fbe5a779625c..971d635291b144 100644
--- a/var/spack/repos/builtin/packages/snappy/package.py
+++ b/var/spack/repos/builtin/packages/snappy/package.py
@@ -24,6 +24,21 @@ class Snappy(CMakePackage):
 
     patch("link_gtest.patch", when="@:1.1.8")
 
+    # Version 1.1.9 makes use of an assembler feature that is not necessarily available when the
+    # __GNUC__ preprocessor macro is defined. Version 1.1.10 switched to the correct macro
+    # __GCC_ASM_FLAG_OUTPUTS__, which we also do for the version 1.1.9 by applying the patch from
+    # the upstream repo (see the commit message of the patch for more details).
+    patch(
+        "https://github.com/google/snappy/commit/8dd58a519f79f0742d4c68fbccb2aed2ddb651e8.patch?full_index=1",
+        sha256="debcdf182c046a30e9afea99ebbff280dd1fbb203e89abce6a05d3d17c587768",
+        when="@1.1.9",
+    )
+
+    # nvhpc@:22.3 does not know flag '-fno-rtti'
+    # nvhpc@:22.7 fails to compile snappy.cc: line 126: error: excessive recursion at instantiation
+    #   of class "snappy::::make_index_sequence<
+    conflicts("@1.1.9:", when="%nvhpc@:22.7")
+
     def cmake_args(self):
         return [
             self.define("CMAKE_INSTALL_LIBDIR", self.prefix.lib),
diff --git a/var/spack/repos/builtin/packages/sniffles/package.py b/var/spack/repos/builtin/packages/sniffles/package.py
index cc66f693ba839a..ffa92ae83810a2 100644
--- a/var/spack/repos/builtin/packages/sniffles/package.py
+++ b/var/spack/repos/builtin/packages/sniffles/package.py
@@ -15,7 +15,7 @@ class Sniffles(CMakePackage):
     version("1.0.7", sha256="03fa703873bdf9c32055c584448e1eece45f94b4bc68e60c9624cf3841e6d8a9")
     version("1.0.5", sha256="386c6536bdaa4637579e235bac48444c08297337c490652d1e165accd34b258f")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("bamtools", type="link")
 
     patch("unused_libs.patch")
diff --git a/var/spack/repos/builtin/packages/soapdenovo-trans/package.py b/var/spack/repos/builtin/packages/soapdenovo-trans/package.py
index a332bb9e009fd5..6dd83ef30e2905 100644
--- a/var/spack/repos/builtin/packages/soapdenovo-trans/package.py
+++ b/var/spack/repos/builtin/packages/soapdenovo-trans/package.py
@@ -18,7 +18,7 @@ class SoapdenovoTrans(MakefilePackage):
     version("1.0.5", sha256="c1903c0d81142270db95916e2833400f72c4841b5c9194f182c19ebda418936f")
     version("1.0.4", sha256="378a54cde0ebe240fb515ba67197c053cf95393645c1ae1399b3a611be2a9795")
 
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("samtools@0.1.8", type="link", when="target=aarch64:")
 
     build_directory = "src"
diff --git a/var/spack/repos/builtin/packages/soapdenovo2/package.py b/var/spack/repos/builtin/packages/soapdenovo2/package.py
index 4f5708bd89b3ee..57f84d00c79dca 100644
--- a/var/spack/repos/builtin/packages/soapdenovo2/package.py
+++ b/var/spack/repos/builtin/packages/soapdenovo2/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import re
+
 from spack.package import *
 
 
@@ -15,16 +17,19 @@ class Soapdenovo2(MakefilePackage):
 
     homepage = "https://github.com/aquaskyline/SOAPdenovo2"
     url = "https://github.com/aquaskyline/SOAPdenovo2/archive/r240.tar.gz"
+    maintainers("snehring")
 
     version("242", sha256="a0043ceb41bc17a1c3fd2b8abe4f9029a60ad3edceb2b15af3c2cfabd36aa11b")
     version("240", sha256="cc9e9f216072c0bbcace5efdead947e1c3f41f09baec5508c7b90f933a090909")
 
     def flag_handler(self, name, flags):
-        if self.spec.satisfies("%gcc@10:"):
-            if name == "cflags" or name == "CFLAGS":
-                flags.append("-fcommon")
-            if name == "cxxflags" or name == "CXXFLAGS":
-                flags.append("-fcommon")
+        if name.lower() == "cflags" and self.spec.satisfies("%gcc@10:"):
+            flags.append("-fcommon")
+        if name.lower() in ["cflags", "cxxflags", "cppflags"]:
+            opt_flag = re.compile("-O.*")
+            # This package cannot compile with any optimization flag other
+            # than the -O3 specified in the makefile
+            flags = [f for f in flags if not opt_flag.match(f)]
         return (flags, None, None)
 
     def install(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/sortmerna/package.py b/var/spack/repos/builtin/packages/sortmerna/package.py
index 68b351b89c882d..8a9cec6c261108 100644
--- a/var/spack/repos/builtin/packages/sortmerna/package.py
+++ b/var/spack/repos/builtin/packages/sortmerna/package.py
@@ -15,7 +15,7 @@ class Sortmerna(CMakePackage):
 
     version("2017-07-13", commit="8bde6fa113a5d99a23ae81b48eeea6760e966094")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("sse2neon", when="target=aarch64:")
 
     patch("for_aarch64.patch", when="target=aarch64:")
diff --git a/var/spack/repos/builtin/packages/sos/package.py b/var/spack/repos/builtin/packages/sos/package.py
index 0f7c5b5e144490..9234c7ab9d5d7a 100644
--- a/var/spack/repos/builtin/packages/sos/package.py
+++ b/var/spack/repos/builtin/packages/sos/package.py
@@ -15,6 +15,7 @@ class Sos(AutotoolsPackage):
     # notify when the package is updated.
     maintainers("rscohn2")
 
+    version("1.5.2", sha256="c9df8c6ab43890e5d8970467c188ae2fad736845875ca4c370ff047dbb37d017")
     version("1.5.1", sha256="0a6303dcbdd713ef2d83c617c1eb821227603c98cb9816c53585fd993da8a984")
     version("1.5.0", sha256="02679da6085cca2919f900022c46bad48479690586cb4e7f971ec3a735bab4d4")
     version("1.4.5", sha256="42778ba3cedb632ac3fbbf8917f415a804f8ca3b67fb3da6d636e6c50c501906")
diff --git a/var/spack/repos/builtin/packages/sosflow/package.py b/var/spack/repos/builtin/packages/sosflow/package.py
index 86e326e8eb32ff..35794faa3da49b 100644
--- a/var/spack/repos/builtin/packages/sosflow/package.py
+++ b/var/spack/repos/builtin/packages/sosflow/package.py
@@ -14,7 +14,7 @@ class Sosflow(CMakePackage):
     homepage = "https://github.com/cdwdirect/sos_flow/wiki"
     git = "https://github.com/cdwdirect/sos_flow.git"
 
-    version("spack", tag="spack-build-v0.9901")
+    version("spack", tag="spack-build-v0.9901", commit="57c77f18af88b01615d5900124f9cfd1770c8241")
 
     depends_on("libevpath")
     depends_on("sqlite@3:")
diff --git a/var/spack/repos/builtin/packages/sourmash/package.py b/var/spack/repos/builtin/packages/sourmash/package.py
new file mode 100644
index 00000000000000..2c02ce865d502c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/sourmash/package.py
@@ -0,0 +1,48 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Sourmash(PythonPackage):
+    """Sourmash: Quickly search, compare, and analyze genomic and metagenomic data sets with
+    k-mer sketches."""
+
+    homepage = "https://sourmash.bio/"
+    pypi = "sourmash/sourmash-4.8.2.tar.gz"
+
+    version("4.8.2", sha256="e0df78032e53ed88977445933ba3481dd10c7d3bd26d019511a6a4e6d7518475")
+
+    depends_on("python@3.8:", type=("build", "run"))
+    # build-only
+    depends_on("py-maturin@0.14.13:0.14", type="build")
+    depends_on("rust", type="build")
+    # general
+    depends_on("py-screed@1.1.2:1", type=("build", "run"))
+    depends_on("py-cffi@1.14.0:", type=("build", "run"))
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-matplotlib", type=("build", "run"))
+    depends_on("py-scipy", type=("build", "run"))
+    depends_on("py-deprecation@2.0.6:", type=("build", "run"))
+    depends_on("py-cachetools@4:5", type=("build", "run"))
+    depends_on("py-bitstring@3.1.9:4", type=("build", "run"))
+    depends_on("py-importlib_metadata@3.6:", when="^python@:3.9", type=("build", "run"))
+
+    def install(self, spec, prefix):
+        # build rust libs
+        cargo = Executable("cargo")
+        cargo("build", "--release")
+        # install python package
+        args = std_pip_args + ["--prefix=" + prefix, "."]
+        pip(*args)
+        # move sourmash.so into expected place
+        site_packages = join_path(python_platlib, "sourmash")
+        lib_ext = "dylib" if spec.platform == "Darwin" else "so"
+        install(
+            join_path("target", "release", "libsourmash.{}".format(lib_ext)),
+            join_path(site_packages, "_lowlevel__lib.so"),
+        )
+        # patch invalid read mode
+        filter_file(r"'(.*)'\), 130\)", r"'\1'))", join_path(site_packages, "_lowlevel.py"))
diff --git a/var/spack/repos/builtin/packages/sowing/package.py b/var/spack/repos/builtin/packages/sowing/package.py
index 6cdc78ac716181..2b21b54dbdab76 100644
--- a/var/spack/repos/builtin/packages/sowing/package.py
+++ b/var/spack/repos/builtin/packages/sowing/package.py
@@ -12,9 +12,12 @@ class Sowing(AutotoolsPackage):
     and MPICH.
     """
 
-    homepage = "https://www.mcs.anl.gov/petsc/index.html"
+    homepage = "https://bitbucket.org/petsc/pkg-sowing"
     url = "https://ftp.mcs.anl.gov/pub/petsc/externalpackages/sowing-1.1.23-p1.tar.gz"
+    git = "https://bitbucket.org/petsc/pkg-sowing"
+    maintainers("balay")
 
+    version("1.1.26-p8", sha256="2ef4d6db2075230247306e3d0194ffb3e4591aeb05bd244647c72dc1a4e52994")
     version("1.1.26-p1", sha256="fa0bd07295e5d768f2c33c8ab32205cc6411d42cb40bde0700fb57f9ee31c3d9")
     version("1.1.25-p1", sha256="c3a5bb170fffeeb1405ec4c3a048744a528d2bef24de29b6ac5e970cfeaddab5")
     version("1.1.23-p1", sha256="3e36f59e06fccbbf7b78d185c5654edaf70cf76f1c584bcbf08c39d7f29125e8")
diff --git a/var/spack/repos/builtin/packages/sp/package.py b/var/spack/repos/builtin/packages/sp/package.py
index 493e041c41b746..ad96799da4d4da 100644
--- a/var/spack/repos/builtin/packages/sp/package.py
+++ b/var/spack/repos/builtin/packages/sp/package.py
@@ -13,21 +13,33 @@ class Sp(CMakePackage):
 
     homepage = "https://noaa-emc.github.io/NCEPLIBS-sp"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-sp/archive/refs/tags/v2.3.3.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-sp"
 
-    maintainers("t-brown", "AlexanderRichert-NOAA", "edwardhartnett", "Hang-Lei-NOAA")
+    maintainers("AlexanderRichert-NOAA", "edwardhartnett", "Hang-Lei-NOAA")
 
+    version("develop", branch="develop")
     version("2.5.0", sha256="aec475ccb5ccf7c5a758dfb699626f2be78a22729a9d8d5e0a286db6a3213a51")
     version("2.4.0", sha256="dbb4280e622d2683b68a28f8e3837744adf9bbbb1e7940856e8f4597f481c708")
     version("2.3.3", sha256="c0d465209e599de3c0193e65671e290e9f422f659f1da928505489a3edeab99f")
 
-    variant("openmp", default=True, description="builds with OpenMP support")
     variant("shared", default=False, description="Build shared library", when="@2.4:")
     variant("pic", default=True, description="Build with position-independent-code")
+    variant("openmp", default=False, description="Use OpenMP threading")
+    variant(
+        "precision",
+        default=("4", "d"),
+        values=("4", "d", "8"),
+        multi=True,
+        description="Library versions: 4=4-byte reals, d=8-byte reals, 8=8-byte ints and reals",
+        when="@2.4:",
+    )
 
     def setup_run_environment(self, env):
-        suffixes = ["4", "d"]
-        if self.spec.satisfies("@:2.3"):
-            suffixes += ["8"]
+        if self.spec.satisfies("@2.4:"):
+            suffixes = self.spec.variants["precision"].value
+        else:
+            suffixes = ("4", "d", "8")
+
         for suffix in suffixes:
             lib = find_libraries(
                 "libsp_" + suffix,
@@ -39,9 +51,18 @@ def setup_run_environment(self, env):
             env.set("SP_INC" + suffix, "include_" + suffix)
 
     def cmake_args(self):
-        args = [
-            self.define_from_variant("OPENMP", "openmp"),
-            self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
-            self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
-        ]
+        args = []
+        args.append(self.define_from_variant("BUILD_SHARED_LIBS", "shared"))
+        args.append(self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"))
+        args.append(self.define_from_variant("OPENMP", "openmp"))
+        args.append(self.define("BUILD_4", self.spec.satisfies("precision=4")))
+        args.append(self.define("BUILD_D", self.spec.satisfies("precision=d")))
+        args.append(self.define("BUILD_8", self.spec.satisfies("precision=8")))
+        args.append(self.define("BUILD_DEPRECATED", False))
+        if self.spec.satisfies("@2.4:"):
+            args.append(self.define("BUILD_TESTING", self.run_tests))
         return args
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/spack/package.py b/var/spack/repos/builtin/packages/spack/package.py
index e2a6c1d83052a9..ba1e026e81c69c 100644
--- a/var/spack/repos/builtin/packages/spack/package.py
+++ b/var/spack/repos/builtin/packages/spack/package.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 from spack.package import *
+from spack.variant import DisjointSetsOfValues
 
 
 class Spack(Package):
@@ -21,6 +22,11 @@ class Spack(Package):
     maintainers("haampie")
 
     version("develop", branch="develop")
+    version("0.20.1", sha256="141be037b56e4b095840a95ac51c428c29dad078f7f88140ae6355b2a1b32dc3")
+    version("0.20.0", sha256="a189b4e8173eefdf76617445125b329d912f730767048846c38c8a2637396a7d")
+    version("0.19.2", sha256="4978b37da50f5690f4e1aa0cfe3975a89ccef85d96c68d417ea0716a8ce3aa98")
+    version("0.19.1", sha256="c9666f0b22ccf3cbda2736104d5d4e3b9cad5b4b4f01874a501e97d2c9477452")
+    version("0.19.0", sha256="b4225daf4f365a15caa58ef465d125b0d108ac5430b74d53ca4e807777943daf")
     version("0.18.1", sha256="d1491374ce280653ee0bc48cd80527d06860b886af8b0d4a7cf1d0a2309191b7")
     version("0.18.0", sha256="7b8d1e6bb49cd4f46f79a93fa577e00336dafeb5452712e36efeafd02711d38e")
     version("0.17.3", sha256="e9bf38917fa3b5231a65930aa657ef19fd380bebcc9ee44204167b1593f6fa06")
@@ -32,12 +38,38 @@ class Spack(Package):
     version("0.16.1", sha256="8d893036b24d9ee0feee41ac33dd66e4fc68d392918f346f8a7a36a69c567567")
     version("0.16.0", sha256="064b2532c70916c7684d4c7c973416ac32dd2ea15f5c392654c75258bfc8c6c2")
 
-    variant("development_tools", default=True, description="Build development dependencies")
-
-    # Python (with spack python -i ipython support)
+    variant("development_tools", default=False, description="Build development dependencies")
+    variant(
+        "fetchers",
+        # TODO: make Spack support default=... with any_combination_of :(
+        values=DisjointSetsOfValues(
+            ("none",), ("curl", "git", "mercurial", "subversion", "s3")
+        ).with_default("git"),
+        description="Fetchers for sources and binaries. "
+        "By default, urllib is used since Spack 0.17",
+    )
+    variant(
+        "modules",
+        # TODO: make Spack support default=... with any_combination_of :(
+        values=DisjointSetsOfValues(("none",), ("environment-modules", "lmod")).with_default(
+            "environment-modules,lmod"
+        ),
+        description="This variant makes Spack install the specified module system; "
+        "notice that Spack can still generate module files even if modules=none is selected.",
+    )
+
+    # This should be read as "require at least curl", not "require curl".
+    requires("fetchers=curl", when="@:0.16", msg="Curl is required for Spack < 0.17")
+
+    # Python
     depends_on("python@2.6.0:2.7,3.5:", type="run")
     depends_on("python@2.7.0:2.7,3.5:", type="run", when="@0.18.0:")
     depends_on("python@2.7.0:2.7,3.6:", type="run", when="@0.19.0:")
+
+    # Old Spack unfortunately depends on distutils, removed in Python 3.12
+    depends_on("python@:3.12", type="run", when="@0.18:0.20.1")
+
+    # spack python -i ipython support
     depends_on("py-ipython", type="run")
 
     # Concretizer
@@ -59,24 +91,26 @@ class Spack(Package):
     depends_on("ccache", type="run")
 
     # Fetchers
-    depends_on("curl", type="run")
-    depends_on("git", type="run")
-    depends_on("mercurial", type="run")
-    depends_on("subversion", type="run")
+    depends_on("curl", type="run", when="fetchers=curl")
+    depends_on("git", type="run", when="fetchers=git")
+    depends_on("mercurial", type="run", when="fetchers=mercurial")
+    depends_on("subversion", type="run", when="fetchers=subversion")
+    depends_on("py-boto3", type="run", when="fetchers=s3")
 
     # Modules
-    depends_on("tcl", type="run")
-    depends_on("lmod", type="run")
-    # Spack 0.18 uses lmod's depends_on function, which was introduced in v7.5.12
-    depends_on("lmod@7.5.12:", type="run", when="@0.18:")
+    depends_on("environment-modules", type="run", when="modules=environment-modules")
+
+    with when("modules=lmod"):
+        depends_on("lmod", type="run")
+        # Spack 0.18 uses lmod's depends_on function, which was introduced in v7.5.12
+        depends_on("lmod@7.5.12:", type="run", when="@0.18:")
 
     # Buildcache
-    # We really just need the 'strings' from binutils
-    depends_on("binutils", type="run")
+    # We really just need the 'strings' from binutils for older versions of spack
+    depends_on("binutils", type="run", when="@:0.20")
     depends_on("gnupg", type="run")
     depends_on("patchelf", type="run", when="platform=linux")
     depends_on("patchelf", type="run", when="platform=cray")
-    depends_on("py-boto3", type="run")
 
     # See https://github.com/spack/spack/pull/24686
     # and #25595, #25726, #25853, #25923, #25924 upstream in python/cpython
diff --git a/var/spack/repos/builtin/packages/spades/package.py b/var/spack/repos/builtin/packages/spades/package.py
index 73b56f151b104e..d257645a91e900 100644
--- a/var/spack/repos/builtin/packages/spades/package.py
+++ b/var/spack/repos/builtin/packages/spades/package.py
@@ -23,7 +23,7 @@ class Spades(CMakePackage):
     version("3.10.1", sha256="d49dd9eb947767a14a9896072a1bce107fb8bf39ed64133a9e2f24fb1f240d96")
 
     depends_on("python", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("bzip2")
 
     # SPAdes will explicitly not compile with gcc < 5.3.0
diff --git a/var/spack/repos/builtin/packages/spath/package.py b/var/spack/repos/builtin/packages/spath/package.py
index 3cc4346ce9c70a..fb4441bd1f10c0 100644
--- a/var/spack/repos/builtin/packages/spath/package.py
+++ b/var/spack/repos/builtin/packages/spath/package.py
@@ -23,7 +23,7 @@ class Spath(CMakePackage):
     version("0.0.2", sha256="7a65be59c3d27e92ed4718fba1a97a4a1c68e0a552b54de13d58afe3d8199cf7")
     version("0.0.1", sha256="f41c0ac74e6fb8acfd0c072d756db0fc9c00441f22be492cc4ad25f7fb596a24")
 
-    depends_on("zlib", type="link", when="@:0.0.2")
+    depends_on("zlib-api", type="link", when="@:0.0.2")
 
     variant("mpi", default=True, description="Build with MPI support")
     depends_on("mpi", when="+mpi")
diff --git a/var/spack/repos/builtin/packages/spdk/package.py b/var/spack/repos/builtin/packages/spdk/package.py
index 1f3c3f6c3c6e74..0dab950c55ebe9 100644
--- a/var/spack/repos/builtin/packages/spdk/package.py
+++ b/var/spack/repos/builtin/packages/spdk/package.py
@@ -21,7 +21,9 @@ class Spdk(AutotoolsPackage):
     maintainers("hyoklee")
 
     version("master", branch="master", submodules=True)
-    version("23.01", tag="v23.01", submodules=True)
+    version(
+        "23.01", tag="v23.01", commit="10edc60aa8b5f1b04d6496fea976dec75e276a95", submodules=True
+    )
 
     variant("crypto", default=False, description="Build vbdev crypto module")
     variant("dpdk", default=False, description="Build with dpdk")
diff --git a/var/spack/repos/builtin/packages/spdlog/package.py b/var/spack/repos/builtin/packages/spdlog/package.py
index de933739bbf5eb..36dc0a19b9a8ee 100644
--- a/var/spack/repos/builtin/packages/spdlog/package.py
+++ b/var/spack/repos/builtin/packages/spdlog/package.py
@@ -12,6 +12,7 @@ class Spdlog(CMakePackage):
     homepage = "https://github.com/gabime/spdlog"
     url = "https://github.com/gabime/spdlog/archive/v0.9.0.tar.gz"
 
+    version("1.12.0", sha256="4dccf2d10f410c1e2feaff89966bfc49a1abb29ef6f08246335b110e001e09a9")
     version("1.11.0", sha256="ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb")
     version("1.10.0", sha256="697f91700237dbae2326b90469be32b876b2b44888302afbc7aceb68bcfe8224")
     version("1.9.2", sha256="6fff9215f5cb81760be4cc16d033526d1080427d236e86d70bb02994f85e3d38")
diff --git a/var/spack/repos/builtin/packages/spectre/package.py b/var/spack/repos/builtin/packages/spectre/package.py
index c183b63b8a7ac1..df0ff30acd88db 100644
--- a/var/spack/repos/builtin/packages/spectre/package.py
+++ b/var/spack/repos/builtin/packages/spectre/package.py
@@ -29,6 +29,18 @@ class Spectre(CMakePackage):
     generator("ninja")
 
     version("develop", branch="develop")
+    version(
+        "2023.10.11", sha256="f25d17bc80cc49ebdd81726326701fe9ecd2b6705d86e6e3d48d9e4a458c8aff"
+    )
+    version(
+        "2023.09.07", sha256="2375117df09d99a2716d445ff51d151422467bd42cd38b5f1177d2d40cb90916"
+    )
+    version(
+        "2023.08.18", sha256="bdeb7da707d51d0e3b2a29b1d28646c5a64cba15844612e7b3726e8a28b37692"
+    )
+    version(
+        "2023.07.29", sha256="134668b81b8e89e3fd02b8b1415a1198889d7fb90f04ca6556458d3ce4489e43"
+    )
     version(
         "2023.06.19", sha256="f1140dfca1a9cf58f04acfe853c5597fa19c463d52b3643428e379496bff1236"
     )
diff --git a/var/spack/repos/builtin/packages/sperr/package.py b/var/spack/repos/builtin/packages/sperr/package.py
index 131a6a7fdadc77..5def42991f7d5b 100644
--- a/var/spack/repos/builtin/packages/sperr/package.py
+++ b/var/spack/repos/builtin/packages/sperr/package.py
@@ -12,23 +12,32 @@ class Sperr(CMakePackage):
 
     # Package info
     homepage = "https://github.com/NCAR/SPERR"
-    url = "https://github.com/NCAR/SPERR/archive/refs/tags/v0.6.2.tar.gz"
+    url = "https://github.com/NCAR/SPERR/archive/refs/tags/v0.7.1.tar.gz"
     git = "https://github.com/NCAR/SPERR.git"
     maintainers("shaomeng", "robertu94")
 
     # Versions
     version("main", branch="main")
+    version("0.7.1", sha256="1c3f46200be365427d1f57f5873f1b0b6dbcd297de4603a47a7fa3f41b273d79")
     version("0.6.2", sha256="d986997e2d79a1f27146ad02c623359976a1e72a1ab0d957e128d430cda3782d")
     version("0.5", sha256="20ad48c0e7599d3e5866e024d0c49648eb817f72ad5459f5468122cf14a97171")
 
-    depends_on("git", type="build")
-    depends_on("zstd", type=("build", "link"), when="+zstd")
-    depends_on("pkgconfig", type=("build"), when="+zstd")
-
+    # Variants
     variant("shared", description="build shared libaries", default=True)
-    variant("zstd", description="use zstd for more compression", default=True)
-    variant("openmp", description="use openmp in 3D inputs", default=True)
+    variant("openmp", description="use OpenMP in 3D inputs", default=True)
     variant("utilities", description="build SPERR CLI utilities", default=True)
+    variant("zstd", description="use ZSTD for more compression", default=True, when="@:0.6.2")
+    variant(
+        "bundle_zstd",
+        description="Use SPERR bundled ZSTD. Keep it off in SPACK builds.",
+        default=False,
+        when="@:0.6.2",
+    )
+
+    # Depend ons
+    depends_on("git", type="build")
+    depends_on("pkgconfig", type=("build"), when="+zstd")
+    depends_on("zstd", type=("build", "link"), when="@:0.6.2+zstd")
 
     def cmake_args(self):
         # ensure the compiler supports OpenMP if it is used
@@ -37,11 +46,11 @@ def cmake_args(self):
 
         args = [
             self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
-            self.define_from_variant("USE_ZSTD", "zstd"),
             self.define_from_variant("USE_OMP", "openmp"),
             self.define_from_variant("BUILD_CLI_UTILITIES", "utilities"),
+            self.define_from_variant("USE_ZSTD", "zstd"),
+            self.define_from_variant("USE_BUNDLED_ZSTD", "bundle_zstd"),
             "-DSPERR_PREFER_RPATH=OFF",
-            "-DUSE_BUNDLED_ZSTD=OFF",
             "-DBUILD_UNIT_TESTS=OFF",
         ]
         return args
diff --git a/var/spack/repos/builtin/packages/spglib/package.py b/var/spack/repos/builtin/packages/spglib/package.py
index 130c195ec08124..1d00091c13158c 100644
--- a/var/spack/repos/builtin/packages/spglib/package.py
+++ b/var/spack/repos/builtin/packages/spglib/package.py
@@ -18,6 +18,7 @@ class Spglib(CMakePackage):
     # patch by Krishnendu Ghosh
     patch("fix_cpp.patch", when="@:1.10.3")
 
+    version("2.1.0", sha256="31bca273a1bc54e1cff4058eebe7c0a35d5f9b489579e84667d8e005c73dcc13")
     version("2.0.2", sha256="10e44a35099a0a5d0fc6ee0cdb39d472c23cb98b1f5167c0e2b08f6069f3db1e")
     version("2.0.1", sha256="d7407c0d67174a0c5e41a82ed62948c43fcaf1b5529f97238d7fadd1123ffe22")
     version("2.0.0", sha256="426c4004e84fdb732d86aa5fcada5257ca8bc7a6915c06ced27565176c16ee96")
@@ -46,11 +47,16 @@ class Spglib(CMakePackage):
     version("1.10.1", sha256="8ed979cda82f6d440567197ec191bffcb82ee83c5bfe8a484c5a008dd00273f0")
     version("1.10.0", sha256="117fff308731784bea2ddaf3d076f0ecbf3981b31ea1c1bfd5ce4f057a5325b1")
 
-    variant("openmp", default=True, when="@1.16.2:")
+    variant("openmp", default=True, description="Build with OpenMP support", when="@1.16.2:")
+    variant("fortran", default=True, description="Build Fortran interface", when="@1.16.4:")
 
     @property
     def libs(self):
         return find_libraries("libsymspg", root=self.prefix, shared=True, recursive=True)
 
     def cmake_args(self):
-        return [self.define_from_variant("USE_OMP", "openmp")]
+        pfx = "SPGLIB_" if self.spec.satisfies("@2.1.0:") else ""
+        return [
+            self.define_from_variant(pfx + "USE_OMP", "openmp"),
+            self.define_from_variant(pfx + "WITH_Fortran", "fortran"),
+        ]
diff --git a/var/spack/repos/builtin/packages/spiner/package.py b/var/spack/repos/builtin/packages/spiner/package.py
index 74ce0b2ce1364f..a1ef3d637dacd9 100644
--- a/var/spack/repos/builtin/packages/spiner/package.py
+++ b/var/spack/repos/builtin/packages/spiner/package.py
@@ -17,6 +17,8 @@ class Spiner(CMakePackage, CudaPackage):
     maintainers("rbberger")
 
     version("main", branch="main")
+    version("1.6.2", sha256="91fb403ce3b151fbdf8b6ff5aed0d8dde1177749f5633951027b100ebc7080d3")
+    version("1.6.1", sha256="52774322571d3b9b0dc3c6b255257de9af0e8e6170834360f2252c1ac272cbe7")
     version("1.6.0", sha256="afa5526d87c78c1165ead06c09c5c2b9e4a913687443e5adff7b709ea4dd7edf")
     version(
         "1.5.1",
@@ -49,7 +51,8 @@ class Spiner(CMakePackage, CudaPackage):
     depends_on("cmake@3.23:", when="@1.6.0:", type="build")
     depends_on("catch2@2.13.4:2.13.9", type="test")
     depends_on("ports-of-call@1.2.0:", when="@:1.5.1")
-    depends_on("ports-of-call@1.3.0:", when="@1.6.0:")
+    depends_on("ports-of-call@1.5.1:", when="@1.6.0:")
+    depends_on("ports-of-call@main", when="@main")
 
     # Currently the raw cuda backend of ports-of-call is not supported.
     depends_on("ports-of-call portability_strategy=Kokkos", when="@:1.5.1 +kokkos")
@@ -58,10 +61,7 @@ class Spiner(CMakePackage, CudaPackage):
         depends_on("kokkos@3.3.00: cuda_arch=" + _flag, when="+cuda+kokkos cuda_arch=" + _flag)
     for _flag in ("~cuda", "+cuda", "~openmp", "+openmp"):
         depends_on("kokkos@3.3.00: " + _flag, when="+kokkos" + _flag)
-    depends_on(
-        "kokkos@3.3.00: ~shared+wrapper+cuda_lambda+cuda_constexpr+cuda_relocatable_device_code",
-        when="+cuda+kokkos",
-    )
+    depends_on("kokkos@3.3.00: ~shared+wrapper+cuda_lambda+cuda_constexpr", when="+cuda+kokkos")
 
     depends_on("hdf5+hl~mpi", when="+hdf5~mpi")
     depends_on("hdf5+hl+mpi", when="+hdf5+mpi")
diff --git a/var/spack/repos/builtin/packages/spiral-package-fftx/package.py b/var/spack/repos/builtin/packages/spiral-package-fftx/package.py
index 0f64fb4db9e536..e7bb9bc13377c7 100644
--- a/var/spack/repos/builtin/packages/spiral-package-fftx/package.py
+++ b/var/spack/repos/builtin/packages/spiral-package-fftx/package.py
@@ -12,22 +12,22 @@ class SpiralPackageFftx(Package):
     Transform as well as higher-level operations composed of linear operations
     combined with DFT transforms."""
 
-    homepage = "https://spiral.net"
-    url = "https://github.com/spiral-software/spiral-package-fftx/archive/refs/tags/1.0.0.tar.gz"
+    homepage = "https://spiralgen.com"
+    url = "https://github.com/spiral-software/spiral-package-fftx/archive/refs/tags/1.2.2.tar.gz"
     git = "https://github.com/spiral-software/spiral-package-fftx.git"
 
     maintainers("spiralgen")
-    extends("spiral-software")
+    # Although this package 'extends("spiral-software")' don't declare it as
+    # such.  If this package is required spiral-software should be installed
+    # with the +fftx variant active
 
     version("develop", branch="develop")
     version("main", branch="main")
+    version("1.2.2", sha256="18dacc3f974c4bd58295be2ea61f8ae0aada9a239f27b93d7806df564612cf22")
+    version("1.2.1", sha256="3f15aa5949c1b09eb59257cf1c5f6fcddc6e46f77ae9d5fce8acd8b9f99ce941")
     version("1.1.1", sha256="99ec7fab9274d378524b4933917fae23f9590255518c7a124cb46bd5e8d9af37")
-    version("1.1.0", sha256="979d7e59fc39e7e5423bce64628cea467079667d75ce885febee7c42fa7164aa")
-    version("1.0.0", sha256="9ed352049fcaab31a1a898149d16438c95a1656a2d24df6dee14e3b61efacb5c")
 
-    # FFTX package is an extension for Spiral (spec: spiral-software).  Spiral finds
-    # extensions in the "namespaces/packages" folder.  Install the tree in a similarly
-    # named folder so that when activated it'll get symlinked to the correct place.
+    # FFTX package is an extension for Spiral (spec: spiral-software).
 
     def install(self, spec, prefix):
         spiral_pkgs = join_path(prefix, "namespaces", "packages", "fftx")
diff --git a/var/spack/repos/builtin/packages/spiral-package-hcol/package.py b/var/spack/repos/builtin/packages/spiral-package-hcol/package.py
index 678e8e070ed65e..07ea2a3c46ec9b 100644
--- a/var/spack/repos/builtin/packages/spiral-package-hcol/package.py
+++ b/var/spack/repos/builtin/packages/spiral-package-hcol/package.py
@@ -10,20 +10,19 @@ class SpiralPackageHcol(Package):
     """This is the SPIRAL package for the Hybrid Control Operator Language
     (HCOL)."""
 
-    homepage = "https://spiral.net"
+    homepage = "https://spiralgen.com"
     url = "https://github.com/spiral-software/spiral-package-hcol/archive/refs/tags/1.0.0.tar.gz"
     git = "https://github.com/spiral-software/spiral-package-hcol.git"
 
     maintainers("spiralgen")
-    extends("spiral-software")
+    # Although this package 'extends("spiral-software")' don't declare it as
+    # such.  If this package is required spiral-software should be installed
+    # with the +hcol variant active
 
     version("master", branch="master")
     version("1.0.0", sha256="18ae6f0a090de03723612a6c91ca17cf62971129540936d8c2738bd8f807a511")
 
-    # HCOL package is an extension for Spiral (spec: spiral-software).  Extensions
-    # packages for Spiral are intended to be installed in the spiral-software prefix,
-    # in the "namespaces/packages" folder.  Install the tree in that folder under the
-    # name 'hcol'.
+    # HCOL package is an extension for Spiral (spec: spiral-software).
 
     def install(self, spec, prefix):
         spiral_pkgs = join_path(prefix, "namespaces", "packages", "hcol")
diff --git a/var/spack/repos/builtin/packages/spiral-package-jit/package.py b/var/spack/repos/builtin/packages/spiral-package-jit/package.py
new file mode 100644
index 00000000000000..c0d37abfbedc61
--- /dev/null
+++ b/var/spack/repos/builtin/packages/spiral-package-jit/package.py
@@ -0,0 +1,32 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class SpiralPackageJit(Package):
+    """This is the SPIRAL package for Just-In-Time (JIT) or Real-Time
+    Compilation (RTC)."""
+
+    homepage = "https://spiralgen.com"
+    url = "https://github.com/spiral-software/spiral-package-jit/archive/refs/tags/1.0.3.tar.gz"
+    git = "https://github.com/spiral-software/spiral-package-jit.git"
+
+    maintainers("spiralgen")
+    # Although this package 'extends("spiral-software")' don't declare it as
+    # such.  If this package is required spiral-software should be installed
+    # with the +jit variant active
+
+    version("develop", branch="develop")
+    version("main", branch="main")
+    version("1.0.3", sha256="97ff0d7d46ed4e53b1971ca279a30b27f0d9b328c70585d4cc0c56dfe6701894")
+    version("1.0.2", sha256="d7fac0493ac406a8b1874491223c3a9a1c6727ea1aa39de7ef4694c59aac9d26")
+    version("1.0.1", sha256="acf22db04e705276f06642d7f2ebf161f6c347f93bb1bdd6e3ddcfc4b7be5707")
+
+    # JIT package is an extension for Spiral (spec: spiral-software).
+
+    def install(self, spec, prefix):
+        spiral_pkgs = join_path(prefix, "namespaces", "packages", "jit")
+        install_tree(".", spiral_pkgs)
diff --git a/var/spack/repos/builtin/packages/spiral-package-mpi/package.py b/var/spack/repos/builtin/packages/spiral-package-mpi/package.py
index ed1b187834c5a3..6c4ae70d5a21c9 100644
--- a/var/spack/repos/builtin/packages/spiral-package-mpi/package.py
+++ b/var/spack/repos/builtin/packages/spiral-package-mpi/package.py
@@ -9,20 +9,21 @@
 class SpiralPackageMpi(Package):
     """This is the SPIRAL package for MPI."""
 
-    homepage = "https://spiral.net"
-    url = "https://github.com/spiral-software/spiral-package-mpi/archive/refs/tags/1.0.0.tar.gz"
+    homepage = "https://spiralgen.com"
+    url = "https://github.com/spiral-software/spiral-package-mpi/archive/refs/tags/1.1.0.tar.gz"
     git = "https://github.com/spiral-software/spiral-package-mpi.git"
 
     maintainers("spiralgen")
-    extends("spiral-software")
+    # Although this package 'extends("spiral-software")' don't declare it as
+    # such.  If this package is required spiral-software should be installed
+    # with the +mpi variant active
 
     version("develop", branch="develop")
     version("main", branch="main")
+    version("1.1.0", sha256="baf3c9dac7fee330e4bb4adbd24cc7e55f27fc27417644c0b216124f9052f1f5")
     version("1.0.0", sha256="64896a82aacce9cc8abe88b921e09ba7a5fceb8262e490f60a7088583c2c2151")
 
-    # MPI package is an extension for Spiral (spec: spiral-software).  Spiral finds
-    # extensions in the "namespaces/packages" folder.  Install the tree in a similarly
-    # named folder so that when activated it'll get symlinked to the correct place.
+    # MPI package is an extension for Spiral (spec: spiral-software).
 
     def install(self, spec, prefix):
         spiral_pkgs = join_path(prefix, "namespaces", "packages", "mpi")
diff --git a/var/spack/repos/builtin/packages/spiral-package-simt/package.py b/var/spack/repos/builtin/packages/spiral-package-simt/package.py
index 8203df47b5a83f..9dc727e0f0ca55 100644
--- a/var/spack/repos/builtin/packages/spiral-package-simt/package.py
+++ b/var/spack/repos/builtin/packages/spiral-package-simt/package.py
@@ -10,21 +10,21 @@ class SpiralPackageSimt(Package):
     """This is the SPIRAL package for SIMT:  SIMT, single instruction multiple
     threads, is used to generate code for GPUs and multi-threading aplications."""
 
-    homepage = "https://spiral.net"
-    url = "https://github.com/spiral-software/spiral-package-simt/archive/refs/tags/1.0.0.tar.gz"
+    homepage = "https://spiralgen.com"
+    url = "https://github.com/spiral-software/spiral-package-simt/archive/refs/tags/1.1.0.tar.gz"
     git = "https://github.com/spiral-software/spiral-package-simt.git"
 
     maintainers("spiralgen")
-    extends("spiral-software")
+    # Although this package 'extends("spiral-software")' don't declare it as
+    # such.  If this package is required spiral-software should be installed
+    # with the +simt variant active
 
     version("develop", branch="develop")
     version("main", branch="main")
     version("1.1.0", sha256="4d6a5e586889b9e000968c99f3068ba86a12cc389665c6deadc4734117ef7a95")
     version("1.0.0", sha256="888ca01aa8fd5df80d6ae1bd64eb1b1e70240b6a36bc3437eb48f5a4b59c2d07")
 
-    # SIMT package is an extension for Spiral (spec: spiral-software).  Spiral finds
-    # extensions in the "namespaces/packages" folder.  Install the tree in a similarly
-    # named folder so that when activated it'll get symlinked to the correct place.
+    # SIMT package is an extension for Spiral (spec: spiral-software).
 
     def install(self, spec, prefix):
         spiral_pkgs = join_path(prefix, "namespaces", "packages", "simt")
diff --git a/var/spack/repos/builtin/packages/spiral-software/package.py b/var/spack/repos/builtin/packages/spiral-software/package.py
index 0fd4f474230d06..7f97bca8b89051 100644
--- a/var/spack/repos/builtin/packages/spiral-software/package.py
+++ b/var/spack/repos/builtin/packages/spiral-software/package.py
@@ -11,31 +11,66 @@ class SpiralSoftware(CMakePackage):
     mathematical functions that produces very high performance code for a wide
     spectrum of hardware platforms."""
 
-    homepage = "https://spiral.net"
-    url = "https://github.com/spiral-software/spiral-software/archive/refs/tags/8.4.0.tar.gz"
+    homepage = "https://spiralgen.com"
+    url = "https://github.com/spiral-software/spiral-software/archive/refs/tags/8.5.0.tar.gz"
     git = "https://github.com/spiral-software/spiral-software.git"
 
     maintainers("spiralgen")
 
     version("develop", branch="develop")
     version("master", branch="master")
+    version("8.5.0", sha256="829345b8ca3ab0069a1a6e230f60ab03257060a8f05c021cee022e294eef592d")
     version("8.4.0", sha256="d0c58de65c678130eeee6b8b8b48061bbe463468990f66d9b452225ce46dee19")
     version("8.3.0", sha256="41cf0e7f14f9497e98353baa1ef4ca6204ce5ca525db8093f5bb44e89992abdf")
-    version("8.2.1", sha256="78d7bb1c22a5b2d216eac7b6ddedd20b601ba40227e64f743cbb54d4e5a7794d")
-    version("8.2.0", sha256="983f38d270ae2cb753c88cbce3f412e307c773807ad381acedeb9275afc0be32")
 
     extendable = True
 
-    # No dependencies.
+    # No dependencies.  Spiral pacakges are listed here as variants.  If a
+    # variant (i.e., spiral-package) is enabled then spiral-software depends
+    # on the package, so dependencies may be added during the install process.
+
+    variant("fftx", default=False, description="Install Spiral package FFTX.")
+    variant(
+        "simt",
+        default=False,
+        description="Install Spiral package for Single Instruction, Multiple Threads"
+        " (SIMT) to generate code for GPUs.",
+    )
+    variant(
+        "mpi",
+        default=False,
+        description="Install Spiral package for Message Passing Interface (MPI).",
+    )
+    variant(
+        "jit",
+        default=False,
+        description="Install Spiral supporting Just-In-Time (aka RTC) Compilation.",
+    )
+    variant(
+        "hcol",
+        default=False,
+        description="Install Spiral package for the Hybrid Control Operator Language (HCOL).",
+    )
+
+    # Dependencies
+    for pkg in ["fftx", "simt", "mpi", "jit", "hcol"]:
+        depends_on(f"spiral-package-{pkg}", when=f"+{pkg}")
 
     def build(self, spec, prefix):
         with working_dir(self.build_directory):
             make("all")
             make("install")
 
+    def spiral_package_install(self, spec, prefix, pkg):
+        pkg_name = "spiral-package-" + pkg
+        pkg_prefix = spec[pkg_name].prefix
+        dest = join_path(prefix, "namespaces", "packages", pkg)
+        src = join_path(pkg_prefix, "namespaces", "packages", pkg)
+        install_tree(src, dest)
+
     def install(self, spec, prefix):
         with working_dir(self.stage.source_path):
-            files = ("LICENSE", "README.md", "ReleaseNotes.md")
+            files = ("LICENSE", "README.md", "ReleaseNotes.md", "Contributing.md")
             for fil in files:
                 install(fil, prefix)
 
@@ -61,6 +96,10 @@ def install(self, spec, prefix):
             install_tree("grp", prefix.gap.grp)
             install_tree("bin", prefix.gap.bin)
 
+        for pkg in ["fftx", "simt", "mpi", "jit", "hcol"]:
+            if f"+{pkg}" in spec:
+                self.spiral_package_install(spec, prefix, pkg)
+
     def setup_dependent_build_environment(self, env, dependent_spec):
         env.set("SPIRAL_HOME", self.prefix)
 
diff --git a/var/spack/repos/builtin/packages/sqlcipher/package.py b/var/spack/repos/builtin/packages/sqlcipher/package.py
index 501166e1909573..82b29b96c753cc 100644
--- a/var/spack/repos/builtin/packages/sqlcipher/package.py
+++ b/var/spack/repos/builtin/packages/sqlcipher/package.py
@@ -27,7 +27,7 @@ class Sqlcipher(AutotoolsPackage):
 
     depends_on("openssl")
     depends_on("tcl", type=["build"])
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def configure_args(self):
         args = []
diff --git a/var/spack/repos/builtin/packages/sqlite/package.py b/var/spack/repos/builtin/packages/sqlite/package.py
index 10af40ef2c98af..19890314707597 100644
--- a/var/spack/repos/builtin/packages/sqlite/package.py
+++ b/var/spack/repos/builtin/packages/sqlite/package.py
@@ -17,6 +17,7 @@ class Sqlite(AutotoolsPackage):
 
     homepage = "https://www.sqlite.org"
 
+    version("3.43.2", sha256="6d422b6f62c4de2ca80d61860e3a3fb693554d2f75bb1aaca743ccc4d6f609f0")
     version("3.42.0", sha256="7abcfd161c6e2742ca5c6c0895d1f853c940f203304a0b49da4e1eca5d088ca6")
     version("3.40.1", sha256="2c5dea207fa508d765af1ef620b637dcb06572afa6f01f0815bd5bbf864b33d9")
     version("3.40.0", sha256="0333552076d2700c75352256e91c78bf5cd62491589ba0c69aed0a81868980e7")
@@ -59,7 +60,7 @@ class Sqlite(AutotoolsPackage):
     variant("rtree", default=True, description="Build with Rtree module")
 
     depends_on("readline")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # See https://blade.tencent.com/magellan/index_en.html
     conflicts("+fts", when="@:3.25")
diff --git a/var/spack/repos/builtin/packages/squashfs/package.py b/var/spack/repos/builtin/packages/squashfs/package.py
index b32765627e75a5..d71da4bc5e23f3 100644
--- a/var/spack/repos/builtin/packages/squashfs/package.py
+++ b/var/spack/repos/builtin/packages/squashfs/package.py
@@ -13,6 +13,7 @@ class Squashfs(MakefilePackage):
     url = "https://downloads.sourceforge.net/project/squashfs/squashfs/squashfs4.3/squashfs4.3.tar.gz"
 
     # version      sha1
+    version("4.6.1", sha256="94201754b36121a9f022a190c75f718441df15402df32c2b520ca331a107511c")
     version(
         "4.5.1",
         sha256="277b6e7f75a4a57f72191295ae62766a10d627a4f5e5f19eadfbc861378deea7",
@@ -61,6 +62,7 @@ class Squashfs(MakefilePackage):
         multi=False,
         description="Default compression algorithm",
     )
+    variant("static", default=False, description="Build fully static mksquashfs executable")
 
     conflicts(
         "squashfs~gzip default_compression=gzip",
@@ -83,11 +85,15 @@ class Squashfs(MakefilePackage):
         msg="Cannot set default compression to missing algorithm",
     )
 
-    depends_on("zlib", when="+gzip")
+    depends_on("zlib-api", when="+gzip")
     depends_on("lz4", when="+lz4")
+    depends_on("lz4 libs=static", when="+lz4 +static")
     depends_on("lzo", when="+lzo")
+    depends_on("lzo libs=static", when="+lzo +static")
     depends_on("xz", when="+xz")
+    depends_on("xz libs=static", when="+xz +static")
     depends_on("zstd", when="+zstd")
+    depends_on("zstd libs=static", when="+zstd +static")
 
     # patch from
     # https://github.com/plougher/squashfs-tools/commit/fe2f5da4b0f8994169c53e84b7cb8a0feefc97b5.patch
@@ -103,6 +109,7 @@ def make_options(self, spec):
             "XZ_SUPPORT={0}".format(1 if "+xz" in spec else 0),
             "ZSTD_SUPPORT={0}".format(1 if "+zstd" in spec else 0),
             "COMP_DEFAULT={0}".format(default),
+            "EXTRA_LDFLAGS={0}".format("-static" if "+static" in spec else ""),
         ]
 
     def build(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/squashfuse/package.py b/var/spack/repos/builtin/packages/squashfuse/package.py
index 8c06506828b0d6..85b7c03c8a8a42 100644
--- a/var/spack/repos/builtin/packages/squashfuse/package.py
+++ b/var/spack/repos/builtin/packages/squashfuse/package.py
@@ -16,6 +16,7 @@ class Squashfuse(AutotoolsPackage):
     maintainers("haampie")
 
     version("master", branch="master")
+    version("0.5.0", sha256="d7602c7a3b1d0512764547d27cb8cc99d1b21181e1c9819e76461ee96c2ab4d9")
     version("0.1.104", sha256="aa52460559e0d0b1753f6b1af5c68cfb777ca5a13913285e93f4f9b7aa894b3a")
     version("0.1.103", sha256="42d4dfd17ed186745117cfd427023eb81effff3832bab09067823492b6b982e7")
 
@@ -40,7 +41,7 @@ class Squashfuse(AutotoolsPackage):
     depends_on("pkgconfig", type="build")
 
     # compression libs
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("lz4", when="+lz4")
     depends_on("lzo", when="+lzo")
     depends_on("xz", when="+xz")
@@ -63,7 +64,10 @@ def configure_args(self):
         args = ["--disable-demo"]
         args += self.enable_or_disable("shared")
         args += self.enable_or_disable("static")
-        args += self.with_or_without("zlib", activation_value="prefix")
+        if "+zlib" in self.spec:
+            args.append("--with-zlib=%s" % self.spec["zlib-api"].prefix)
+        else:
+            args.append("--without-zlib")
         args += self.with_or_without("lz4", activation_value="prefix")
         args += self.with_or_without("lzo", activation_value="prefix")
         args += self.with_or_without("xz", activation_value="prefix")
diff --git a/var/spack/repos/builtin/packages/sra-tools/package.py b/var/spack/repos/builtin/packages/sra-tools/package.py
index fc360a22d6e755..432adf002a52e1 100644
--- a/var/spack/repos/builtin/packages/sra-tools/package.py
+++ b/var/spack/repos/builtin/packages/sra-tools/package.py
@@ -13,8 +13,8 @@ class SraTools(CMakePackage):
     homepage = "https://github.com/ncbi/sra-tools"
     git = "https://github.com/ncbi/sra-tools.git"
 
-    version("3.0.3", tag="3.0.3")
-    version("3.0.0", tag="3.0.0")
+    version("3.0.3", tag="3.0.3", commit="01f0aa21bb20b84c68ea34404d43da680811e27a")
+    version("3.0.0", tag="3.0.0", commit="bd2053a1049e64207e75f4395fd1be7f1572a5aa")
 
     depends_on("openjdk")
     depends_on("flex@2.6:")
diff --git a/var/spack/repos/builtin/packages/srcml-identifier-getter-tool/package.py b/var/spack/repos/builtin/packages/srcml-identifier-getter-tool/package.py
index 3e442f1729a403..0724c7cd527d1d 100644
--- a/var/spack/repos/builtin/packages/srcml-identifier-getter-tool/package.py
+++ b/var/spack/repos/builtin/packages/srcml-identifier-getter-tool/package.py
@@ -17,7 +17,7 @@ class SrcmlIdentifierGetterTool(CMakePackage):
     version("2022-10-17", commit="01394c247ae6f61cc5864a9697e72e3623d8e7fb", submodules=True)
 
     depends_on("libxml2")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("lzma")
 
     def install(self, spec, prefix):
diff --git a/var/spack/repos/builtin/packages/sse2neon/package.py b/var/spack/repos/builtin/packages/sse2neon/package.py
index 1192cf8da6fe81..4ad956212651df 100644
--- a/var/spack/repos/builtin/packages/sse2neon/package.py
+++ b/var/spack/repos/builtin/packages/sse2neon/package.py
@@ -13,8 +13,12 @@ class Sse2neon(Package):
 
     homepage = "https://github.com/DLTcollab/sse2neon"
     git = "https://github.com/DLTcollab/sse2neon.git"
+    url = "https://github.com/DLTcollab/sse2neon/archive/refs/tags/v1.6.0.tar.gz"
 
     version("master", branch="master")
+    version("1.6.0", sha256="06f4693219deccb91b457135d836fc514a1c0a57e9fa66b143982901d2d19677")
+    version("1.5.1", sha256="4001e2dfb14fcf3831211581ed83bcc83cf6a3a69f638dcbaa899044a351bb2a")
+    version("1.5.0", sha256="92ab852aac6c8726a615f77438f2aa340f168f9f6e70c72033d678613e97b65a")
 
     def install(self, spec, prefix):
         mkdirp(prefix.include)
diff --git a/var/spack/repos/builtin/packages/sst-core/package.py b/var/spack/repos/builtin/packages/sst-core/package.py
index a2fba7728f9f42..860d30d0b38289 100644
--- a/var/spack/repos/builtin/packages/sst-core/package.py
+++ b/var/spack/repos/builtin/packages/sst-core/package.py
@@ -65,7 +65,7 @@ class SstCore(AutotoolsPackage):
     depends_on("mpi", when="+pdes_mpi")
     depends_on("zoltan", when="+zoltan")
     depends_on("hdf5", when="+hdf5")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
 
     depends_on("autoconf@1.68:", type="build")
     depends_on("automake@1.11.1:", type="build")
@@ -87,7 +87,7 @@ def configure_args(self):
         if "+hdf5" in self.spec:
             args.append("--with-hdf5=%s" % self.spec["hdf5"].prefix)
         if "+zlib" in self.spec:
-            args.append("--with-zlib=%s" % self.spec["zlib"].prefix)
+            args.append("--with-zlib=%s" % self.spec["zlib-api"].prefix)
 
         if "+pdes_mpi" in self.spec:
             args.append("--enable-mpi")
diff --git a/var/spack/repos/builtin/packages/sst-elements/package.py b/var/spack/repos/builtin/packages/sst-elements/package.py
index 948725c7403d42..49677daf049f7b 100644
--- a/var/spack/repos/builtin/packages/sst-elements/package.py
+++ b/var/spack/repos/builtin/packages/sst-elements/package.py
@@ -76,7 +76,7 @@ class SstElements(AutotoolsPackage):
     depends_on("otf", when="+otf")
     depends_on("otf2", when="+otf2")
     depends_on("gettext")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     depends_on("autoconf@1.68:", type="build")
     depends_on("automake@1.11.1:", type="build")
diff --git a/var/spack/repos/builtin/packages/sst-macro/package.py b/var/spack/repos/builtin/packages/sst-macro/package.py
index adfa95cb9a5587..df129cfb4f7549 100644
--- a/var/spack/repos/builtin/packages/sst-macro/package.py
+++ b/var/spack/repos/builtin/packages/sst-macro/package.py
@@ -46,7 +46,7 @@ class SstMacro(AutotoolsPackage):
     depends_on("m4", type="build", when="@master:")
 
     depends_on("binutils", type="build")
-    depends_on("zlib", type=("build", "link"))
+    depends_on("zlib-api", type=("build", "link"))
     depends_on("otf2", when="+otf2")
     depends_on("llvm+clang@5:9", when="+skeletonizer")
     depends_on("mpi", when="+pdes_mpi")
diff --git a/var/spack/repos/builtin/packages/stacks/package.py b/var/spack/repos/builtin/packages/stacks/package.py
index 6c938ab39fb05a..eebc9aacf43800 100644
--- a/var/spack/repos/builtin/packages/stacks/package.py
+++ b/var/spack/repos/builtin/packages/stacks/package.py
@@ -21,7 +21,7 @@ class Stacks(AutotoolsPackage):
 
     depends_on("perl", type=("build", "run"))
     depends_on("sparsehash", when="+sparsehash")
-    depends_on("zlib", when="@2.3b:")
+    depends_on("zlib-api", when="@2.3b:")
     conflicts("%gcc@:4.9.0", when="@2.3b:")
 
     def configure_args(self):
diff --git a/var/spack/repos/builtin/packages/staden-io-lib/package.py b/var/spack/repos/builtin/packages/staden-io-lib/package.py
index 007fb6910566f7..402e448735ea63 100644
--- a/var/spack/repos/builtin/packages/staden-io-lib/package.py
+++ b/var/spack/repos/builtin/packages/staden-io-lib/package.py
@@ -19,7 +19,7 @@ class StadenIoLib(AutotoolsPackage):
 
     variant("curl", default=False, description="Build with curl support")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("curl", when="+curl")
 
     def configure_args(self):
diff --git a/var/spack/repos/builtin/packages/star/package.py b/var/spack/repos/builtin/packages/star/package.py
index 492e4a710fde0e..18c7f9450c51bb 100644
--- a/var/spack/repos/builtin/packages/star/package.py
+++ b/var/spack/repos/builtin/packages/star/package.py
@@ -33,7 +33,7 @@ class Star(MakefilePackage):
         url="https://github.com/alexdobin/STAR/archive/STAR_2.4.2a.tar.gz",
     )
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     build_directory = "source"
 
diff --git a/var/spack/repos/builtin/packages/static-analysis-suite/package.py b/var/spack/repos/builtin/packages/static-analysis-suite/package.py
index 118a33398274aa..1699138296f284 100644
--- a/var/spack/repos/builtin/packages/static-analysis-suite/package.py
+++ b/var/spack/repos/builtin/packages/static-analysis-suite/package.py
@@ -13,9 +13,21 @@ class StaticAnalysisSuite(CMakePackage):
     homepage = "https://github.com/dpiparo/SAS"
     url = "https://github.com/dpiparo/SAS/archive/0.1.3.tar.gz"
 
-    version("0.2.0", sha256="a369e56f8edc61dbf59ae09dbb11d98bc05fd337c5e47e13af9c913bf7bfc538")
-    version("0.1.4", sha256="9b2a3436efe3c8060ee4882f3ed37d848ee79a63d6055a71a23fad6409559f40")
-    version("0.1.3", sha256="93c3194bb7d518c215e79436bfb43304683832b3cc66bfc838f6195ce4574943")
+    version(
+        "0.2.0",
+        sha256="a369e56f8edc61dbf59ae09dbb11d98bc05fd337c5e47e13af9c913bf7bfc538",
+        deprecated=True,
+    )
+    version(
+        "0.1.4",
+        sha256="9b2a3436efe3c8060ee4882f3ed37d848ee79a63d6055a71a23fad6409559f40",
+        deprecated=True,
+    )
+    version(
+        "0.1.3",
+        sha256="93c3194bb7d518c215e79436bfb43304683832b3cc66bfc838f6195ce4574943",
+        deprecated=True,
+    )
 
     depends_on("python@2.7:")
     depends_on("llvm@3.5:")
diff --git a/var/spack/repos/builtin/packages/stress-ng/package.py b/var/spack/repos/builtin/packages/stress-ng/package.py
index b76967e6f9124d..44a82def7eb0ba 100644
--- a/var/spack/repos/builtin/packages/stress-ng/package.py
+++ b/var/spack/repos/builtin/packages/stress-ng/package.py
@@ -22,7 +22,7 @@ class StressNg(MakefilePackage):
     depends_on("libbsd")
     depends_on("judy")
     depends_on("libatomic-ops")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("keyutils")
     depends_on("libgcrypt")
     depends_on("libcap")
diff --git a/var/spack/repos/builtin/packages/stringtie/package.py b/var/spack/repos/builtin/packages/stringtie/package.py
index 91c1b6fb527b08..6fe5170ba463fb 100644
--- a/var/spack/repos/builtin/packages/stringtie/package.py
+++ b/var/spack/repos/builtin/packages/stringtie/package.py
@@ -18,7 +18,7 @@ class Stringtie(MakefilePackage):
     version("1.3.4a", sha256="6164a5fa9bf8807ef68ec89f47e3a61fe57fa07fe858f52fb6627f705bf71add")
     version("1.3.3b", sha256="30e8a3a29b474f0abeef1540d9b4624a827d8b29d7347226d86a38afea28bc0f")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def build(self, spec, prefix):
         make("release")
diff --git a/var/spack/repos/builtin/packages/strumpack/package.py b/var/spack/repos/builtin/packages/strumpack/package.py
index 5f9853c7292d5a..a82b3784b49a32 100644
--- a/var/spack/repos/builtin/packages/strumpack/package.py
+++ b/var/spack/repos/builtin/packages/strumpack/package.py
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+from platform import machine
+
 from spack.package import *
 from spack.util.environment import set_env
 
@@ -29,6 +31,7 @@ class Strumpack(CMakePackage, CudaPackage, ROCmPackage):
     test_requires_compiler = True
 
     version("master", branch="master")
+    version("7.2.0", sha256="6988c00c3213f13e53d75fb474102358f4fecf07a4b4304b7123d86fdc784639")
     version("7.1.3", sha256="c951f38ee7af20da3ff46429e38fcebd57fb6f12619b2c56040d6da5096abcb0")
     version("7.1.2", sha256="262a0193fa1682d0eaa90363f739e0be7a778d5deeb80e4d4ae12446082a39cc")
     version("7.1.1", sha256="56481a22955c2eeb40932777233fc227347743c75683d996cb598617dd2a8635")
@@ -61,6 +64,7 @@ class Strumpack(CMakePackage, CudaPackage, ROCmPackage):
     variant("count_flops", default=False, description="Build with flop counters")
     variant("task_timers", default=False, description="Build with timers for internal routines")
     variant("slate", default=True, description="Build with SLATE support")
+    variant("magma", default=False, description="Build with MAGMA support")
 
     depends_on("cmake@3.11:", when="@:6.2.9", type="build")
     depends_on("cmake@3.17:", when="@6.3.0:", type="build")
@@ -84,6 +88,8 @@ class Strumpack(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("rocsolver", when="+rocm")
     depends_on("rocthrust", when="+rocm")
     depends_on("slate", when="+slate")
+    depends_on("magma+cuda", when="+magma+cuda")
+    depends_on("magma+rocm", when="+magma+rocm")
     depends_on("slate+cuda", when="+cuda+slate")
     depends_on("slate+rocm", when="+rocm+slate")
     for val in ROCmPackage.amdgpu_targets:
@@ -100,6 +106,7 @@ class Strumpack(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("+rocm", when="+cuda")
     conflicts("+slate", when="@:5.1.1")
     conflicts("+slate", when="~mpi")
+    conflicts("+magma", when="~rocm~cuda")
 
     patch("intel-19-compile.patch", when="@3.1.1")
     patch("shared-rocm.patch", when="@5.1.1")
@@ -118,6 +125,7 @@ def cmake_args(self):
             self.define_from_variant("TPL_ENABLE_PARMETIS", "parmetis"),
             self.define_from_variant("TPL_ENABLE_SCOTCH", "scotch"),
             self.define_from_variant("TPL_ENABLE_BPACK", "butterflypack"),
+            self.define_from_variant("TPL_ENABLE_MAGMA", "magma"),
             self.define_from_variant("STRUMPACK_COUNT_FLOPS", "count_flops"),
             self.define_from_variant("STRUMPACK_TASK_TIMERS", "task_timers"),
             "-DTPL_BLAS_LIBRARIES=%s" % spec["blas"].libs.joined(";"),
@@ -167,7 +175,7 @@ def cmake_args(self):
 
         if "%cce" in spec:
             # Assume the proper Cray CCE module (cce) is loaded:
-            craylibs_path = env["CRAYLIBS_" + env["MACHTYPE"].capitalize()]
+            craylibs_path = env["CRAYLIBS_" + machine().upper()]
             env.setdefault("LDFLAGS", "")
             env["LDFLAGS"] += " -Wl,-rpath," + craylibs_path
 
diff --git a/var/spack/repos/builtin/packages/su2/package.py b/var/spack/repos/builtin/packages/su2/package.py
index 3d62477f4031f3..3e835484cdb58b 100644
--- a/var/spack/repos/builtin/packages/su2/package.py
+++ b/var/spack/repos/builtin/packages/su2/package.py
@@ -52,8 +52,8 @@ class Su2(MesonPackage):
 
     depends_on("meson@0.61.1:", type=("build"))
     depends_on("python@3:", type=("build", "run"))
-    depends_on("zlib")
-    depends_on("pkg-config")
+    depends_on("zlib-api")
+    depends_on("pkgconfig")
     depends_on("mpi", when="+mpi")
     depends_on("swig", type="build", when="+pywrapper")
     depends_on("py-mpi4py", when="+pywrapper")
diff --git a/var/spack/repos/builtin/packages/sublime-text/package.py b/var/spack/repos/builtin/packages/sublime-text/package.py
index 30636a7a8e2e99..312658e3bf53dc 100644
--- a/var/spack/repos/builtin/packages/sublime-text/package.py
+++ b/var/spack/repos/builtin/packages/sublime-text/package.py
@@ -11,10 +11,11 @@ class SublimeText(Package):
     prose."""
 
     homepage = "https://www.sublimetext.com/"
-    url = "https://download.sublimetext.com/sublime_text_build_4143_x64.tar.xz"
+    url = "https://download.sublimetext.com/sublime_text_build_4152_x64.tar.xz"
 
     maintainers("LRWeber")
 
+    version("4.4152", sha256="6ede3c83519959897041c6506e850753c19962603b71bd9f73a625ae1e4d3554")
     version("4.4143", sha256="7de862c38d19367414117110328dded754ac709fed54c8cc5cb0737c894c073c")
     version(
         "3.2.2.3211", sha256="0b3c8ca5e6df376c3c24a4b9ac2e3b391333f73b229bc6e87d0b4a5f636d74ee"
diff --git a/var/spack/repos/builtin/packages/subread/package.py b/var/spack/repos/builtin/packages/subread/package.py
index b694713b648511..ae63d69d02f68d 100644
--- a/var/spack/repos/builtin/packages/subread/package.py
+++ b/var/spack/repos/builtin/packages/subread/package.py
@@ -14,6 +14,7 @@ class Subread(MakefilePackage):
 
     homepage = "https://subread.sourceforge.net/"
     url = "https://sourceforge.net/projects/subread/files/subread-1.5.2/subread-1.5.2-source.tar.gz/download"
+    maintainers("snehring")
 
     version("2.0.6", sha256="f0fdda6b98634d2946028948c220253e10a0f27c7fa5f24913b65b3ac6cbb045")
     version("2.0.4", sha256="c54b37ed83b34318d8f119b5c02fb9d0a65c811195bcc9e1745df6daf74ca2db")
@@ -24,19 +25,18 @@ class Subread(MakefilePackage):
     version("1.6.0", sha256="31251ec4c134e3965d25ca3097890fb37e2c7a4163f6234515534fd325b1002a")
     version("1.5.2", sha256="a8c5f0e09ed3a105f01866517a89084c7302ff70c90ef8714aeaa2eab181a0aa")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def build(self, spec, prefix):
         plat = sys.platform
         with working_dir("src"):
             if plat.startswith("linux"):
                 filter_file("CC_EXEC = gcc", "CC_EXEC = {0}".format(spack_cc), "Makefile.Linux")
-                if spec.target.family == "aarch64":
-                    filter_file("-mtune=core2", "", "Makefile.Linux")
-                    if spec.satisfies("@1.6.2:2.0.0"):
-                        filter_file("-mtune=core2", "", "longread-one/Makefile")
-                    elif spec.satisfies("@1.6.0"):
-                        filter_file("-mtune=core2", "", "longread-mapping/Makefile")
+                filter_file("-mtune=core2", "", "Makefile.Linux")
+                if spec.satisfies("@1.6.2:"):
+                    filter_file("-mtune=core2", "", "longread-one/Makefile")
+                if spec.satisfies("@1.6.0"):
+                    filter_file("-mtune=core2", "", "longread-mapping/Makefile")
                 make("-f", "Makefile.Linux")
             elif plat.startswith("darwin"):
                 make("-f", "Makefile.MacOS")
diff --git a/var/spack/repos/builtin/packages/subversion/package.py b/var/spack/repos/builtin/packages/subversion/package.py
index 1795c74eb99c5f..f5bbe55b0cc95b 100644
--- a/var/spack/repos/builtin/packages/subversion/package.py
+++ b/var/spack/repos/builtin/packages/subversion/package.py
@@ -42,7 +42,7 @@ class Subversion(AutotoolsPackage):
 
     depends_on("apr")
     depends_on("apr-util")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("sqlite@3.8.2:")
     depends_on("expat")
     depends_on("lz4", when="@1.10:")
@@ -68,7 +68,7 @@ def configure_args(self):
                 spec["expat"].libs.directories[0],
                 spec["expat"].libs.names[0],
             ),
-            "--with-zlib={0}".format(spec["zlib"].prefix),
+            "--with-zlib={0}".format(spec["zlib-api"].prefix),
             "--without-apxs",
             "--without-trang",
             "--without-doxygen",
diff --git a/var/spack/repos/builtin/packages/sundials/package.py b/var/spack/repos/builtin/packages/sundials/package.py
index 2fc0604d5672c7..71ae9186a00578 100644
--- a/var/spack/repos/builtin/packages/sundials/package.py
+++ b/var/spack/repos/builtin/packages/sundials/package.py
@@ -27,6 +27,8 @@ class Sundials(CMakePackage, CudaPackage, ROCmPackage):
     # Versions
     # ==========================================================================
     version("develop", branch="develop")
+    version("6.6.1", sha256="21f71e4aef95b18f954c8bbdc90b62877443950533d595c68051ab768b76984b")
+    version("6.6.0", sha256="f90029b8da846c8faff5530fd1fa4847079188d040554f55c1d5d1e04743d29d")
     version("6.5.1", sha256="4252303805171e4dbdd19a01e52c1dcfe0dafc599c3cfedb0a5c2ffb045a8a75")
     version("6.5.0", sha256="4e0b998dff292a2617e179609b539b511eb80836f5faacf800e688a886288502")
     version("6.4.1", sha256="7bf10a8d2920591af3fba2db92548e91ad60eb7241ab23350a9b1bc51e05e8d0")
@@ -290,6 +292,12 @@ class Sundials(CMakePackage, CudaPackage, ROCmPackage):
     # fix issues with exported PETSc target(s) in SUNDIALSConfig.cmake
     patch("sundials-v5.8.0.patch", when="@5.8.0")
 
+    def flag_handler(self, name, flags):
+        if name == "cxxflags":
+            if self.spec.satisfies("+sycl"):
+                flags.append("-fsycl")
+        return (flags, None, None)
+
     # ==========================================================================
     # SUNDIALS Settings
     # ==========================================================================
diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py
index 7af573699337ed..241dc4b552c7ff 100644
--- a/var/spack/repos/builtin/packages/superlu-dist/package.py
+++ b/var/spack/repos/builtin/packages/superlu-dist/package.py
@@ -134,8 +134,6 @@ def flag_handler(self, name, flags):
         flags = list(flags)
         if name == "cxxflags":
             flags.append(self.compiler.cxx11_flag)
-        if name == "cflags" and "%pgi" not in self.spec:
-            flags.append("-std=c99")
         if (
             name == "cflags"
             and (self.spec.satisfies("%xl") or self.spec.satisfies("%xl_r"))
diff --git a/var/spack/repos/builtin/packages/superlu/package.py b/var/spack/repos/builtin/packages/superlu/package.py
index fd7f66d4d0d43a..9ad86ab21932f6 100644
--- a/var/spack/repos/builtin/packages/superlu/package.py
+++ b/var/spack/repos/builtin/packages/superlu/package.py
@@ -46,6 +46,8 @@ class Superlu(CMakePackage, Package):
         conditional("cmake", when="@5:"), conditional("generic", when="@:4"), default="cmake"
     )
 
+    requires("build_system=cmake", when="platform=windows")
+
     variant("pic", default=True, description="Build with position independent code")
 
     depends_on("blas")
diff --git a/var/spack/repos/builtin/packages/survey/package.py b/var/spack/repos/builtin/packages/survey/package.py
index 79bac929665e8e..1fc4c550f0d37e 100644
--- a/var/spack/repos/builtin/packages/survey/package.py
+++ b/var/spack/repos/builtin/packages/survey/package.py
@@ -19,7 +19,7 @@ class Survey(CMakePackage):
     available for tools inside current MPI implementations including:
     MPICH, MVAPICH, MPT, and OpenMPI. It also supports multiple
     architectures and has been tested on machines based on Intel,
-    AMD, ARM, and IBM P8/9 processors and integrated GPUs.
+    AMD, ARM, and IBM P8/9 processors and integrated NVIDIA GPUs.
 
     Survey is a licensed product with the source not openly available.
     To access the survey source and build with spack please contact:
@@ -33,7 +33,8 @@ class Survey(CMakePackage):
     maintainers("jgalarowicz")
 
     version("master", branch="master")
-    version("1.0.8", branch="1.0.8")
+    version("1.0.9", branch="1.0.9")
+    version("1.0.8", tag="1.0.8")
     version("1.0.7", tag="1.0.7")
     version("1.0.6", tag="1.0.6")
     version("1.0.5", tag="1.0.5")
@@ -45,6 +46,7 @@ class Survey(CMakePackage):
     version("1.0.0", branch="1.0.0")
 
     variant("mpi", default=False, description="Enable mpi, build MPI data collector")
+    variant("debug", default=False, description="Build a debug survey version")
 
     variant(
         "tls_model",
@@ -61,9 +63,10 @@ class Survey(CMakePackage):
     depends_on("libmonitor@2021.11.08+commrank", type=("build", "link", "run"), when="@1.0.3:")
 
     depends_on("papi@5:", type=("build", "link", "run"))
-    depends_on("gotcha@master", type=("build", "link", "run"))
-    depends_on("llvm-openmp@9.0.0", type=("build", "link", "run"), when="@:1.0.2")
-    depends_on("llvm-openmp@12.0.1", type=("build", "link", "run"), when="@1.0.3:")
+    depends_on("gotcha@master", type=("build", "link"), when="@:1.0.7")
+    depends_on("gotcha@1.0.4", type=("build", "link"), when="@1.0.8:")
+    depends_on("llvm-openmp@9.0.0", type=("build", "link"), when="@:1.0.2")
+    depends_on("llvm-openmp@12.0.1", type=("build", "link"), when="@1.0.3:")
 
     # MPI Installation
     depends_on("mpi", when="+mpi")
@@ -81,6 +84,10 @@ class Survey(CMakePackage):
     depends_on("py-more-itertools", type=("build", "run"), when="@1.0.4:")
     depends_on("py-versioneer", type=("build", "run"), when="@1.0.5:")
     depends_on("py-filelock", type=("build", "run"), when="@1.0.7:")
+    depends_on("py-zipp", type=("build", "run"), when="@1.0.7:")
+    depends_on("py-humanize", type=("build", "run"), when="@1.0.8:")
+    depends_on("py-importlib-resources", type=("build", "run"), when="@1.0.8:")
+    depends_on("py-gitpython", type=("build", "run"), when="@1.0.9:")
 
     extends("python")
 
@@ -117,6 +124,11 @@ def cmake_args(self):
             mpi_options = self.get_mpi_cmake_options(spec)
             cmake_args.extend(mpi_options)
 
+        if "+debug" in spec:
+            cmake_args.append("-DCMAKE_C_FLAGS=-g -O2")
+            cmake_args.append("-DCMAKE_CXX_FLAGS=-g -O2")
+            cmake_args.append("-DCMAKE_BUILD_TYPE=Custom")
+
         return cmake_args
 
     def setup_run_environment(self, env):
diff --git a/var/spack/repos/builtin/packages/sw4/package.py b/var/spack/repos/builtin/packages/sw4/package.py
index a31543a818ca9f..c5f601ff883c91 100644
--- a/var/spack/repos/builtin/packages/sw4/package.py
+++ b/var/spack/repos/builtin/packages/sw4/package.py
@@ -18,12 +18,12 @@ class Sw4(MakefilePackage):
 
     version("master", branch="master")
     version("developer", branch="developer")
-    version("3.0-beta2", tag="v3.0-beta2")
+    version("3.0", tag="v3.0", commit="13e6d431976f7fc49124c997bf87353aa7afd35e")
 
     variant("openmp", default=True, description="build with OpenMP")
     variant("hdf5", default=True, description="build with HDF5")
     variant("proj", default=True, description="build with proj")
-    variant("zfp", default=True, description="build with ZFP")
+    variant("zfp", default=False, description="build with ZFP")
     variant("fftw", default=True, description="build with FFTW")
 
     depends_on("mpi")
diff --git a/var/spack/repos/builtin/packages/swig/package.py b/var/spack/repos/builtin/packages/swig/package.py
index bd1ae02c512850..b45813a21d88cb 100644
--- a/var/spack/repos/builtin/packages/swig/package.py
+++ b/var/spack/repos/builtin/packages/swig/package.py
@@ -69,7 +69,7 @@ class Swig(AutotoolsPackage, SourceforgePackage):
 
     depends_on("pcre", when="@:4.0")
     depends_on("pcre2", when="@4.1:")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     AUTOCONF_VERSIONS = "@" + ",".join(
         [
diff --git a/var/spack/repos/builtin/packages/swipl/package.py b/var/spack/repos/builtin/packages/swipl/package.py
index 41ff42a60b6235..3afeac5fcecfab 100644
--- a/var/spack/repos/builtin/packages/swipl/package.py
+++ b/var/spack/repos/builtin/packages/swipl/package.py
@@ -45,7 +45,7 @@ class Swipl(CMakePackage):
     depends_on("unwind", when="+unwind")
     depends_on("unixodbc", when="+odbc")
     depends_on("openssl", when="+ssl")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
 
     depends_on("libxt", when="+xpce")
     depends_on("libx11", when="+xpce")
diff --git a/var/spack/repos/builtin/packages/sz/package.py b/var/spack/repos/builtin/packages/sz/package.py
index ef4592f0589e42..587c24120b010d 100644
--- a/var/spack/repos/builtin/packages/sz/package.py
+++ b/var/spack/repos/builtin/packages/sz/package.py
@@ -66,7 +66,7 @@ class Sz(CMakePackage, AutotoolsPackage):
     # with Fujitsu compiler.
     patch("fix_optimization.patch", when="@2.0.2.0:%fj")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("zstd")
 
     extends("python", when="+python")
@@ -82,6 +82,12 @@ class Sz(CMakePackage, AutotoolsPackage):
 
     patch("ctags-only-if-requested.patch", when="@2.1.8.1:2.1.8.3")
 
+    def flag_handler(self, name, flags):
+        if name == "cflags":
+            if self.spec.satisfies("%oneapi"):
+                flags.append("-Wno-error=implicit-function-declaration")
+        return (flags, None, None)
+
     def setup_run_environment(self, env):
         if "+hdf5" in self.spec:
             env.prepend_path("HDF5_PLUGIN_PATH", self.prefix.lib64)
diff --git a/var/spack/repos/builtin/packages/t8code/package.py b/var/spack/repos/builtin/packages/t8code/package.py
new file mode 100644
index 00000000000000..a922c9a3485696
--- /dev/null
+++ b/var/spack/repos/builtin/packages/t8code/package.py
@@ -0,0 +1,75 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+
+
+class T8code(AutotoolsPackage):
+    """t8code is a C/C++ library to manage parallel adaptive meshes with various element types.
+    t8code uses a collection (a forest) of multiple connected adaptive space-trees in parallel
+    and scales to at least one million MPI ranks and over one Trillion mesh elements."""
+
+    homepage = "https://github.com/DLR-AMR/t8code"
+    url = "https://github.com/DLR-AMR/t8code/releases/download/v1.4.1/t8-1.4.1.tar.gz"
+
+    maintainers = ["Davknapp", "melven"]
+
+    version("1.4.1", sha256="b0ec0c9b4a182f8ac7e930ba80cd20e6dc5baefc328630e4a9dac8c688749e9a")
+
+    variant("mpi", default=True, description="Enable MPI parallel code")
+    variant("vtk", default=False, description="Enable vtk-dependent code")
+    variant("petsc", default=False, description="Enable PETSc-dependent code")
+    variant("netcdf", default=False, description="Enable NetCDF-dependent code")
+    variant("metis", default=False, description="Enable metis-dependent code")
+
+    depends_on("mpi", when="+mpi")
+    depends_on("vtk@9.1:", when="+vtk")
+    # t8code@1.4.1 doesn't build with petsc@3.19.1
+    depends_on("petsc@3.18", when="+petsc")
+    depends_on("netcdf-c~mpi", when="+netcdf~mpi")
+    depends_on("netcdf-c+mpi", when="+netcdf+mpi")
+    depends_on("metis", when="+metis")
+
+    # Per default, t8code uses hardcoded zlib library from vtk package
+    # The configure command is overwritten to choose the integrated spack package
+    def patch(self):
+        if "+vtk" in self.spec:
+            filter_file(r"vtkzlib-\$t8_vtk_version", "z", "configure")
+
+    def configure_args(self):
+        args = ["CFLAGS=-O3", "CXXFLAGS=-O3"]
+        spec = self.spec
+
+        if "+mpi" in spec:
+            args.append("--enable-mpi")
+            args.append("CC=mpicc")
+            args.append("CXX=mpicxx")
+        else:
+            args.append("--disable-mpi")
+
+        if "+vtk" in spec:
+            args.append("--with-vtk")
+            vtk_ver = spec["vtk"].version.up_to(2)
+            include_dir = os.path.join(spec["vtk"].headers.directories[0], f"vtk-{vtk_ver}")
+            lib_dir = spec["vtk"].prefix.lib
+
+            # vtk paths need to be passed to configure command
+            args.append(f"CPPFLAGS=-I{include_dir}")
+            args.append(f"LDFLAGS=-L{lib_dir}")
+            # Chosen vtk version number is needed for t8code to find the right version
+            args.append(f"--with-vtk_version_number={vtk_ver}")
+
+        if "+petsc" in spec:
+            args.append(f"--with-petsc={spec['petsc'].prefix}")
+
+        if "+netcdf" in spec:
+            args.append("--with-netcdf")
+
+        if "+metis" in spec:
+            args.append(f"--with-metis={spec['metis'].prefix}")
+
+        return args
diff --git a/var/spack/repos/builtin/packages/tabix/package.py b/var/spack/repos/builtin/packages/tabix/package.py
index 8f4f94f60ead36..06d3140e878e2d 100644
--- a/var/spack/repos/builtin/packages/tabix/package.py
+++ b/var/spack/repos/builtin/packages/tabix/package.py
@@ -16,7 +16,7 @@ class Tabix(MakefilePackage):
 
     depends_on("perl", type=("build", "run"))
     depends_on("python", type=("build", "run"))
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/tandem/package.py b/var/spack/repos/builtin/packages/tandem/package.py
index 22c7081568a88e..4bb56cc557302e 100644
--- a/var/spack/repos/builtin/packages/tandem/package.py
+++ b/var/spack/repos/builtin/packages/tandem/package.py
@@ -18,14 +18,24 @@ class Tandem(CMakePackage):
     version("main", branch="main", submodules=True)
 
     # we cannot use the tar.gz file because it does not contains submodules
-    version("1.0", tag="v1.0", submodules=True)
+    version("1.0", tag="v1.0", commit="eccab10cbdf5842ed9903fac7a023be5e2779f36", submodules=True)
     patch("fix_v1.0_compilation.diff", when="@1.0")
 
     maintainers("dmay23", "Thomas-Ulrich")
-    variant("polynomial_degree", default="2")
-    variant("domain_dimension", default="2", values=("2", "3"), multi=False)
-    variant("min_quadrature_order", default="0")
-    variant("libxsmm", default=False, description="installs libxsmm-generator")
+    variant("polynomial_degree", default="2", description="Polynomial degree")
+    variant(
+        "domain_dimension",
+        default="2",
+        description="Dimension of the domain",
+        values=("2", "3"),
+        multi=False,
+    )
+    variant(
+        "min_quadrature_order",
+        default="0",
+        description="Minimum order of quadrature rule, 0 = automatic",
+    )
+    variant("libxsmm", default=False, description="Install libxsmm-generator")
 
     depends_on("mpi")
     depends_on("parmetis +int64 +shared")
@@ -33,8 +43,9 @@ class Tandem(CMakePackage):
     depends_on("libxsmm@1.17 +generator", when="+libxsmm target=x86_64:")
     depends_on("lua@5.3.2:5.4.4")
     depends_on("eigen@3.4.0")
-    depends_on("zlib@1.2.8:1.2.13")
-    depends_on("petsc@3.14.6:3.18.5 +int64 +mumps +scalapack")
+
+    depends_on("zlib-api")
+    depends_on("petsc@3.14.6:3.18.5 +int64 +mumps +scalapack memalign=32")
     depends_on("petsc@3.14.6:3.18.5 +int64 +mumps +scalapack +knl", when="target=skylake:")
     # see https://github.com/TEAR-ERC/tandem/issues/45
     conflicts("%intel")
diff --git a/var/spack/repos/builtin/packages/tasmanian/addons70.patch b/var/spack/repos/builtin/packages/tasmanian/addons70.patch
deleted file mode 100644
index 8d983c6308b730..00000000000000
--- a/var/spack/repos/builtin/packages/tasmanian/addons70.patch
+++ /dev/null
@@ -1,25 +0,0 @@
-diff --git a/Addons/CMakeLists.txt b/Addons/CMakeLists.txt
-index 1279ada..0b6d9be 100644
---- a/Addons/CMakeLists.txt
-+++ b/Addons/CMakeLists.txt
-@@ -49,19 +49,7 @@ endif()
- 
- # The Tasmanian MPI capabilities are templated into the Addons
- if (Tasmanian_ENABLE_MPI)
--    target_link_libraries(Tasmanian_addons INTERFACE ${MPI_CXX_LIBRARIES})
--
--    if (DEFINED MPI_CXX_INCLUDE_PATH)
--        target_include_directories(Tasmanian_addons INTERFACE "${MPI_CXX_INCLUDE_PATH}")
--    endif()
--
--    if(DEFINED MPI_CXX_COMPILE_FLAGS)
--        target_compile_options(Tasmanian_addons INTERFACE "${MPI_CXX_COMPILE_FLAGS}")
--    endif()
--
--    if(DEFINED MPI_CXX_LINK_FLAGS)
--        set_target_properties(Tasmanian_addons PROPERTIES INTERFACE_LINK_OPTIONS "${MPI_CXX_LINK_FLAGS}")
--    endif()
-+    target_link_libraries(Tasmanian_addons INTERFACE MPI::MPI_CXX)
- 
-     add_executable(Tasmanian_mpitester testMPI.cpp testMPI.hpp testMPIDream.hpp)
-     set_target_properties(Tasmanian_mpitester PROPERTIES OUTPUT_NAME "mpitester")
diff --git a/var/spack/repos/builtin/packages/tasmanian/package.py b/var/spack/repos/builtin/packages/tasmanian/package.py
index f7176d31ed45ee..b4d4ead7bf3842 100644
--- a/var/spack/repos/builtin/packages/tasmanian/package.py
+++ b/var/spack/repos/builtin/packages/tasmanian/package.py
@@ -12,7 +12,7 @@ class Tasmanian(CMakePackage, CudaPackage, ROCmPackage):
     interpolation as well as parameter calibration."""
 
     homepage = "https://ornl.github.io/TASMANIAN/stable/"
-    url = "https://github.com/ORNL/TASMANIAN/archive/v7.9.tar.gz"
+    url = "https://github.com/ORNL/TASMANIAN/archive/v8.0.tar.gz"
     git = "https://github.com/ORNL/TASMANIAN.git"
 
     tags = ["e4s"]
@@ -22,38 +22,15 @@ class Tasmanian(CMakePackage, CudaPackage, ROCmPackage):
 
     version("develop", branch="master")
 
+    version("8.0", sha256="248c941346150bf6cfb386ba86b69bd4697f4fc93bff0e8d5f57e555614fd534")
     version("7.9", sha256="decba62e6bbccf1bc26c6e773a8d4fd51d7f3e3e534ddd386ec41300694ce5cc")
     version("7.7", sha256="85fb3a7b302ea21a3b700712767a59a623d9ab93da03308fa47d4413654c3878")
     version("7.5", sha256="d621bd36dced4db86ef638693ba89b336762e7a3d7fedb3b5bcefb03390712b3")
-    version("7.3", sha256="5bd1dd89cc5c84506f6900b6569b17e50becd73eb31ec85cfa11d6f1f912c4fa")
 
-    # API is very stable since 7.0, but the refactoring made 7.0 and 7.1 rocky
+    # Tasmanian is backwards compatible, no need to use 7.3 from back in 2020
     version(
-        "7.1",
-        sha256="9c24a591506a478745b802f1fa5c557da7bc80b12d8070855de6bc7aaca7547a",
-        deprecated=True,
-    )
-    version(
-        "7.0",
-        sha256="4094ba4ee2f1831c575d00368c8471d3038f813398be2e500739cef5c7c4a47b",
-        deprecated=True,
-    )  # use for xsdk-0.5.0
-    # 5.0, 5.1 and 6.0 use older API from 2018, all users have moved up by now
-    version(
-        "6.0",
-        sha256="ceab842e9fbce2f2de971ba6226967caaf1627b3e5d10799c3bd2e7c3285ba8b",
-        deprecated=True,
-    )  # use for xsdk-0.4.0
-    version(
-        "5.1",
-        sha256="b0c1be505ce5f8041984c63edca9100d81df655733681858f5cc10e8c0c72711",
-        deprecated=True,
-    )
-
-    version(
-        "5.0",
-        sha256="2540bb63dea987ab205f7b375aff41f320b1de9bd7f1d1064ef96b22eeda1251",
-        url="https://tasmanian.ornl.gov/documents/Tasmanian_v5.0.zip",
+        "7.3",
+        sha256="5bd1dd89cc5c84506f6900b6569b17e50becd73eb31ec85cfa11d6f1f912c4fa",
         deprecated=True,
     )
 
@@ -73,7 +50,7 @@ class Tasmanian(CMakePackage, CudaPackage, ROCmPackage):
 
     variant("python", default=False, description="add Python binding for Tasmanian")
 
-    variant("fortran", default=False, description="add Fortran 90/95 interface to Tasmanian")
+    variant("fortran", default=False, description="add Fortran 2003 interface to Tasmanian")
 
     variant(
         "build_type",
@@ -82,12 +59,10 @@ class Tasmanian(CMakePackage, CudaPackage, ROCmPackage):
         values=("Debug", "Release"),
     )
 
-    depends_on("cmake@2.8:", type="build")
-    depends_on("cmake@3.5:", type="build", when="@6.0:")
     depends_on("cmake@3.10:", type=("build", "run"), when="@7.0:")
-    depends_on("cmake@3.22:", type=("build", "run"), when="@develop")
+    depends_on("cmake@3.22:", type=("build", "run"), when="@8.0:")
 
-    depends_on("python@2.7:", when="+python", type=("build", "run"))
+    depends_on("python@3.0:", when="+python", type=("build", "run"))
     depends_on("py-numpy", when="+python", type=("build", "run"))
 
     extends("python", when="+python", type=("build", "run"))
@@ -97,29 +72,24 @@ class Tasmanian(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("blas", when="+blas", type=("build", "run"))  # openblas 0.2.18 or newer
     depends_on("lapack", when="+blas @7.1:", type=("build", "run"))  # lapack used since 7.1
 
-    depends_on("cuda@8.0.61:", when="+cuda", type=("build", "run"))
-    depends_on("cuda@8.0.61:", when="+magma", type=("build", "run"))
+    depends_on("cuda@10.0:", when="+cuda", type=("build", "run"))
+    depends_on("cuda@10.0:", when="+magma", type=("build", "run"))
 
-    depends_on("hip@3.8:", when="+rocm", type=("build", "run"))
-    depends_on("rocblas@3.8:", when="+rocm", type=("build", "run"))
-    depends_on("rocsparse@3.8:", when="+rocm", type=("build", "run"))
-    depends_on("rocsolver@3.8:", when="+rocm", type=("build", "run"))
+    depends_on("hip@5.0:", when="+rocm", type=("build", "run"))
+    depends_on("rocblas@5.0:", when="+rocm", type=("build", "run"))
+    depends_on("rocsparse@5.0:", when="+rocm", type=("build", "run"))
+    depends_on("rocsolver@5.0:", when="+rocm", type=("build", "run"))
 
-    depends_on("magma@2.4.0:", when="+magma @6.0:", type=("build", "run"))
     depends_on("magma@2.5.0:", when="+magma @7.0:", type=("build", "run"))
 
+    # https://github.com/spack/spack/issues/39536#issuecomment-1685161942
+    conflicts("^cuda@12", when="@:7.9 +cuda")
+
     conflicts("+magma", when="~cuda~rocm")  # currently MAGMA only works with CUDA
     conflicts("+cuda", when="+rocm")  # can pick CUDA or ROCm, not both
 
-    # old versions
-    conflicts("+rocm", when="@:7.3")  # ROCm was added in 7.3, tested in 7.5
-    conflicts("+magma", when="@:5.1")  # magma does not work prior to 6.0
-    conflicts("+mpi", when="@:5.1")  # MPI is broken prior to 6.0
-    conflicts("+xsdkflags", when="@:5.1")  # 6.0 is the first version included in xSDK
-
-    # patching some bugs
-    patch("addons70.patch", when="@7.0")
-    patch("packageconf70.patch", when="@7.0")
+    # patching a bug in the interpretation of the C++ standard
+    patch("tas80_clang17.patch", when="@8.0")
 
     def setup_build_environment(self, env):
         # needed for the hipcc compiler
@@ -129,29 +99,16 @@ def setup_build_environment(self, env):
     def cmake_args(self):
         spec = self.spec
 
-        # 7.1 is the last version to use xSDK legacy build options
-        if "+xsdkflags" in spec and spec.satisfies("@:7.1"):
-            args = [
-                "-DUSE_XSDK_DEFAULTS:BOOL=ON",
-                self.define_from_variant("XSDK_ENABLE_PYTHON", "python"),
-                self.define_from_variant("TPL_ENABLE_MPI", "mpi"),
-                self.define_from_variant("XSDK_ENABLE_OPENMP", "openmp"),
-                self.define_from_variant("TPL_ENABLE_BLAS", "blas"),
-                self.define_from_variant("XSDK_ENABLE_CUDA", "cuda"),
-                self.define_from_variant("TPL_ENABLE_MAGMA", "magma"),
-                self.define_from_variant("XSDK_ENABLE_FORTRAN", "fortran"),
-            ]
-        else:
-            args = [
-                self.define_from_variant("Tasmanian_ENABLE_OPENMP", "openmp"),
-                self.define_from_variant("Tasmanian_ENABLE_BLAS", "blas"),
-                self.define_from_variant("Tasmanian_ENABLE_PYTHON", "python"),
-                self.define_from_variant("Tasmanian_ENABLE_MPI", "mpi"),
-                self.define_from_variant("Tasmanian_ENABLE_CUDA", "cuda"),
-                self.define_from_variant("Tasmanian_ENABLE_HIP", "rocm"),
-                self.define_from_variant("Tasmanian_ENABLE_MAGMA", "magma"),
-                self.define_from_variant("Tasmanian_ENABLE_FORTRAN", "fortran"),
-            ]
+        args = [
+            self.define_from_variant("Tasmanian_ENABLE_OPENMP", "openmp"),
+            self.define_from_variant("Tasmanian_ENABLE_BLAS", "blas"),
+            self.define_from_variant("Tasmanian_ENABLE_PYTHON", "python"),
+            self.define_from_variant("Tasmanian_ENABLE_MPI", "mpi"),
+            self.define_from_variant("Tasmanian_ENABLE_CUDA", "cuda"),
+            self.define_from_variant("Tasmanian_ENABLE_HIP", "rocm"),
+            self.define_from_variant("Tasmanian_ENABLE_MAGMA", "magma"),
+            self.define_from_variant("Tasmanian_ENABLE_FORTRAN", "fortran"),
+        ]
 
         if spec.satisfies("+blas"):
             args.append("-DBLAS_LIBRARIES={0}".format(spec["blas"].libs.joined(";")))
@@ -162,15 +119,6 @@ def cmake_args(self):
                 "-DPYTHON_EXECUTABLE:FILEPATH={0}".format(self.spec["python"].command.path)
             )
 
-        # See https://github.com/ROCmSoftwarePlatform/rocFFT/issues/322
-        if self.spec.satisfies("+rocm") and self.spec.satisfies("^cmake@3.21:"):
-            args.append(self.define("__skip_rocmclang", "ON"))
-
-        # _CUBLAS and _CUDA were separate options prior to 6.0
-        # skipping _CUBLAS leads to peformance regression
-        if spec.satisfies("@:5.1"):
-            args.append(self.define_from_variant("Tasmanian_ENABLE_CUBLAS", "cuda"))
-
         return args
 
     @run_after("install")
@@ -186,12 +134,14 @@ def test_make_test(self):
 
         options = [cmake_dir]
         if "+rocm" in self.spec:
+            options.append(f"-Dhip_DIR={self.spec['hip'].prefix.lib.cmake.hip}")
             options.append(
                 f"-DAMDDeviceLibs_DIR={self.spec['llvm-amdgpu'].prefix.lib.cmake.AMDDeviceLibs}"
             )
             options.append(f"-Damd_comgr_DIR={self.spec['comgr'].prefix.lib.cmake.amd_comgr}")
             options.append(
-                f"-Dhsa-runtime64_DIR={self.spec['hsa-rocr-dev'].prefix.lib.cmake.hsa-runtime64}"
+                "-Dhsa-runtime64_DIR="
+                + join_path(self.spec["hsa-rocr-dev"].prefix.lib.cmake, "hsa-runtime64")
             )
             options.append(f"-DHSA_HEADER={self.spec['hsa-rocr-dev'].prefix.include}")
             options.append(f"-DCMAKE_INCLUDE_PATH={self.spec['hsa-rocr-dev'].prefix.include.hsa}")
diff --git a/var/spack/repos/builtin/packages/tasmanian/packageconf70.patch b/var/spack/repos/builtin/packages/tasmanian/packageconf70.patch
deleted file mode 100644
index c53255687f08b6..00000000000000
--- a/var/spack/repos/builtin/packages/tasmanian/packageconf70.patch
+++ /dev/null
@@ -1,15 +0,0 @@
-diff --git a/Config/TasmanianConfig.in.cmake b/Config/TasmanianConfig.in.cmake
-index 8912e4c..df54aaf 100644
---- a/Config/TasmanianConfig.in.cmake
-+++ b/Config/TasmanianConfig.in.cmake
-@@ -7,6 +7,10 @@ cmake_minimum_required(VERSION 3.10)
- # but this doesn't seem to work, not sure if this is a "relocatable package" (low concern)
- include("@CMAKE_INSTALL_PREFIX@/lib/@CMAKE_PROJECT_NAME@/@CMAKE_PROJECT_NAME@.cmake")
- 
-+if (@Tasmanian_ENABLE_MPI@)
-+    find_package(MPI REQUIRED)
-+endif()
-+
- add_executable(Tasmanian::tasgrid IMPORTED)
- set_property(TARGET Tasmanian::tasgrid PROPERTY IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/bin/tasgrid${CMAKE_EXECUTABLE_SUFFIX_CXX}")
- 
diff --git a/var/spack/repos/builtin/packages/tasmanian/tas80_clang17.patch b/var/spack/repos/builtin/packages/tasmanian/tas80_clang17.patch
new file mode 100644
index 00000000000000..241789cddd4939
--- /dev/null
+++ b/var/spack/repos/builtin/packages/tasmanian/tas80_clang17.patch
@@ -0,0 +1,101 @@
+diff --git a/SparseGrids/tsgGridFourier.cpp b/SparseGrids/tsgGridFourier.cpp
+index 31e75a87..438b0631 100644
+--- a/SparseGrids/tsgGridFourier.cpp
++++ b/SparseGrids/tsgGridFourier.cpp
+@@ -961,7 +961,7 @@ std::vector GridFourier::getCandidateConstructionPoints(std::functionaddTensor(new_tensors.getIndex(i), [&](int l)->int{ return wrapper.getNumPoints(l); }, tweights[i]);
+ 
+-    return MultiIndexManipulations::indexesToNodes(dynamic_values->getNodesIndexes(), wrapper);
++    return MultiIndexManipulations::getIndexesToNodes(dynamic_values->getNodesIndexes(), wrapper);
+ }
+ std::vector GridFourier::getMultiIndex(const double x[]){
+     std::vector p(num_dimensions);
+diff --git a/SparseGrids/tsgGridGlobal.cpp b/SparseGrids/tsgGridGlobal.cpp
+index dd81ace0..01aa4fa3 100644
+--- a/SparseGrids/tsgGridGlobal.cpp
++++ b/SparseGrids/tsgGridGlobal.cpp
+@@ -473,7 +473,7 @@ std::vector GridGlobal::getCandidateConstructionPoints(std::functionaddTensor(new_tensors.getIndex(i), [&](int l)->int{ return wrapper.getNumPoints(l); }, tweights[i]);
+ 
+-    return MultiIndexManipulations::indexesToNodes(dynamic_values->getNodesIndexes(), wrapper);
++    return MultiIndexManipulations::getIndexesToNodes(dynamic_values->getNodesIndexes(), wrapper);
+ }
+ std::vector GridGlobal::getMultiIndex(const double x[]){
+     std::vector p(num_dimensions);
+diff --git a/SparseGrids/tsgGridLocalPolynomial.cpp b/SparseGrids/tsgGridLocalPolynomial.cpp
+index f2cf6809..176736c3 100644
+--- a/SparseGrids/tsgGridLocalPolynomial.cpp
++++ b/SparseGrids/tsgGridLocalPolynomial.cpp
+@@ -576,7 +576,7 @@ void GridLocalPolynomial::expandGrid(const std::vector &point, const std::v
+         surpluses = Data2D(num_outputs, 1, std::vector(value)); // one value is its own surplus
+     }else{ // merge with existing points
+         // compute the surplus for the point
+-        std::vector xnode = MultiIndexManipulations::indexesToNodes(point, *rule);
++        std::vector xnode = MultiIndexManipulations::getIndexesToNodes(point, *rule);
+         std::vector approximation(num_outputs), surp(num_outputs);
+         evaluate(xnode.data(), approximation.data());
+         std::transform(approximation.begin(), approximation.end(), value.begin(), surp.begin(), [&](double e, double v)->double{ return v - e; });
+@@ -755,7 +755,7 @@ void GridLocalPolynomial::updateSurpluses(MultiIndexSet const &work, int max_lev
+         for(int s=0; s x = MultiIndexManipulations::indexesToNodes(work.getIndex(i), num_dimensions, *rule);
++            std::vector x = MultiIndexManipulations::getIndexesToNodes(work.getIndex(i), num_dimensions, *rule);
+             double *surpi = surpluses.getStrip(i);
+ 
+             std::vector monkey_count(max_level + 1);
+@@ -818,7 +818,7 @@ void GridLocalPolynomial::applyTransformationTransposed(double weights[], const
+     for(int l=active_top_level; l>0; l--){
+         for(size_t i=0; i node = MultiIndexManipulations::indexesToNodes(work.getIndex(active_points[i]), num_dimensions, *rule);
++                std::vector node = MultiIndexManipulations::getIndexesToNodes(work.getIndex(active_points[i]), num_dimensions, *rule);
+ 
+                 std::fill(used.begin(), used.end(), false);
+ 
+@@ -1071,7 +1071,7 @@ void GridLocalPolynomial::getQuadratureWeights(double *weights) const{
+     for(int l=top_level; l>0; l--){
+         for(int i=0; i node = MultiIndexManipulations::indexesToNodes(work.getIndex(i), num_dimensions, *rule);
++                std::vector node = MultiIndexManipulations::getIndexesToNodes(work.getIndex(i), num_dimensions, *rule);
+ 
+                 std::vector used(work.getNumIndexes(), false);
+ 
+diff --git a/SparseGrids/tsgGridWavelet.cpp b/SparseGrids/tsgGridWavelet.cpp
+index b043d077..d2f8115c 100644
+--- a/SparseGrids/tsgGridWavelet.cpp
++++ b/SparseGrids/tsgGridWavelet.cpp
+@@ -415,7 +415,7 @@ void GridWavelet::buildInterpolationMatrix() const{
+     for(int b=0; b xi = MultiIndexManipulations::indexesToNodes(work.getIndex(i), (size_t) num_dimensions, rule1D);
++            std::vector xi = MultiIndexManipulations::getIndexesToNodes(work.getIndex(i), (size_t) num_dimensions, rule1D);
+ 
+             // loop over the basis functions to see if supported
+             int numpntr = 0;
+diff --git a/SparseGrids/tsgIndexManipulator.hpp b/SparseGrids/tsgIndexManipulator.hpp
+index 16a1321f..0c27a4cd 100644
+--- a/SparseGrids/tsgIndexManipulator.hpp
++++ b/SparseGrids/tsgIndexManipulator.hpp
+@@ -562,7 +562,7 @@ OutputIteratorLike indexesToNodes(IteratorLike ibegin, size_t num_entries, RuleL
+  * \brief Overload that returns the result in a vector.
+  */
+ template
+-std::vector indexesToNodes(IndexList const &list, RuleLike const &rule){
++std::vector getIndexesToNodes(IndexList const &list, RuleLike const &rule){
+     std::vector result(std::distance(list.begin(), list.end()));
+     indexesToNodes(list, rule, result.begin());
+     return result;
+@@ -573,7 +573,7 @@ std::vector indexesToNodes(IndexList const &list, RuleLike const &rule){
+  * \brief Overload that returns the result in a vector.
+  */
+ template
+-std::vector indexesToNodes(IteratorLike ibegin, size_t num_entries, RuleLike const &rule){
++std::vector getIndexesToNodes(IteratorLike ibegin, size_t num_entries, RuleLike const &rule){
+     std::vector result(num_entries);
+     indexesToNodes(ibegin, num_entries, rule, result.begin());
+     return result;
diff --git a/var/spack/repos/builtin/packages/tau/package.py b/var/spack/repos/builtin/packages/tau/package.py
index 8bea62ce9aff3a..56cf5f1d721a7b 100644
--- a/var/spack/repos/builtin/packages/tau/package.py
+++ b/var/spack/repos/builtin/packages/tau/package.py
@@ -26,6 +26,8 @@ class Tau(Package):
     tags = ["e4s"]
 
     version("master", branch="master")
+    version("2.33", sha256="04d9d67adb495bc1ea56561f33c5ce5ba44f51cc7f64996f65bd446fac5483d9")
+    version("2.32.1", sha256="0eec3de46b0873846dfc639270c5e30a226b463dd6cb41aa12e975b7563f0eeb")
     version("2.32", sha256="ee774a06e30ce0ef0f053635a52229152c39aba4f4933bed92da55e5e13466f3")
     version("2.31.1", sha256="bf445b9d4fe40a5672a7b175044d2133791c4dfb36a214c1a55a931aebc06b9d")
     version("2.31", sha256="27e73c395dd2a42b91591ce4a76b88b1f67663ef13aa19ef4297c68f45d946c2")
@@ -84,6 +86,7 @@ class Tau(Package):
     variant("io", default=True, description="Activates POSIX I/O support")
     variant("adios2", default=False, description="Activates ADIOS2 output support")
     variant("sqlite", default=False, description="Activates SQLite3 output support")
+    variant("syscall", default=False, description="Activates syscall wrapper")
     variant(
         "profileparam",
         default=False,
@@ -98,9 +101,10 @@ class Tau(Package):
     variant(
         "x86_64", default=False, description="Force build for x86 Linux instead of auto-detect"
     )
+    variant("dyninst", default=False, description="Activates dyninst support")
 
     depends_on("cmake@3.14:", type="build", when="%clang")
-    depends_on("zlib", type="link")
+    depends_on("zlib-api", type="link")
     depends_on("pdt", when="+pdt")  # Required for TAU instrumentation
     depends_on("scorep", when="+scorep")
     depends_on("otf2@2.1:2.3", when="+otf2")
@@ -124,8 +128,10 @@ class Tau(Package):
     depends_on("rocprofiler-dev", when="+rocprofiler")
     depends_on("roctracer-dev", when="+roctracer")
     depends_on("hsa-rocr-dev", when="+rocm")
+    depends_on("rocm-smi-lib", when="@2.32.1: +rocm")
     depends_on("java", type="run")  # for paraprof
     depends_on("oneapi-level-zero", when="+level_zero")
+    depends_on("dyninst@12.3.0:", when="+dyninst")
 
     # Elf only required from 2.28.1 on
     conflicts("+elf", when="@:2.28.0")
@@ -134,6 +140,7 @@ class Tau(Package):
     # ADIOS2, SQLite only available from 2.29.1 on
     conflicts("+adios2", when="@:2.29.1")
     conflicts("+sqlite", when="@:2.29.1")
+    conflicts("+dyninst", when="@:2.32.1")
 
     patch("unwind.patch", when="@2.29.0")
 
@@ -186,7 +193,7 @@ def set_compiler_options(self, spec):
         return compiler_options
 
     def setup_build_environment(self, env):
-        env.prepend_path("LIBRARY_PATH", self.spec["zlib"].prefix.lib)
+        env.prepend_path("LIBRARY_PATH", self.spec["zlib-api"].prefix.lib)
         env.prepend_path("LIBRARY_PATH", self.spec["hwloc"].prefix.lib)
 
     def install(self, spec, prefix):
@@ -241,6 +248,9 @@ def install(self, spec, prefix):
         if "+io" in spec:
             options.append("-iowrapper")
 
+        if "+syscall" in spec:
+            options.append("-syscall")
+
         if "+binutils" in spec:
             options.append("-bfd=%s" % spec["binutils"].prefix)
 
@@ -259,8 +269,9 @@ def install(self, spec, prefix):
         if "+mpi" in spec:
             env["CC"] = spec["mpi"].mpicc
             env["CXX"] = spec["mpi"].mpicxx
-            env["F77"] = spec["mpi"].mpif77
-            env["FC"] = spec["mpi"].mpifc
+            if "+fortran" in spec:
+                env["F77"] = spec["mpi"].mpif77
+                env["FC"] = spec["mpi"].mpifc
             options.append("-mpiinc=%s" % spec["mpi"].prefix.include)
             options.append("-mpilib=%s" % spec["mpi"].prefix.lib)
 
@@ -288,6 +299,8 @@ def install(self, spec, prefix):
 
         if "+rocm" in spec:
             options.append("-rocm=%s" % spec["hsa-rocr-dev"].prefix)
+            if spec.satisfies("@2.32.1"):
+                options.append("-rocmsmi=%s" % spec["rocm-smi-lib"].prefix)
 
         if "+rocprofiler" in spec:
             options.append("-rocprofiler=%s" % spec["rocprofiler-dev"].prefix)
@@ -332,6 +345,15 @@ def install(self, spec, prefix):
                     break
             options.append("-pythonlib=%s" % lib_path)
 
+        if "+dyninst" in spec:
+            options.append("-dyninst=%s" % spec["dyninst"].prefix)
+            if "+tbb" not in spec:
+                options.append("-tbb=%s" % spec["intel-tbb"].prefix)
+            if "+boost" not in spec:
+                options.append("-boost=%s" % spec["boost"].prefix)
+            if "+elf" not in spec:
+                options.append("-elf=%s" % spec["elfutils"].prefix)
+
         compiler_specific_options = self.set_compiler_options(spec)
         options.extend(compiler_specific_options)
         configure(*options)
diff --git a/var/spack/repos/builtin/packages/tcl/package.py b/var/spack/repos/builtin/packages/tcl/package.py
index 951c9b88115a11..dee78161bb1149 100644
--- a/var/spack/repos/builtin/packages/tcl/package.py
+++ b/var/spack/repos/builtin/packages/tcl/package.py
@@ -33,10 +33,12 @@ class Tcl(AutotoolsPackage, SourceforgePackage):
 
     extendable = True
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     configure_directory = "unix"
 
+    filter_compiler_wrappers("tclConfig.sh", relative_root="lib")
+
     def install(self, spec, prefix):
         with working_dir(self.build_directory):
             make("install")
@@ -151,13 +153,12 @@ def setup_dependent_build_environment(self, env, dependent_spec):
         # https://core.tcl-lang.org/tk/tktview/447bd3e4abe17452d19a80e6840dcc8a2603fcbc
         env.prepend_path("TCLLIBPATH", self.spec["tcl"].libs.directories[0], separator=" ")
 
-        for d in dependent_spec.traverse(deptype=("build", "run", "test")):
-            if d.package.extends(self.spec):
-                # Tcl libraries may be installed in lib or lib64, see #19546
-                for lib in ["lib", "lib64"]:
-                    tcllibpath = join_path(d.prefix, lib)
-                    if os.path.exists(tcllibpath):
-                        env.prepend_path("TCLLIBPATH", tcllibpath, separator=" ")
+        if dependent_spec.package.extends(self.spec):
+            # Tcl libraries may be installed in lib or lib64, see #19546
+            for lib in ["lib", "lib64"]:
+                tcllibpath = join_path(dependent_spec.prefix, lib)
+                if os.path.exists(tcllibpath):
+                    env.prepend_path("TCLLIBPATH", tcllibpath, separator=" ")
 
     def setup_dependent_run_environment(self, env, dependent_spec):
         """Set TCLLIBPATH to include the tcl-shipped directory for
@@ -167,10 +168,9 @@ def setup_dependent_run_environment(self, env, dependent_spec):
 
         * https://wiki.tcl-lang.org/page/TCLLIBPATH
         """
-        for d in dependent_spec.traverse(deptype=("build", "run", "test")):
-            if d.package.extends(self.spec):
-                # Tcl libraries may be installed in lib or lib64, see #19546
-                for lib in ["lib", "lib64"]:
-                    tcllibpath = join_path(d.prefix, lib)
-                    if os.path.exists(tcllibpath):
-                        env.prepend_path("TCLLIBPATH", tcllibpath, separator=" ")
+        if dependent_spec.package.extends(self.spec):
+            # Tcl libraries may be installed in lib or lib64, see #19546
+            for lib in ["lib", "lib64"]:
+                tcllibpath = join_path(dependent_spec.prefix, lib)
+                if os.path.exists(tcllibpath):
+                    env.prepend_path("TCLLIBPATH", tcllibpath, separator=" ")
diff --git a/var/spack/repos/builtin/packages/tecio/package.py b/var/spack/repos/builtin/packages/tecio/package.py
new file mode 100644
index 00000000000000..d54636eefb35eb
--- /dev/null
+++ b/var/spack/repos/builtin/packages/tecio/package.py
@@ -0,0 +1,59 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+
+
+class Tecio(CMakePackage):
+    """The TecIO library allows third-party applications to read and write
+    Tecplot file format."""
+
+    homepage = "https://www.tecplot.com/products/tecio-library/"
+    url = "file://{}/tecio.tgz".format(os.getcwd())
+
+    manual_download = True
+
+    maintainers("snehring")
+
+    version("2022R2", sha256="35a8137800611da3a413da09223e2d1683b54e195bc52f8fc4ec131cfce7cb23")
+
+    depends_on("cmake@3.0.2:", type="build")
+
+    depends_on("boost@1.69.0:+system", type="build")
+    depends_on("mpi", when="+mpi")
+
+    variant("mpi", default=False, description="Build tecio with mpi support.")
+
+    @property
+    def root_cmakelists_dir(self):
+        if self.spec.satisfies("+mpi"):
+            return "teciompisrc"
+        else:
+            return "teciosrc"
+
+    def install(self, spec, prefix):
+        includes = [
+            "StandardIntegralTypes.h",
+            "TECIO.h",
+            "tecio_Exports.h",
+            "tecio.inc",
+            "tecio.for",
+            "tecio.f90",
+        ]
+        mkdirp(join_path(prefix, "include"))
+        mkdirp(join_path(prefix, "lib"))
+        mkdirp(join_path(prefix, "bin"))
+        with working_dir(self.root_cmakelists_dir):
+            for f in includes:
+                install(f, prefix.include)
+            install("tecio_license_agreement.txt", prefix)
+        with working_dir(self.build_directory):
+            install("szcombine", prefix.bin)
+            if spec.satisfies("+mpi"):
+                install("libteciompi.a", prefix.lib)
+            else:
+                install("libtecio.a", prefix.lib)
diff --git a/var/spack/repos/builtin/packages/teckit/package.py b/var/spack/repos/builtin/packages/teckit/package.py
index 318258cb7a26a1..ab0c13f9405088 100644
--- a/var/spack/repos/builtin/packages/teckit/package.py
+++ b/var/spack/repos/builtin/packages/teckit/package.py
@@ -28,7 +28,7 @@ class Teckit(AutotoolsPackage):
     version("2.5.6", commit="41c20be2793e1afcbb8de6339af89d1eeab84fe8")
     version("2.5.5", commit="2733fd9895819e3697257550cc39b8e419c1ee7e")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
     depends_on("libtool", type="build")
diff --git a/var/spack/repos/builtin/packages/tecplot/package.py b/var/spack/repos/builtin/packages/tecplot/package.py
index bc4a96392def20..a877c9da76d95f 100644
--- a/var/spack/repos/builtin/packages/tecplot/package.py
+++ b/var/spack/repos/builtin/packages/tecplot/package.py
@@ -19,6 +19,11 @@ class Tecplot(Package):
 
     maintainers("LRWeber")
 
+    version(
+        "2023r1",
+        sha256="58e7f4de875e65047f4edd684013d0ff538df6246f00c059458989f281be4c93",
+        expand=False,
+    )
     version(
         "2022r2",
         sha256="e30cb7bf894e7cd568a2b24beb4bf667f1781ae27b59bb73410fafe12ddfdcdf",
diff --git a/var/spack/repos/builtin/packages/test-drive/package.py b/var/spack/repos/builtin/packages/test-drive/package.py
new file mode 100644
index 00000000000000..b5221570db9c00
--- /dev/null
+++ b/var/spack/repos/builtin/packages/test-drive/package.py
@@ -0,0 +1,17 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class TestDrive(MesonPackage):
+    """Simple testing framework for Fortran packages"""
+
+    homepage = "https://github.com/fortran-lang/test-drive"
+    url = "https://github.com/fortran-lang/test-drive/releases/download/v0.4.0/test-drive-0.4.0.tar.xz"
+
+    maintainers("awvwgk")
+
+    version("0.4.0", "effabe5d46ea937a79f3ea8d37eea43caf38f9f1377398bad0ca02784235e54a")
diff --git a/var/spack/repos/builtin/packages/texlive/package.py b/var/spack/repos/builtin/packages/texlive/package.py
index 138134f202f8d9..6046dd006a11c3 100644
--- a/var/spack/repos/builtin/packages/texlive/package.py
+++ b/var/spack/repos/builtin/packages/texlive/package.py
@@ -116,7 +116,7 @@ class Texlive(AutotoolsPackage):
     depends_on("pixman", when="@2019:")
     depends_on("poppler@:0.84", when="@2019:")
     depends_on("teckit", when="@2019:")
-    depends_on("zlib", when="@2019:")
+    depends_on("zlib-api", when="@2019:")
     depends_on("zziplib", when="@2019:")
 
     build_directory = "spack-build"
diff --git a/var/spack/repos/builtin/packages/the-silver-searcher/package.py b/var/spack/repos/builtin/packages/the-silver-searcher/package.py
index 8276f1ab74d6dc..b99fabdd767fe7 100644
--- a/var/spack/repos/builtin/packages/the-silver-searcher/package.py
+++ b/var/spack/repos/builtin/packages/the-silver-searcher/package.py
@@ -19,5 +19,5 @@ class TheSilverSearcher(AutotoolsPackage):
 
     depends_on("pcre")
     depends_on("xz")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("pkgconfig", type="build")
diff --git a/var/spack/repos/builtin/packages/thepeg/package.py b/var/spack/repos/builtin/packages/thepeg/package.py
index 3c09f7e47ab0ba..f1fe8300f0a447 100644
--- a/var/spack/repos/builtin/packages/thepeg/package.py
+++ b/var/spack/repos/builtin/packages/thepeg/package.py
@@ -73,7 +73,7 @@ class Thepeg(AutotoolsPackage):
     depends_on("automake", type="build")
     depends_on("libtool", type="build")
     depends_on("m4", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     variant("hepmc", default="2", values=("2", "3"), description="HepMC interface to build ")
     variant("rivet", default=True, description="Add rivet integration")
@@ -95,7 +95,7 @@ def flag_handler(self, name, flags):
 
     def configure_args(self):
         args = ["--with-gsl=" + self.spec["gsl"].prefix, "--without-javagui"]
-        args += ["--with-zlib=" + self.spec["zlib"].prefix]
+        args += ["--with-zlib=" + self.spec["zlib-api"].prefix]
 
         if self.spec.satisfies("@:1.8"):
             args += ["--with-LHAPDF=" + self.spec["lhapdf"].prefix]
diff --git a/var/spack/repos/builtin/packages/thrift/package.py b/var/spack/repos/builtin/packages/thrift/package.py
index b6a43aadd788af..af4510cf9695e6 100644
--- a/var/spack/repos/builtin/packages/thrift/package.py
+++ b/var/spack/repos/builtin/packages/thrift/package.py
@@ -53,7 +53,7 @@ class Thrift(Package):
     depends_on("openssl")
 
     # Variant dependencies
-    depends_on("zlib", when="+c")
+    depends_on("zlib-api", when="+c")
     depends_on("libevent", when="+c")
 
     depends_on("java@7:", when="+java")
diff --git a/var/spack/repos/builtin/packages/tiled-mm/package.py b/var/spack/repos/builtin/packages/tiled-mm/package.py
index c10b7a74fee972..e44e92fb321de4 100644
--- a/var/spack/repos/builtin/packages/tiled-mm/package.py
+++ b/var/spack/repos/builtin/packages/tiled-mm/package.py
@@ -10,18 +10,19 @@ class TiledMm(CMakePackage, CudaPackage, ROCmPackage):
     """Matrix multiplication on GPUs for matrices stored on a CPU. Similar to cublasXt,
     but ported to both NVIDIA and AMD GPUs."""
 
-    maintainers = ["mtaillefumier", "simonpintarelli"]
     homepage = "https://github.com/eth-cscs/Tiled-MM/"
     url = "https://github.com/eth-cscs/Tiled-MM/archive/refs/tags/v2.0.tar.gz"
     git = "https://github.com/eth-cscs/Tiled-MM.git"
 
+    maintainers("mtaillefumier", "simonpintarelli", "RMeli")
+
     version("master", branch="master")
     version("2.2", sha256="6d0b49c9588ece744166822fd44a7bc5bec3dc666b836de8bf4bf1a7bb675aac")
     version("2.0", sha256="ea554aea8c53d7c8e40044e6d478c0e8137d7e8b09d7cb9650703430d92cf32e")
 
-    variant("shared", default=True)
-    variant("examples", default=False)
-    variant("tests", default=False)
+    variant("shared", default=True, description="Build shared libraries")
+    variant("examples", default=False, description="Enable examples")
+    variant("tests", default=False, description="Enable tests")
 
     depends_on("rocblas", when="+rocm")
     depends_on("cxxopts", when="+tests")
@@ -33,8 +34,8 @@ class TiledMm(CMakePackage, CudaPackage, ROCmPackage):
     def cmake_args(self):
         args = [
             self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
-            self.define_from_variant("TIELDMM_WITH_EXAMPLES", "examples"),
-            self.define_from_variant("TIELDMM_WITH_TESTS", "tests"),
+            self.define_from_variant("TILEDMM_WITH_EXAMPLES", "examples"),
+            self.define_from_variant("TILEDMM_WITH_TESTS", "tests"),
         ]
 
         if "+rocm" in self.spec:
diff --git a/var/spack/repos/builtin/packages/tinygltf/package.py b/var/spack/repos/builtin/packages/tinygltf/package.py
index 5aced4c5ad3d79..f00d79fdbdc576 100644
--- a/var/spack/repos/builtin/packages/tinygltf/package.py
+++ b/var/spack/repos/builtin/packages/tinygltf/package.py
@@ -11,7 +11,12 @@ class Tinygltf(CMakePackage):
 
     homepage = "https://github.com/syoyo/tinygltf"
     url = "https://github.com/syoyo/tinygltf/archive/refs/tags/v2.5.0.tar.gz"
+    git = "https://github.com/syoyo/tinygltf/"
 
+    version("release", branch="release")
+    version("2.8.14", sha256="63cd43746c9ddfe5777494500422e831a312299e386fbf80922839dc1a5575f8")
+    version("2.7.0", sha256="a1bbc0b831719e3a809a1bb01ce299a60e80b4e15221f58e822303ba22a69d45")
+    version("2.6.3", sha256="f61e4a501baa7fbf31b18ea0f6815a59204ad0de281f7b04f0168f6bbd17c340")
     version("2.5.0", sha256="5d85bd556b60b1b69527189293cfa4902957d67fabb8582b6532f23a5ef27ec1")
 
     depends_on("cmake@3.6:", type="build")
diff --git a/var/spack/repos/builtin/packages/tippecanoe/package.py b/var/spack/repos/builtin/packages/tippecanoe/package.py
index 4f112c7051375c..015ce91807bf90 100644
--- a/var/spack/repos/builtin/packages/tippecanoe/package.py
+++ b/var/spack/repos/builtin/packages/tippecanoe/package.py
@@ -17,7 +17,7 @@ class Tippecanoe(MakefilePackage):
     version("1.34.3", sha256="7a2dd2376a93d66a82c8253a46dbfcab3eaaaaca7bf503388167b9ee251bee54")
 
     depends_on("sqlite")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def edit(self, spec, prefix):
         makefile = FileFilter("Makefile")
diff --git a/var/spack/repos/builtin/packages/tkrzw/package.py b/var/spack/repos/builtin/packages/tkrzw/package.py
index 8a10d659c33284..e1541d60a66ea2 100644
--- a/var/spack/repos/builtin/packages/tkrzw/package.py
+++ b/var/spack/repos/builtin/packages/tkrzw/package.py
@@ -65,7 +65,7 @@ class Tkrzw(AutotoolsPackage):
         description="List of supported compression backends",
     )
 
-    depends_on("zlib", when="compression=zlib")
+    depends_on("zlib-api", when="compression=zlib")
     depends_on("lz4", when="compression=lz4")
     depends_on("xz", when="compression=lzma")  # lzma.h is in the xz package, not in lzma
     depends_on("zstd", when="compression=zstd")
diff --git a/var/spack/repos/builtin/packages/tmalign/package.py b/var/spack/repos/builtin/packages/tmalign/package.py
index c27a79a3d898dc..5ea2557490c0db 100644
--- a/var/spack/repos/builtin/packages/tmalign/package.py
+++ b/var/spack/repos/builtin/packages/tmalign/package.py
@@ -27,7 +27,7 @@ class Tmalign(Package):
         deprecated=True,
     )
 
-    variant("fast-math", default=False, when="@20220412:")
+    variant("fast-math", default=False, description="Enable fast math", when="@20220412:")
 
     with when("@20220412:"):
         phases = ["build", "install"]
diff --git a/var/spack/repos/builtin/packages/tmscore/package.py b/var/spack/repos/builtin/packages/tmscore/package.py
index 334b2cecc317da..a69e335413996b 100644
--- a/var/spack/repos/builtin/packages/tmscore/package.py
+++ b/var/spack/repos/builtin/packages/tmscore/package.py
@@ -21,7 +21,7 @@ class Tmscore(Package):
         expand=False,
     )
 
-    variant("fast-math", default=False)
+    variant("fast-math", default=False, description="Enable fast math")
 
     phases = ["build", "install"]
 
diff --git a/var/spack/repos/builtin/packages/topaz/package.py b/var/spack/repos/builtin/packages/topaz/package.py
new file mode 100644
index 00000000000000..855cba4d6c90b1
--- /dev/null
+++ b/var/spack/repos/builtin/packages/topaz/package.py
@@ -0,0 +1,27 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Topaz(PythonPackage):
+    """topaz: Pipeline for particle picking in cryo-electron microscopy images using
+    convolutional neural networks trained from positive and unlabeled examples. Also
+    featuring micrograph and tomogram denoising with DNNs."""
+
+    homepage = "https://topaz-em.readthedocs.io/"
+    pypi = "topaz-em/topaz-em-0.2.5.tar.gz"
+
+    version("0.2.5", sha256="002a6eb775598b6c4df0225f3a488bfe6a6da9246e8ca42eb4e7d58f694c25cc")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-torch@1:", type=("build", "run"))
+    depends_on("py-torchvision", type=("build", "run"))
+    depends_on("py-numpy@1.11:", type=("build", "run"))
+    depends_on("py-pandas", type=("build", "run"))
+    depends_on("py-scikit-learn@0.19.0:", type=("build", "run"))
+    depends_on("py-scipy@0.17.0:", type=("build", "run"))
+    depends_on("py-pillow@6.2.0:", type=("build", "run"))
+    depends_on("py-future", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/tracer/package.py b/var/spack/repos/builtin/packages/tracer/package.py
index 6c814a9baba08e..2569349e674633 100644
--- a/var/spack/repos/builtin/packages/tracer/package.py
+++ b/var/spack/repos/builtin/packages/tracer/package.py
@@ -15,7 +15,7 @@ class Tracer(MakefilePackage):
     maintainers("bhatele")
 
     version("develop", branch="master")
-    version("2.2", tag="v2.2")
+    version("2.2", tag="v2.2", commit="fdd1b07a1a0faca14aac53dcbcbccc44237ae7cb")
 
     variant("otf2", default=True, description="Use OTF2 traces for simulation")
 
diff --git a/var/spack/repos/builtin/packages/tracy-client/package.py b/var/spack/repos/builtin/packages/tracy-client/package.py
index 0d3e3e9fe5f343..dd219f31ee039a 100644
--- a/var/spack/repos/builtin/packages/tracy-client/package.py
+++ b/var/spack/repos/builtin/packages/tracy-client/package.py
@@ -15,6 +15,7 @@ class TracyClient(CMakePackage):
     maintainers("msimberg")
 
     version("master", git="https://github.com/wolfpld/tracy.git", branch="master")
+    version("0.10", sha256="a76017d928f3f2727540fb950edd3b736caa97b12dbb4e5edce66542cbea6600")
     version("0.9", sha256="93a91544e3d88f3bc4c405bad3dbc916ba951cdaadd5fcec1139af6fa56e6bfc")
     version("0.8.2", sha256="4784eddd89c17a5fa030d408392992b3da3c503c872800e9d3746d985cfcc92a")
     version("0.8.1", sha256="004992012b2dc879a9f6d143cbf94d7ea30e88135db3ef08951605d214892891")
diff --git a/var/spack/repos/builtin/packages/tracy/package.py b/var/spack/repos/builtin/packages/tracy/package.py
index 572e5d879b11a7..111b4a86534600 100644
--- a/var/spack/repos/builtin/packages/tracy/package.py
+++ b/var/spack/repos/builtin/packages/tracy/package.py
@@ -15,6 +15,7 @@ class Tracy(MakefilePackage):
     maintainers("msimberg")
 
     version("master", git="https://github.com/wolfpld/tracy.git", branch="master")
+    version("0.10", sha256="a76017d928f3f2727540fb950edd3b736caa97b12dbb4e5edce66542cbea6600")
     version("0.9", sha256="93a91544e3d88f3bc4c405bad3dbc916ba951cdaadd5fcec1139af6fa56e6bfc")
     version("0.8.2", sha256="4784eddd89c17a5fa030d408392992b3da3c503c872800e9d3746d985cfcc92a")
     version("0.8.1", sha256="004992012b2dc879a9f6d143cbf94d7ea30e88135db3ef08951605d214892891")
diff --git a/var/spack/repos/builtin/packages/transdecoder/package.py b/var/spack/repos/builtin/packages/transdecoder/package.py
index ec1228d458fff7..173a6e9b4342a1 100644
--- a/var/spack/repos/builtin/packages/transdecoder/package.py
+++ b/var/spack/repos/builtin/packages/transdecoder/package.py
@@ -15,6 +15,7 @@ class Transdecoder(MakefilePackage):
     homepage = "https://transdecoder.github.io/"
     url = "https://github.com/TransDecoder/TransDecoder/archive/TransDecoder-v5.5.0.tar.gz"
 
+    version("5.7.1", sha256="41dd5e95f6ba946ff21340417d867e5e99f123b4035779b25d3cffd20b828a30")
     version("5.7.0", sha256="421b50dd08b12a88f2f09922e20c50903e335f26947843d9f925f5c0e9ddd79f")
     version("5.5.0", sha256="c800d9226350817471e9f51267c91f7cab99dbc9b26c980527fc1019e7d90a76")
     version(
diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py
index 9f63ae9e826dc4..1681ac35d2e9a2 100644
--- a/var/spack/repos/builtin/packages/trilinos/package.py
+++ b/var/spack/repos/builtin/packages/trilinos/package.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import os
+import pathlib
 import sys
 
 from spack.build_environment import dso_suffix
@@ -41,6 +42,8 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage):
 
     version("master", branch="master")
     version("develop", branch="develop")
+    version("14.4.0", sha256="8e7d881cf6677aa062f7bfea8baa1e52e8956aa575d6a4f90f2b6f032632d4c6")
+    version("14.2.0", sha256="c96606e5cd7fc9d25b9dc20719cd388658520d7cbbd2b4de77a118440d1e0ccb")
     version("14.0.0", sha256="054d2fabdf70fce0dfaeb20eed265bd7894045d3e00c3d1ddb72d1c77c339ca1")
     version("13.4.1", sha256="5465cbff3de7ef4ac7d40eeff9d99342c00d9d20eee0a5f64f0a523093f5f1b3")
     version("13.4.0", sha256="39550006e059043b7e2177f10467ae2f77fe639901aee91cbc1e359516ff8d3e")
@@ -71,7 +74,9 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage):
     variant("complex", default=False, description="Enable complex numbers in Trilinos")
     variant("cuda_rdc", default=False, description="Turn on RDC for CUDA build")
     variant("rocm_rdc", default=False, description="Turn on RDC for ROCm build")
-    variant("cxxstd", default="14", values=["11", "14", "17"], multi=False)
+    variant(
+        "cxxstd", default="14", description="C++ standard", values=["11", "14", "17"], multi=False
+    )
     variant("debug", default=False, description="Enable runtime safety and debug checks")
     variant(
         "explicit_template_instantiation",
@@ -143,6 +148,7 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage):
     variant("stratimikos", default=False, description="Compile with Stratimikos")
     variant("teko", default=False, description="Compile with Teko")
     variant("tempus", default=False, description="Compile with Tempus")
+    variant("test", default=False, description="Enable testing")
     variant("thyra", default=False, description="Compile with Thyra")
     variant("tpetra", default=True, description="Compile with Tpetra")
     variant("trilinoscouplings", default=False, description="Compile with TrilinosCouplings")
@@ -338,13 +344,11 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("gotype=all", when="@12.15:")
 
     # CUDA without wrapper requires clang
-    for _compiler in spack.compilers.supported_compilers():
-        if _compiler != "clang":
-            conflicts(
-                "+cuda",
-                when="~wrapper %" + _compiler,
-                msg="trilinos~wrapper+cuda can only be built with the " "Clang compiler",
-            )
+    requires(
+        "%clang",
+        when="+cuda~wrapper",
+        msg="trilinos~wrapper+cuda can only be built with the Clang compiler",
+    )
     conflicts("+cuda_rdc", when="~cuda")
     conflicts("+rocm_rdc", when="~rocm")
     conflicts("+wrapper", when="~cuda")
@@ -354,6 +358,11 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("@:13.0.1 +cuda", when="^cuda@11:")
     # Build hangs with CUDA 11.6 (see #28439)
     conflicts("+cuda +stokhos", when="^cuda@11.6:")
+    # superlu-dist defines a macro EMPTY which conflicts with a header in cuda
+    # used when building stokhos
+    # Fix: https://github.com/xiaoyeli/superlu_dist/commit/09cb1430f7be288fd4d75b8ed461aa0b7e68fefe
+    # is not tagged yet. See discussion here https://github.com/trilinos/Trilinos/issues/11839
+    conflicts("+cuda +stokhos +superlu-dist")
     # Cuda UVM must be enabled prior to 13.2
     # See https://github.com/spack/spack/issues/28869
     conflicts("~uvm", when="@:13.1 +cuda")
@@ -362,8 +371,35 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("+stokhos", when="%xl")
     conflicts("+stokhos", when="%xl_r")
 
+    # Current Windows support, only have serial static builds
+    conflicts(
+        "+shared",
+        when="platform=windows",
+        msg="Only static builds are supported on Windows currently.",
+    )
+    conflicts(
+        "+mpi",
+        when="platform=windows",
+        msg="Only serial builds are supported on Windows currently.",
+    )
+
     # ###################### Dependencies ##########################
 
+    # External Kokkos
+    depends_on("kokkos@4.1.00", when="@14.4.0: +kokkos")
+    depends_on("kokkos +wrapper", when="trilinos@14.4.0: +kokkos +wrapper")
+    depends_on("kokkos ~wrapper", when="trilinos@14.4.0: +kokkos ~wrapper")
+
+    for a in CudaPackage.cuda_arch_values:
+        arch_str = "+cuda cuda_arch=" + a
+        kokkos_spec = "kokkos@4.1.00 " + arch_str
+        depends_on(kokkos_spec, when="@14.4.0 +kokkos " + arch_str)
+
+    for a in ROCmPackage.amdgpu_targets:
+        arch_str = "+rocm amdgpu_target={0}".format(a)
+        kokkos_spec = "kokkos@4.1.00 {0}".format(arch_str)
+        depends_on(kokkos_spec, when="@14.4.0 +kokkos {0}".format(arch_str))
+
     depends_on("adios2", when="+adios2")
     depends_on("blas")
     depends_on("boost+graph+math+exception+stacktrace", when="+boost")
@@ -372,8 +408,11 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage):
 
     #
     depends_on("cgns", when="+exodus")
+    depends_on("cmake@3.23:", type="build", when="@14.0.0:")
     depends_on("hdf5+hl", when="+hdf5")
-    depends_on("hypre~internal-superlu~int64", when="+hypre")
+    for plat in ["cray", "darwin", "linux"]:
+        depends_on("hypre~internal-superlu~int64", when="+hypre platform=%s" % plat)
+    depends_on("hypre-cmake~int64", when="+hypre platform=windows")
     depends_on("kokkos-nvcc-wrapper", when="+wrapper")
     depends_on("lapack")
     # depends_on('perl', type=('build',)) # TriBITS finds but doesn't use...
@@ -397,7 +436,7 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("superlu-dist", when="+superlu-dist")
     depends_on("superlu@4.3 +pic", when="+superlu")
     depends_on("swig", when="+python")
-    depends_on("zlib", when="+zoltan")
+    depends_on("zlib-api", when="+zoltan")
 
     # Trilinos' Tribits config system is limited which makes it very tricky to
     # link Amesos with static MUMPS, see
@@ -427,6 +466,8 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage):
 
     # ###################### Patches ##########################
 
+    patch("shylu-node-optional.patch", when="@13:14.4.0 +shylu")
+
     patch("umfpack_from_suitesparse.patch", when="@11.14.1:12.8.1")
     for _compiler in ["xl", "xl_r", "clang"]:
         patch("xlf_seacas.patch", when="@12.10.1:12.12.1 %" + _compiler)
@@ -579,6 +620,12 @@ def define_enable(suffix, value=None):
             ]
         )
 
+        if "+test" in spec:
+            options.append(define_trilinos_enable("TESTS", True))
+            options.append(define("BUILD_TESTING", True))
+        else:
+            options.append(define_trilinos_enable("TESTS", False))
+
         if spec.version >= Version("13"):
             options.append(define_from_variant("CMAKE_CXX_STANDARD", "cxxstd"))
         else:
@@ -763,7 +810,7 @@ def define_tpl(trilinos_name, spack_name, have_dep):
             ("METIS", "metis"),
             ("Netcdf", "netcdf-c"),
             ("SCALAPACK", "scalapack"),
-            ("Zlib", "zlib"),
+            ("Zlib", "zlib-api"),
         ]
         if spec.satisfies("@12.12.1:"):
             tpl_dep_map.append(("Pnetcdf", "parallel-netcdf"))
@@ -773,6 +820,10 @@ def define_tpl(trilinos_name, spack_name, have_dep):
         for tpl_name, dep_name in tpl_dep_map:
             define_tpl(tpl_name, dep_name, dep_name in spec)
 
+        # External Kokkos
+        if spec.satisfies("@14.4.0 +kokkos"):
+            options.append(define_tpl_enable("Kokkos"))
+
         # MPI settings
         options.append(define_tpl_enable("MPI"))
         if "+mpi" in spec:
@@ -785,7 +836,7 @@ def define_tpl(trilinos_name, spack_name, have_dep):
                     define("CMAKE_C_COMPILER", spec["mpi"].mpicc),
                     define("CMAKE_CXX_COMPILER", spec["mpi"].mpicxx),
                     define("CMAKE_Fortran_COMPILER", spec["mpi"].mpifc),
-                    define("MPI_BASE_DIR", spec["mpi"].prefix),
+                    define("MPI_BASE_DIR", str(pathlib.PurePosixPath(spec["mpi"].prefix))),
                 ]
             )
 
@@ -936,7 +987,7 @@ def filter_python(self):
         # https://github.com/trilinos/Trilinos/issues/866
         # A workaround is to remove PyTrilinos from the COMPONENTS_LIST
         # and to remove -lpytrilonos from Makefile.export.Trilinos
-        if "+python" in self.spec:
+        if self.spec.satisfies("@:13.0.1 +python"):
             filter_file(
                 r"(SET\(COMPONENTS_LIST.*)(PyTrilinos;)(.*)",
                 (r"\1\3"),
diff --git a/var/spack/repos/builtin/packages/trilinos/shylu-node-optional.patch b/var/spack/repos/builtin/packages/trilinos/shylu-node-optional.patch
new file mode 100644
index 00000000000000..491b8aa490e86a
--- /dev/null
+++ b/var/spack/repos/builtin/packages/trilinos/shylu-node-optional.patch
@@ -0,0 +1,10 @@
+diff -ruN spack-src/packages/shylu/cmake/Dependencies.cmake spack-src-patched/packages/shylu/cmake/Dependencies.cmake
+--- spack-src/packages/shylu/cmake/Dependencies.cmake	2023-08-01 15:08:54.000000000 -0700
++++ spack-src-patched/packages/shylu/cmake/Dependencies.cmake	2023-09-22 12:53:28.755219076 -0700
+@@ -1,4 +1,5 @@
+-SET(LIB_REQUIRED_DEP_PACKAGES ShyLU_DD ShyLU_Node)
++SET(LIB_REQUIRED_DEP_PACKAGES ShyLU_DD)
++SET(LIB_OPTIONAL_DEP_PACKAGES ShyLU_Node)
+ SET(LIB_OPTIONAL_DEP_PACKAGES)
+ SET(TEST_REQUIRED_DEP_PACKAGES)
+ SET(TEST_OPTIONAL_DEP_PACKAGES)
diff --git a/var/spack/repos/builtin/packages/trompeloeil/package.py b/var/spack/repos/builtin/packages/trompeloeil/package.py
new file mode 100644
index 00000000000000..13abdb3dc39a97
--- /dev/null
+++ b/var/spack/repos/builtin/packages/trompeloeil/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Trompeloeil(CMakePackage):
+    """Trompeloeil is a thread-safe header-only mocking framework for C++11/14
+    using the Boost Software License 1.0"""
+
+    homepage = "https://github.com/rollbear/trompeloeil"
+    url = "https://github.com/rollbear/trompeloeil/archive/v43.tar.gz"
+    git = "https://github.com/rollbear/trompeloeil.git"
+
+    version("master", branch="master")
+    version("45", sha256="124b0aa45d84415193719376b6557fc1f1180cbfebf4dc4f7ca247cb404d6bd8")
+    version("44", sha256="004877db6ba22f24c7867e112e081eeb68858122f55ebe7c7dd9d8d9e3b46c88")
+    version("43", sha256="86a0afa2e97347202a0a883ab43da78c1d4bfff0d6cb93205cfc433d0d9eb9eb")
diff --git a/var/spack/repos/builtin/packages/truchas/package.py b/var/spack/repos/builtin/packages/truchas/package.py
index 684f89fca24c8b..80d51aa740fe2f 100644
--- a/var/spack/repos/builtin/packages/truchas/package.py
+++ b/var/spack/repos/builtin/packages/truchas/package.py
@@ -21,9 +21,10 @@ class Truchas(CMakePackage):
     url = "https://gitlab.com/truchas/truchas/-/archive/22.04.1/truchas-22.04.1.tar.bz2"
     git = "https://gitlab.com/truchas/truchas.git"
 
-    maintainers("pbrady")
+    maintainers("pbrady", "zjibben")
 
     version("develop", branch="master")
+    version("23.06", sha256="a786caba5129d7e33ba42a06751d6c570bd3b9697e3404276a56216d27820c68")
     version("22.04.1", sha256="ed2000f27ee5c4bd3024063a374023878c61e8a3c76c37542fffd341d1226dc1")
 
     # ------------------------------------------------------------ #
@@ -70,6 +71,7 @@ class Truchas(CMakePackage):
     # Solvers
     # ------------------------------------------------------------ #
     depends_on("hypre@2.20: ~fortran")
+    depends_on("netlib-lapack")
 
     # ------------------------------------------------------------ #
     # Mapping
diff --git a/var/spack/repos/builtin/packages/tulip/package.py b/var/spack/repos/builtin/packages/tulip/package.py
index 28dcaca3e9df40..135c6ba3db3faf 100644
--- a/var/spack/repos/builtin/packages/tulip/package.py
+++ b/var/spack/repos/builtin/packages/tulip/package.py
@@ -29,7 +29,7 @@ class Tulip(CMakePackage):
     depends_on("qt")
     depends_on("qhull")
     depends_on("freetype")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("glew")
     depends_on("jpeg")
     depends_on("libpng")
diff --git a/var/spack/repos/builtin/packages/turbine/package.py b/var/spack/repos/builtin/packages/turbine/package.py
index 26e4e64eb844b1..c2e9758c902892 100644
--- a/var/spack/repos/builtin/packages/turbine/package.py
+++ b/var/spack/repos/builtin/packages/turbine/package.py
@@ -13,7 +13,6 @@ class Turbine(AutotoolsPackage):
     homepage = "http://swift-lang.org/Swift-T"
     url = "https://swift-lang.github.io/swift-t-downloads/spack/turbine-1.3.0.tar.gz"
     git = "https://github.com/swift-lang/swift-t.git"
-    configure_directory = "turbine/code"
 
     version("master", branch="master")
     version("1.3.0", sha256="9709e5dada91a7dce958a7967d6ff2bd39ccc9e7da62d05a875324b5089da393")
diff --git a/var/spack/repos/builtin/packages/ucsc-bedclip/package.py b/var/spack/repos/builtin/packages/ucsc-bedclip/package.py
new file mode 100644
index 00000000000000..d22328725edaa7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/ucsc-bedclip/package.py
@@ -0,0 +1,47 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class UcscBedclip(Package):
+    """Remove lines from bed file that refer to off-chromosome locations."""
+
+    homepage = "http://hgdownload.cse.ucsc.edu/admin/exe/"
+    url = "http://hgdownload.cse.ucsc.edu/admin/exe/userApps.archive/userApps.v449.src.tgz"
+    maintainers("pabloaledo")
+
+    version("377", sha256="932f149c19641064a9cd3f2382cbb54b45a9292b8444792872d531346925d676")
+    version("449", sha256="b5a86863d6cfe2120f6c796a13b1572ad05b22622f6534b95c9d26ccbede09b7")
+
+    depends_on("libpng")
+    depends_on("libuuid")
+    depends_on("gmake")
+    depends_on("mysql-connector-c")
+    depends_on("openssl")
+    depends_on("zlib-api")
+
+    def setup_build_environment(self, env):
+        env.set("MYSQLLIBS", "-lmysqlclient")
+        env.set("L", "-lssl")
+        env.set("BINDIR", "bin")
+
+    def install(self, spec, prefix):
+        with working_dir("kent/src/lib"):
+            make()
+        with working_dir("kent/src/htslib"):
+            make()
+        with working_dir("kent/src/jkOwnLib"):
+            make()
+        with working_dir("kent/src/hg/lib"):
+            make()
+        with working_dir("kent/src/hg/lib"):
+            make()
+        with working_dir("kent/src/utils/bedClip"):
+            mkdirp("bin")
+            mkdirp(prefix.bin)
+            make()
+            make("install")
+            install("bin/bedClip", prefix.bin)
diff --git a/var/spack/repos/builtin/packages/ucsc-bedgraphtobigwig/package.py b/var/spack/repos/builtin/packages/ucsc-bedgraphtobigwig/package.py
new file mode 100644
index 00000000000000..8935078445040d
--- /dev/null
+++ b/var/spack/repos/builtin/packages/ucsc-bedgraphtobigwig/package.py
@@ -0,0 +1,51 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class UcscBedgraphtobigwig(Package):
+    """Convert a bedGraph file to bigWig format."""
+
+    homepage = "http://hgdownload.cse.ucsc.edu/admin/exe/"
+    url = "https://hgdownload.cse.ucsc.edu/admin/exe/userApps.archive/userApps.v445.src.tgz"
+    maintainers("pabloaledo")
+
+    version("449", sha256="b5a86863d6cfe2120f6c796a13b1572ad05b22622f6534b95c9d26ccbede09b7")
+    version("445", sha256="c7abb5db6a5e16a79aefcee849d2b59dbc71ee112ca1e41fea0afb25229cf56c")
+
+    depends_on("libpng")
+    depends_on("libuuid")
+    depends_on("gmake")
+    depends_on("openssl")
+    depends_on("zlib-api")
+    # This package has known issues installing with the latest MySQL because
+    # MySQL removed the type my_bool, while mariadb didn't.
+    # https://groups.google.com/a/soe.ucsc.edu/g/genome/c/mIT6fe9l99g
+    depends_on("mysql-client")
+    conflicts("^mysql@8.0.0:")
+
+    def setup_build_environment(self, env):
+        env.set("MYSQLLIBS", "-lmysqlclient")
+        env.set("L", "-lssl")
+        env.set("BINDIR", "bin")
+
+    def install(self, spec, prefix):
+        with working_dir("kent/src/lib"):
+            make()
+        with working_dir("kent/src/htslib"):
+            make()
+        with working_dir("kent/src/jkOwnLib"):
+            make()
+        with working_dir("kent/src/hg/lib"):
+            make()
+        with working_dir("kent/src/hg/lib"):
+            make()
+        with working_dir("kent/src/utils/bedGraphToBigWig"):
+            mkdirp(prefix.bin)
+            mkdirp("bin")
+            make()
+            make("install")
+            install("bin/bedGraphToBigWig", prefix.bin)
diff --git a/var/spack/repos/builtin/packages/ucx/package.py b/var/spack/repos/builtin/packages/ucx/package.py
index b76f0c8dbc7455..4cbbda730fb240 100644
--- a/var/spack/repos/builtin/packages/ucx/package.py
+++ b/var/spack/repos/builtin/packages/ucx/package.py
@@ -87,6 +87,7 @@ class Ucx(AutotoolsPackage, CudaPackage):
     variant("rocm", default=False, description="Enable ROCm support")
     variant(
         "simd",
+        description="SIMD features",
         values=disjoint_sets(("auto",), simd_values)
         .with_default("auto")
         .with_non_feature_values("auto"),
diff --git a/var/spack/repos/builtin/packages/udunits/package.py b/var/spack/repos/builtin/packages/udunits/package.py
index 2b92c71d472c78..75a41c45b85a88 100644
--- a/var/spack/repos/builtin/packages/udunits/package.py
+++ b/var/spack/repos/builtin/packages/udunits/package.py
@@ -12,6 +12,8 @@ class Udunits(AutotoolsPackage):
     homepage = "https://www.unidata.ucar.edu/software/udunits"
     url = "https://artifacts.unidata.ucar.edu/repository/downloads-udunits/2.2.28/udunits-2.2.28.tar.gz"
 
+    maintainers("AlexanderRichert-NOAA")
+
     # Unidata now only provides the latest version of each X.Y branch.
     # Older 2.2 versions have been deprecated accordingly but are still
     # available in the build cache.
@@ -35,10 +37,18 @@ class Udunits(AutotoolsPackage):
     depends_on("expat")
 
     variant("shared", default=True, description="Build shared library")
+    variant(
+        "pic", default=True, description="Enable position-independent code (PIC)", when="~shared"
+    )
 
     @property
     def libs(self):
-        return find_libraries(["libudunits2"], root=self.prefix, recursive=True, shared=True)
+        return find_libraries(
+            "libudunits2", root=self.prefix, recursive=True, shared=self.spec.satisfies("+shared")
+        )
 
     def configure_args(self):
-        return self.enable_or_disable("shared")
+        config_args = []
+        config_args.extend(self.enable_or_disable("shared"))
+        config_args.extend(self.with_or_without("pic"))
+        return config_args
diff --git a/var/spack/repos/builtin/packages/ufs-utils/package.py b/var/spack/repos/builtin/packages/ufs-utils/package.py
index 7ac622fccb54c5..50380bfe5889b2 100644
--- a/var/spack/repos/builtin/packages/ufs-utils/package.py
+++ b/var/spack/repos/builtin/packages/ufs-utils/package.py
@@ -18,10 +18,36 @@ class UfsUtils(CMakePackage):
 
     maintainers("t-brown", "edwardhartnett", "AlexanderRichert-NOAA", "Hang-Lei-NOAA")
 
-    version("1.10.0", tag="ufs_utils_1_10_0", submodules=True)
-    version("1.9.0", tag="ufs_utils_1_9_0", submodules=True)
-    version("1.8.0", tag="ufs_utils_1_8_0", submodules=True)
-    version("1.7.0", tag="ufs_utils_1_7_0", submodules=True)
+    version(
+        "1.11.0",
+        tag="ufs_utils_1_11_0",
+        commit="72701ab45165ae67a1c4b4d855e763bf5674dbd2",
+        submodules=True,
+    )
+    version(
+        "1.10.0",
+        tag="ufs_utils_1_10_0",
+        commit="d1e928bca221361a62d747964826bf80775db6af",
+        submodules=True,
+    )
+    version(
+        "1.9.0",
+        tag="ufs_utils_1_9_0",
+        commit="7b1f169b54c6697f1a1b105dae217b4da5fab199",
+        submodules=True,
+    )
+    version(
+        "1.8.0",
+        tag="ufs_utils_1_8_0",
+        commit="735e2bad1f11cb9c5924bda82150494548a97164",
+        submodules=True,
+    )
+    version(
+        "1.7.0",
+        tag="ufs_utils_1_7_0",
+        commit="1730d3718603ae83a2c77cb335464507d6dd7f59",
+        submodules=True,
+    )
 
     depends_on("mpi")
     depends_on("cmake@3.23:")
@@ -39,7 +65,7 @@ class UfsUtils(CMakePackage):
     depends_on("sigio")
     depends_on("sp")
     depends_on("w3emc")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def cmake_args(self):
         return [
diff --git a/var/spack/repos/builtin/packages/ufs-weather-model/package.py b/var/spack/repos/builtin/packages/ufs-weather-model/package.py
index 5ceb1f1b8d3030..dcc1e47e901a14 100644
--- a/var/spack/repos/builtin/packages/ufs-weather-model/package.py
+++ b/var/spack/repos/builtin/packages/ufs-weather-model/package.py
@@ -22,8 +22,18 @@ class UfsWeatherModel(CMakePackage):
     maintainers("t-brown", "AlexanderRichert-NOAA")
 
     version("develop", branch="develop", submodules=True, commit="ea0b6e4")
-    version("2.0.0", tag="ufs-v2.0.0", submodules=True)
-    version("1.1.0", tag="ufs-v1.1.0", submodules=True)
+    version(
+        "2.0.0",
+        tag="ufs-v2.0.0",
+        commit="e3cb92f1cd8941c019ee5ef7da5c9aef67d55cf8",
+        submodules=True,
+    )
+    version(
+        "1.1.0",
+        tag="ufs-v1.1.0",
+        commit="5bea16b6d41d810dc2e45cba0fa3841f45ea7c7a",
+        submodules=True,
+    )
 
     variant("mpi", default=True, description="Enable MPI")
     variant(
@@ -98,7 +108,11 @@ class UfsWeatherModel(CMakePackage):
     )
     variant("mom6solo", default=False, description="Build MOM6 solo executable", when="@develop")
 
-    variant("cmake_platform", default="auto", description="Override value for CMAKE_Platform env variable ('linux.intel', 'hera.gnu', 'acorn', etc.)")
+    variant(
+        "cmake_platform",
+        default="auto",
+        description="Override CMAKE_Platform env variable ('linux.intel', 'hera.gnu', etc.)",
+    )
 
     variant("app", default="ATM", description="UFS application", when="@develop")
 
diff --git a/var/spack/repos/builtin/packages/umpire/package.py b/var/spack/repos/builtin/packages/umpire/package.py
index a69bd3c85d54cf..c64bfdf256db78 100644
--- a/var/spack/repos/builtin/packages/umpire/package.py
+++ b/var/spack/repos/builtin/packages/umpire/package.py
@@ -24,35 +24,105 @@ class Umpire(CachedCMakePackage, CudaPackage, ROCmPackage):
 
     version("develop", branch="develop", submodules=False)
     version("main", branch="main", submodules=False)
-    version("2022.03.1", tag="v2022.03.1", submodules=False)
-    version("2022.03.0", tag="v2022.03.0", submodules=False)
-    version("6.0.0", tag="v6.0.0", submodules=True)
-    version("5.0.1", tag="v5.0.1", submodules=True)
-    version("5.0.0", tag="v5.0.0", submodules=True)
-    version("4.1.2", tag="v4.1.2", submodules=True)
-    version("4.1.1", tag="v4.1.1", submodules=True)
-    version("4.1.0", tag="v4.1.0", submodules=True)
-    version("4.0.1", tag="v4.0.1", submodules=True)
-    version("4.0.0", tag="v4.0.0", submodules=True)
-    version("3.0.0", tag="v3.0.0", submodules=True)
-    version("2.1.0", tag="v2.1.0", submodules=True)
-    version("2.0.0", tag="v2.0.0", submodules=True)
-    version("1.1.0", tag="v1.1.0", submodules=True)
-    version("1.0.1", tag="v1.0.1", submodules=True)
-    version("1.0.0", tag="v1.0.0", submodules=True)
-    version("0.3.5", tag="v0.3.5", submodules=True)
-    version("0.3.4", tag="v0.3.4", submodules=True)
-    version("0.3.3", tag="v0.3.3", submodules=True)
-    version("0.3.2", tag="v0.3.2", submodules=True)
-    version("0.3.1", tag="v0.3.1", submodules=True)
-    version("0.3.0", tag="v0.3.0", submodules=True)
-    version("0.2.4", tag="v0.2.4", submodules=True)
-    version("0.2.3", tag="v0.2.3", submodules=True)
-    version("0.2.2", tag="v0.2.2", submodules=True)
-    version("0.2.1", tag="v0.2.1", submodules=True)
-    version("0.2.0", tag="v0.2.0", submodules=True)
-    version("0.1.4", tag="v0.1.4", submodules=True)
-    version("0.1.3", tag="v0.1.3", submodules=True)
+    version(
+        "2022.10.0",
+        tag="v2022.10.0",
+        commit="93b1441aaa258c1dcd211a552b75cff6461a2a8a",
+        submodules=False,
+    )
+    version(
+        "2022.03.1",
+        tag="v2022.03.1",
+        commit="6bf231bdbbc797df70d60027ddb714ac2ef7c0a1",
+        submodules=False,
+    )
+    version(
+        "2022.03.0",
+        tag="v2022.03.0",
+        commit="2db6224ae0c3f3e0bbd6722e95c1167b7f79be7b",
+        submodules=False,
+    )
+    version(
+        "6.0.0", tag="v6.0.0", commit="5f886b4299496b7cb6f9d62dc1372ce6d3832fbc", submodules=True
+    )
+    version(
+        "5.0.1", tag="v5.0.1", commit="5201a47a35e3844160dcbecd0916f8c96aa7dd07", submodules=True
+    )
+    version(
+        "5.0.0", tag="v5.0.0", commit="2196615500057e068f2d93597b4f8da89d582afb", submodules=True
+    )
+    version(
+        "4.1.2", tag="v4.1.2", commit="447f4640eff7b8f39d3c59404f3b03629b90c021", submodules=True
+    )
+    version(
+        "4.1.1", tag="v4.1.1", commit="df1830b5ea04185f93fc229ed667da62d1d2d6e3", submodules=True
+    )
+    version(
+        "4.1.0", tag="v4.1.0", commit="62f146d9c6b291cd79b29386dcb84b30f7b4212e", submodules=True
+    )
+    version(
+        "4.0.1", tag="v4.0.1", commit="06d8692d084a88b15b0ef2794a1da779197da747", submodules=True
+    )
+    version(
+        "4.0.0", tag="v4.0.0", commit="bdd598512516bdc4238502f180c8a7e145c6e68f", submodules=True
+    )
+    version(
+        "3.0.0", tag="v3.0.0", commit="657676087574f61f9d90b996a3bdbf4e1cdfc92e", submodules=True
+    )
+    version(
+        "2.1.0", tag="v2.1.0", commit="52e10c05cd40dfdfde186c1e63213695f5aeaf65", submodules=True
+    )
+    version(
+        "2.0.0", tag="v2.0.0", commit="0dc8b4736357645b99632ee7c17a3dc6af771fbb", submodules=True
+    )
+    version(
+        "1.1.0", tag="v1.1.0", commit="3db26e6a2626ee8c0cfa5c9769cfac6e33587122", submodules=True
+    )
+    version(
+        "1.0.1", tag="v1.0.1", commit="a6741073431cab3a7a2434f9119a54d18e9978f4", submodules=True
+    )
+    version(
+        "1.0.0", tag="v1.0.0", commit="82482fd7450ab378db110f06f7e0302112c22c05", submodules=True
+    )
+    version(
+        "0.3.5", tag="v0.3.5", commit="a283977bb548cbaa0221bdb6c9832f7834f69e74", submodules=True
+    )
+    version(
+        "0.3.4", tag="v0.3.4", commit="20a77408d8ae467af21d5802d14afe54f1253694", submodules=True
+    )
+    version(
+        "0.3.3", tag="v0.3.3", commit="715a8bd003eb1d9db1e2ac7ff2c6251cfd445c27", submodules=True
+    )
+    version(
+        "0.3.2", tag="v0.3.2", commit="06f37f2011fa4d9482f15e04fc206e2e7b7aa9e2", submodules=True
+    )
+    version(
+        "0.3.1", tag="v0.3.1", commit="aef223065fdfe85d1e46bab95e3874821702891a", submodules=True
+    )
+    version(
+        "0.3.0", tag="v0.3.0", commit="529004f9e88fbb49ee93a97465ff904be249039c", submodules=True
+    )
+    version(
+        "0.2.4", tag="v0.2.4", commit="f774afae69b6f2e5c99ea8bf5660ccf68bd5436d", submodules=True
+    )
+    version(
+        "0.2.3", tag="v0.2.3", commit="af158291f574701aabb6a2b16e6536aefaf4496e", submodules=True
+    )
+    version(
+        "0.2.2", tag="v0.2.2", commit="68f4b86fd877c9ca00c9438c603e5dbc40d5f219", submodules=True
+    )
+    version(
+        "0.2.1", tag="v0.2.1", commit="c22df368e2f52398351f49fbe2522bd1150ad171", submodules=True
+    )
+    version(
+        "0.2.0", tag="v0.2.0", commit="7910b8d4dbfe83faacf65e864304ca916e34b86c", submodules=True
+    )
+    version(
+        "0.1.4", tag="v0.1.4", commit="c2848289ba9d8c85346610d25af9531b82c50fc3", submodules=True
+    )
+    version(
+        "0.1.3", tag="v0.1.3", commit="cc347edeb17f5f30f694aa47f395d17369a2e449", submodules=True
+    )
 
     patch("std-filesystem-pr784.patch", when="@2022.03.1 +rocm ^blt@0.5.2:")
     patch("camp_target_umpire_3.0.0.patch", when="@3.0.0")
@@ -66,6 +136,20 @@ class Umpire(CachedCMakePackage, CudaPackage, ROCmPackage):
         when="@:5.0.1 ^blt@0.4:",
     )
 
+    # https://github.com/LLNL/Umpire/pull/805
+    patch(
+        "https://github.com/LLNL/Umpire/pull/805/commits/47ff0aa1f7a01a917c3b7ac618e8a9e44a10fd25.patch?full_index=1",
+        sha256="7ed5d2c315a3b31e339f664f6108e32d7cb4cb8e9f22e5c78a65ba02625ccc09",
+        when="@2022.10.0",
+    )
+
+    # https://github.com/LLNL/Umpire/pull/816
+    patch(
+        "https://github.com/LLNL/Umpire/pull/816/commits/2292d1d6078f6d9523b7ad0886ffa053644569d5.patch?full_index=1",
+        sha256="0f43cad7cdaec3c225ab6414ab9f81bd405a1157abf5a508e515bcb6ca53326d",
+        when="@2022.10.0",
+    )
+
     variant("fortran", default=False, description="Build C/Fortran API")
     variant("c", default=True, description="Build C API")
     variant("numa", default=False, description="Enable NUMA support")
@@ -84,12 +168,14 @@ class Umpire(CachedCMakePackage, CudaPackage, ROCmPackage):
 
     depends_on("cmake@3.8:", type="build")
     depends_on("cmake@3.9:", when="+cuda", type="build")
-    depends_on("cmake@3.14:", when="@2022.03.0:")
+    depends_on("cmake@3.14:", when="@2022.03.0:", type="build")
 
+    depends_on("blt@0.5.2:", type="build", when="@2022.10.0:")
     depends_on("blt@0.5.0:", type="build", when="@2022.03.0:")
     depends_on("blt@0.4.1", type="build", when="@6.0.0")
     depends_on("blt@0.4.0:", type="build", when="@4.1.3:5.0.1")
     depends_on("blt@0.3.6:", type="build", when="@:4.1.2")
+    conflicts("^blt@:0.3.6", when="+rocm")
 
     depends_on("camp", when="@5.0.0:")
     depends_on("camp@0.2.2:0.2.3", when="@6.0.0")
diff --git a/var/spack/repos/builtin/packages/unblur/package.py b/var/spack/repos/builtin/packages/unblur/package.py
index 00c57e291e2bbb..040f45e2451c72 100644
--- a/var/spack/repos/builtin/packages/unblur/package.py
+++ b/var/spack/repos/builtin/packages/unblur/package.py
@@ -18,7 +18,7 @@ class Unblur(AutotoolsPackage):
     variant("openmp", default=True, description="Enable OpenMP support")
     variant("shared", default=True, description="Dynamic linking")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("jpeg")
     depends_on("libtiff")
     depends_on("gsl")
diff --git a/var/spack/repos/builtin/packages/uncrustify/package.py b/var/spack/repos/builtin/packages/uncrustify/package.py
index 50331ad508c734..b6bd6ad244e2cb 100644
--- a/var/spack/repos/builtin/packages/uncrustify/package.py
+++ b/var/spack/repos/builtin/packages/uncrustify/package.py
@@ -16,19 +16,19 @@ class Uncrustify(CMakePackage, AutotoolsPackage):
     maintainers("gmaurel")
 
     version("master", branch="master")
-    version("0.74", commit="62048b")
-    version("0.73", commit="25b765")
-    version("0.72", commit="1d3d8f")
-    version("0.71", commit="64d82f")
-    version("0.70", commit="51f64d")
-    version("0.69", commit="a7a8fb")
-    version("0.68", commit="86bc34")
-    version("0.67", commit="00321a")
-    version("0.66", commit="80f549")
-    version("0.65", commit="905676")
-    version("0.64", commit="1d7d97")
-    version("0.63", commit="44ce0f")
-    version("0.62", commit="5987f2")
+    version("0.74", commit="62048b01507304653ea98a74b31e0edbadaf7496")
+    version("0.73", commit="25b765b4ccf1fc50302df3779188ccd402962ee0")
+    version("0.72", commit="1d3d8fa5e81bece0fac4b81316b0844f7cc35926")
+    version("0.71", commit="64d82fd66f9eeeba14a591aa0939d495eccd1bc6")
+    version("0.70", commit="51f64d6e62f5ea84f2c428c0478d01b1fbf6948c")
+    version("0.69", commit="a7a8fb35d653e0b49e1c86f2eb8a2970025d5989")
+    version("0.68", commit="86bc346e01c16c96e4baff8132e024ca13772ce9")
+    version("0.67", commit="00321aa37802ae9ae78459957498a3c933b8254f")
+    version("0.66", commit="80f549b6f026d0b4cf14eae3a1ba8a7389642e45")
+    version("0.65", commit="9056763eb1c8c3837fd718eba03facdd4d8c179d")
+    version("0.64", commit="1d7d97fb637dcb05ebc5fe57ee1020e2a659210d")
+    version("0.63", commit="44ce0f156396b79ddf3ed9242023a14e9665b76f")
+    version("0.62", commit="5987f2223f16b993dbece1360363eef9515fe5e8")
     version("0.61", sha256="1df0e5a2716e256f0a4993db12f23d10195b3030326fdf2e07f8e6421e172df9")
 
     build_system(
diff --git a/var/spack/repos/builtin/packages/unifyfs/package.py b/var/spack/repos/builtin/packages/unifyfs/package.py
index 4263d7aceba3bb..9cc1b3db64af90 100644
--- a/var/spack/repos/builtin/packages/unifyfs/package.py
+++ b/var/spack/repos/builtin/packages/unifyfs/package.py
@@ -38,24 +38,24 @@ class Unifyfs(AutotoolsPackage):
 
     variant(
         "auto-mount",
-        default="True",
+        default=True,
         description="Enable automatic mount/unmount in MPI_Init/Finalize",
     )
     variant(
         "boostsys",
-        default="False",
+        default=False,
         description="Have Mercury use preprocessor headers from boost dependency",
     )
-    variant("fortran", default="True", description="Build with gfortran support")
-    variant("pmi", default="False", description="Enable PMI2 build options")
-    variant("pmix", default="False", description="Enable PMIx build options")
+    variant("fortran", default=True, description="Build with gfortran support")
+    variant("pmi", default=False, description="Enable PMI2 build options")
+    variant("pmix", default=False, description="Enable PMIx build options")
     variant(
         "preload",
-        default="False",
+        default=False,
         when="@1.0.1:",
         description="Enable support for LD_PRELOAD library",
     )
-    variant("spath", default="True", description="Use spath library to normalize relative paths")
+    variant("spath", default=True, description="Use spath library to normalize relative paths")
 
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
@@ -73,7 +73,10 @@ class Unifyfs(AutotoolsPackage):
     # after v0.13.1.
     depends_on("mochi-margo@0.10:", when="@1.1:")
     depends_on("mpi")
-    depends_on("openssl@:1")
+
+    # unifyfs@:1.1 uses MD5 functions that are deprecated in OpenSSL 3, and
+    # likely to be removed in the next major release.
+    depends_on("openssl@:3")
 
     # Mochi-Margo dependencies
     depends_on("mercury@1.0.1+bmi", when="@:0.9.1")
@@ -94,9 +97,6 @@ class Unifyfs(AutotoolsPackage):
     patch("unifyfs-sysio.c.patch", when="@0.9.1")
     patch("include-sys-sysmacros.h.patch", when="@0.9.1:0.9.2")
 
-    # Parallel disabled to prevent tests from being run out-of-order when
-    # installed with the --test={root, all} option.
-    parallel = False
     debug_build = False
     build_directory = "spack-build"
 
@@ -124,6 +124,15 @@ def setup_build_environment(self, env):
         if self.spec.satisfies("%oneapi"):
             env.append_flags("CFLAGS", "-Wno-unused-function")
 
+    @when("%cce@11.0.3:")
+    def patch(self):
+        filter_file("-Werror", "", "client/src/Makefile.in")
+        filter_file("-Werror", "", "client/src/Makefile.am")
+
+    @when("@develop")
+    def autoreconf(self, spec, prefix):
+        Executable("./autogen.sh")()
+
     def configure_args(self):
         spec = self.spec
         args = ["--with-gotcha=%s" % spec["gotcha"].prefix]
@@ -142,12 +151,6 @@ def configure_args(self):
 
         return args
 
-    @when("@develop")
-    def autoreconf(self, spec, prefix):
-        sh = which("sh")
-        sh("./autogen.sh")
-
-    @when("%cce@11.0.3:")
-    def patch(self):
-        filter_file("-Werror", "", "client/src/Makefile.in")
-        filter_file("-Werror", "", "client/src/Makefile.am")
+    def check(self):
+        with working_dir(self.build_directory):
+            make("check", parallel=False)
diff --git a/var/spack/repos/builtin/packages/unzip/configure-cflags.patch b/var/spack/repos/builtin/packages/unzip/configure-cflags.patch
new file mode 100644
index 00000000000000..f73b39f3228233
--- /dev/null
+++ b/var/spack/repos/builtin/packages/unzip/configure-cflags.patch
@@ -0,0 +1,37 @@
+--- ./unix/configure.old	2009-04-16 19:25:12.000000000 +0000
++++ ./unix/configure	2023-07-14 09:35:33.735000149 +0000
+@@ -383,7 +383,7 @@
+ do
+   echo Check for $func
+   echo "int main(){ $func(); return 0; }" > conftest.c
+-  $CC $BFLAG -o conftest conftest.c >/dev/null 2>/dev/null
++  $CC $BFLAG $CFLAGS -o conftest conftest.c >/dev/null 2>/dev/null
+   [ $? -ne 0 ] && CFLAGSR="${CFLAGSR} -DNO_`echo $func | tr '[a-z]' '[A-Z]'`"
+ done
+
+@@ -395,14 +395,14 @@
+   echo "int main() { lchmod(\"${temp_file}\", 0666); }" \
+ ) > conftest.c
+ ln -s "${temp_link}" "${temp_file}" && \
+- $CC $BFLAG -o conftest conftest.c >/dev/null 2>/dev/null && \
++ $CC $CFLAGS $BFLAG -o conftest conftest.c >/dev/null 2>/dev/null && \
+  ./conftest
+ [ $? -ne 0 ] && CFLAGSR="${CFLAGSR} -DNO_LCHMOD"
+ rm -f "${temp_file}"
+
+ echo Check for memset
+ echo "int main(){ char k; memset(&k,0,0); return 0; }" > conftest.c
+-$CC -o conftest conftest.c >/dev/null 2>/dev/null
++$CC $CFLAGS -o conftest conftest.c >/dev/null 2>/dev/null
+ [ $? -ne 0 ] && CFLAGSR="${CFLAGSR} -DZMEM"
+
+ echo Check for errno declaration
+@@ -422,7 +422,7 @@
+ int main() { return closedir(opendir(".")); }
+ _EOF_
+
+-$CC -o conftest conftest.c >/dev/null 2>/dev/null
++$CC $CFLAGS -o conftest conftest.c >/dev/null 2>/dev/null
+ if [ $? -ne 0 ]; then
+   OPT=""
+   for lib in ndir dir ucb bsd BSD PW x dirent
diff --git a/var/spack/repos/builtin/packages/unzip/package.py b/var/spack/repos/builtin/packages/unzip/package.py
index 3d77899b2fb9b5..f9399cbaa6fd5b 100644
--- a/var/spack/repos/builtin/packages/unzip/package.py
+++ b/var/spack/repos/builtin/packages/unzip/package.py
@@ -14,21 +14,33 @@ class Unzip(MakefilePackage):
 
     version("6.0", sha256="036d96991646d0449ed0aa952e4fbe21b476ce994abc276e49d30e686708bd37")
 
+    patch("configure-cflags.patch", when="%clang@16:")
+
     # The Cray cc wrapper doesn't handle the '-s' flag (strip) cleanly.
     @when("platform=cray")
     def patch(self):
         filter_file(r"^LFLAGS2=.*", "LFLAGS2=", join_path("unix", "configure"))
 
-    make_args = ["-f", join_path("unix", "Makefile"), "LOC=-DLARGE_FILE_SUPPORT"]
+    def get_make_args(self):
+        make_args = ["-f", join_path("unix", "Makefile")]
+
+        cflags = []
+        if self.spec.satisfies("%clang@16:"):
+            cflags.append("-Wno-error=implicit-function-declaration")
+            cflags.append("-Wno-error=implicit-int")
+        cflags.append("-DLARGE_FILE_SUPPORT")
+
+        make_args.append('LOC="{}"'.format(" ".join(cflags)))
+        return make_args
 
     @property
     def build_targets(self):
         target = "macosx" if "platform=darwin" in self.spec else "generic"
-        return self.make_args + [target]
+        return self.get_make_args() + [target]
 
     def url_for_version(self, version):
         return "http://downloads.sourceforge.net/infozip/unzip{0}.tar.gz".format(version.joined)
 
     @property
     def install_targets(self):
-        return self.make_args + ["prefix={0}".format(self.prefix), "install"]
+        return self.get_make_args() + ["prefix={0}".format(self.prefix), "install"]
diff --git a/var/spack/repos/builtin/packages/upcxx/package.py b/var/spack/repos/builtin/packages/upcxx/package.py
index 96f9426082a5bb..5d8ba9e018b888 100644
--- a/var/spack/repos/builtin/packages/upcxx/package.py
+++ b/var/spack/repos/builtin/packages/upcxx/package.py
@@ -116,7 +116,7 @@ class Upcxx(Package, CudaPackage, ROCmPackage):
     depends_on("mpi", when="+mpi")
     depends_on("python@2.7.5:", type=("build", "run"))
 
-    conflicts("hip@:4.4.0", when="+rocm")
+    conflicts("^hip@:4.4.0", when="+rocm")
 
     depends_on("oneapi-level-zero@1.8.0:", when="+level_zero")
 
diff --git a/var/spack/repos/builtin/packages/upp/package.py b/var/spack/repos/builtin/packages/upp/package.py
index 5a9e33e4542e02..8bdb1187921da4 100644
--- a/var/spack/repos/builtin/packages/upp/package.py
+++ b/var/spack/repos/builtin/packages/upp/package.py
@@ -19,19 +19,36 @@ class Upp(CMakePackage):
 
     maintainers("AlexanderRichert-NOAA", "edwardhartnett", "Hang-Lei-NOAA")
 
+    version("develop", branch="develop")
+    version(
+        "11.0.0",
+        tag="upp_v11.0.0",
+        commit="6b5c589c7650132c6f13a729a2853676a7b93bbb",
+        submodules=True,
+    )
     version("10.0.10", sha256="0c96a88d0e79b554d5fcee9401efcf4d6273da01d15e3413845274f73d70b66e")
-    version("10.0.9", tag="upp_v10.0.9", submodules=True)
-    version("10.0.8", tag="upp_v10.0.8", submodules=True)
+    version(
+        "10.0.9",
+        tag="upp_v10.0.9",
+        commit="a49af0549958def4744cb3903c7315476fe44530",
+        submodules=True,
+    )
+    version(
+        "10.0.8",
+        tag="upp_v10.0.8",
+        commit="ce989911a7a09a2e2a0e61b3acc87588b5b9fc26",
+        submodules=True,
+    )
     version("8.2.0", sha256="38de2178dc79420f42aa3fb8b85796fc49d43d66f90e5276e47ab50c282627ac")
 
-    variant("openmp", default=True)
-    variant("postexec", default=True)
-    variant("wrf-io", default=False)
-    variant("docs", default=False)
+    variant("openmp", default=True, description="Use OpenMP threading")
+    variant("postexec", default=True, description="Build NCEPpost executable")
+    variant("wrf-io", default=False, description="Build with WRF-IO library")
+    variant("docs", default=False, description="Build Doxygen documentation")
 
     depends_on("mpi")
     depends_on("netcdf-fortran")
-    depends_on("bacio")
+    depends_on("bacio@2.4.1")
     depends_on("crtm")
     depends_on("g2")
     depends_on("g2tmpl")
diff --git a/var/spack/repos/builtin/packages/usalign/package.py b/var/spack/repos/builtin/packages/usalign/package.py
index 47f08e1f8a1a91..70d79b8b23f810 100644
--- a/var/spack/repos/builtin/packages/usalign/package.py
+++ b/var/spack/repos/builtin/packages/usalign/package.py
@@ -26,11 +26,11 @@ class Usalign(Package):
     # date assumed from paper publication date
     version(
         "20220829",
-        sha256="9ee129017a68125c22ce89123ecbac9421add87ee077cd1994c6e8a39a8a8b21",
+        sha256="fdc447a0c48813f91cf3045b9c19bd0c6b98fc6a3ad23780ef75fac8d4662e8c",
         expand=False,
     )
 
-    variant("fast-math", default=False)
+    variant("fast-math", default=False, description="Enable fast math")
 
     phases = ["build", "install"]
 
diff --git a/var/spack/repos/builtin/packages/util-linux/package.py b/var/spack/repos/builtin/packages/util-linux/package.py
index 6c5c950cfb48c9..b0c04e0f70efc8 100644
--- a/var/spack/repos/builtin/packages/util-linux/package.py
+++ b/var/spack/repos/builtin/packages/util-linux/package.py
@@ -37,7 +37,7 @@ class UtilLinux(AutotoolsPackage):
     depends_on("python@2.7:", type="build")
     depends_on("pkgconfig", type="build")
     depends_on("ncurses", type="link")
-    depends_on("zlib", type="link")  # sbin/mkfs.cramfs
+    depends_on("zlib-api", type="link")  # sbin/mkfs.cramfs
     depends_on("libxcrypt", type="link")  # sbin/sulogin
 
     variant("bash", default=False, description="Install bash completion scripts")
diff --git a/var/spack/repos/builtin/packages/vacuumms/package.py b/var/spack/repos/builtin/packages/vacuumms/package.py
index c4e798a30a8c6d..d3a7736738bc7a 100644
--- a/var/spack/repos/builtin/packages/vacuumms/package.py
+++ b/var/spack/repos/builtin/packages/vacuumms/package.py
@@ -23,7 +23,7 @@ class Vacuumms(CMakePackage):
     maintainers("frankwillmore")
 
     version("master", branch="master")
-    version("1.1.1", tag="v1.1.1")
+    version("1.1.1", tag="v1.1.1", commit="a395930c87f5b52ab0ab8db278bd2bde592e8f12")
     version(
         "1.0.0",
         sha256="c18fe52f5041880da7f50d3808d37afb3e9c936a56f80f67838d045bf7af372f",
diff --git a/var/spack/repos/builtin/packages/vampirtrace/package.py b/var/spack/repos/builtin/packages/vampirtrace/package.py
index 37973070d8822c..2d4f7587fd3655 100644
--- a/var/spack/repos/builtin/packages/vampirtrace/package.py
+++ b/var/spack/repos/builtin/packages/vampirtrace/package.py
@@ -21,7 +21,7 @@ class Vampirtrace(AutotoolsPackage):
     depends_on("mpi", when="+mpi")
     depends_on("otf")
     depends_on("papi")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # VampirTrace fails to build with newer versions of MPICH due to
     # https://github.com/pmodels/mpich/commit/c3dbc09ae20a503ac4b870893e3e330d52ea5a3b
@@ -45,7 +45,7 @@ def configure_args(self):
             "--with-extern-otf",
             "--with-extern-otf-dir={0}".format(spec["otf"].prefix),
             "--with-papi-dir={0}".format(spec["papi"].prefix),
-            "--with-zlib-dir={0}".format(spec["zlib"].prefix),
+            "--with-zlib-dir={0}".format(spec["zlib-api"].prefix),
             "--with-wrapper-cc-compiler={0}".format(compiler.cc),
             "--with-wrapper-cc-cpp={0} -E".format(compiler.cc),
             "--with-wrapper-cxx-compiler={0}".format(compiler.cxx),
diff --git a/var/spack/repos/builtin/packages/variorum/package.py b/var/spack/repos/builtin/packages/variorum/package.py
index d9512e362c5aa8..9d4a385d0ed118 100644
--- a/var/spack/repos/builtin/packages/variorum/package.py
+++ b/var/spack/repos/builtin/packages/variorum/package.py
@@ -17,13 +17,14 @@ class Variorum(CMakePackage):
 
     maintainers("slabasan", "rountree")
 
+    version("0.7.0", sha256="36ec0219379ea2b7c8f9770b3271335c776ff5a3de71585714c33356345b2f0c")
     version("0.6.0", sha256="c0928a0e6901808ee50142d1034de15edc2c90d7d1b9fbce43757226e7c04306")
     version("0.5.0", sha256="de331762e7945ee882d08454ff9c66436e2b6f87f761d2b31c6ab3028723bfed")
     version("0.4.1", sha256="be7407b856bc2239ecaa27d3df80aee2f541bb721fbfa183612bd9c0ce061f28")
     version("0.4.0", sha256="70ff1c5a3ae15d0bd07d409ab6f3c128e69528703a829cb18ecb4a50adeaea34")
     version("0.3.0", sha256="f79563f09b8fe796283c879b05f7730c36d79ca0346c12995b7bccc823653f42")
     version("0.2.0", sha256="b8c010b26aad8acc75d146c4461532cf5d9d3d24d6fc30ee68f6330a68e65744")
-    version("0.1.0", tag="v0.1.0")
+    version("0.1.0", tag="v0.1.0", commit="7747ee48cc60567bb3f09e732f24c041ecac894d")
 
     ############
     # Variants #
diff --git a/var/spack/repos/builtin/packages/vcftools/package.py b/var/spack/repos/builtin/packages/vcftools/package.py
index ce0b38bc9aac32..e592728113f0cb 100644
--- a/var/spack/repos/builtin/packages/vcftools/package.py
+++ b/var/spack/repos/builtin/packages/vcftools/package.py
@@ -16,17 +16,26 @@ class Vcftools(AutotoolsPackage):
     homepage = "https://vcftools.github.io/"
     url = "https://github.com/vcftools/vcftools/releases/download/v0.1.14/vcftools-0.1.14.tar.gz"
 
+    version("0.1.16", sha256="dbfc774383c106b85043daa2c42568816aa6a7b4e6abc965eeea6c47dde914e3")
     # this is "a pre-release"
     # version('0.1.15', sha256='31e47afd5be679d89ece811a227525925b6907cce4af2c86f10f465e080383e3')
     version("0.1.14", sha256="76d799dd9afcb12f1ed42a07bc2886cd1a989858a4d047f24d91dcf40f608582")
 
     depends_on("perl", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # this needs to be in sync with what setup_run_environment adds to
     # PERL5LIB below
     def configure_args(self):
-        return ["--with-pmdir={0}".format(self.prefix.lib)]
+        args = []
+        # between 0.1.16 and 14 the behavior of the configure script
+        # wrt the perl lib dir changed and it became relative to the
+        # install directory, if you specify the whole prefix in
+        # it now you end up with a nasty recreation of the
+        # prefix tree in self.prefix.
+        if self.spec.satisfies("@:0.1.14"):
+            args.append(f"--with-pmdir={self.prefix.lib}")
+        return args
 
     @run_before("install")
     def filter_sbang(self):
diff --git a/var/spack/repos/builtin/packages/veccore/package.py b/var/spack/repos/builtin/packages/veccore/package.py
index afcf1cba0877b7..41a8116b348856 100644
--- a/var/spack/repos/builtin/packages/veccore/package.py
+++ b/var/spack/repos/builtin/packages/veccore/package.py
@@ -31,3 +31,11 @@ class Veccore(CMakePackage):
     version("0.4.1", sha256="59ffe668c061acde89afb33749f4eb8bab35dd5f6e51f632758794c1a745aabf")
     version("0.4.0", sha256="0a38b958c92647c30b5709d17edaf39d241b92b988f1040c0fbe24932b42927e")
     version("0.3.2", sha256="d72b03df00f5e94b2d07f78ab3af6d9d956c19e9a1fae07267b48f6fc8d7713f")
+
+    variant("vc", default=False, description="Enable Vc backend")
+
+    depends_on("vc@1.2.0:", when="@0.2.0: +vc")
+    depends_on("vc@1.3.3:", when="@0.6.0: +vc")
+
+    def cmake_args(self):
+        return [self.define_from_variant("VC", "vc")]
diff --git a/var/spack/repos/builtin/packages/vecgeom/package.py b/var/spack/repos/builtin/packages/vecgeom/package.py
index a5cfd426b9b38f..7a403bdd9ee93b 100644
--- a/var/spack/repos/builtin/packages/vecgeom/package.py
+++ b/var/spack/repos/builtin/packages/vecgeom/package.py
@@ -5,6 +5,7 @@
 
 
 from spack.package import *
+from spack.variant import _ConditionalVariantValues
 
 
 class Vecgeom(CMakePackage, CudaPackage):
@@ -20,7 +21,17 @@ class Vecgeom(CMakePackage, CudaPackage):
     maintainers("drbenmorgan", "sethrj")
 
     version("master", branch="master")
-    version("1.2.3", sha256="703e52d78b5b78e9f595bc76771659ab0cb09898ea32c50cfbde07d6d09ef1e1")
+    version("1.2.5", sha256="af76f0aac34ec3748120969b0fca0f899d91b25cb5727f2c022a6e8304e91327")
+    version(
+        "1.2.4",
+        sha256="4f5d43a9cd34a5e0200c41547a438cbb1ed4439f5bb757857c5a225d708590ce",
+        deprecated=True,
+    )
+    version(
+        "1.2.3",
+        sha256="703e52d78b5b78e9f595bc76771659ab0cb09898ea32c50cfbde07d6d09ef1e1",
+        deprecated=True,
+    )
     version(
         "1.2.2",
         sha256="887134d40fc9731138189299f0bd5e73485fbb95a96eb4124ce0854e4672291f",
@@ -128,7 +139,7 @@ class Vecgeom(CMakePackage, CudaPackage):
         deprecated=True,
     )
 
-    _cxxstd_values = ("11", "14", "17")
+    _cxxstd_values = (conditional("11", "14", when="@:1.1"), "17", conditional("20", when="@1.2:"))
     variant(
         "cxxstd",
         default="17",
@@ -148,8 +159,6 @@ class Vecgeom(CMakePackage, CudaPackage):
     depends_on("veccore@0.4.2", when="@:1.0")
 
     conflicts("+cuda", when="@:1.1.5")
-    conflicts("cxxstd=14", when="@1.2:")
-    conflicts("cxxstd=11", when="@1.2:")
 
     # Fix missing CMAKE_CUDA_STANDARD
     patch(
@@ -164,10 +173,18 @@ class Vecgeom(CMakePackage, CudaPackage):
         when="@1.1.18 +cuda ^cuda@:11.4",
     )
 
-    for std in _cxxstd_values:
-        depends_on("geant4 cxxstd=" + std, when="+geant4 cxxstd=" + std)
-        depends_on("root cxxstd=" + std, when="+root cxxstd=" + std)
-        depends_on("xerces-c cxxstd=" + std, when="+gdml cxxstd=" + std)
+    def std_when(values):
+        for v in values:
+            if isinstance(v, _ConditionalVariantValues):
+                for c in v:
+                    yield (c.value, c.when)
+            else:
+                yield (v, "")
+
+    for _std, _when in std_when(_cxxstd_values):
+        depends_on(f"geant4 cxxstd={_std}", when=f"{_when} +geant4 cxxstd={_std}")
+        depends_on(f"root cxxstd={_std}", when=f"{_when} +root cxxstd={_std}")
+        depends_on(f"xerces-c cxxstd={_std}", when=f"{_when} +gdml cxxstd={_std}")
 
     def cmake_args(self):
         spec = self.spec
diff --git a/var/spack/repos/builtin/packages/veloc/package.py b/var/spack/repos/builtin/packages/veloc/package.py
index 4ca315364b32e6..b470582aa2b25c 100644
--- a/var/spack/repos/builtin/packages/veloc/package.py
+++ b/var/spack/repos/builtin/packages/veloc/package.py
@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 from spack.package import *
-from spack.pkg.builtin.boost import Boost
 
 
 class Veloc(CMakePackage):
@@ -12,12 +11,14 @@ class Veloc(CMakePackage):
     checkpoint-restart runtime for HPC supercomputing infrastructures"""
 
     homepage = "https://github.com/ECP-VeloC/VELOC"
-    url = "https://github.com/ECP-VeloC/VELOC/archive/1.5.tar.gz"
+    url = "https://github.com/ECP-VeloC/VELOC/archive/veloc-1.7.tar.gz"
     git = "https://github.com/ecp-veloc/veloc.git"
 
+    maintainers = ["bnicolae"]
     tags = ["e4s"]
 
     version("main", branch="main")
+    version("1.7", sha256="ca50cb300830ea7f7e5679d32e3671ce4ea250eac384e7231def8dbb90e0cf31")
     version("1.6", sha256="451b46ad13e360270044c0dba09d8e4fbd64149f8e8d71310fdb520424c5eeaa")
     version("1.5", sha256="892f3623c73254d40fbbb8cbc3056219a31510e37aae2ede4100c04743701a5c")
     version("1.4", sha256="d5d12aedb9e97f079c4428aaa486bfa4e31fe1db547e103c52e76c8ec906d0a8")
@@ -26,40 +27,29 @@ class Veloc(CMakePackage):
     version("1.1", sha256="2bbdacf3e0ce4e7c9e360874d8d85b405525bdc7bd992bdb1f1ba49218072160")
     version("1.0", sha256="d594b73d6549a61fce8e67b8984a17cebc3e766fc520ed1636ae3683cdde77cb")
 
-    # TODO: replace this with an explicit list of components of Boost,
-    # for instance depends_on('boost +filesystem')
-    # See https://github.com/spack/spack/pull/22303 for reference
-    depends_on(Boost.with_default_variants)
     depends_on("libpthread-stubs")
     depends_on("mpi")
+    depends_on("boost")
     depends_on("er")
     depends_on("axl@:0.3.0", when="@:1.5")
-    depends_on("openssl")  # Relies on the OpenSSL crypto library for checksums
-    depends_on("pdsh", when="@master")
-    depends_on("cmake@3.9:", type="build")
+    depends_on("axl@0.5.0:", when="@1.6:")
+    depends_on("openssl")
+    depends_on("cmake@3.10:", type="build")
 
-    with when("@1.6:"):
-        depends_on("axl@0.5.0:")
-        depends_on("redset")
-        depends_on("rankstr")
-        depends_on("shuffile")
-        depends_on("kvtree")
-
-    conflicts("%gcc@:4.9.3")
-
-    # requires C++11
     def flag_handler(self, name, flags):
         flags = list(flags)
         if name == "cxxflags":
-            flags.append(self.compiler.cxx11_flag)
+            if self.spec.satisfies("@:1.5"):
+                flags.append(self.compiler.cxx11_flag)
+            else:
+                flags.append(self.compiler.cxx17_flag)
         return (None, None, flags)
 
     def cmake_args(self):
         args = [
             "-DWITH_AXL_PREFIX=%s" % self.spec["axl"].prefix,
             "-DWITH_ER_PREFIX=%s" % self.spec["er"].prefix,
-            "-DBOOST_ROOT=%s" % self.spec["boost"].prefix,
             "-DMPI_CXX_COMPILER=%s" % self.spec["mpi"].mpicxx,
+            "-DBOOST_ROOT=%s" % self.spec["boost"].prefix,
         ]
-
         return args
diff --git a/var/spack/repos/builtin/packages/velvet/package.py b/var/spack/repos/builtin/packages/velvet/package.py
index 57902b4fa96a3c..55c76fee335792 100644
--- a/var/spack/repos/builtin/packages/velvet/package.py
+++ b/var/spack/repos/builtin/packages/velvet/package.py
@@ -6,6 +6,13 @@
 from spack.package import *
 
 
+def is_positive_int(x):
+    if x.isdigit() and int(x) > 0:
+        return True
+    else:
+        return False
+
+
 class Velvet(MakefilePackage):
     """Velvet is a de novo genomic assembler specially designed for short read
     sequencing technologies."""
@@ -13,14 +20,54 @@ class Velvet(MakefilePackage):
     homepage = "https://www.ebi.ac.uk/~zerbino/velvet/"
     url = "https://www.ebi.ac.uk/~zerbino/velvet/velvet_1.2.10.tgz"
 
+    maintainers("snehring")
+
     version("1.2.10", sha256="884dd488c2d12f1f89cdc530a266af5d3106965f21ab9149e8cb5c633c977640")
 
-    depends_on("zlib")
+    variant(
+        "categories",
+        default="2",
+        description="Number of channels which can be handled independently",
+        values=is_positive_int,
+    )
+    variant(
+        "maxkmerlength",
+        default="31",
+        description="Longest kmer size you can use in an analysis",
+        values=is_positive_int,
+    )
+    variant("bigassembly", default=False, description="Allow assemblies with more than 2^31 reads")
+    variant(
+        "vbigassembly",
+        default=False,
+        description="Allow unsigned 64-bit array index values (also enables bigassembly)",
+    )
+    variant(
+        "longsequences", default=False, description="Allow assembling contigs longer than 32kb"
+    )
+    variant("openmp", default=False, description="Enable multithreading")
+    variant("single_cov_cat", default=False, description="Per-library coverage")
+
+    depends_on("zlib-api")
 
     def edit(self, spec, prefix):
+        makefile = FileFilter("Makefile")
         if spec.target.family == "aarch64":
-            makefile = FileFilter("Makefile")
             makefile.filter("-m64", "")
+        maxkmerlength = self.spec.variants["maxkmerlength"].value
+        categories = self.spec.variants["categories"].value
+        makefile.filter(r"^MAXKMERLENGTH\s*=\s*.*", f"MAXKMERLENGTH = {maxkmerlength}")
+        makefile.filter(r"^CATEGORIES\s*=\s*.*", f"CATEGORIES = {categories}")
+        if "+bigassembly" in self.spec:
+            makefile.filter("^ifdef BIGASSEMBLY", "BIGASSEMBLY=1\nifdef BIGASSEMBLY")
+        if "+vbigassembly" in self.spec:
+            makefile.filter("^ifdef VBIGASSEMBLY", "VBIGASSEMBLY=1\nifdef VBIGASSEMBLY")
+        if "+longsequences" in self.spec:
+            makefile.filter("^ifdef LONGSEQUENCES", "LONGSEQUENCES=1\nifdef LONGSEQUENCES")
+        if "+openmp" in self.spec:
+            makefile.filter("^ifdef OPENMP", "OPENMP=1\nifdef OPENMP")
+        if "+single_cov_cat" in self.spec:
+            makefile.filter("^ifdef SINGLE_COV_CAT", "SINGLE_COV_CAT=1\nifdef SINGLE_COV_CAT")
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/vifi/package.py b/var/spack/repos/builtin/packages/vifi/package.py
index db5acababfef96..d5d26aedfe7623 100644
--- a/var/spack/repos/builtin/packages/vifi/package.py
+++ b/var/spack/repos/builtin/packages/vifi/package.py
@@ -13,7 +13,7 @@ class Vifi(Package):
     homepage = "https://github.com/namphuon/ViFi"
     git = "https://github.com/namphuon/ViFi.git"
 
-    version("master", tag="master")
+    version("master", branch="master")
 
     depends_on("perl", type="run")
     depends_on("python", type="run")
diff --git a/var/spack/repos/builtin/packages/vigra/package.py b/var/spack/repos/builtin/packages/vigra/package.py
index 7357524428c454..a64418d766ec85 100644
--- a/var/spack/repos/builtin/packages/vigra/package.py
+++ b/var/spack/repos/builtin/packages/vigra/package.py
@@ -67,7 +67,7 @@ def cmake_args(self):
                     "-DWITH_HDF5=ON",
                     "-DHDF5_CORE_LIBRARY={0}".format(spec["hdf5"].libs.libraries[0]),
                     "-DHDF5_INCLUDE_DIR={0}".format(spec["hdf5"].prefix.include),
-                    "-DHDF5_Z_LIBRARY={0}".format(spec["zlib"].libs.libraries[0]),
+                    "-DHDF5_Z_LIBRARY={0}".format(spec["zlib-api"].libs.libraries[0]),
                 ]
             )
         else:
diff --git a/var/spack/repos/builtin/packages/visit/package.py b/var/spack/repos/builtin/packages/visit/package.py
index 290280e17dc891..d26f8b6fa02907 100644
--- a/var/spack/repos/builtin/packages/visit/package.py
+++ b/var/spack/repos/builtin/packages/visit/package.py
@@ -78,6 +78,7 @@ class Visit(CMakePackage):
     variant("osmesa", default=False, description="Use OSMesa for off-screen CPU rendering")
     variant("adios2", default=True, description="Enable ADIOS2 file format")
     variant("hdf5", default=True, description="Enable HDF5 file format")
+    variant("netcdf", default=True, description="Enable NetCDF file format")
     variant("silo", default=True, description="Enable Silo file format")
     variant("python", default=True, description="Enable Python support")
     variant("mpi", default=True, description="Enable parallel engine")
@@ -98,6 +99,9 @@ class Visit(CMakePackage):
     # Fix pthread and librt link errors
     patch("visit32-missing-link-libs.patch", when="@3.2")
 
+    # Fix const-correctness in VTK interface
+    patch("vtk-8.2-constcorrect.patch", when="@3.3.3 ^vtk@8.2.1a")
+
     # Exactly one of 'gui' or 'osmesa' has to be enabled
     conflicts("+gui", when="+osmesa")
 
@@ -129,7 +133,8 @@ class Visit(CMakePackage):
     depends_on("qwt+opengl", when="+gui")
 
     # python@3.8 doesn't work with VisIt.
-    depends_on("python@3.2:3.7,3.9:", when="+python")
+    depends_on("python@3.2:3.7,3.9:", when="@:3.2 +python")
+    depends_on("python@3.2:", when="@3.3: +python")
     extends("python", when="+python")
 
     # VisIt uses the hdf5 1.8 api
@@ -138,6 +143,11 @@ class Visit(CMakePackage):
     depends_on("hdf5+mpi", when="+hdf5+mpi")
     depends_on("hdf5~mpi", when="+hdf5~mpi")
 
+    # Enable netCDF library based on MPI variant and OLD C++ interface
+    depends_on("netcdf-c+mpi", when="+netcdf+mpi")
+    depends_on("netcdf-c~mpi", when="+netcdf~mpi")
+    depends_on("netcdf-cxx", when="+netcdf")
+
     # VisIt uses Silo's 'ghost zone' data structures, which are only available
     # in v4.10+ releases: https://wci.llnl.gov/simulation/computer-codes/silo/releases/release-notes-4.10
     depends_on("silo@4.10: +shared", when="+silo")
@@ -175,7 +185,7 @@ class Visit(CMakePackage):
     # vtk-m operations are performed.
     depends_on("vtk-m", patches=[patch("vtk-m_transport_tag_topology_field_in.patch")])
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     @when("@3:,develop")
     def patch(self):
@@ -185,6 +195,11 @@ def patch(self):
         for filename in find("src", "CMakeLists.txt"):
             filter_file(r"\bvtk(tiff|jpeg|png)", r"${vtk\1_LIBRARIES}", filename)
 
+        # NetCDF components are in separate directories using Spack, which is
+        # not what Visit's CMake logic expects
+        if "+netcdf" in self.spec:
+            filter_file(r"(set\(NETCDF_CXX_DIR)", r"#\1", "src/CMake/FindNetcdf.cmake")
+
     def flag_handler(self, name, flags):
         if name in ("cflags", "cxxflags"):
             # NOTE: This is necessary in order to allow VisIt to compile a couple
@@ -212,7 +227,7 @@ def cmake_args(self):
             self.define("VTK_MAJOR_VERSION", spec["vtk"].version[0]),
             self.define("VTK_MINOR_VERSION", spec["vtk"].version[1]),
             self.define("VISIT_VTK_DIR", spec["vtk"].prefix),
-            self.define("VISIT_ZLIB_DIR", spec["zlib"].prefix),
+            self.define("VISIT_ZLIB_DIR", spec["zlib-api"].prefix),
             self.define("VISIT_JPEG_DIR", spec["jpeg"].prefix),
             self.define("VISIT_USE_GLEW", False),
             self.define("VISIT_CONFIG_SITE", "NONE"),
@@ -288,6 +303,14 @@ def cmake_args(self):
             if "+mpi" in spec and "+mpi" in spec["hdf5"]:
                 args.append(self.define("VISIT_HDF5_MPI_DIR", spec["hdf5"].prefix))
 
+        if "+netcdf" in spec:
+            args.extend(
+                [
+                    self.define("NETCDF_DIR", spec["netcdf-c"].prefix),
+                    self.define("NETCDF_CXX_DIR", spec["netcdf-cxx"].prefix),
+                ]
+            )
+
         if "+silo" in spec:
             args.append(self.define("VISIT_SILO_DIR", spec["silo"].prefix))
 
diff --git a/var/spack/repos/builtin/packages/visit/vtk-8.2-constcorrect.patch b/var/spack/repos/builtin/packages/visit/vtk-8.2-constcorrect.patch
new file mode 100644
index 00000000000000..2fed026d04e91d
--- /dev/null
+++ b/var/spack/repos/builtin/packages/visit/vtk-8.2-constcorrect.patch
@@ -0,0 +1,83 @@
+diff --git a/src/visit_vtk/full/vtkCSGCell.C b/src/visit_vtk/full/vtkCSGCell.C
+index fa89df8325..28e82a0b86 100644
+--- a/src/visit_vtk/full/vtkCSGCell.C
++++ b/src/visit_vtk/full/vtkCSGCell.C
+@@ -11,7 +11,7 @@
+ 
+ vtkStandardNewMacro(vtkCSGCell);
+ 
+-int vtkCSGCell::EvaluatePosition(double  vtkNotUsed(x)[3], 
++int vtkCSGCell::EvaluatePosition(const double  vtkNotUsed(x)[3], 
+                                   double  vtkNotUsed(closestPoint)[3],
+                                   int&   vtkNotUsed(subId), 
+                                   double  vtkNotUsed(pcoords)[3], 
+@@ -22,14 +22,14 @@ int vtkCSGCell::EvaluatePosition(double  vtkNotUsed(x)[3],
+ }
+ 
+ void vtkCSGCell::EvaluateLocation(int&  vtkNotUsed(subId), 
+-                                   double vtkNotUsed(pcoords)[3],
++                                   const double vtkNotUsed(pcoords)[3],
+                                    double vtkNotUsed(x)[3],
+                                    double *vtkNotUsed(weights))
+ {
+ }
+ 
+ int vtkCSGCell::CellBoundary(int vtkNotUsed(subId), 
+-                            double vtkNotUsed(pcoords)[3], 
++                            const double vtkNotUsed(pcoords)[3], 
+                             vtkIdList *vtkNotUsed(pts))
+ {
+   return 0;
+@@ -47,8 +47,8 @@ vtkCSGCell::Clip(double, vtkDataArray*, vtkIncrementalPointLocator*, vtkCellArra
+ 
+ // Project point on line. If it lies between 0<=t<=1 and distance off line
+ // is less than tolerance, intersection detected.
+-int vtkCSGCell::IntersectWithLine(double vtkNotUsed(p1)[3], 
+-                                   double vtkNotUsed(p2)[3], 
++int vtkCSGCell::IntersectWithLine(const double vtkNotUsed(p1)[3], 
++                                   const double vtkNotUsed(p2)[3], 
+                                    double vtkNotUsed(tol), 
+                                    double& vtkNotUsed(t),
+                                    double vtkNotUsed(x)[3], 
+@@ -69,8 +69,8 @@ int vtkCSGCell::Triangulate(int vtkNotUsed(index),
+ }
+ 
+ void vtkCSGCell::Derivatives(int vtkNotUsed(subId), 
+-                            double vtkNotUsed(pcoords)[3], 
+-                            double *vtkNotUsed(values), 
++                            const double vtkNotUsed(pcoords)[3], 
++                            const double *vtkNotUsed(values), 
+                             int vtkNotUsed(dim), 
+                             double *vtkNotUsed(derivs))
+ {
+diff --git a/src/visit_vtk/full/vtkCSGCell.h b/src/visit_vtk/full/vtkCSGCell.h
+index f8543cd822..b19f209f1e 100644
+--- a/src/visit_vtk/full/vtkCSGCell.h
++++ b/src/visit_vtk/full/vtkCSGCell.h
+@@ -35,21 +35,21 @@ public:
+   int GetNumberOfFaces() override {return 0;};
+   vtkCell *GetEdge(int) override {return 0;};
+   vtkCell *GetFace(int) override {return 0;};
+-  int CellBoundary(int subId, double pcoords[3], vtkIdList *pts) override;
++  int CellBoundary(int subId, const double pcoords[3], vtkIdList *pts) override;
+ 
+   virtual void Contour(double, vtkDataArray*, vtkIncrementalPointLocator*, vtkCellArray*, vtkCellArray*, vtkCellArray*, vtkPointData*, vtkPointData*, vtkCellData*, vtkIdType, vtkCellData*) override;
+ 
+   virtual void Clip(double, vtkDataArray*, vtkIncrementalPointLocator*, vtkCellArray*, vtkPointData*, vtkPointData*, vtkCellData*, vtkIdType, vtkCellData*, int) override;
+ 
+-  int EvaluatePosition(double x[3], double* closestPoint, 
++  int EvaluatePosition(const double x[3], double* closestPoint, 
+                        int& subId, double pcoords[3], 
+                        double& dist2, double *weights) override;
+-  void EvaluateLocation(int& subId, double pcoords[3], double x[3],
++  void EvaluateLocation(int& subId, const double pcoords[3], double x[3],
+                         double *weights) override;
+-  int IntersectWithLine(double p1[3], double p2[3], double tol, double& t,
++  int IntersectWithLine(const double p1[3], const double p2[3], double tol, double& t,
+                         double x[3], double pcoords[3], int& subId) override;
+   int Triangulate(int index, vtkIdList *ptIds, vtkPoints *pts) override;
+-  void Derivatives(int subId, double pcoords[3], double *values, 
++  void Derivatives(int subId, const double pcoords[3], const double *values, 
+                    int dim, double *derivs) override;
+ 
+ protected:
diff --git a/var/spack/repos/builtin/packages/vizglow/package.py b/var/spack/repos/builtin/packages/vizglow/package.py
index 7e0e28e09000b7..99899d3d525c49 100644
--- a/var/spack/repos/builtin/packages/vizglow/package.py
+++ b/var/spack/repos/builtin/packages/vizglow/package.py
@@ -48,7 +48,7 @@ class Vizglow(Package):
     )
 
     # depends_on('mesa')  # TODO: mesa build doesn't work for me
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("freetype")
     depends_on("fontconfig")
     depends_on("libxrender")
@@ -100,7 +100,7 @@ def filter_ld_library_path(self, spec, prefix):
 
         ld_library_path = ":".join(
             [
-                spec["zlib"].prefix.lib,
+                spec["zlib-api"].prefix.lib,
                 spec["freetype"].prefix.lib,
                 spec["fontconfig"].prefix.lib,
                 spec["libxrender"].prefix.lib,
diff --git a/var/spack/repos/builtin/packages/votca/package.py b/var/spack/repos/builtin/packages/votca/package.py
index aa5a5d0b3fe243..db9d260f86530d 100644
--- a/var/spack/repos/builtin/packages/votca/package.py
+++ b/var/spack/repos/builtin/packages/votca/package.py
@@ -28,6 +28,7 @@ class Votca(CMakePackage):
         "new-gmx", default=False, description="Build against gromacs>2019 - no tabulated kernels"
     )
     variant("xtp", default=True, description="Build xtp parts of votca")
+
     conflicts("votca-tools")
     conflicts("votca-csg")
     conflicts("votca-xtp")
diff --git a/var/spack/repos/builtin/packages/vsearch/package.py b/var/spack/repos/builtin/packages/vsearch/package.py
index cfe7855a0380a0..0c28ce5a2b88a7 100644
--- a/var/spack/repos/builtin/packages/vsearch/package.py
+++ b/var/spack/repos/builtin/packages/vsearch/package.py
@@ -23,4 +23,4 @@ class Vsearch(AutotoolsPackage):
     depends_on("automake", type="build")
     depends_on("bzip2")
     depends_on("libtool", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/vt/package.py b/var/spack/repos/builtin/packages/vt/package.py
index 8142ae8ff440f7..284fa50be01026 100644
--- a/var/spack/repos/builtin/packages/vt/package.py
+++ b/var/spack/repos/builtin/packages/vt/package.py
@@ -15,7 +15,7 @@ class Vt(MakefilePackage):
     version("0.5772", sha256="b147520478a2f7c536524511e48133d0360e88282c7159821813738ccbda97e7")
     version("0.577", sha256="009e2592e787ab37e471b4e8a66520141bb2791ca78142ca1767d27036f460d0")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, spack):
         mkdirp(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/vtk-m/package.py b/var/spack/repos/builtin/packages/vtk-m/package.py
index ee30d22b1c6084..ce58cc1d6f0ad7 100644
--- a/var/spack/repos/builtin/packages/vtk-m/package.py
+++ b/var/spack/repos/builtin/packages/vtk-m/package.py
@@ -29,6 +29,7 @@ class VtkM(CMakePackage, CudaPackage, ROCmPackage):
 
     version("master", branch="master")
     version("release", branch="release")
+    version("2.1.0-rc2", sha256="94631fff9f668f40c9c797f03cf32a0d22d57111e309b1e8133c2a3f292b4af1")
     version(
         "2.0.0",
         sha256="32643cf3564fa77f8e2a2a5456a574b6b2355bb68918eb62ccde493993ade1a3",
@@ -53,18 +54,23 @@ class VtkM(CMakePackage, CudaPackage, ROCmPackage):
     variant("shared", default=False, description="build shared libs")
 
     variant("doubleprecision", default=True, description="enable double precision")
-    variant("logging", default=False, description="build logging support")
-    variant("ascent_types", default=True, description="build support for ascent types")
+    variant("logging", default=False, when="@1.3:", description="build logging support")
+    variant(
+        "ascent_types",
+        default=True,
+        when="~64bitids",
+        description="build support for ascent types",
+    )
     variant(
         "virtuals",
         default=False,
         description="enable support for deprecated virtual functions",
         when="@:1.9",
     )
-    variant("mpi", default=False, description="build mpi support")
+    variant("mpi", default=False, when="@1.3:", description="build mpi support")
     variant("rendering", default=True, description="build rendering support")
     variant("64bitids", default=False, description="enable 64 bits ids")
-    variant("testlib", default=False, description="build test library")
+    variant("testlib", default=False, when="@1.7:", description="build test library")
     variant("fpic", default=False, description="build fpic support")
     variant("examples", default=True, when="@1.8:", description="Install builtin examples")
 
@@ -75,7 +81,12 @@ class VtkM(CMakePackage, CudaPackage, ROCmPackage):
     variant(
         "cuda_native", default=True, description="build using native cuda backend", when="+cuda"
     )
-    variant("openmp", default=(sys.platform != "darwin"), description="build openmp support")
+    variant(
+        "openmp",
+        default=(sys.platform != "darwin"),
+        when="@1.3:",
+        description="build openmp support",
+    )
     variant("tbb", default=(sys.platform == "darwin"), description="build TBB support")
 
     depends_on("cmake@3.12:", type="build")  # CMake >= 3.12
@@ -91,6 +102,7 @@ class VtkM(CMakePackage, CudaPackage, ROCmPackage):
 
     # VTK-m uses the default Kokkos backend
     depends_on("kokkos", when="+kokkos")
+    depends_on("kokkos@3.7:3.9", when="@2.0 +kokkos")
     # VTK-m native CUDA and Kokkos CUDA backends are not compatible
     depends_on("kokkos ~cuda", when="+kokkos +cuda +cuda_native")
     depends_on("kokkos +cuda", when="+kokkos +cuda ~cuda_native")
@@ -128,13 +140,13 @@ class VtkM(CMakePackage, CudaPackage, ROCmPackage):
 
     conflicts("+cuda", when="cuda_arch=none", msg="vtk-m +cuda requires that cuda_arch be set")
 
-    conflicts(
-        "+ascent_types", when="+64bitids", msg="Ascent types requires 32 bit IDs for compatibility"
-    )
-
     # Patch
     patch("diy-include-cstddef.patch", when="@1.5.3:1.8.0")
 
+    # VTK-M PR#2972
+    # https://gitlab.kitware.com/vtk/vtk-m/-/merge_requests/2972
+    patch("vtkm-cuda-swap-conflict-pr2972.patch", when="@1.9 +cuda ^cuda@12:")
+
     def cmake_args(self):
         spec = self.spec
         options = []
@@ -155,73 +167,28 @@ def cmake_args(self):
             "86": "ampere",
         }
         with working_dir("spack-build", create=True):
-            options = ["-DVTKm_ENABLE_TESTING:BOOL=OFF"]
-            # shared vs static libs logic
-            # force building statically with cuda
-            if "+shared" in spec:
-                options.append("-DBUILD_SHARED_LIBS=ON")
-            else:
-                options.append("-DBUILD_SHARED_LIBS=OFF")
+            is_release = spec.variants["build_type"].value == "Release"
+            options = [
+                self.define("VTKm_ENABLE_TESTING", False),
+                self.define("VTKm_NO_ASSERT", is_release),
+                self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
+                self.define_from_variant("VTKm_ENABLE_KOKKOS", "kokkos"),
+                self.define_from_variant("VTKm_ENABLE_LOGGING", "logging"),
+                self.define_from_variant("VTKm_ENABLE_MPI", "mpi"),
+                self.define_from_variant("VTKm_ENABLE_OPENMP", "openmp"),
+                self.define_from_variant("VTKm_ENABLE_RENDERING", "rendering"),
+                self.define_from_variant("VTKm_ENABLE_TBB", "tbb"),
+                self.define_from_variant("VTKm_ENABLE_TESTING_LIBRARY", "testlib"),
+                self.define_from_variant("VTKm_INSTALL_EXAMPLES", "examples"),
+                self.define_from_variant("VTKm_NO_DEPRECATED_VIRTUAL", "virtuals"),
+                self.define_from_variant("VTKm_USE_64BIT_IDS", "64bitids"),
+                self.define_from_variant("VTKm_USE_DEFAULT_TYPES_FOR_ASCENT", "ascent_types"),
+                self.define_from_variant("VTKm_USE_DOUBLE_PRECISION", "doubleprecision"),
+            ]
 
-            # double precision
-            if "+doubleprecision" in spec:
-                options.append("-DVTKm_USE_DOUBLE_PRECISION:BOOL=ON")
-            else:
-                options.append("-DVTKm_USE_DOUBLE_PRECISION:BOOL=OFF")
-
-            # logging support
-            if "+logging" in spec:
-                if not spec.satisfies("@1.3.0:"):
-                    raise InstallError(
-                        "logging is not supported for\
-                            vtkm version lower than 1.3"
-                    )
-                options.append("-DVTKm_ENABLE_LOGGING:BOOL=ON")
-            else:
-                options.append("-DVTKm_ENABLE_LOGGING:BOOL=OFF")
-
-            # mpi support
-            if "+mpi" in spec:
-                if not spec.satisfies("@1.3.0:"):
-                    raise InstallError(
-                        "mpi is not supported for\
-                            vtkm version lower than 1.3"
-                    )
-                options.append("-DVTKm_ENABLE_MPI:BOOL=ON")
-            else:
-                options.append("-DVTKm_ENABLE_MPI:BOOL=OFF")
-
-            # rendering support
-            if "+rendering" in spec:
-                options.append("-DVTKm_ENABLE_RENDERING:BOOL=ON")
-            else:
-                options.append("-DVTKm_ENABLE_RENDERING:BOOL=OFF")
-
-            # Support for ascent types
-            if "+ascent_types" in spec:
-                options.append("-DVTKm_USE_DEFAULT_TYPES_FOR_ASCENT:BOOL=ON")
-            else:
-                options.append("-DVTKm_USE_DEFAULT_TYPES_FOR_ASCENT:BOOL=OFF")
-
-            # Support for deprecated virtual functions
-            if "+virtuals" in spec:
-                options.append("-DVTKm_NO_DEPRECATED_VIRTUAL:BOOL=OFF")
-            else:
-                options.append("-DVTKm_NO_DEPRECATED_VIRTUAL:BOOL=ON")
-
-            # 64 bit ids
-            if "+64bitids" in spec:
-                options.append("-DVTKm_USE_64BIT_IDS:BOOL=ON")
-                print("64 bit ids enabled")
-            else:
-                options.append("-DVTKm_USE_64BIT_IDS:BOOL=OFF")
-
-            # Support for the testing header files
-            if "+testlib" in spec and spec.satisfies("@1.7.0:"):
-                options.append("-DVTKm_ENABLE_TESTING_LIBRARY:BOOL=ON")
-
-            if spec.variants["build_type"].value != "Release":
-                options.append("-DVTKm_NO_ASSERT:BOOL=ON")
+            if "+tbb" in spec:
+                # vtk-m detectes tbb via TBB_ROOT env var
+                os.environ["TBB_ROOT"] = spec["tbb"].prefix
 
             # Support for relocatable code
             if "~shared" in spec and "+fpic" in spec:
@@ -231,57 +198,31 @@ def cmake_args(self):
             if "+cuda_native" in spec:
                 options.append("-DVTKm_ENABLE_CUDA:BOOL=ON")
                 options.append("-DCMAKE_CUDA_HOST_COMPILER={0}".format(env["SPACK_CXX"]))
-                if "cuda_arch" in spec.variants:
-                    cuda_value = spec.variants["cuda_arch"].value
-                    cuda_arch = cuda_value[0]
-                    if cuda_arch in gpu_name_table:
-                        vtkm_cuda_arch = gpu_name_table[cuda_arch]
-                        options.append("-DVTKm_CUDA_Architecture={0}".format(vtkm_cuda_arch))
+
+                if spec.satisfies("@1.9.0:") and spec.satisfies("^cmake@3.18:"):
+                    options.append(self.builder.define_cuda_architectures(self))
+
                 else:
-                    # this fix is necessary if compiling platform has cuda, but
-                    # no devices (this is common for front end nodes on hpc
-                    # clusters). We choose volta as a lowest common denominator
-                    options.append("-DVTKm_CUDA_Architecture=volta")
+                    # VTKm_CUDA_Architecture only accepts a single CUDA arch
+                    num_cuda_arch = spec.variants["cuda_arch"].value[0]
+                    str_cuda_arch = str()
+
+                    try:
+                        str_cuda_arch = gpu_name_table[num_cuda_arch]
+                    except KeyError:
+                        raise InstallError(
+                            f"cuda_arch={num_cuda_arch} needs cmake>=3.18 & VTK-m>=1.9.0"
+                        )
+                    options.append(f"-DVTKm_CUDA_Architecture={str_cuda_arch}")
+
             else:
                 options.append("-DVTKm_ENABLE_CUDA:BOOL=OFF")
 
             # hip support
             if "+rocm" in spec:
-                archs = ",".join(self.spec.variants["amdgpu_target"].value)
-                options.append("-DCMAKE_HIP_ARCHITECTURES:STRING={0}".format(archs))
-
-            # openmp support
-            if "+openmp" in spec:
-                # openmp is added since version 1.3.0
-                if not spec.satisfies("@1.3.0:"):
-                    raise InstallError(
-                        "OpenMP is not supported for\
-                            vtkm version lower than 1.3"
-                    )
-                options.append("-DVTKm_ENABLE_OPENMP:BOOL=ON")
-            else:
-                options.append("-DVTKm_ENABLE_OPENMP:BOOL=OFF")
-
-            if "+kokkos" in spec:
-                options.append("-DVTKm_ENABLE_KOKKOS:BOOL=ON")
-            else:
-                options.append("-DVTKm_ENABLE_KOKKOS:BOOL=OFF")
-
-            # tbb support
-            if "+tbb" in spec:
-                # vtk-m detectes tbb via TBB_ROOT env var
-                os.environ["TBB_ROOT"] = spec["tbb"].prefix
-                options.append("-DVTKm_ENABLE_TBB:BOOL=ON")
-            else:
-                options.append("-DVTKm_ENABLE_TBB:BOOL=OFF")
-
-            # Install examples
-            if "+examples" in spec:
-                options.append("-DVTKm_INSTALL_EXAMPLES:BOOL=ON")
-            else:
-                options.append("-DVTKm_INSTALL_EXAMPLES:BOOL=OFF")
+                options.append(self.builder.define_hip_architectures(self))
 
-            return options
+        return options
 
     # Delegate in the vtk-m built smoke test
     def smoke_test(self):
diff --git a/var/spack/repos/builtin/packages/vtk-m/vtkm-cuda-swap-conflict-pr2972.patch b/var/spack/repos/builtin/packages/vtk-m/vtkm-cuda-swap-conflict-pr2972.patch
new file mode 100644
index 00000000000000..2fe942b63521fb
--- /dev/null
+++ b/var/spack/repos/builtin/packages/vtk-m/vtkm-cuda-swap-conflict-pr2972.patch
@@ -0,0 +1,63 @@
+diff -ruN spack-src/vtkm/exec/cuda/internal/ExecutionPolicy.h spack-src-patched/vtkm/exec/cuda/internal/ExecutionPolicy.h
+--- spack-src/vtkm/exec/cuda/internal/ExecutionPolicy.h	2022-10-11 12:07:59.000000000 -0700
++++ spack-src-patched/vtkm/exec/cuda/internal/ExecutionPolicy.h	2023-07-06 17:23:35.898388363 -0700
+@@ -17,6 +17,7 @@
+ #include 
+ VTKM_THIRDPARTY_PRE_INCLUDE
+ #include 
++#include 
+ #include 
+ #include 
+ VTKM_THIRDPARTY_POST_INCLUDE
+diff -ruN spack-src/vtkm/Swap.h spack-src-patched/vtkm/Swap.h
+--- spack-src/vtkm/Swap.h	2022-10-11 12:07:59.000000000 -0700
++++ spack-src-patched/vtkm/Swap.h	2023-07-06 17:25:31.623393290 -0700
+@@ -24,21 +24,31 @@
+ 
+ /// Performs a swap operation. Safe to call from cuda code.
+ #if defined(VTKM_CUDA)
++// CUDA 12 adds a `cub::Swap` function that creates ambiguity with `vtkm::Swap`.
++// This happens when a function from the `cub` namespace is called with an object of a class
++// defined in the `vtkm` namespace as an argument. If that function has an unqualified call to
++// `Swap`, it results in ADL being used, causing the templated functions `cub::Swap` and
++// `vtkm::Swap` to conflict.
++#if defined(VTKM_CUDA_VERSION_MAJOR) && (VTKM_CUDA_VERSION_MAJOR >= 12) && \
++  defined(VTKM_CUDA_DEVICE_PASS)
++using cub::Swap;
++#else
+ template 
+-VTKM_EXEC_CONT void Swap(T& a, T& b)
++VTKM_EXEC_CONT inline void Swap(T& a, T& b)
+ {
+-  using namespace thrust;
++  using thrust::swap;
+   swap(a, b);
+ }
++#endif
+ #elif defined(VTKM_HIP)
+ template 
+-__host__ void Swap(T& a, T& b)
++__host__ inline void Swap(T& a, T& b)
+ {
+-  using namespace std;
++  using std::swap;
+   swap(a, b);
+ }
+ template 
+-__device__ void Swap(T& a, T& b)
++__device__ inline void Swap(T& a, T& b)
+ {
+   T temp = a;
+   a = b;
+@@ -46,9 +56,9 @@
+ }
+ #else
+ template 
+-VTKM_EXEC_CONT void Swap(T& a, T& b)
++VTKM_EXEC_CONT inline void Swap(T& a, T& b)
+ {
+-  using namespace std;
++  using std::swap;
+   swap(a, b);
+ }
+ #endif
diff --git a/var/spack/repos/builtin/packages/vtk/package.py b/var/spack/repos/builtin/packages/vtk/package.py
index c7bec82c742e3e..d73bb332594ea4 100644
--- a/var/spack/repos/builtin/packages/vtk/package.py
+++ b/var/spack/repos/builtin/packages/vtk/package.py
@@ -27,6 +27,15 @@ class Vtk(CMakePackage):
     version("9.0.3", sha256="bc3eb9625b2b8dbfecb6052a2ab091fc91405de4333b0ec68f3323815154ed8a")
     version("9.0.1", sha256="1b39a5e191c282861e7af4101eaa8585969a2de05f5646c9199a161213a622c7")
     version("9.0.0", sha256="15def4e6f84d72f82386617fe595ec124dda3cbd13ea19a0dcd91583197d8715")
+    # v8.2.1a is a compatability version of VTK to allow VisIt to build in CI and contains
+    # patches that were not tested by VTK CI or for a VTK release
+    # - Python 3.8 compatability
+    # - VisIt 3.3.3 compatability
+    version(
+        "8.2.1a",
+        url="https://www.vtk.org/files/release/8.2/VTK-8.2.0.tar.gz",
+        sha256="34c3dc775261be5e45a8049155f7228b6bd668106c72a3c435d95730d17d57bb",
+    )
     version("8.2.0", sha256="34c3dc775261be5e45a8049155f7228b6bd668106c72a3c435d95730d17d57bb")
     version("8.1.2", sha256="0995fb36857dd76ccfb8bb07350c214d9f9099e80b1e66b4a8909311f24ff0db")
     version("8.1.1", sha256="71a09b4340f0a9c58559fe946dc745ab68a866cf20636a41d97b6046cb736324")
@@ -63,11 +72,14 @@ class Vtk(CMakePackage):
     # We cannot build with both osmesa and qt in spack
     conflicts("+osmesa", when="+qt")
 
+    conflicts("%gcc@13", when="@9.2")
+
     with when("+python"):
         # Depend on any Python, add bounds below.
         extends("python@2.7:", type=("build", "run"))
-        # Python 3.8 support from vtk 9
-        depends_on("python@:3.7", when="@:8", type=("build", "run"))
+        depends_on("python@:3.7", when="@:8.2.0", type=("build", "run"))
+        # Python 3.8 support from vtk 9 and patched 8.2
+        depends_on("python@:3.8", when="@:8.2.1a", type=("build", "run"))
         # Python 3.10 support from vtk 9.2
         depends_on("python@:3.9", when="@:9.1", type=("build", "run"))
 
@@ -86,7 +98,15 @@ class Vtk(CMakePackage):
 
     # Fix IOADIOS2 module to work with kits
     # https://gitlab.kitware.com/vtk/vtk/-/merge_requests/8653
-    patch("vtk-adios2-module-no-kit.patch", when="@8.2.0:9.0.3")
+    patch("vtk-adios2-module-no-kit.patch", when="@9:9.0.3")
+
+    # Python 3.8 compatibility for VTK 8.2
+    # https://gitlab.kitware.com/vtk/vtk/-/merge_requests/6269
+    # https://gitlab.kitware.com/vtk/vtk/-/merge_requests/6275
+    patch("vtk82_python38.patch", when="@8.2.1a")
+
+    # Fix link error in exodusII
+    patch("vtk-8.2-exodusII-gcc11.patch", when="@8.2.1a")
 
     # The use of the OpenGL2 backend requires at least OpenGL Core Profile
     # version 3.2 or higher.
@@ -133,22 +153,34 @@ class Vtk(CMakePackage):
     depends_on("netcdf-cxx")
     depends_on("libpng")
     depends_on("libtiff")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("eigen", when="@8.2.0:")
     depends_on("double-conversion", when="@8.2.0:")
     depends_on("sqlite", when="@8.2.0:")
-    depends_on("pugixml", when="@9:")
+    depends_on("pugixml", when="@8.2.0:")
     depends_on("libogg")
     depends_on("libtheora")
     depends_on("utf8cpp", when="@9:")
     depends_on("gl2ps", when="@8.1:")
     depends_on("gl2ps@1.4.1:", when="@9:")
-    depends_on("proj@4", when="@8.2")
+    depends_on("proj@4", when="@8.2.0")
     depends_on("proj@4:7", when="@9:")
     depends_on("cgns@4.1.1:+mpi", when="@9.1: +mpi")
     depends_on("cgns@4.1.1:~mpi", when="@9.1: ~mpi")
-    depends_on("seacas@2021-05-12:+mpi", when="@9.1: +mpi")
-    depends_on("seacas@2021-05-12:~mpi", when="@9.1: ~mpi")
+    with when("@9.1:"):
+        depends_on("seacas+mpi", when="+mpi")
+        depends_on("seacas~mpi", when="~mpi")
+        depends_on("seacas@2021-05-12:")
+
+    # seacas@2023-05-30 does not provide needed SEACASIoss_INCLUDE_DIRS:
+    # CMake Error at CMake/vtkModule.cmake:5552 (message):
+    # The variable `SEACASIoss_INCLUDE_DIRS` was expected to have been available,
+    # but was not defined:
+    conflicts("seacas@2023-05-30", when="@:9.2")
+
+    # vtk@9.2: need Ioss::Utils::get_debug_stream() which only 2022-10-14 provides,
+    # and to be safe against other issues, make them build with this version only:
+    depends_on("seacas@2022-10-14", when="@9.2:")
     depends_on("nlohmann-json", when="@9.2:")
 
     # For finding Fujitsu-MPI wrapper commands
@@ -163,11 +195,18 @@ class Vtk(CMakePackage):
     )
 
     patch(
-        "https://gitlab.kitware.com/vtk/vtk/-/commit/5a1c96e12e9b4a660d326be3bed115a2ceadb573.patch",
-        sha256="65175731c080961f85d779d613ac1f6bce89783745e54e864edec7637b03b18a",
+        "https://gitlab.kitware.com/vtk/vtk/-/commit/5a1c96e12e9b4a660d326be3bed115a2ceadb573.diff",
+        sha256="c446a90459b108082db5b28d9aeda99d030e636325e01929beba062cafb16b76",
         when="@9.1",
     )
 
+    @when("@9.2:")
+    def patch(self):
+        # provide definition for Ioss::Init::Initializer::Initializer(),
+        # required on macOS, as "-undefined error" is the default,
+        # but not on Linux, as undefined symbols are tolerated
+        filter_file("TARGETS Ioss", "TARGETS Ioss Ionit", "ThirdParty/ioss/CMakeLists.txt")
+
     def url_for_version(self, version):
         url = "http://www.vtk.org/files/release/{0}/VTK-{1}.tar.gz"
         return url.format(version.up_to(2), version)
@@ -201,6 +240,11 @@ def cmake_args(self):
             "-DVTK_ALL_NEW_OBJECT_FACTORY:BOOL=ON",
         ]
 
+        # Version 8.2.1a using internal libproj/pugixml for compatability
+        if spec.satisfies("@8.2.1a"):
+            cmake_args.append("-DVTK_USE_SYSTEM_LIBPROJ:BOOL=OFF")
+            cmake_args.append("-DVTK_USE_SYSTEM_PUGIXML:BOOL=OFF")
+
         # Disable wrappers for other languages.
         cmake_args.append("-DVTK_WRAP_JAVA=OFF")
         if spec.satisfies("@:8.1"):
diff --git a/var/spack/repos/builtin/packages/vtk/vtk-8.2-exodusII-gcc11.patch b/var/spack/repos/builtin/packages/vtk/vtk-8.2-exodusII-gcc11.patch
new file mode 100644
index 00000000000000..7208d8167c8355
--- /dev/null
+++ b/var/spack/repos/builtin/packages/vtk/vtk-8.2-exodusII-gcc11.patch
@@ -0,0 +1,11 @@
+diff --git a/ThirdParty/exodusII/vtkexodusII/src/ex_create_par.c b/ThirdParty/exodusII/vtkexodusII/src/ex_create_par.c
+index bf5bb44711..1fe5cd9737 100644
+--- a/ThirdParty/exodusII/vtkexodusII/src/ex_create_par.c
++++ b/ThirdParty/exodusII/vtkexodusII/src/ex_create_par.c
+@@ -614,5 +614,5 @@ int ex_create_par_int(const char *path, int cmode, int *comp_ws, int *io_ws, MPI
+  * Prevent warning in some versions of ranlib(1) because the object
+  * file has no symbols.
+  */
+-const char exodus_unused_symbol_dummy_1;
++extern const char exodus_unused_symbol_dummy_1;
+ #endif
diff --git a/var/spack/repos/builtin/packages/vtk/vtk82_python38.patch b/var/spack/repos/builtin/packages/vtk/vtk82_python38.patch
new file mode 100644
index 00000000000000..aea5b31c8b9461
--- /dev/null
+++ b/var/spack/repos/builtin/packages/vtk/vtk82_python38.patch
@@ -0,0 +1,231 @@
+diff --git a/Wrapping/PythonCore/PyVTKMethodDescriptor.cxx b/Wrapping/PythonCore/PyVTKMethodDescriptor.cxx
+index 2b0d443537..945767acc0 100644
+--- a/Wrapping/PythonCore/PyVTKMethodDescriptor.cxx
++++ b/Wrapping/PythonCore/PyVTKMethodDescriptor.cxx
+@@ -186,7 +186,11 @@ PyTypeObject PyVTKMethodDescriptor_Type = {
+   sizeof(PyMethodDescrObject),           // tp_basicsize
+   0,                                     // tp_itemsize
+   PyVTKMethodDescriptor_Delete,          // tp_dealloc
++#if PY_VERSION_HEX >= 0x03080000
++  0,                                     // tp_vectorcall_offseta
++#else
+   nullptr,                               // tp_print
++#endif
+   nullptr,                               // tp_getattr
+   nullptr,                               // tp_setattr
+   nullptr,                               // tp_compare
+diff --git a/Wrapping/PythonCore/PyVTKNamespace.cxx b/Wrapping/PythonCore/PyVTKNamespace.cxx
+index 71ee2a3516..ccc57db893 100644
+--- a/Wrapping/PythonCore/PyVTKNamespace.cxx
++++ b/Wrapping/PythonCore/PyVTKNamespace.cxx
+@@ -49,7 +49,11 @@ PyTypeObject PyVTKNamespace_Type = {
+   0,                                     // tp_basicsize
+   0,                                     // tp_itemsize
+   PyVTKNamespace_Delete,                 // tp_dealloc
++#if PY_VERSION_HEX >= 0x03080000
++  0,                                     // tp_vectorcall_offseta
++#else
+   nullptr,                               // tp_print
++#endif
+   nullptr,                               // tp_getattr
+   nullptr,                               // tp_setattr
+   nullptr,                               // tp_compare
+diff --git a/Wrapping/PythonCore/PyVTKReference.cxx b/Wrapping/PythonCore/PyVTKReference.cxx
+index 943ac71080..6e42309323 100644
+--- a/Wrapping/PythonCore/PyVTKReference.cxx
++++ b/Wrapping/PythonCore/PyVTKReference.cxx
+@@ -1010,7 +1010,11 @@ PyTypeObject PyVTKReference_Type = {
+   sizeof(PyVTKReference),                // tp_basicsize
+   0,                                     // tp_itemsize
+   PyVTKReference_Delete,                 // tp_dealloc
++#if PY_VERSION_HEX >= 0x03080000
++  0,                                     // tp_vectorcall_offseta
++#else
+   nullptr,                               // tp_print
++#endif
+   nullptr,                               // tp_getattr
+   nullptr,                               // tp_setattr
+   nullptr,                               // tp_compare
+@@ -1067,7 +1071,11 @@ PyTypeObject PyVTKNumberReference_Type = {
+   sizeof(PyVTKReference),                // tp_basicsize
+   0,                                     // tp_itemsize
+   PyVTKReference_Delete,                 // tp_dealloc
++#if PY_VERSION_HEX >= 0x03080000
++  0,                                     // tp_vectorcall_offseta
++#else
+   nullptr,                               // tp_print
++#endif
+   nullptr,                               // tp_getattr
+   nullptr,                               // tp_setattr
+   nullptr,                               // tp_compare
+@@ -1124,7 +1132,11 @@ PyTypeObject PyVTKStringReference_Type = {
+   sizeof(PyVTKReference),                // tp_basicsize
+   0,                                     // tp_itemsize
+   PyVTKReference_Delete,                 // tp_dealloc
++#if PY_VERSION_HEX >= 0x03080000
++  0,                                     // tp_vectorcall_offseta
++#else
+   nullptr,                               // tp_print
++#endif
+   nullptr,                               // tp_getattr
+   nullptr,                               // tp_setattr
+   nullptr,                               // tp_compare
+@@ -1181,7 +1193,11 @@ PyTypeObject PyVTKTupleReference_Type = {
+   sizeof(PyVTKReference),                // tp_basicsize
+   0,                                     // tp_itemsize
+   PyVTKReference_Delete,                 // tp_dealloc
++#if PY_VERSION_HEX >= 0x03080000
++  0,                                     // tp_vectorcall_offseta
++#else
+   nullptr,                               // tp_print
++#endif
+   nullptr,                               // tp_getattr
+   nullptr,                               // tp_setattr
+   nullptr,                               // tp_compare
+diff --git a/Wrapping/PythonCore/PyVTKTemplate.cxx b/Wrapping/PythonCore/PyVTKTemplate.cxx
+index be200985b3..73993e6ad7 100644
+--- a/Wrapping/PythonCore/PyVTKTemplate.cxx
++++ b/Wrapping/PythonCore/PyVTKTemplate.cxx
+@@ -268,7 +268,11 @@ PyTypeObject PyVTKTemplate_Type = {
+   0,                                     // tp_basicsize
+   0,                                     // tp_itemsize
+   nullptr,                               // tp_dealloc
++#if PY_VERSION_HEX >= 0x03080000
++  0,                                     // tp_vectorcall_offseta
++#else
+   nullptr,                               // tp_print
++#endif
+   nullptr,                               // tp_getattr
+   nullptr,                               // tp_setattr
+   nullptr,                               // tp_compare
+diff --git a/Wrapping/Tools/vtkWrapPythonClass.c b/Wrapping/Tools/vtkWrapPythonClass.c
+index b1e45f8e80..f37ef1d17b 100644
+--- a/Wrapping/Tools/vtkWrapPythonClass.c
++++ b/Wrapping/Tools/vtkWrapPythonClass.c
+@@ -521,7 +521,11 @@ void vtkWrapPython_GenerateObjectType(
+     "  sizeof(PyVTKObject), // tp_basicsize\n"
+     "  0, // tp_itemsize\n"
+     "  PyVTKObject_Delete, // tp_dealloc\n"
++    "#if PY_VERSION_HEX >= 0x03080000\n"
++    "  0, // tp_vectorcall_offset\n"
++    "#else\n"
+     "  nullptr, // tp_print\n"
++    "#endif\n"
+     "  nullptr, // tp_getattr\n"
+     "  nullptr, // tp_setattr\n"
+     "  nullptr, // tp_compare\n"
+diff --git a/Wrapping/Tools/vtkWrapPythonEnum.c b/Wrapping/Tools/vtkWrapPythonEnum.c
+index b933702242..a6d1a96384 100644
+--- a/Wrapping/Tools/vtkWrapPythonEnum.c
++++ b/Wrapping/Tools/vtkWrapPythonEnum.c
+@@ -145,7 +145,11 @@ void vtkWrapPython_GenerateEnumType(
+     "  sizeof(PyIntObject), // tp_basicsize\n"
+     "  0, // tp_itemsize\n"
+     "  nullptr, // tp_dealloc\n"
++    "#if PY_VERSION_HEX >= 0x03080000\n"
++    "  0, // tp_vectorcall_offset\n"
++    "#else\n"
+     "  nullptr, // tp_print\n"
++    "#endif\n"
+     "  nullptr, // tp_getattr\n"
+     "  nullptr, // tp_setattr\n"
+     "  nullptr, // tp_compare\n"
+diff --git a/Wrapping/Tools/vtkWrapPythonInit.c b/Wrapping/Tools/vtkWrapPythonInit.c
+index d1eb3b890c..f89501f322 100644
+--- a/Wrapping/Tools/vtkWrapPythonInit.c
++++ b/Wrapping/Tools/vtkWrapPythonInit.c
+@@ -54,7 +54,7 @@ static void CreateImplFile(const char *libName,
+   fprintf(fout,"#include \"vtkPython.h\"\n");
+   fprintf(fout,"#include \"vtkPythonCompatibility.h\"\n");
+   fprintf(fout,"#include \"vtkSystemIncludes.h\"\n");
+-  fprintf(fout,"#include \n");
++  fprintf(fout,"#include \n");
+   fprintf(fout,"// Handle compiler warning messages, etc.\n"
+           "#if defined( _MSC_VER ) && !defined(VTK_DISPLAY_WIN32_WARNINGS)\n"
+           "#pragma warning ( disable : 4706 )\n"
+diff --git a/Wrapping/Tools/vtkWrapPythonMethod.c b/Wrapping/Tools/vtkWrapPythonMethod.c
+index 17d1a8cb2e..61a639ed38 100644
+--- a/Wrapping/Tools/vtkWrapPythonMethod.c
++++ b/Wrapping/Tools/vtkWrapPythonMethod.c
+@@ -659,7 +659,7 @@ void vtkWrapPython_ReturnValue(
+       fprintf(fp,
+             "      if (result && PyVTKObject_Check(result))\n"
+             "      {\n"
+-            "        PyVTKObject_GetObject(result)->UnRegister(0);\n"
++            "        PyVTKObject_GetObject(result)->UnRegister(nullptr);\n"
+             "        PyVTKObject_SetFlag(result, VTK_PYTHON_IGNORE_UNREGISTER, 1);\n"
+             "      }\n");
+     }
+@@ -700,9 +700,7 @@ void vtkWrapPython_ReturnValue(
+   }
+   else if (vtkWrap_IsArray(val))
+   {
+-    fprintf(fp,
+-            "      result = %sBuildTuple(tempr, sizer);\n",
+-            prefix);
++    fprintf(fp, "      result = vtkPythonArgs::BuildTuple(tempr, sizer);\n");
+   }
+   else if (vtkWrap_IsStdVector(val))
+   {
+@@ -809,9 +807,8 @@ void vtkWrapPython_SaveArgs(FILE *fp, FunctionInfo *currentFunction)
+     {
+       noneDone = 0;
+ 
+-      fprintf(fp,
+-              "    ap.Save(%.*stemp%d, %.*ssave%d, ",
+-              (n-1), asterisks, i, (n-1), asterisks, i);
++      fprintf(fp, "    vtkPythonArgs::Save(%.*stemp%d, %.*ssave%d, ", (n - 1), asterisks, i,
++        (n - 1), asterisks, i);
+ 
+       if (vtkWrap_IsNArray(arg))
+       {
+@@ -1096,9 +1093,8 @@ static void vtkWrapPython_WriteBackToArgs(
+              !vtkWrap_IsConst(arg) &&
+              !vtkWrap_IsSetVectorMethod(currentFunction))
+     {
+-      fprintf(fp,
+-              "    if (ap.HasChanged(%.*stemp%d, %.*ssave%d, ",
+-              (n-1), asterisks, i, (n-1), asterisks, i);
++      fprintf(fp, "    if (vtkPythonArgs::HasChanged(%.*stemp%d, %.*ssave%d, ", (n - 1), asterisks,
++        i, (n - 1), asterisks, i);
+ 
+       if (vtkWrap_IsNArray(arg))
+       {
+@@ -1140,7 +1136,7 @@ static void vtkWrapPython_WriteBackToArgs(
+               "    {\n"
+               "      PyObject *vec = (temp%d.size() == 0 ?\n"
+               "        PyTuple_New(0) :\n"
+-              "        ap.BuildTuple(temp%d.data(), temp%d.size()));\n"
++              "        vtkPythonArgs::BuildTuple(temp%d.data(), temp%d.size()));\n"
+               "      ap.SetContents(%d, vec);\n"
+               "      Py_DECREF(vec);\n"
+               "    }\n"
+diff --git a/Wrapping/Tools/vtkWrapPythonMethodDef.c b/Wrapping/Tools/vtkWrapPythonMethodDef.c
+index 8e9735f312..05cee2f740 100644
+--- a/Wrapping/Tools/vtkWrapPythonMethodDef.c
++++ b/Wrapping/Tools/vtkWrapPythonMethodDef.c
+@@ -1038,7 +1038,7 @@ static void vtkWrapPython_CustomMethods(
+             "    if (tempr != nullptr)\n"
+             "    {\n"
+             "      result = vtkPythonArgs::BuildVTKObject(tempr);\n"
+-            "      PyVTKObject_GetObject(result)->UnRegister(0);\n"
++            "      PyVTKObject_GetObject(result)->UnRegister(nullptr);\n"
+             "    }\n"
+             "  }\n"
+             "\n"
+diff --git a/Wrapping/Tools/vtkWrapPythonType.c b/Wrapping/Tools/vtkWrapPythonType.c
+index 744cb1b9d3..b7a7ea0d15 100644
+--- a/Wrapping/Tools/vtkWrapPythonType.c
++++ b/Wrapping/Tools/vtkWrapPythonType.c
+@@ -709,7 +709,11 @@ void vtkWrapPython_GenerateSpecialType(
+     "  sizeof(PyVTKSpecialObject), // tp_basicsize\n"
+     "  0, // tp_itemsize\n"
+     "  Py%s_Delete, // tp_dealloc\n"
++    "#if PY_VERSION_HEX >= 0x03080000\n"
++    "  0, // tp_vectorcall_offset\n"
++    "#else\n"
+     "  nullptr, // tp_print\n"
++    "#endif\n"
+     "  nullptr, // tp_getattr\n"
+     "  nullptr, // tp_setattr\n"
+     "  nullptr, // tp_compare\n"
diff --git a/var/spack/repos/builtin/packages/w3emc/package.py b/var/spack/repos/builtin/packages/w3emc/package.py
index 983b3ff6fec9f9..335e9caa5e6fa8 100644
--- a/var/spack/repos/builtin/packages/w3emc/package.py
+++ b/var/spack/repos/builtin/packages/w3emc/package.py
@@ -14,9 +14,12 @@ class W3emc(CMakePackage):
 
     homepage = "https://noaa-emc.github.io/NCEPLIBS-w3emc/"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-w3emc/archive/refs/tags/v2.9.0.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-w3emc"
 
-    maintainers("t-brown", "AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
+    maintainers("AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
 
+    version("develop", branch="develop")
+    version("2.11.0", sha256="53a03d03421c5da699b026ca220512ed494a531b83284693f66d2579d570c43b")
     version("2.10.0", sha256="366b55a0425fc3e729ecb9f3b236250349399fe4c8e19f325500463043fd2f18")
     version("2.9.3", sha256="9ca1b08dd13dfbad4a955257ae0cf38d2e300ccd8d983606212bc982370a29bc")
     version("2.9.2", sha256="eace811a1365f69b85fdf2bcd93a9d963ba72de5a7111e6fa7c0e6578b69bfbc")
@@ -26,9 +29,25 @@ class W3emc(CMakePackage):
 
     variant("pic", default=True, description="Build with position-independent-code")
     variant("bufr", default=False, description="Build with BUFR routines", when="@2.10:")
+    variant(
+        "precision",
+        default=("4", "d"),
+        values=("4", "d", "8"),
+        multi=True,
+        description="Set precision (_4/_d/_8 library versions)",
+        when="@2.10:",
+    )
+    variant("shared", default=False, description="Build shared library", when="@2.10: +pic")
+    variant(
+        "extradeps",
+        default=False,
+        description="Build w3emc with subprograms which call unknown dependencies",
+        when="@2.10:",
+    )
 
-    depends_on("bufr", when="@2.10: +bufr")
+    conflicts("+shared +extradeps", msg="Shared library cannot be built with unknown dependencies")
 
+    depends_on("bufr", when="@2.10: +bufr")
     depends_on("bacio", when="@2.9.2:")
 
     # w3emc 2.7.3 contains gblevents which has these dependencies
@@ -36,21 +55,34 @@ class W3emc(CMakePackage):
     depends_on("sigio", when="@2.7.3")
     depends_on("netcdf-fortran", when="@2.7.3")
 
-    def cmake_args(self):
-        args = [
-            self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
-            self.define_from_variant("BUILD_WITH_BUFR", "bufr"),
-        ]
-
-        return args
-
     def setup_run_environment(self, env):
-        suffixes = ["4", "d"]
         if self.spec.satisfies("@:2.9"):
-            suffixes += ["8"]
+            suffixes = ("4", "d", "8")
+            shared = False
+        else:
+            suffixes = self.spec.variants["precision"].value
+            shared = self.spec.satisfies("+shared")
+
         for suffix in suffixes:
             lib = find_libraries(
-                "libw3emc_" + suffix, root=self.prefix, shared=False, recursive=True
+                "libw3emc_" + suffix, root=self.prefix, shared=shared, recursive=True
             )
             env.set("W3EMC_LIB" + suffix, lib[0])
             env.set("W3EMC_INC" + suffix, join_path(self.prefix, "include_" + suffix))
+
+    def cmake_args(self):
+        args = [
+            self.define_from_variant("BUILD_WITH_BUFR", "bufr"),
+            self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
+            self.define("BUILD_4", self.spec.satisfies("precision=4")),
+            self.define("BUILD_D", self.spec.satisfies("precision=d")),
+            self.define("BUILD_8", self.spec.satisfies("precision=8")),
+            self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
+            self.define_from_variant("BUILD_WITH_EXTRA_DEPS", "extradeps"),
+        ]
+
+        return args
+
+    def check(self):
+        with working_dir(self.builder.build_directory):
+            make("test")
diff --git a/var/spack/repos/builtin/packages/wannier90/package.py b/var/spack/repos/builtin/packages/wannier90/package.py
index a354f7a2f112ef..f20333581e6056 100644
--- a/var/spack/repos/builtin/packages/wannier90/package.py
+++ b/var/spack/repos/builtin/packages/wannier90/package.py
@@ -17,7 +17,9 @@ class Wannier90(MakefilePackage):
 
     homepage = "http://wannier.org"
     url = "https://github.com/wannier-developers/wannier90/archive/v3.1.0.tar.gz"
+    git = "https://github.com/wannier-developers/wannier90.git"
 
+    version("develop", branch="develop")
     version("3.1.0", sha256="40651a9832eb93dec20a8360dd535262c261c34e13c41b6755fa6915c936b254")
     version("3.0.0", sha256="f196e441dcd7b67159a1d09d2d7de2893b011a9f03aab6b30c4703ecbf20fe5b")
     version("2.1.0", sha256="ee90108d4bc4aa6a1cf16d72abebcb3087cf6c1007d22dda269eb7e7076bddca")
diff --git a/var/spack/repos/builtin/packages/warpx/package.py b/var/spack/repos/builtin/packages/warpx/package.py
index 3386027a0209c0..ab883f2c8fcb9d 100644
--- a/var/spack/repos/builtin/packages/warpx/package.py
+++ b/var/spack/repos/builtin/packages/warpx/package.py
@@ -17,7 +17,7 @@ class Warpx(CMakePackage):
     """
 
     homepage = "https://ecp-warpx.github.io"
-    url = "https://github.com/ECP-WarpX/WarpX/archive/refs/tags/23.06.tar.gz"
+    url = "https://github.com/ECP-WarpX/WarpX/archive/refs/tags/23.08.tar.gz"
     git = "https://github.com/ECP-WarpX/WarpX.git"
 
     maintainers("ax3l", "dpgrote", "MaxThevenet", "RemiLehe")
@@ -25,6 +25,8 @@ class Warpx(CMakePackage):
 
     # NOTE: if you update the versions here, also see py-warpx
     version("develop", branch="development")
+    version("23.08", sha256="67695ff04b83d1823ea621c19488e54ebaf268532b0e5eb4ea8ad293d7ab3ddc")
+    version("23.07", sha256="511633f94c0d0205013609bde5bbf92a29c2e69f6e69b461b80d09dc25602945")
     version("23.06", sha256="75fcac949220c44dce04de581860c9a2caa31a0eee8aa7d49455fa5fc928514b")
     version("23.05", sha256="34306a98fdb1f5f44ab4fb92f35966bfccdcf1680a722aa773af2b59a3060d73")
     version("23.04", sha256="e5b285c73e13a0d922eba5d83760c168d4fd388e54a519830003b2e692dab823")
diff --git a/var/spack/repos/builtin/packages/watch/package.py b/var/spack/repos/builtin/packages/watch/package.py
index a32c208db8be48..000bb81905a9b2 100644
--- a/var/spack/repos/builtin/packages/watch/package.py
+++ b/var/spack/repos/builtin/packages/watch/package.py
@@ -17,7 +17,7 @@ class Watch(AutotoolsPackage):
     git = "https://gitlab.com/procps-ng/procps.git"
 
     version("master", branch="master")
-    version("3.3.15", tag="v3.3.15")
+    version("3.3.15", tag="v3.3.15", commit="7bb949bcba13c107fa0f45d2d0298b1ad6b6d6cc")
 
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
diff --git a/var/spack/repos/builtin/packages/wayland/package.py b/var/spack/repos/builtin/packages/wayland/package.py
index 03c276a98090c5..baee2fcc817433 100644
--- a/var/spack/repos/builtin/packages/wayland/package.py
+++ b/var/spack/repos/builtin/packages/wayland/package.py
@@ -27,6 +27,8 @@ class Wayland(MesonPackage, AutotoolsPackage):
         default="meson",
     )
 
+    variant("doc", default=False, description="Build documentation")
+
     version("1.22.0", sha256="bbca9c906a8fb8992409ebf51812f19e2a784b2c169d4b784cdd753b4bb448ef")
     version("1.21.0", sha256="53b7fa67142e653820030ec049971bcb5e84ac99e05cba5bcb9cb55f43fae4b3")
     version("1.20.0", sha256="20523cd6f2c18c3c86725467157c6221e19de76fbfad944042a2d494af3c7a92")
@@ -45,11 +47,28 @@ class Wayland(MesonPackage, AutotoolsPackage):
         depends_on("meson@0.56.0:", type="build")
 
     depends_on("pkgconfig", type="build")
-    depends_on("doxygen", type="build")
-    depends_on("xmlto", type="build")
-    depends_on("libxslt", type="build")
-    depends_on("docbook-xsl", type="build")
     depends_on("libxml2")
     depends_on("chrpath")
     depends_on("expat")
     depends_on("libffi")
+
+    with when("+doc"):
+        depends_on("docbook-xsl", type="build")
+        depends_on("doxygen", type="build")
+        depends_on("xmlto", type="build")
+        depends_on("libxslt", type="build")
+        depends_on("graphviz+libgd", type="build")
+
+    @when("build_system=autotools")
+    def configure_args(self):
+        args = []
+        args.extend(self.enable_or_disable("documentation", variant="doc"))
+        return args
+
+    @when("build_system=meson")
+    def meson_args(self):
+        spec = self.spec
+        opt_bool = lambda c, o: "-D%s=%s" % (o, str(c).lower())
+        args = []
+        args.append(opt_bool("+doc" in spec, "documentation"))
+        return args
diff --git a/var/spack/repos/builtin/packages/weechat/package.py b/var/spack/repos/builtin/packages/weechat/package.py
index ff34b175d6027a..9410c181544452 100644
--- a/var/spack/repos/builtin/packages/weechat/package.py
+++ b/var/spack/repos/builtin/packages/weechat/package.py
@@ -30,7 +30,7 @@ class Weechat(CMakePackage):
     depends_on("aspell")
     depends_on("libgcrypt")
     depends_on("curl")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("ruby@1.9.1:", when="+ruby")
     depends_on("tcl@8.5:", when="+tcl")
     depends_on("perl", when="+perl")
diff --git a/var/spack/repos/builtin/packages/wget/package.py b/var/spack/repos/builtin/packages/wget/package.py
index df8be50adebdcb..5e0dad3573239b 100644
--- a/var/spack/repos/builtin/packages/wget/package.py
+++ b/var/spack/repos/builtin/packages/wget/package.py
@@ -40,7 +40,7 @@ class Wget(AutotoolsPackage, GNUMirrorPackage):
     depends_on("gettext", type="build")
     depends_on("python@3:", type="build", when="+python")
 
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("libpsl", when="+libpsl")
     depends_on("pcre", when="+pcre")
 
diff --git a/var/spack/repos/builtin/packages/wgrib2/package.py b/var/spack/repos/builtin/packages/wgrib2/package.py
index 4cd400c992f9c8..45bbb2daa7948d 100644
--- a/var/spack/repos/builtin/packages/wgrib2/package.py
+++ b/var/spack/repos/builtin/packages/wgrib2/package.py
@@ -86,6 +86,8 @@ class Wgrib2(MakefilePackage):
     conflicts("+openmp", when="%apple-clang")
 
     depends_on("wget", type=("build"), when="+netcdf4")
+    # makefile behavior with shell commands/character escapes breaks with gmake@4.3:
+    depends_on("gmake@:4.2")
 
     variant_map = {
         "netcdf3": "USE_NETCDF3",
diff --git a/var/spack/repos/builtin/packages/wgsim/package.py b/var/spack/repos/builtin/packages/wgsim/package.py
index 30935a31879087..f142338d4c57ac 100644
--- a/var/spack/repos/builtin/packages/wgsim/package.py
+++ b/var/spack/repos/builtin/packages/wgsim/package.py
@@ -20,7 +20,7 @@ class Wgsim(Package):
 
     version("2011.10.17", commit="a12da3375ff3b51a5594d4b6fa35591173ecc229")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         cc = Executable(spack_cc)
diff --git a/var/spack/repos/builtin/packages/whizard/package.py b/var/spack/repos/builtin/packages/whizard/package.py
index 9d580ca8a7c75c..3297c2eddbe5f1 100644
--- a/var/spack/repos/builtin/packages/whizard/package.py
+++ b/var/spack/repos/builtin/packages/whizard/package.py
@@ -12,7 +12,10 @@ class Whizard(AutotoolsPackage):
     and simulated event samples."""
 
     homepage = "whizard.hepforge.org"
-    url = "https://whizard.hepforge.org/downloads/?f=whizard-2.8.3.tar.gz"
+    urls = [
+        "https://launchpad.net/whizard/3.1.x/3.1.2/+download/whizard-3.1.2.tar.gz",
+        "https://whizard.hepforge.org/downloads/?f=whizard-2.8.3.tar.gz",
+    ]
     git = "https://gitlab.tp.nt.uni-siegen.de/whizard/public.git"
 
     tags = ["hep"]
@@ -20,6 +23,8 @@ class Whizard(AutotoolsPackage):
     maintainers("vvolkl")
 
     version("master", branch="master")
+    version("3.1.2", sha256="4f706f8ef02a580ae4dba867828691dfe0b3f9f9b8982b617af72eb8cd4c6fa3")
+    version("3.1.1", sha256="dd48e4e39b8a4990be47775ec6171f89d8147cb2e9e293afc7051a7dbc5a23ef")
     version("3.1.0", sha256="9dc5e6d1a25d2fc708625f85010cb81b63559ff02cceb9b35024cf9f426c0ad9")
     version("3.0.3", sha256="20f2269d302fc162a6aed8e781b504ba5112ef0711c078cdb08b293059ed67cf")
     version("3.0.2", sha256="f1db92cd95a0281f6afbf4ac32ab027670cb97a57ad8f5139c0d1f61593d66ec")
@@ -64,7 +69,7 @@ class Whizard(AutotoolsPackage):
         when="+openloops",
     )
     depends_on("texlive", when="+latex")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     # Fix for https://github.com/key4hep/key4hep-spack/issues/71
     # NOTE: This will become obsolete in a future release of whizard, so once
@@ -77,7 +82,7 @@ class Whizard(AutotoolsPackage):
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
     depends_on("libtool", type="build")
-    depends_on("pkgconf", type="build")
+    depends_on("pkgconfig", type="build")
 
     conflicts(
         "%gcc@:5.0",
@@ -98,6 +103,8 @@ def setup_build_environment(self, env):
         # and seems incompatible with
         # filter_compiler_wrappers, thus the
         # actual compilers need to be used to build
+        if self.spec.satisfies("+lcio"):
+            env.set("LCIO", self.spec["lcio"].prefix)
         env.set("CC", self.compiler.cc)
         env.set("CXX", self.compiler.cxx)
         env.set("FC", self.compiler.fc)
diff --git a/var/spack/repos/builtin/packages/win-sdk/package.py b/var/spack/repos/builtin/packages/win-sdk/package.py
index aeab5be2b7d7d3..e40b85898c161c 100644
--- a/var/spack/repos/builtin/packages/win-sdk/package.py
+++ b/var/spack/repos/builtin/packages/win-sdk/package.py
@@ -35,7 +35,9 @@ class WinSdk(Package):
     version("10.0.10586")
     version("10.0.26639")
 
-    variant("plat", values=("x64", "x86", "arm", "arm64"), default="x64")
+    variant(
+        "plat", values=("x64", "x86", "arm", "arm64"), default="x64", description="Toolchain arch"
+    )
 
     # WinSDK versions depend on compatible compilers
     # WDK versions do as well, but due to their one to one dep on the SDK
diff --git a/var/spack/repos/builtin/packages/win-wdk/package.py b/var/spack/repos/builtin/packages/win-wdk/package.py
index ad3309c3a06b75..07f2a5901877dc 100644
--- a/var/spack/repos/builtin/packages/win-wdk/package.py
+++ b/var/spack/repos/builtin/packages/win-wdk/package.py
@@ -66,7 +66,9 @@ class WinWdk(Package):
         expand=False,
     )
 
-    variant("plat", values=("x64", "x86", "arm", "arm64"), default="x64")
+    variant(
+        "plat", values=("x64", "x86", "arm", "arm64"), default="x64", description="Toolchain arch"
+    )
 
     # need one to one dep on SDK per https://github.com/MicrosoftDocs/windows-driver-docs/issues/1550
     # additionally, the WDK needs to be paired with a version of the Windows SDK
diff --git a/var/spack/repos/builtin/packages/wiredtiger/package.py b/var/spack/repos/builtin/packages/wiredtiger/package.py
index 40a91502c4bbc6..f03f2c267e85e2 100644
--- a/var/spack/repos/builtin/packages/wiredtiger/package.py
+++ b/var/spack/repos/builtin/packages/wiredtiger/package.py
@@ -19,7 +19,7 @@ class Wiredtiger(AutotoolsPackage):
     depends_on("swig", type=("build", "run"), when="+python")
     depends_on("lz4", when="+lz4")
     depends_on("snappy", when="+snappy")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
     depends_on("zstd", when="+zstd")
     depends_on("rsync", type="build")
 
diff --git a/var/spack/repos/builtin/packages/wise2/package.py b/var/spack/repos/builtin/packages/wise2/package.py
new file mode 100644
index 00000000000000..153305896befc8
--- /dev/null
+++ b/var/spack/repos/builtin/packages/wise2/package.py
@@ -0,0 +1,58 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Wise2(MakefilePackage):
+    """The Wise2 package is now a rather stately bioinformatics package that
+    has be around for a while. Its key programs are genewise, a program
+    for aligning proteins or protein HMMs to DNA, and dynamite a rather
+    cranky "macro language" which automates the production of dynamic
+    programming."""
+
+    homepage = "https://www.ebi.ac.uk/~birney/wise2/"
+    url = "https://www.ebi.ac.uk/~birney/wise2/wise2.4.1.tar.gz"
+
+    maintainers("snehring")
+
+    version("2.4.1", sha256="240e2b12d6cd899040e2efbcb85b0d3c10245c255f3d07c1db45d0af5a4d5fa1")
+
+    depends_on("gettext")
+    depends_on("glib")
+    depends_on("libiconv")
+    depends_on("pcre2")
+
+    build_directory = "src"
+
+    build_targets = ["all"]
+
+    def edit(self, spec, prefix):
+        glib_include_include = join_path(
+            spec["glib"].prefix.include, "glib-" + str(spec["glib"].version[0]) + ".0"
+        )
+        glib_lib_include = join_path(
+            spec["glib"].prefix.lib, "glib-" + str(spec["glib"].version[0]) + ".0", "include"
+        )
+        glib_lib = spec["glib"].prefix.lib
+        glib_config_files = ["src/makefile", "src/network/makefile", "src/models/makefile"]
+        for f in glib_config_files:
+            filter_file(
+                "`glib-config --cflags`",
+                f"-I{glib_include_include} -I{glib_lib_include}",
+                f,
+                string=True,
+            )
+            filter_file("`glib-config --libs`", f"-L{glib_lib} -lglib-2.0", f, string=True)
+        filter_file('"glib.h"', "", "src/dynlibsrc/subseqhash.h", string=True)
+        filter_file("getline", "getlineseq", "src/HMMer2/sqio.c", string=True)
+        filter_file("isnumber", "isdigit", "src/models/phasemodel.c", string=True)
+        filter_file(r".*welcome.csh.*", "", "src/makefile")
+
+    def install(self, spec, prefix):
+        with working_dir("src"):
+            install_tree("bin", prefix.bin)
+        mkdirp(prefix.share.wise2)
+        install_tree("wisecfg", prefix.share.wise2)
diff --git a/var/spack/repos/builtin/packages/wps/package.py b/var/spack/repos/builtin/packages/wps/package.py
index d7711a19eb6f59..71cb8d0a6291da 100644
--- a/var/spack/repos/builtin/packages/wps/package.py
+++ b/var/spack/repos/builtin/packages/wps/package.py
@@ -26,6 +26,7 @@ class Wps(Package):
     variant(
         "build_type",
         default="serial",
+        description="Build type",
         values=("serial", "serial_NO_GRIB2", "dmpar", "dmpar_NO_GRIB2"),
     )
 
diff --git a/var/spack/repos/builtin/packages/wrf-io/package.py b/var/spack/repos/builtin/packages/wrf-io/package.py
index 71e85be873bfd6..c0608a582b640f 100644
--- a/var/spack/repos/builtin/packages/wrf-io/package.py
+++ b/var/spack/repos/builtin/packages/wrf-io/package.py
@@ -15,9 +15,11 @@ class WrfIo(CMakePackage):
 
     homepage = "https://noaa-emc.github.io/NCEPLIBS-wrf_io"
     url = "https://github.com/NOAA-EMC/NCEPLIBS-wrf_io/archive/refs/tags/v1.2.0.tar.gz"
+    git = "https://github.com/NOAA-EMC/NCEPLIBS-wrf_io"
 
-    maintainers("t-brown", "AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
+    maintainers("AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")
 
+    version("develop", branch="develop")
     version("1.2.0", sha256="000cf5294a2c68460085258186e1f36c86d3d0d9c433aa969a0f92736b745617")
 
     variant("openmp", default=False, description="Enable multithreading with OpenMP")
diff --git a/var/spack/repos/builtin/packages/wrf/package.py b/var/spack/repos/builtin/packages/wrf/package.py
index 60030c906ec2a2..1fcc8226d7d44e 100644
--- a/var/spack/repos/builtin/packages/wrf/package.py
+++ b/var/spack/repos/builtin/packages/wrf/package.py
@@ -69,6 +69,11 @@ class Wrf(Package):
     maintainers("MichaelLaufer", "ptooley")
     tags = ["windows"]
 
+    version(
+        "4.5.1",
+        sha256="9d557c34c105db4d41e727843ecb19199233c7cf82c5369b34a2ce8efe65e2d1",
+        url="https://github.com/wrf-model/WRF/releases/download/v4.5.1/v4.5.1.tar.gz",
+    )
     version(
         "4.5.0",
         sha256="14fd78abd4e32c1d99e2e97df0370030a5c58ec84c343591bdc5e74f163c5525",
@@ -96,15 +101,22 @@ class Wrf(Package):
         url="https://github.com/wrf-model/WRF/archive/V3.9.1.1.tar.gz",
     )
 
-    variant("build_type", default="dmpar", values=("serial", "smpar", "dmpar", "dm+sm"))
+    variant(
+        "build_type",
+        default="dmpar",
+        description="Build type",
+        values=("serial", "smpar", "dmpar", "dm+sm"),
+    )
     variant(
         "nesting",
         default="basic",
+        description="Nesting",
         values=("no_nesting", "basic", "preset_moves", "vortex_following"),
     )
     variant(
         "compile_type",
         default="em_real",
+        description="Compile type",
         values=(
             "em_real",
             "em_quarter_ss",
@@ -156,11 +168,19 @@ class Wrf(Package):
     patch("patches/4.2/var.gen_be.Makefile.patch", when="@4.2:")
     patch("patches/4.2/Makefile.patch", when="@4.2")
     patch("patches/4.2/tirpc_detect.patch", when="@4.2")
-    patch("patches/4.2/add_aarch64.patch", when="@4.2:")
+    patch("patches/4.2/add_aarch64.patch", when="@4.2:4.3.1 %gcc target=aarch64:")
+    patch("patches/4.2/add_aarch64_acfl.patch", when="@4.2:4.3.1 %arm target=aarch64:")
     patch("patches/4.2/configure_aocc_2.3.patch", when="@4.2 %aocc@:2.4.0")
     patch("patches/4.2/configure_aocc_3.0.patch", when="@4.2: %aocc@3.0.0:3.2.0")
     patch("patches/4.2/hdf5_fix.patch", when="@4.2: %aocc")
     patch("patches/4.2/derf_fix.patch", when="@4.2 %aocc")
+    patch(
+        "patches/4.2/add_tools_flags_acfl2304.patch",
+        when="@4.2:4.4.2 %arm@23.04.1: target=aarch64:",
+    )
+
+    patch("patches/4.3/add_aarch64.patch", when="@4.3.2:4.4.2 %gcc target=aarch64:")
+    patch("patches/4.3/add_aarch64_acfl.patch", when="@4.3.2:4.4.2 %arm target=aarch64:")
 
     patch("patches/4.4/arch.postamble.patch", when="@4.4:")
     patch("patches/4.4/configure.patch", when="@4.4:4.4.2")
@@ -217,7 +237,7 @@ class Wrf(Package):
     depends_on("netcdf-fortran")
     depends_on("jasper")
     depends_on("libpng")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("perl")
     depends_on("jemalloc", when="%aocc")
     # not sure if +fortran is required, but seems like a good idea
diff --git a/var/spack/repos/builtin/packages/wrf/patches/4.2/add_aarch64.patch b/var/spack/repos/builtin/packages/wrf/patches/4.2/add_aarch64.patch
index 065c131b576862..58f3085ea278bc 100644
--- a/var/spack/repos/builtin/packages/wrf/patches/4.2/add_aarch64.patch
+++ b/var/spack/repos/builtin/packages/wrf/patches/4.2/add_aarch64.patch
@@ -1,17 +1,17 @@
-From 9eb0c715cfcefb3df204200d44f5e8526bd14e31 Mon Sep 17 00:00:00 2001
-From: Phil Tooley 
-Date: Fri, 16 Oct 2020 13:20:24 +0100
-Subject: [PATCH] add aarch64 gcc support
-
----
- arch/configure.defaults | 43 +++++++++++++++++++++++++++++++++++++++++
- 1 file changed, 43 insertions(+)
-
 diff --git a/arch/configure.defaults b/arch/configure.defaults
-index 7767a81e..bfd2a802 100644
+index 6e98941a..17a94e48 100644
 --- a/arch/configure.defaults
 +++ b/arch/configure.defaults
-@@ -1975,6 +1975,49 @@ LIB_BUNDLED     = \
+@@ -44,7 +44,7 @@ RLFLAGS		=
+ CC_TOOLS        =      cc 
+ 
+ ###########################################################
+-#ARCH    Linux i486 i586 i686 armv7l aarch64, gfortran compiler with gcc #serial smpar dmpar dm+sm
++#ARCH    Linux i486 i586 i686, gfortran compiler with gcc #serial smpar dmpar dm+sm
+ #
+ DESCRIPTION     =       GNU ($SFC/$SCC)
+ DMPARALLEL      =       # 1
+@@ -1981,6 +1981,49 @@ LIB_BUNDLED     = \
                       $(WRF_SRC_ROOT_DIR)/frame/module_internal_header_util.o \
                       $(WRF_SRC_ROOT_DIR)/frame/pack_utils.o
  
@@ -26,7 +26,7 @@ index 7767a81e..bfd2a802 100644
 +SFC             =       gfortran
 +SCC             =       gcc
 +CCOMP           =       gcc
-+DM_FC           =       mpif90 
++DM_FC           =       mpif90
 +DM_CC           =       mpicc -DMPI2_SUPPORT
 +FC              =       CONFIGURE_FC
 +CC              =       CONFIGURE_CC
@@ -34,33 +34,30 @@ index 7767a81e..bfd2a802 100644
 +RWORDSIZE       =       CONFIGURE_RWORDSIZE
 +PROMOTION       =       #-fdefault-real-8
 +ARCH_LOCAL      =       -DNONSTANDARD_SYSTEM_SUBR  -DWRF_USE_CLM
-+CFLAGS_LOCAL    =       -w -O3 -c -march=native -mtune=native
-+DFLAGS_LOCAL   =       
-+CPLUSPLUSLIB    =       
++CFLAGS_LOCAL    =       -w -O3 -c
++DFLAGS_LOCAL   =
++CPLUSPLUSLIB    =
 +ESMF_LDFLAG     =       $(CPLUSPLUSLIB)
-+FCOPTIM         =       -O3 -ftree-vectorize -funroll-loops -march=native -mtune=native
-+FCREDUCEDOPT	=       $(FCOPTIM)
-+FCNOOPT		=       -O0
++FCOPTIM         =       -Ofast -fno-expensive-optimizations -fno-reciprocal-math -fsigned-zeros -fno-unsafe-math-optimizations -funroll-loops
++FCREDUCEDOPT   =       $(FCOPTIM)
++FCNOOPT                =       -O0
 +FCDEBUG         =       -g # -g $(FCNOOPT) # -ggdb -fbacktrace -fcheck=bounds,do,mem,pointer -ffpe-trap=invalid,zero,overflow
 +FORMAT_FIXED    =       -ffixed-form
 +FORMAT_FREE     =       -ffree-form -ffree-line-length-none
-+FCSUFFIX        =       
++FCCOMPAT        =
 +BYTESWAPIO      =       -fconvert=big-endian -frecord-marker=4
-+FCBASEOPTS_NO_G =       -w $(FORMAT_FREE) $(BYTESWAPIO)
++FCBASEOPTS_NO_G =       -w $(FORMAT_FREE) $(BYTESWAPIO) $(FCCOMPAT)
 +FCBASEOPTS      =       $(FCBASEOPTS_NO_G) $(FCDEBUG)
-+MODULE_SRCH_FLAG =     
++MODULE_SRCH_FLAG =
 +TRADFLAG        =      CONFIGURE_TRADFLAG
 +CPP             =      /lib/cpp CONFIGURE_CPPFLAGS
 +AR              =      ar
 +ARFLAGS         =      ru
 +M4              =      m4 -G
 +RANLIB          =      ranlib
-+RLFLAGS		=	
-+CC_TOOLS        =      $(SCC) 
++RLFLAGS                =
++CC_TOOLS        =      $(SCC)
 +
  #insert new stanza here
  
  ###########################################################
--- 
-2.28.0
-
diff --git a/var/spack/repos/builtin/packages/wrf/patches/4.2/add_aarch64_acfl.patch b/var/spack/repos/builtin/packages/wrf/patches/4.2/add_aarch64_acfl.patch
new file mode 100644
index 00000000000000..be2db85ec85dc0
--- /dev/null
+++ b/var/spack/repos/builtin/packages/wrf/patches/4.2/add_aarch64_acfl.patch
@@ -0,0 +1,63 @@
+diff --git a/arch/configure.defaults b/arch/configure.defaults
+index 6e98941a..85d96019 100644
+--- a/arch/configure.defaults
++++ b/arch/configure.defaults
+@@ -44,7 +44,7 @@ RLFLAGS		=
+ CC_TOOLS        =      cc 
+ 
+ ###########################################################
+-#ARCH    Linux i486 i586 i686 armv7l aarch64, gfortran compiler with gcc #serial smpar dmpar dm+sm
++#ARCH    Linux i486 i586 i686, gfortran compiler with gcc #serial smpar dmpar dm+sm
+ #
+ DESCRIPTION     =       GNU ($SFC/$SCC)
+ DMPARALLEL      =       # 1
+@@ -1981,6 +1981,49 @@ LIB_BUNDLED     = \
+                      $(WRF_SRC_ROOT_DIR)/frame/module_internal_header_util.o \
+                      $(WRF_SRC_ROOT_DIR)/frame/pack_utils.o
+ 
++###########################################################
++#ARCH    Linux aarch64, armflang compiler  #serial smpar dmpar dm+sm
++#
++DESCRIPTION     =       Arm GNU ($SFC/$SCC): Aarch64
++DMPARALLEL      =       # 1
++OMPCPP          =       # -D_OPENMP
++OMP             =       # -fopenmp
++OMPCC           =       # -fopenmp
++SFC             =       armflang
++SCC             =       armclang
++CCOMP           =       armclang
++DM_FC           =       mpif90
++DM_CC           =       mpicc -DMPI2_SUPPORT
++FC              =       CONFIGURE_FC
++CC              =       CONFIGURE_CC
++LD              =       $(FC)
++RWORDSIZE       =       CONFIGURE_RWORDSIZE
++PROMOTION       =       #-fdefault-real-8
++ARCH_LOCAL      =       -DNONSTANDARD_SYSTEM_SUBR  -DWRF_USE_CLM
++CFLAGS_LOCAL    =       -w -O3 -c -Wno-error=implicit-function-declaration -Wno-error=implicit-int
++DFLAGS_LOCAL   =
++CPLUSPLUSLIB    =
++ESMF_LDFLAG     =       $(CPLUSPLUSLIB)
++FCOPTIM         =       -Ofast -funroll-loops
++FCREDUCEDOPT   =       $(FCOPTIM)
++FCNOOPT                =       -O0
++FCDEBUG         =       -g # -g $(FCNOOPT) # -ggdb -fbacktrace -fcheck=bounds,do,mem,pointer -ffpe-trap=invalid,zero,overflow
++FORMAT_FIXED    =       -ffixed-form
++FORMAT_FREE     =       -ffree-form -ffree-line-length-0
++FCCOMPAT        =
++BYTESWAPIO      =       -fconvert=big-endian
++FCBASEOPTS_NO_G =       -w $(OMP) $(FORMAT_FREE) $(BYTESWAPIO) $(FCCOMPAT)
++FCBASEOPTS      =       $(FCBASEOPTS_NO_G) $(FCDEBUG)
++MODULE_SRCH_FLAG =
++TRADFLAG        =      CONFIGURE_TRADFLAG
++CPP             =      /lib/cpp CONFIGURE_CPPFLAGS
++AR              =      ar
++ARFLAGS         =      ru
++M4              =      m4 -G
++RANLIB          =      ranlib
++RLFLAGS                =
++CC_TOOLS        =      $(SCC)
++
+ #insert new stanza here
+ 
+ ###########################################################
diff --git a/var/spack/repos/builtin/packages/wrf/patches/4.2/add_tools_flags_acfl2304.patch b/var/spack/repos/builtin/packages/wrf/patches/4.2/add_tools_flags_acfl2304.patch
new file mode 100644
index 00000000000000..da9aaec4ccdd2f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/wrf/patches/4.2/add_tools_flags_acfl2304.patch
@@ -0,0 +1,19 @@
+*** a/arch/postamble	2023-04-01 15:35:41.729081823 +0000
+--- b/arch/postamble	2023-04-02 07:30:17.031503636 +0000
+*************** INCLUDE_MODULES =    $(MODULE_SRCH_FLAG)
+*** 56,62 ****
+                        -I$(NETCDFFPATH)/include \
+                        CONFIGURE_RTTOV_INC CONFIGURE_CTSM_INC
+  REGISTRY        =    Registry
+! CC_TOOLS_CFLAGS = CONFIGURE_NMM_CORE
+  
+  LIB             =    $(LIB_BUNDLED) $(LIB_EXTERNAL) $(LIB_LOCAL) $(LIB_WRF_HYDRO) CONFIGURE_CTSM_LIB
+  LDFLAGS         =    $(OMP) $(FCFLAGS) $(LDFLAGS_LOCAL) CONFIGURE_LDFLAGS
+--- 56,62 ----
+                        -I$(NETCDFFPATH)/include \
+                        CONFIGURE_RTTOV_INC CONFIGURE_CTSM_INC
+  REGISTRY        =    Registry
+! CC_TOOLS_CFLAGS = CONFIGURE_NMM_CORE $(CFLAGS_LOCAL)
+  
+  LIB             =    $(LIB_BUNDLED) $(LIB_EXTERNAL) $(LIB_LOCAL) $(LIB_WRF_HYDRO) CONFIGURE_CTSM_LIB
+  LDFLAGS         =    $(OMP) $(FCFLAGS) $(LDFLAGS_LOCAL) CONFIGURE_LDFLAGS
diff --git a/var/spack/repos/builtin/packages/wrf/patches/4.3/add_aarch64.patch b/var/spack/repos/builtin/packages/wrf/patches/4.3/add_aarch64.patch
new file mode 100644
index 00000000000000..e24d8009383d01
--- /dev/null
+++ b/var/spack/repos/builtin/packages/wrf/patches/4.3/add_aarch64.patch
@@ -0,0 +1,72 @@
+diff --git a/arch/configure.defaults b/arch/configure.defaults
+index 6aa210d7..eead95fb 100644
+--- a/arch/configure.defaults
++++ b/arch/configure.defaults
+@@ -45,7 +45,7 @@ CC_TOOLS        =      cc
+ NETCDFPAR_BUILD	=      CONFIGURE_NETCDFPAR_BUILD
+
+ ###########################################################
+-#ARCH    Linux i486 i586 i686 armv7l aarch64, gfortran compiler with gcc #serial smpar dmpar dm+sm
++#ARCH    Linux i486 i586 i686, gfortran compiler with gcc #serial smpar dmpar dm+sm
+ #
+ DESCRIPTION     =       GNU ($SFC/$SCC)
+ DMPARALLEL      =       # 1
+@@ -2023,7 +2023,7 @@ LIB_BUNDLED     = \
+                      $(WRF_SRC_ROOT_DIR)/frame/pack_utils.o
+
+ ###########################################################
+-#ARCH   Linux   armv7l aarch64, gnu OpenMPI #serial smpar dmpar dm+sm
++#ARCH   Linux   , gnu OpenMPI #serial smpar dmpar dm+sm
+ #
+ DESCRIPTION     =       GNU ($SFC/$SCC)
+ DMPARALLEL      =       # 1
+@@ -2066,4 +2066,47 @@ RLFLAGS         =
+ CC_TOOLS        =      $(SCC)
+ NETCDFPAR_BUILD	=      CONFIGURE_NETCDFPAR_BUILD
+
++###########################################################
++#ARCH    Linux aarch64, gfortran compiler with gcc  #serial smpar dmpar dm+sm
++#
++DESCRIPTION     =       Arm GNU ($SFC/$SCC): Aarch64
++DMPARALLEL      =       # 1
++OMPCPP          =       # -D_OPENMP
++OMP             =       # -fopenmp
++OMPCC           =       # -fopenmp
++SFC             =       gfortran
++SCC             =       gcc
++CCOMP           =       gcc
++DM_FC           =       mpif90
++DM_CC           =       mpicc -DMPI2_SUPPORT
++FC              =       CONFIGURE_FC
++CC              =       CONFIGURE_CC
++LD              =       $(FC)
++RWORDSIZE       =       CONFIGURE_RWORDSIZE
++PROMOTION       =       #-fdefault-real-8
++ARCH_LOCAL      =       -DNONSTANDARD_SYSTEM_SUBR  -DWRF_USE_CLM
++CFLAGS_LOCAL    =       -w -O3 -c
++DFLAGS_LOCAL   =
++CPLUSPLUSLIB    =
++ESMF_LDFLAG     =       $(CPLUSPLUSLIB)
++FCOPTIM         =       -Ofast -fno-expensive-optimizations -fno-reciprocal-math -fsigned-zeros -fno-unsafe-math-optimizations -funroll-loops
++FCREDUCEDOPT	=       $(FCOPTIM)
++FCNOOPT		=       -O0
++FCDEBUG         =       -g # -g $(FCNOOPT) # -ggdb -fbacktrace -fcheck=bounds,do,mem,pointer -ffpe-trap=invalid,zero,overflow
++FORMAT_FIXED    =       -ffixed-form
++FORMAT_FREE     =       -ffree-form -ffree-line-length-none
++FCCOMPAT        =
++BYTESWAPIO      =       -fconvert=big-endian -frecord-marker=4
++FCBASEOPTS_NO_G =       -w $(FORMAT_FREE) $(BYTESWAPIO) $(FCCOMPAT)
++FCBASEOPTS      =       $(FCBASEOPTS_NO_G) $(FCDEBUG)
++MODULE_SRCH_FLAG =
++TRADFLAG        =      CONFIGURE_TRADFLAG
++CPP             =      /lib/cpp CONFIGURE_CPPFLAGS
++AR              =      ar
++ARFLAGS         =      ru
++M4              =      m4 -G
++RANLIB          =      ranlib
++RLFLAGS		=
++CC_TOOLS        =      $(SCC)
++
+ #insert new stanza here
+
+ ###########################################################
diff --git a/var/spack/repos/builtin/packages/wrf/patches/4.3/add_aarch64_acfl.patch b/var/spack/repos/builtin/packages/wrf/patches/4.3/add_aarch64_acfl.patch
new file mode 100644
index 00000000000000..0632ac8be91a96
--- /dev/null
+++ b/var/spack/repos/builtin/packages/wrf/patches/4.3/add_aarch64_acfl.patch
@@ -0,0 +1,72 @@
+diff --git a/arch/configure.defaults b/arch/configure.defaults
+index 6aa210d7..45630015 100644
+--- a/arch/configure.defaults
++++ b/arch/configure.defaults
+@@ -45,7 +45,7 @@ CC_TOOLS        =      cc
+ NETCDFPAR_BUILD	=      CONFIGURE_NETCDFPAR_BUILD
+
+ ###########################################################
+-#ARCH    Linux i486 i586 i686 armv7l aarch64, gfortran compiler with gcc #serial smpar dmpar dm+sm
++#ARCH    Linux i486 i586 i686, gfortran compiler with gcc #serial smpar dmpar dm+sm
+ #
+ DESCRIPTION     =       GNU ($SFC/$SCC)
+ DMPARALLEL      =       # 1
+@@ -2023,7 +2023,7 @@ LIB_BUNDLED     = \
+                      $(WRF_SRC_ROOT_DIR)/frame/pack_utils.o
+
+ ###########################################################
+-#ARCH   Linux   armv7l aarch64, gnu OpenMPI #serial smpar dmpar dm+sm
++#ARCH   Linux   , gnu OpenMPI #serial smpar dmpar dm+sm
+ #
+ DESCRIPTION     =       GNU ($SFC/$SCC)
+ DMPARALLEL      =       # 1
+@@ -2066,4 +2066,47 @@ RLFLAGS         =
+ CC_TOOLS        =      $(SCC)
+ NETCDFPAR_BUILD	=      CONFIGURE_NETCDFPAR_BUILD
+
++###########################################################
++#ARCH    Linux aarch64, armflang compiler #serial smpar dmpar dm+sm
++#
++DESCRIPTION     =       Arm GNU ($SFC/$SCC): Aarch64
++DMPARALLEL      =       # 1
++OMPCPP          =       # -D_OPENMP
++OMP             =       # -fopenmp
++OMPCC           =       # -fopenmp
++SFC             =       armflang
++SCC             =       armclang
++CCOMP           =       armclang
++DM_FC           =       mpif90
++DM_CC           =       mpicc -DMPI2_SUPPORT
++FC              =       CONFIGURE_FC
++CC              =       CONFIGURE_CC
++LD              =       $(FC)
++RWORDSIZE       =       CONFIGURE_RWORDSIZE
++PROMOTION       =       #-fdefault-real-8
++ARCH_LOCAL      =       -DNONSTANDARD_SYSTEM_SUBR  -DWRF_USE_CLM
++CFLAGS_LOCAL    =       -w -O3 -c -Wno-error=implicit-function-declaration -Wno-error=implicit-int
++DFLAGS_LOCAL   =
++CPLUSPLUSLIB    =
++ESMF_LDFLAG     =       $(CPLUSPLUSLIB)
++FCOPTIM         =       -Ofast -funroll-loops
++FCREDUCEDOPT	=       $(FCOPTIM)
++FCNOOPT		=       -O0
++FCDEBUG         =       -g # -g $(FCNOOPT) # -ggdb -fbacktrace -fcheck=bounds,do,mem,pointer -ffpe-trap=invalid,zero,overflow
++FORMAT_FIXED    =       -ffixed-form
++FORMAT_FREE     =       -ffree-form -ffree-line-length-0
++FCCOMPAT        =
++BYTESWAPIO      =       -fconvert=big-endian
++FCBASEOPTS_NO_G =       -w $(OMP) $(FORMAT_FREE) $(BYTESWAPIO) $(FCCOMPAT)
++FCBASEOPTS      =       $(FCBASEOPTS_NO_G) $(FCDEBUG)
++MODULE_SRCH_FLAG =
++TRADFLAG        =      CONFIGURE_TRADFLAG
++CPP             =      /lib/cpp CONFIGURE_CPPFLAGS
++AR              =      ar
++ARFLAGS         =      ru
++M4              =      m4 -G
++RANLIB          =      ranlib
++RLFLAGS		=
++CC_TOOLS        =      $(SCC)
++
+ #insert new stanza here
+
+ ###########################################################
diff --git a/var/spack/repos/builtin/packages/wt/package.py b/var/spack/repos/builtin/packages/wt/package.py
index 6af01b57ea4d79..9043912eeaf219 100644
--- a/var/spack/repos/builtin/packages/wt/package.py
+++ b/var/spack/repos/builtin/packages/wt/package.py
@@ -59,7 +59,7 @@ class Wt(CMakePackage):
     depends_on("mariadb", when="+mariadb")
     depends_on("postgresql", when="+postgresql")
     depends_on("pango", when="+pango")
-    depends_on("zlib", when="+zlib")
+    depends_on("zlib-api", when="+zlib")
 
     def cmake_args(self):
         cmake_args = [
diff --git a/var/spack/repos/builtin/packages/wtdbg2/package.py b/var/spack/repos/builtin/packages/wtdbg2/package.py
index 55148062dbf21a..9795f1a4516818 100644
--- a/var/spack/repos/builtin/packages/wtdbg2/package.py
+++ b/var/spack/repos/builtin/packages/wtdbg2/package.py
@@ -14,7 +14,7 @@ class Wtdbg2(MakefilePackage):
 
     version("2.3", sha256="fb61d38a4c60a39b3b194e63b855141c05ddcbe71cf244ae613766a9b0a56621")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("sse2neon", when="target=aarch64:")
 
     patch("for_aarch64.patch", when="target=aarch64:")
diff --git a/var/spack/repos/builtin/packages/wxparaver/package.py b/var/spack/repos/builtin/packages/wxparaver/package.py
index 210752836d3a4d..1018fffa3be9bc 100644
--- a/var/spack/repos/builtin/packages/wxparaver/package.py
+++ b/var/spack/repos/builtin/packages/wxparaver/package.py
@@ -29,7 +29,7 @@ class Wxparaver(AutotoolsPackage):
     depends_on("wxwidgets@2.8:")  # NOTE: using external for this one is usually simpler
     depends_on("wxpropgrid@1.4:")
     depends_on("libxml2")
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def configure_args(self):
         spec = self.spec
diff --git a/var/spack/repos/builtin/packages/xapian-core/package.py b/var/spack/repos/builtin/packages/xapian-core/package.py
index aa13eec964d013..9c138d0a5e5afd 100644
--- a/var/spack/repos/builtin/packages/xapian-core/package.py
+++ b/var/spack/repos/builtin/packages/xapian-core/package.py
@@ -19,4 +19,4 @@ class XapianCore(AutotoolsPackage):
     version("1.4.11", sha256="9f16b2f3e2351a24034d7636f73566ab74c3f0729e9e0492934e956b25c5bc07")
     version("1.4.3", sha256="7d5295511ca2de70463a29e75f6a2393df5dc1485bf33074b778c66e1721e475")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
diff --git a/var/spack/repos/builtin/packages/xcb-util-xrm/package.py b/var/spack/repos/builtin/packages/xcb-util-xrm/package.py
index cd966daa2ccd04..0e2503535cc9af 100644
--- a/var/spack/repos/builtin/packages/xcb-util-xrm/package.py
+++ b/var/spack/repos/builtin/packages/xcb-util-xrm/package.py
@@ -15,7 +15,7 @@ class XcbUtilXrm(AutotoolsPackage):
 
     # This GitHub project includes some git submodules, which must be fetched
     # in order to build it.
-    version("1.2", tag="v1.2", submodules=True)
+    version("1.2", tag="v1.2", commit="a45b3d0bbaa94bf8a68405ab8c5c61404da464ce", submodules=True)
 
     depends_on("autoconf", type="build")
     depends_on("automake", type="build")
diff --git a/var/spack/repos/builtin/packages/xdmf3/fix_hdf5_hid_t.diff b/var/spack/repos/builtin/packages/xdmf3/fix_hdf5_hid_t.diff
new file mode 100644
index 00000000000000..8323ddda26def7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/xdmf3/fix_hdf5_hid_t.diff
@@ -0,0 +1,40 @@
+diff --git a/core/XdmfHDF5Controller.hpp b/core/XdmfHDF5Controller.hpp
+index c5c15d0a..496cc80d 100644
+--- a/core/XdmfHDF5Controller.hpp
++++ b/core/XdmfHDF5Controller.hpp
+@@ -27,13 +27,14 @@
+ // C Compatible Includes
+ #include "XdmfCore.hpp"
+ #include "XdmfHeavyDataController.hpp"
++#include 
+ 
+ // So that hdf5 does not need to be included in the header files
+ // It would add a dependancy to programs that use Xdmf
+ #ifndef _H5Ipublic_H
+   #ifndef XDMF_HID_T
+   #define XDMF_HID_T
+-    typedef int hid_t;
++    typedef int64_t hid_t;
+   #endif
+ #endif
+ 
+diff --git a/core/XdmfHDF5Writer.hpp b/core/XdmfHDF5Writer.hpp
+index cfbec6f4..f83aa0de 100644
+--- a/core/XdmfHDF5Writer.hpp
++++ b/core/XdmfHDF5Writer.hpp
+@@ -28,13 +28,14 @@
+ #include "XdmfCore.hpp"
+ #include "XdmfHeavyDataWriter.hpp"
+ #include "XdmfHeavyDataController.hpp"
++#include 
+ 
+ // So that hdf5 does not need to be included in the header files
+ // It would add a dependancy to programs that use Xdmf
+ #ifndef _H5Ipublic_H
+   #ifndef XDMF_HID_T
+   #define XDMF_HID_T
+-    typedef int hid_t;
++    typedef int64_t hid_t;
+   #endif
+ #endif
+ 
diff --git a/var/spack/repos/builtin/packages/xdmf3/package.py b/var/spack/repos/builtin/packages/xdmf3/package.py
index 8a84aa27f10238..ba54eed8413081 100644
--- a/var/spack/repos/builtin/packages/xdmf3/package.py
+++ b/var/spack/repos/builtin/packages/xdmf3/package.py
@@ -30,8 +30,10 @@ class Xdmf3(CMakePackage):
     # See https://github.com/spack/spack/pull/22303 for reference
     depends_on(Boost.with_default_variants)
     depends_on("mpi", when="+mpi")
-    depends_on("hdf5+mpi", when="+mpi")
-    depends_on("hdf5~mpi", when="~mpi")
+    depends_on("hdf5@1.10:+mpi", when="+mpi")
+    depends_on("hdf5@1.10:~mpi", when="~mpi")
+    # motivated by discussion in https://gitlab.kitware.com/xdmf/xdmf/-/issues/28
+    patch("fix_hdf5_hid_t.diff")
 
     def cmake_args(self):
         """Populate cmake arguments for XDMF."""
@@ -42,7 +44,7 @@ def cmake_args(self):
             "-DXDMF_BUILD_UTILS=ON",
             "-DXDMF_WRAP_JAVA=OFF",
             "-DXDMF_WRAP_PYTHON=OFF",
-            "-DXDMF_BUILD_TESTING=ON",
+            "-DXDMF_BUILD_TESTING=OFF",
         ]
 
         return cmake_args
diff --git a/var/spack/repos/builtin/packages/xerces-c/package.py b/var/spack/repos/builtin/packages/xerces-c/package.py
index b31facf98ff710..b31bb5525e57e5 100644
--- a/var/spack/repos/builtin/packages/xerces-c/package.py
+++ b/var/spack/repos/builtin/packages/xerces-c/package.py
@@ -31,7 +31,7 @@ class XercesC(AutotoolsPackage):
     variant(
         "cxxstd",
         default="default",
-        values=("default", "98", "11", "14", "17"),
+        values=("default", "98", "11", "14", "17", "20"),
         multi=False,
         description="Use the specified C++ standard when building",
     )
diff --git a/var/spack/repos/builtin/packages/xgboost/package.py b/var/spack/repos/builtin/packages/xgboost/package.py
index 5996799e9e8079..c5a25c5721f5ba 100644
--- a/var/spack/repos/builtin/packages/xgboost/package.py
+++ b/var/spack/repos/builtin/packages/xgboost/package.py
@@ -22,10 +22,18 @@ class Xgboost(CMakePackage, CudaPackage):
     maintainers("adamjstewart")
 
     version("master", branch="master", submodules=True)
-    version("1.6.2", tag="v1.6.2", submodules=True)
-    version("1.6.1", tag="v1.6.1", submodules=True)
-    version("1.5.2", tag="v1.5.2", submodules=True)
-    version("1.3.3", tag="v1.3.3", submodules=True)
+    version(
+        "1.6.2", tag="v1.6.2", commit="b9934246faa9a25e10a12339685dfbe56d56f70b", submodules=True
+    )
+    version(
+        "1.6.1", tag="v1.6.1", commit="5d92a7d936fc3fad4c7ecb6031c3c1c7da882a14", submodules=True
+    )
+    version(
+        "1.5.2", tag="v1.5.2", commit="742c19f3ecf2135b4e008a4f4a10b59add8b1045", submodules=True
+    )
+    version(
+        "1.3.3", tag="v1.3.3", commit="000292ce6d99ed658f6f9aebabc6e9b330696e7e", submodules=True
+    )
 
     variant("nccl", default=False, description="Build with NCCL to enable distributed GPU support")
     variant("openmp", default=True, description="Build with OpenMP support")
diff --git a/var/spack/repos/builtin/packages/xmlrpc-c/package.py b/var/spack/repos/builtin/packages/xmlrpc-c/package.py
index eb1ed76d94c3c7..95ec6f7f92d2a6 100644
--- a/var/spack/repos/builtin/packages/xmlrpc-c/package.py
+++ b/var/spack/repos/builtin/packages/xmlrpc-c/package.py
@@ -24,3 +24,13 @@ def configure_args(self):
             args.append("--build=arm-linux")
 
         return args
+
+    def build(self, spec, prefix):
+        make()
+        with working_dir("tools"):
+            make()
+
+    def install(self, spec, prefix):
+        make("install")
+        with working_dir("tools"):
+            make("install")
diff --git a/var/spack/repos/builtin/packages/xmlto/package.py b/var/spack/repos/builtin/packages/xmlto/package.py
index 1a018bfa877ccc..ca5748188176fd 100644
--- a/var/spack/repos/builtin/packages/xmlto/package.py
+++ b/var/spack/repos/builtin/packages/xmlto/package.py
@@ -18,5 +18,18 @@ class Xmlto(AutotoolsPackage):
     version("0.0.28", sha256="2f986b7c9a0e9ac6728147668e776d405465284e13c74d4146c9cbc51fd8aad3")
 
     # FIXME: missing a lot of dependencies
-    depends_on("libxslt")
+    depends_on("docbook-xsl", type=("build", "run"))
+    depends_on("libxml2", type=("build", "run"))  # xmllint
+    depends_on("libxslt", type=("build", "run"))  # xsltconf
     depends_on("util-linux", type=("build", "run"))  # getopt with support for longopts
+
+    depends_on("docbook-xml", type="run")
+
+    patch(
+        "https://src.fedoraproject.org/rpms/xmlto/raw/rawhide/f/xmlto-c99-1.patch",
+        sha256="056c8bebc25d8d1488cc6a3724e2bcafc0e5e0df5c50080559cdef99bd377839",
+    )
+    patch(
+        "https://src.fedoraproject.org/rpms/xmlto/raw/rawhide/f/xmlto-c99-2.patch",
+        sha256="50e39b1810bbf22a1d67944086c5681bcd58b8c325dfb251d56ac15d088fc17a",
+    )
diff --git a/var/spack/repos/builtin/packages/xpmem/package.py b/var/spack/repos/builtin/packages/xpmem/package.py
index 9fb7600fda4e66..c8091478d49b37 100644
--- a/var/spack/repos/builtin/packages/xpmem/package.py
+++ b/var/spack/repos/builtin/packages/xpmem/package.py
@@ -64,13 +64,7 @@ class Xpmem(AutotoolsPackage):
     conflicts("+kernel-module", when="platform=darwin")
 
     # All compilers except for gcc are in conflict with +kernel-module:
-    for __compiler in spack.compilers.supported_compilers():
-        if __compiler != "gcc":
-            conflicts(
-                "+kernel-module",
-                when="%{0}".format(__compiler),
-                msg="Linux kernel module must be compiled with gcc",
-            )
+    requires("%gcc", when="+kernel-module", msg="Linux kernel module must be compiled with gcc")
 
     def autoreconf(self, spec, prefix):
         Executable("./autogen.sh")()
diff --git a/var/spack/repos/builtin/packages/xrootd/package.py b/var/spack/repos/builtin/packages/xrootd/package.py
index 5f283e28e78fa2..37255235235b37 100644
--- a/var/spack/repos/builtin/packages/xrootd/package.py
+++ b/var/spack/repos/builtin/packages/xrootd/package.py
@@ -17,6 +17,8 @@ class Xrootd(CMakePackage):
 
     maintainers("gartung", "greenc-FNAL", "marcmengel", "vitodb", "wdconinc")
 
+    version("5.6.1", sha256="9afc48ab0fb3ba69611b1edc1b682a185d49b45caf197323eecd1146d705370c")
+    version("5.6.0", sha256="cda0d32d29f94220be9b6627a80386eb33fac2dcc25c8104569eaa4ea3563009")
     version("5.5.5", sha256="0710caae527082e73d3bf8f9d1dffe95808afd3fcaaaa15ab0b937b8b226bc1f")
     version("5.5.4", sha256="41a8557ea2d118b1950282b17abea9230b252aa5ee1a5959173e2534b7d611d3")
     version("5.5.3", sha256="703829c2460204bd3c7ba8eaa23911c3c9a310f6d436211ba0af487ef7f6a980")
@@ -100,27 +102,29 @@ class Xrootd(CMakePackage):
     conflicts("cxxstd=20", when="@5.0:5.5.2")
     conflicts("cxxstd=17", when="@5 ~client_only")
     conflicts("cxxstd=20", when="@5 ~client_only")
-    conflicts("scitokens-cpp", when="@:5.5.2 +client_only")
+    conflicts("^scitokens-cpp", when="@:5.5.2 +client_only")
 
     depends_on("bzip2")
     depends_on("cmake@2.6:", type="build", when="@3.1.0:")
-    conflicts("cmake@:3.0", when="@5.0.0")
-    conflicts("cmake@:3.15.99", when="@5.5.4:")
+    depends_on("cmake@3.16:", type="build", when="@5.6:")
+    conflicts("^cmake@:3.0", when="@5.0.0")
+    conflicts("^cmake@:3.15.99", when="@5.5.4:5.5")
     depends_on("davix", when="+davix")
     depends_on("libxml2", when="+http")
     depends_on("uuid", when="@4.11.0:")
     depends_on("openssl@:1", when="@:5.4")
     depends_on("openssl")
     depends_on("python", when="+python")
-    depends_on("py-setuptools", type="build", when="+python")
+    depends_on("py-setuptools", type="build", when="@:5.5 +python")
+    depends_on("py-pip", type="build", when="@5.6: +python")
     depends_on("readline", when="+readline")
     depends_on("xz")
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("curl")
     depends_on("krb5", when="+krb5")
     depends_on("json-c")
     depends_on("scitokens-cpp", when="+scitokens-cpp")
-    conflicts("openssl@3:", when="@:5.3.99")
+    conflicts("^openssl@3:", when="@:5.3.99")
 
     extends("python", when="+python")
 
diff --git a/var/spack/repos/builtin/packages/xsdk-examples/package.py b/var/spack/repos/builtin/packages/xsdk-examples/package.py
index bf5ab4f02eb132..1f39bcada70f1f 100644
--- a/var/spack/repos/builtin/packages/xsdk-examples/package.py
+++ b/var/spack/repos/builtin/packages/xsdk-examples/package.py
@@ -18,10 +18,9 @@ class XsdkExamples(CMakePackage, CudaPackage, ROCmPackage):
 
     version("develop", branch="master")
     version("0.4.0", sha256="de54e02e0222420976a2f4cf0a6230e4bb625b443c66500fa1441032db206df9")
-    version("0.3.0", sha256="e7444a403c0a69eeeb34a4068be4d6f4e5b54cbfd275629019b9236a538a739e")
     version(
-        "0.2.0",
-        sha256="cf26e3a16a83eba6fb297fb106b0934046f17cf978f96243b44d9d17ad186db6",
+        "0.3.0",
+        sha256="e7444a403c0a69eeeb34a4068be4d6f4e5b54cbfd275629019b9236a538a739e",
         deprecated=True,
     )
 
@@ -48,7 +47,6 @@ class XsdkExamples(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("xsdk@0.7.0", when="@0.3.0")
     depends_on("xsdk@0.7.0 ^mfem+strumpack", when="@0.3.0 ^xsdk+strumpack")
     depends_on("xsdk@0.7.0 ^sundials+magma", when="@0.3.0 +cuda")
-    depends_on("xsdk@0.6.0", when="@0.2.0")
     depends_on("mpi")
     depends_on("cmake@3.21:", type="build", when="@0.3.0:")
 
diff --git a/var/spack/repos/builtin/packages/xsdk/package.py b/var/spack/repos/builtin/packages/xsdk/package.py
index 51a3c72ca11a46..3e02dbd8b36a61 100644
--- a/var/spack/repos/builtin/packages/xsdk/package.py
+++ b/var/spack/repos/builtin/packages/xsdk/package.py
@@ -86,8 +86,7 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
 
     version("develop")
     version("0.8.0")
-    version("0.7.0")
-    version("0.6.0", deprecated=True)
+    version("0.7.0", deprecated=True)
 
     variant("trilinos", default=True, sticky=True, description="Enable trilinos package build")
     variant("datatransferkit", default=True, description="Enable datatransferkit package build")
@@ -112,7 +111,6 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
     xsdk_depends_on("hypre@develop+superlu-dist+shared", when="@develop", cuda_var="cuda")
     xsdk_depends_on("hypre@2.26.0+superlu-dist+shared", when="@0.8.0", cuda_var="cuda")
     xsdk_depends_on("hypre@2.23.0+superlu-dist+shared", when="@0.7.0", cuda_var="cuda")
-    xsdk_depends_on("hypre@2.20.0+superlu-dist+shared", when="@0.6.0")
 
     xsdk_depends_on(
         "mfem@develop+shared+mpi+superlu-dist+petsc+sundials+examples+miniapps",
@@ -132,16 +130,10 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
         cuda_var="cuda",
         rocm_var="rocm",
     )
-    xsdk_depends_on(
-        "mfem@4.2.0+mpi+superlu-dist+petsc+sundials+examples+miniapps",
-        when="@0.6.0",
-        cuda_var="cuda",
-    )
 
     xsdk_depends_on("superlu-dist@develop", when="@develop")
     xsdk_depends_on("superlu-dist@8.1.2", when="@0.8.0")
     xsdk_depends_on("superlu-dist@7.1.1", when="@0.7.0")
-    xsdk_depends_on("superlu-dist@6.4.0", when="@0.6.0")
     xsdk_depends_on(
         "trilinos@develop+hypre+superlu-dist+hdf5~mumps+boost"
         + "~suite-sparse+tpetra+nox+ifpack2+zoltan+zoltan2+amesos2"
@@ -163,22 +155,13 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
         + " cxxstd=14",
         when="@0.7.0 +trilinos",
     )
-    xsdk_depends_on(
-        "trilinos@13.0.1+hypre+superlu-dist+hdf5~mumps+boost"
-        + "~suite-sparse+tpetra+nox+ifpack2+zoltan+zoltan2+amesos2"
-        + "~exodus~dtk+intrepid2+shards gotype=int"
-        + " cxxstd=11",
-        when="@0.6.0 +trilinos",
-    )
 
     xsdk_depends_on("datatransferkit@master", when="@develop +trilinos +datatransferkit")
     dtk7ver = "3.1-rc2" if sys.platform == "darwin" else "3.1-rc3"
     xsdk_depends_on("datatransferkit@" + dtk7ver, when="@0.8.0 +trilinos +datatransferkit")
     xsdk_depends_on("datatransferkit@" + dtk7ver, when="@0.7.0 +trilinos +datatransferkit")
-    xsdk_depends_on("datatransferkit@3.1-rc2", when="@0.6.0 +trilinos +datatransferkit")
 
-    xsdk_depends_on("petsc +trilinos", when="+trilinos @:0.6.0")
-    xsdk_depends_on("petsc +batch", when="@0.6.0: ^cray-mpich")
+    xsdk_depends_on("petsc +batch", when="@0.7.0: ^cray-mpich")
     xsdk_depends_on(
         "petsc@main+mpi+hypre+superlu-dist+metis+hdf5~mumps+double~int64",
         when="@develop",
@@ -196,11 +179,6 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
         when="@0.7.0",
         cuda_var="cuda",
     )
-    xsdk_depends_on(
-        "petsc@3.14.1+mpi+hypre+superlu-dist+metis+hdf5~mumps+double~int64",
-        when="@0.6.0",
-        cuda_var="cuda",
-    )
 
     xsdk_depends_on("dealii +trilinos~adol-c", when="+trilinos +dealii")
     xsdk_depends_on("dealii ~trilinos", when="~trilinos +dealii")
@@ -219,23 +197,16 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
         + "~netcdf+metis~sundials~ginkgo~symengine~simplex~arborx",
         when="@0.7.0 +dealii",
     )
-    xsdk_depends_on(
-        "dealii@9.2.0~assimp~python~doc~gmsh+petsc+slepc+mpi~int64+hdf5"
-        + "~netcdf+metis~sundials~ginkgo~symengine~simplex~arborx",
-        when="@0.6.0 +dealii",
-    )
 
     xsdk_depends_on("pflotran@develop", when="@develop")
     xsdk_depends_on("pflotran@4.0.1", when="@0.8.0")
     xsdk_depends_on("pflotran@3.0.2", when="@0.7.0")
-    xsdk_depends_on("pflotran@xsdk-0.6.0", when="@0.6.0")
 
-    xsdk_depends_on("alquimia@develop", when="@develop +alquimia")
+    xsdk_depends_on("alquimia@master", when="@develop +alquimia")
     xsdk_depends_on("alquimia@1.0.10", when="@0.8.0 +alquimia")
     xsdk_depends_on("alquimia@1.0.9", when="@0.7.0 +alquimia")
-    xsdk_depends_on("alquimia@xsdk-0.6.0", when="@0.6.0 +alquimia")
 
-    xsdk_depends_on("sundials +trilinos", when="+trilinos @0.6.0:")
+    xsdk_depends_on("sundials +trilinos", when="+trilinos @0.7.0:")
     xsdk_depends_on("sundials +ginkgo", when="+ginkgo @0.8.0:")
     xsdk_depends_on(
         "sundials@develop~int64+hypre+petsc+superlu-dist",
@@ -255,19 +226,14 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
         cuda_var="cuda",
         rocm_var="rocm",
     )
-    xsdk_depends_on(
-        "sundials@5.5.0~int64+hypre+petsc+superlu-dist", when="@0.6.0", cuda_var="cuda"
-    )
 
     xsdk_depends_on("plasma@develop:", when="@develop %gcc@6.0:")
     xsdk_depends_on("plasma@22.9.29:", when="@0.8.0 %gcc@6.0:")
     xsdk_depends_on("plasma@21.8.29:", when="@0.7.0 %gcc@6.0:")
-    xsdk_depends_on("plasma@20.9.20:", when="@0.6.0 %gcc@6.0:")
 
     xsdk_depends_on("magma@master", when="@develop", cuda_var="?cuda", rocm_var="?rocm")
     xsdk_depends_on("magma@2.7.0", when="@0.8.0", cuda_var="?cuda", rocm_var="?rocm")
     xsdk_depends_on("magma@2.6.1", when="@0.7.0", cuda_var="?cuda", rocm_var="?rocm")
-    xsdk_depends_on("magma@2.5.4", when="@0.6.0", cuda_var="?cuda")
 
     xsdk_depends_on(
         "amrex@develop+sundials", when="@develop %intel", cuda_var="cuda", rocm_var="rocm"
@@ -284,32 +250,26 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
     xsdk_depends_on("amrex@21.10+sundials", when="@0.7.0 %intel", cuda_var="cuda", rocm_var="rocm")
     xsdk_depends_on("amrex@21.10+sundials", when="@0.7.0 %gcc", cuda_var="cuda", rocm_var="rocm")
     xsdk_depends_on("amrex@21.10+sundials", when="@0.7.0 %cce", cuda_var="cuda", rocm_var="rocm")
-    xsdk_depends_on("amrex@20.10", when="@0.6.0 %intel")
-    xsdk_depends_on("amrex@20.10", when="@0.6.0 %gcc")
 
     xsdk_depends_on("slepc@main", when="@develop")
     xsdk_depends_on("slepc@3.18.1", when="@0.8.0", cuda_var="cuda", rocm_var="rocm")
     xsdk_depends_on("slepc@3.16.0", when="@0.7.0")
-    xsdk_depends_on("slepc@3.14.0", when="@0.6.0")
 
     xsdk_depends_on("omega-h +trilinos", when="+trilinos +omega-h")
     xsdk_depends_on("omega-h ~trilinos", when="~trilinos +omega-h")
     xsdk_depends_on("omega-h@main", when="@develop +omega-h")
     xsdk_depends_on("omega-h@9.34.13", when="@0.8.0 +omega-h")
     xsdk_depends_on("omega-h@9.34.1", when="@0.7.0 +omega-h")
-    xsdk_depends_on("omega-h@9.32.5", when="@0.6.0 +omega-h")
 
-    xsdk_depends_on("strumpack ~cuda", when="~cuda @0.6.0: +strumpack")
+    xsdk_depends_on("strumpack ~cuda", when="~cuda @0.7.0: +strumpack")
     xsdk_depends_on("strumpack ~slate~openmp", when="~slate @0.8.0: +strumpack")
     xsdk_depends_on("strumpack@master", when="@develop +strumpack", cuda_var=["cuda"])
     xsdk_depends_on("strumpack@7.0.1", when="@0.8.0 +strumpack", cuda_var=["cuda"])
     xsdk_depends_on("strumpack@6.1.0~slate~openmp", when="@0.7.0 +strumpack")
-    xsdk_depends_on("strumpack@5.0.0~slate~openmp", when="@0.6.0 +strumpack")
 
     xsdk_depends_on("pumi@master+shared", when="@develop")
     xsdk_depends_on("pumi@2.2.7+shared", when="@0.8.0")
     xsdk_depends_on("pumi@2.2.6", when="@0.7.0")
-    xsdk_depends_on("pumi@2.2.5", when="@0.6.0")
 
     tasmanian_openmp = "~openmp" if sys.platform == "darwin" else "+openmp"
     xsdk_depends_on(
@@ -328,11 +288,6 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
         when="@0.7.0",
         cuda_var=["cuda", "?magma"],
     )
-    xsdk_depends_on(
-        "tasmanian@7.3+xsdkflags+mpi+blas" + tasmanian_openmp,
-        when="@0.6.0",
-        cuda_var=["cuda", "?magma"],
-    )
 
     xsdk_depends_on("arborx@master", when="@develop +arborx")
     xsdk_depends_on("arborx@1.2", when="@0.8.0 +arborx")
@@ -349,7 +304,6 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
     xsdk_depends_on("phist@develop ~fortran ~scamac ~openmp ~host ~int64", when="@develop +phist")
     xsdk_depends_on("phist@1.11.2 ~fortran ~scamac ~openmp ~host ~int64", when="@0.8.0 +phist")
     xsdk_depends_on("phist@1.9.5 ~fortran ~scamac ~openmp ~host ~int64", when="@0.7.0 +phist")
-    xsdk_depends_on("phist@1.9.3 ~fortran ~scamac ~openmp ~host ~int64", when="@0.6.0 +phist")
 
     xsdk_depends_on(
         "ginkgo@develop +mpi ~openmp", when="@develop +ginkgo", cuda_var="cuda", rocm_var="rocm"
@@ -360,7 +314,6 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
     xsdk_depends_on(
         "ginkgo@1.4.0 ~openmp", when="@0.7.0 +ginkgo", cuda_var="cuda", rocm_var="rocm"
     )
-    xsdk_depends_on("ginkgo@1.3.0 ~openmp", when="@0.6.0 +ginkgo", cuda_var="cuda")
 
     xsdk_depends_on("py-libensemble@develop+petsc4py", when="@develop +libensemble")
     xsdk_depends_on("py-petsc4py@main", when="@develop +libensemble")
@@ -368,20 +321,16 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
     xsdk_depends_on("py-petsc4py@3.18.1", when="@0.8.0 +libensemble")
     xsdk_depends_on("py-libensemble@0.8.0+petsc4py", when="@0.7.0 +libensemble")
     xsdk_depends_on("py-petsc4py@3.16.1", when="@0.7.0 +libensemble")
-    xsdk_depends_on("py-libensemble@0.7.1+petsc4py", when="@0.6.0 +libensemble")
-    xsdk_depends_on("py-petsc4py@3.14.0", when="@0.6.0 +libensemble")
 
     xsdk_depends_on("precice ~petsc", when="+precice ^cray-mpich")
     xsdk_depends_on("precice@develop", when="@develop +precice")
     xsdk_depends_on("precice@2.5.0", when="@0.8.0 +precice")
     xsdk_depends_on("precice@2.3.0", when="@0.7.0 +precice")
-    xsdk_depends_on("precice@2.1.1", when="@0.6.0 +precice")
 
     bfpk_openmp = "~openmp" if sys.platform == "darwin" else "+openmp"
     xsdk_depends_on("butterflypack@master", when="@develop +butterflypack")
     xsdk_depends_on("butterflypack@2.2.2" + bfpk_openmp, when="@0.8.0 +butterflypack")
     xsdk_depends_on("butterflypack@2.0.0", when="@0.7.0 +butterflypack")
-    xsdk_depends_on("butterflypack@1.2.1", when="@0.6.0 +butterflypack")
 
     xsdk_depends_on(
         "heffte@develop+fftw",
@@ -401,12 +350,10 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
         cuda_var=["cuda", "?magma"],
         rocm_var=["rocm", "?magma"],
     )
-    xsdk_depends_on("heffte@2.0.0+fftw", when="@0.6.0 +heffte", cuda_var=["cuda", "?magma"])
 
     xsdk_depends_on("slate@master", when="@develop +slate", cuda_var="cuda")
     xsdk_depends_on("slate@2022.07.00", when="@0.8.0 +slate", cuda_var="cuda")
     xsdk_depends_on("slate@2021.05.02", when="@0.7.0 +slate %gcc@6.0:", cuda_var="cuda")
-    xsdk_depends_on("slate@2020.10.00", when="@0.6.0 +slate %gcc@6.0:", cuda_var="cuda")
 
     xsdk_depends_on("exago@develop~ipopt~hiop~python", when="@develop +exago ~raja")
     xsdk_depends_on("exago@develop~ipopt+hiop+raja", when="@develop +exago +raja", cuda_var="cuda")
diff --git a/var/spack/repos/builtin/packages/xtb/package.py b/var/spack/repos/builtin/packages/xtb/package.py
index 2bf378d6afa282..b93d10497ec650 100644
--- a/var/spack/repos/builtin/packages/xtb/package.py
+++ b/var/spack/repos/builtin/packages/xtb/package.py
@@ -23,9 +23,11 @@ class Xtb(MesonPackage):
     depends_on("meson@0.57.2:", type="build")
     depends_on("mctc-lib")
     depends_on("pkgconfig", type="build")
-    depends_on("tblite", when="@6.6.0:")
+    depends_on("tblite", when="+tblite")
+    depends_on("test-drive", type="build")
 
     variant("openmp", default=True, description="Use OpenMP parallelisation")
+    variant("tblite", default=True, when="@6.6.0:", description="Use tblite for xTB calculations")
 
     def meson_args(self):
         lapack = self.spec["lapack"].libs.names[0]
@@ -34,7 +36,7 @@ def meson_args(self):
         elif lapack != "openblas":
             lapack = "netlib"
 
-        lapack_opt = "-Dlapack={0}" if self.version > Version("6.6.0") else "-Dla_backend={0}"
+        lapack_opt = "-Dlapack={0}" if self.version >= Version("6.6.0") else "-Dla_backend={0}"
 
         return [
             lapack_opt.format(lapack),
diff --git a/var/spack/repos/builtin/packages/xv/package.py b/var/spack/repos/builtin/packages/xv/package.py
new file mode 100644
index 00000000000000..2cdfb1e9bd2cbe
--- /dev/null
+++ b/var/spack/repos/builtin/packages/xv/package.py
@@ -0,0 +1,29 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Xv(CMakePackage):
+    """
+    XV image viewer.
+    The XV software was originally written by John Bradley. John Bradley's web site for the XV software can be found at:
+    http://www.trilon.com/xv
+    """
+
+    homepage = "https://github.com/jasper-software/xv"
+    url = "https://github.com/jasper-software/xv/archive/refs/tags/v4.2.0.tar.gz"
+
+    # Licencing
+    # "... XV IS SHAREWARE FOR PERSONAL USE ONLY ..."
+    # full licencing details can be found at:
+    # https://github.com/jasper-software/xv/blob/main/src/README
+
+    version("4.2.0", sha256="2871338c517a7444fc9d6a3d146bc2c5c7bd98b50c83369b24d24ad49fa0ab87")
+
+    depends_on("libjpeg")
+    depends_on("libpng")
+    depends_on("libtiff")
+    depends_on("libx11")
diff --git a/var/spack/repos/builtin/packages/xxdiff/package.py b/var/spack/repos/builtin/packages/xxdiff/package.py
new file mode 100644
index 00000000000000..f0b2705b2951e7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/xxdiff/package.py
@@ -0,0 +1,35 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Xxdiff(MakefilePackage):
+    """Graphical File And Directories Comparator And Merge Tool."""
+
+    homepage = "https://furius.ca/xxdiff/"
+    git = "https://github.com/blais/xxdiff.git"
+
+    maintainers("vanderwb")
+
+    version("master", branch="master")
+    version("2023-01-10", commit="604300ea9875611726ba885fb14f872b964df579")
+
+    depends_on("flex@2.5.31:", type="build")
+    depends_on("bison", type="build")
+    depends_on("qt@5:", type=("build", "link", "run"))
+
+    def edit(self, spec, prefix):
+        env["QMAKE"] = "qmake"
+
+    def build(self, spec, prefix):
+        with working_dir("src"):
+            # Create the makefile
+            make("-f", "Makefile.bootstrap")
+            make()
+
+    def install(self, spec, prefix):
+        mkdir(prefix.bin)
+        install("bin/xxdiff", prefix.bin)
diff --git a/var/spack/repos/builtin/packages/xyce/450-mpich-xyce.patch b/var/spack/repos/builtin/packages/xyce/450-mpich-xyce.patch
deleted file mode 100644
index e47e273cd5ba01..00000000000000
--- a/var/spack/repos/builtin/packages/xyce/450-mpich-xyce.patch
+++ /dev/null
@@ -1,34 +0,0 @@
-From 8a622dd8204754733c769f0d13b685419e6607db Mon Sep 17 00:00:00 2001
-From: Paul Kuberry 
-Date: Wed, 12 Apr 2023 13:36:28 -0600
-Subject: [PATCH] Remove use of OpenMPI specific struct member
-
-MPITest/testBUG967 could not be built with MPICH because it used
-OpenMPI specific struct member variable `_ucount`.
-
-Replaced use of `_ucount` with call to MPI_Get_count, which is
-compatible with OpenMPI and MPICH.
----
- src/test/MPITest/testBUG967.c | 6 ++++--
- 1 file changed, 4 insertions(+), 2 deletions(-)
-
-diff --git a/src/test/MPITest/testBUG967.c b/src/test/MPITest/testBUG967.c
-index cdf7f918f..26e1b5311 100644
---- a/src/test/MPITest/testBUG967.c
-+++ b/src/test/MPITest/testBUG967.c
-@@ -45,8 +45,10 @@ int main(int narg, char**arg)
-   for (i=0; i < nrecvs; i++){
-     MPI_Wait(req + i, &status);
-     procs_from[i] = status.MPI_SOURCE;
--    printf("%d wait source %d count %lu \n",
--           my_proc, status.MPI_SOURCE, status._ucount);
-+    int count;
-+    MPI_Get_count(&status, MPI_INT, &count);
-+    printf("%d wait source %d count %d \n",
-+           my_proc, status.MPI_SOURCE, count);
-   }
- 
-   for (i = 0; i < nrecvs; i++)
--- 
-2.37.1 (Apple Git-137.1)
-
diff --git a/var/spack/repos/builtin/packages/xyce/454-oneapi-xyce.patch b/var/spack/repos/builtin/packages/xyce/454-oneapi-xyce.patch
new file mode 100644
index 00000000000000..19def5492c22e5
--- /dev/null
+++ b/var/spack/repos/builtin/packages/xyce/454-oneapi-xyce.patch
@@ -0,0 +1,34 @@
+diff --git a/src/LinearAlgebraServicesPKG/ksparse/alloc.c b/src/LinearAlgebraServicesPKG/ksparse/alloc.c
+index 320878d7817273269e8805acaa9f6f7a252443f1..af40b7e800e5e8b573a69608aae49c324f2a0253 100644
+--- a/src/LinearAlgebraServicesPKG/ksparse/alloc.c
++++ b/src/LinearAlgebraServicesPKG/ksparse/alloc.c
+@@ -53,6 +53,7 @@ Copyright 1990 Regents of the University of California.  All rights reserved.
+  */
+ 
+ void bye_bye(i)
++    int i;
+ {
+     printf ("inv = %d\n",1/i);
+ }
+
+diff --git a/cmake/tps.cmake b/cmake/tps.cmake
+index 3732758f6..b3f00bb7a 100644
+--- a/cmake/tps.cmake
++++ b/cmake/tps.cmake
+@@ -211,16 +211,6 @@ endif()
+ if (MSVC)
+      set(CMAKE_REQUIRED_DEFINITIONS "${Trilinos_CXX_COMPILER_FLAGS}")
+ endif()
+-get_target_property(CMAKE_REQUIRED_LIBRARIES Teuchos::all_libs INTERFACE_LINK_LIBRARIES)
+-
+-# Perform an initial check to see if we can compile against Trilinos at all.
+-# This could reveal compiler setup problems and/or Trilinos setup problems.
+-check_include_file_cxx(Teuchos_SerialDenseMatrix.hpp Trilinos_COMPILE_SUCCESS ${OpenMP_CXX_FLAGS})
+-if (NOT Trilinos_COMPILE_SUCCESS)
+-     message(FATAL_ERROR "Unable to compile against Trilinos. It is possible\
+-     Trilinos was not properly configured, or the environment has changed since\
+-     Trilinos was installed. See the CMake log files for more information.")
+-endif()
+ 
+ # After the release of Trilinos 12.12.1, the abstract solver interface in NOX
+ # was changed to include a new method that returns solver statistics.  This
diff --git a/var/spack/repos/builtin/packages/xyce/package.py b/var/spack/repos/builtin/packages/xyce/package.py
index e69f781504025d..743bc870857b19 100644
--- a/var/spack/repos/builtin/packages/xyce/package.py
+++ b/var/spack/repos/builtin/packages/xyce/package.py
@@ -25,6 +25,7 @@ class Xyce(CMakePackage):
     maintainers("kuberry", "tbird2001")
 
     version("master", branch="master")
+    version("7.7.0", sha256="1b95450e1905c3af3c16b42c41d5ef1f8ab0e640f48086d0cb4d52961a90a175")
     version("7.6.0", sha256="fc25557e2edc82adbe0436a15fca2929a2f9ab08ddf91f1a47aab5e8b27ec88c")
     version("7.5.0", sha256="854d7d5e19e0ee2138d1f20f10f8f27f2bebb94ec81c157040955cff7250dacd")
     version("7.4.0", sha256="2d6bc1b7377834b2e0bf50131e96728c5be83dbb3548e765bb48911067c87c91")
@@ -52,7 +53,7 @@ class Xyce(CMakePackage):
     # this defaults to 11, consistent with what will be used,
     # and produces an error if any other value is attempted
     cxxstd_choices = ["11"]
-    variant("cxxstd", default="11", values=cxxstd_choices, multi=False)
+    variant("cxxstd", default="11", description="C++ standard", values=cxxstd_choices, multi=False)
 
     variant("pymi", default=False, description="Enable Python Model Interpreter for Xyce")
     # Downstream dynamic library symbols from pip installed numpy and other
@@ -99,6 +100,7 @@ class Xyce(CMakePackage):
     # Issue #1712 forces explicitly enumerating blas packages to propagate variants
     with when("+pymi_static_tpls"):
         # BLAS
+        depends_on("blas")
         depends_on("openblas~shared", when="^openblas")
         depends_on("netlib-lapack~shared", when="^netlib-lapack~external-blas")
 
@@ -122,9 +124,9 @@ class Xyce(CMakePackage):
 
     # fix MPI issue
     patch(
-        "450-mpich-xyce.patch",
-        sha256="e91063d22afeeff01e6c572cef2ac2e3abea27b2fcb5a7e6ac5f41e4734a556d",
-        when="@:7.6,master",
+        "https://github.com/xyce/xyce/commit/2f95783637a5171a7f65f5d18c24d9a580a7f39e.patch?full_index=1",
+        sha256="1aeaac78830fbc9ae089a50ef61c6cbd89d29ead54ce7fdca258e194fa05b1a3",
+        when="@:7.6",
     )
 
     # fix RPATH issue on mac
@@ -141,6 +143,13 @@ class Xyce(CMakePackage):
         when="@7.4:7.6 +pymi",
     )
 
+    # fix oneapi issue
+    patch(
+        "454-oneapi-xyce.patch",
+        sha256="76a3ff987e43d1657f24d55cfd864b487876a72a9a7c8a37c3151a9b586a21c1",
+        when="%oneapi",
+    )
+
     def cmake_args(self):
         spec = self.spec
 
diff --git a/var/spack/repos/builtin/packages/yade/package.py b/var/spack/repos/builtin/packages/yade/package.py
new file mode 100644
index 00000000000000..95ee8f96e9b4fc
--- /dev/null
+++ b/var/spack/repos/builtin/packages/yade/package.py
@@ -0,0 +1,46 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Yade(CMakePackage):
+    """Yade is an free software for particle based simulations."""
+
+    homepage = "https://gitlab.com/yade-dev/trunk"
+    url = "https://gitlab.com/yade-dev/trunk/-/archive/2023.02a/trunk-2023.02a.tar.gz"
+
+    maintainers("lmagdanello")
+
+    version("2023.02a", sha256="f76b5a0aa7f202716efa94cd730e4bc442ffcb40a99caaf6e579ab8695efb0c1")
+    version("2022.01a", sha256="3b76185b706aba6113d1e932c5b883cd772e8d8c6b4e5230a01f3370e2b6904c")
+    version("2021.01a", sha256="3afab3380e8f5d185af7929213f63341445d6a5ee6bc21bbae102d8ffd93df1d")
+    version("2020.01a", sha256="e4856aaa0141c32404355ce9de4f25076fe3940dbc8d0fdbc8ace8020c5191f1")
+    version("2019.01a", sha256="7cb80b912cdc8752850de54ef3c084768c5ab69ce6af4b85dc3921674d7134a5")
+    version("2018.02b", sha256="d1b2ed3751cd4661af1ad4058196adb16eb227845d874e1c221074a699876634")
+    version("2018.02a", sha256="629a83ab71e2f47f2a7a83fd2c18ab5ce5573bf239445be0d4ff34ce08c11263")
+    version("2017.01a", sha256="cd35caa6b6a017ee82f894e7d6f0826fddc1d921aea04b5896d3f1da95cb649b")
+    version("2016.06a", sha256="6e7374d2dcb7c90026be9229a6b30373f9d82fdefd3dc1f952aa6262924f2579")
+
+    depends_on("cmake", type="build")
+    depends_on("gcc@11.4:", type=("build", "run"))
+    depends_on("boost@1.47:", type=("build", "run"))
+    depends_on("qt", type=("build", "run"))
+    depends_on("freeglut", type=("build", "run"))
+    depends_on("libqglviewer", type=("build", "run"))
+    depends_on("python", type=("build", "run"))
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-ipython", type=("build", "run"))
+    depends_on("py-sphinx", type=("build", "run"))
+    depends_on("py-matplotlib", type=("build", "run"))
+    depends_on("eigen@3.2.1:", type=("build", "run"))
+    depends_on("gdb", type=("build", "run"))
+    depends_on("sqlite", type=("build", "run"))
+
+    def cmake_args(self):
+        args = [self.define("CMAKE_BUILD_TYPE", "Release")]
+
+        args.append("-DCMAKE_INSTALL_PREFIX={0}".format(self.stage.source_path))
+        return args
diff --git a/var/spack/repos/builtin/packages/yaksa/package.py b/var/spack/repos/builtin/packages/yaksa/package.py
index b696c00e1263a6..5d94f42740a41d 100644
--- a/var/spack/repos/builtin/packages/yaksa/package.py
+++ b/var/spack/repos/builtin/packages/yaksa/package.py
@@ -26,6 +26,7 @@ class Yaksa(AutotoolsPackage, CudaPackage, ROCmPackage):
     url = "https://github.com/pmodels/yaksa/archive/refs/tags/v0.2.tar.gz"
     maintainers("raffenet", "yfguo", "hzhou")
 
+    version("0.3", sha256="c9e5291211bee8852831bb464f430ad5ba1541e31db5718a6fa2f2d3329fc2d9")
     version("0.2", sha256="9401cb6153dc8c34ddb9781bbabd418fd26b0a27b5da3294ecc21af7be9c86f2")
 
     depends_on("autoconf", type="build")
@@ -47,6 +48,8 @@ def configure_args(self):
             cuda_archs = spec.variants["cuda_arch"].value
             if "none" not in cuda_archs:
                 config_args.append("--with-cuda-sm={0}".format(",".join(cuda_archs)))
+            if "^cuda+allow-unsupported-compilers" in self.spec:
+                config_args.append("NVCC_FLAGS=-allow-unsupported-compiler")
 
         if "+rocm" in spec:
             config_args.append("--with-hip={0}".format(spec["hip"].prefix))
diff --git a/var/spack/repos/builtin/packages/yoda/package.py b/var/spack/repos/builtin/packages/yoda/package.py
index 6cd22a6853e091..4cf2854566a616 100644
--- a/var/spack/repos/builtin/packages/yoda/package.py
+++ b/var/spack/repos/builtin/packages/yoda/package.py
@@ -68,7 +68,7 @@ class Yoda(AutotoolsPackage):
 
     depends_on("python", type=("build", "link", "run"))
     depends_on("py-future", type=("build", "run"))
-    depends_on("zlib")
+    depends_on("zlib-api")
     depends_on("boost", when="@:1.6.0", type=("build", "run"))
 
     # TODO: replace this with an explicit list of components of Boost,
diff --git a/var/spack/repos/builtin/packages/zig/package.py b/var/spack/repos/builtin/packages/zig/package.py
index 697327a5ff91d7..785d8284210ecc 100644
--- a/var/spack/repos/builtin/packages/zig/package.py
+++ b/var/spack/repos/builtin/packages/zig/package.py
@@ -15,8 +15,11 @@ class Zig(CMakePackage):
 
     maintainers("alalazo")
 
-    version("0.10.1", tag="0.10.1")
-    version("0.9.1", tag="0.9.1", deprecated=True)
+    version("0.11.0", tag="0.11.0", commit="67709b638224ac03820226c6744d8b6ead59184c")
+    version("0.10.1", tag="0.10.1", commit="b57081f039bd3f8f82210e8896e336e3c3a6869b")
+    version(
+        "0.9.1", tag="0.9.1", commit="6d44a6222d6eba600deb7f16c124bfa30628fb60", deprecated=True
+    )
 
     variant(
         "build_type",
@@ -28,6 +31,7 @@ class Zig(CMakePackage):
     depends_on("llvm targets=all")
     depends_on("llvm@13", when="@0.9.1")
     depends_on("llvm@15", when="@0.10.1")
+    depends_on("llvm@16", when="@0.11.0")
 
     depends_on("git", type="build")
     depends_on("ccache")
diff --git a/var/spack/repos/builtin/packages/zlib-ng/package.py b/var/spack/repos/builtin/packages/zlib-ng/package.py
index a15dbdb086590d..8444736856a3c2 100644
--- a/var/spack/repos/builtin/packages/zlib-ng/package.py
+++ b/var/spack/repos/builtin/packages/zlib-ng/package.py
@@ -12,22 +12,49 @@ class ZlibNg(AutotoolsPackage, CMakePackage):
 
     homepage = "https://github.com/zlib-ng/zlib-ng"
     url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz"
+    git = "https://github.com/zlib-ng/zlib-ng.git"
 
-    version("2.1.2", sha256="383560d6b00697c04e8878e26c0187b480971a8bce90ffd26a5a7b0f7ecf1a33")
+    maintainers("haampie")
+
+    version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a")
+    version(
+        "2.1.3",
+        sha256="d20e55f89d71991c59f1c5ad1ef944815e5850526c0d9cd8e504eaed5b24491a",
+        deprecated=True,
+    )
+    version(
+        "2.1.2",
+        sha256="383560d6b00697c04e8878e26c0187b480971a8bce90ffd26a5a7b0f7ecf1a33",
+        deprecated=True,
+    )
     version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200")
     version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")
 
     variant("compat", default=True, description="Enable compatibility API")
     variant("opt", default=True, description="Enable optimizations")
 
+    provides("zlib-api", when="+compat")
+
     # Default to autotools, since cmake would result in circular dependencies if it's not
     # reused.
     build_system("autotools", "cmake", default="autotools")
 
+    # rpath shenanigans, see https://github.com/zlib-ng/zlib-ng/pull/1546
+    with when("@2.1.3"):
+        patch("pr-1546.patch", when="platform=darwin")
+        patch("pr-1542.patch")  # fix sse4.2 detection
+        patch("pr-1561.patch", when="build_system=autotools")  # drop bash dependency
+        patch("pr-1562.patch")  # improve intrinsics detection
+
     with when("build_system=cmake"):
         depends_on("cmake@3.5.1:", type="build")
         depends_on("cmake@3.14.0:", type="build", when="@2.1.0:")
 
+    @property
+    def libs(self):
+        name = "libz" if self.spec.satisfies("+compat") else "libz-ng"
+        return find_libraries(name, root=self.prefix, recursive=True, shared=True)
+
 
 class AutotoolsBuilder(autotools.AutotoolsBuilder):
     def configure_args(self):
diff --git a/var/spack/repos/builtin/packages/zlib-ng/pr-1542.patch b/var/spack/repos/builtin/packages/zlib-ng/pr-1542.patch
new file mode 100644
index 00000000000000..675c2c1a3d0ac5
--- /dev/null
+++ b/var/spack/repos/builtin/packages/zlib-ng/pr-1542.patch
@@ -0,0 +1,224 @@
+From 8c5d5eca51d9e4cd9aa046dba8f939b3f4012256 Mon Sep 17 00:00:00 2001
+From: Hans Kristian Rosbach 
+Date: Fri, 21 Jul 2023 13:43:15 +0200
+Subject: [PATCH 1/3] Clean up SSE4.2 support, and no longer use asm fallback
+ or gcc builtin.
+
+Defines changing meaning:
+X86_SSE42 used to mean the compiler supports crc asm fallback.
+X86_SSE42_CRC_INTRIN used to mean compiler supports SSE4.2 intrinsics.
+
+X86_SSE42 now means compiler supports SSE4.2 intrinsics.
+
+This therefore also fixes the adler32_sse42 checks, since those were depending
+on SSE4.2 intrinsics but was mistakenly checking the X86_SSE42 define.
+Now the X86_SSE42 define actually means what it appears to.
+---
+ CMakeLists.txt                 |  5 +----
+ arch/x86/insert_string_sse42.c | 36 +++++----------------------------
+ cmake/detect-intrinsics.cmake  | 23 +++------------------
+ configure                      | 37 ++++++++--------------------------
+ win32/Makefile.msc             |  1 -
+ 5 files changed, 17 insertions(+), 85 deletions(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 420a5c78..1e42239a 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -827,15 +827,12 @@ if(WITH_OPTIM)
+         endif()
+         if(WITH_SSE42)
+             check_sse42_intrinsics()
+-            if(HAVE_SSE42CRC_INLINE_ASM OR HAVE_SSE42CRC_INTRIN)
++            if(HAVE_SSE42_INTRIN)
+                 add_definitions(-DX86_SSE42)
+                 set(SSE42_SRCS ${ARCHDIR}/adler32_sse42.c ${ARCHDIR}/insert_string_sse42.c)
+                 add_feature_info(SSE42_CRC 1 "Support SSE4.2 optimized CRC hash generation, using \"${SSE42FLAG}\"")
+                 list(APPEND ZLIB_ARCH_SRCS ${SSE42_SRCS})
+                 set_property(SOURCE ${SSE42_SRCS} PROPERTY COMPILE_FLAGS "${SSE42FLAG} ${NOLTOFLAG}")
+-                if(HAVE_SSE42CRC_INTRIN)
+-                    add_definitions(-DX86_SSE42_CRC_INTRIN)
+-                endif()
+             else()
+                 set(WITH_SSE42 OFF)
+             endif()
+diff --git a/arch/x86/insert_string_sse42.c b/arch/x86/insert_string_sse42.c
+index 565d92f9..ae092a7e 100644
+--- a/arch/x86/insert_string_sse42.c
++++ b/arch/x86/insert_string_sse42.c
+@@ -5,38 +5,13 @@
+  *
+  */
+ 
++#ifdef X86_SSE42
+ #include "../../zbuild.h"
+-#include 
+-#ifdef _MSC_VER
+-#  include 
+-#endif
++#include 
+ #include "../../deflate.h"
+ 
+-#ifdef X86_SSE42_CRC_INTRIN
+-#  ifdef _MSC_VER
+-#    define HASH_CALC(s, h, val)\
+-        h = _mm_crc32_u32(h, val)
+-#  else
+-#    define HASH_CALC(s, h, val)\
+-        h = __builtin_ia32_crc32si(h, val)
+-#  endif
+-#else
+-#  ifdef _MSC_VER
+-#    define HASH_CALC(s, h, val) {\
+-        __asm mov edx, h\
+-        __asm mov eax, val\
+-        __asm crc32 eax, edx\
+-        __asm mov h, eax\
+-    }
+-#  else
+-#    define HASH_CALC(s, h, val) \
+-        __asm__ __volatile__ (\
+-            "crc32 %1,%0\n\t"\
+-            : "+r" (h)\
+-            : "r" (val)\
+-        );
+-#  endif
+-#endif
++#define HASH_CALC(s, h, val)\
++    h = _mm_crc32_u32(h, val)
+ 
+ #define HASH_CALC_VAR       h
+ #define HASH_CALC_VAR_INIT  uint32_t h = 0
+@@ -45,6 +20,5 @@
+ #define INSERT_STRING       insert_string_sse42
+ #define QUICK_INSERT_STRING quick_insert_string_sse42
+ 
+-#ifdef X86_SSE42
+-#  include "../../insert_string_tpl.h"
++#include "../../insert_string_tpl.h"
+ #endif
+diff --git a/cmake/detect-intrinsics.cmake b/cmake/detect-intrinsics.cmake
+index 9cbc5908..52c54dc8 100644
+--- a/cmake/detect-intrinsics.cmake
++++ b/cmake/detect-intrinsics.cmake
+@@ -481,35 +481,18 @@ macro(check_sse42_intrinsics)
+             set(SSE42FLAG "-msse4.2")
+         endif()
+     endif()
+-    # Check whether compiler supports SSE4.2 CRC inline asm
++    # Check whether compiler supports SSE4.2 intrinsics
+     set(CMAKE_REQUIRED_FLAGS "${SSE42FLAG} ${NATIVEFLAG}")
+     check_c_source_compile_or_run(
+-        "int main(void) {
+-            unsigned val = 0, h = 0;
+-        #if defined(_MSC_VER)
+-            { __asm mov edx, h __asm mov eax, val __asm crc32 eax, edx __asm mov h, eax }
+-        #else
+-            __asm__ __volatile__ ( \"crc32 %1,%0\" : \"+r\" (h) : \"r\" (val) );
+-        #endif
+-            return (int)h;
+-        }"
+-        HAVE_SSE42CRC_INLINE_ASM
+-    )
+-    # Check whether compiler supports SSE4.2 CRC intrinsics
+-    check_c_source_compile_or_run(
+-        "#include 
++        "#include 
+         int main(void) {
+             unsigned crc = 0;
+             char c = 'c';
+-        #if defined(_MSC_VER)
+             crc = _mm_crc32_u32(crc, c);
+-        #else
+-            crc = __builtin_ia32_crc32qi(crc, c);
+-        #endif
+             (void)crc;
+             return 0;
+         }"
+-        HAVE_SSE42CRC_INTRIN
++        HAVE_SSE42_INTRIN
+     )
+     set(CMAKE_REQUIRED_FLAGS)
+ endmacro()
+diff --git a/configure b/configure
+index 8714590e..6b4e7fff 100755
+--- a/configure
++++ b/configure
+@@ -1431,38 +1431,23 @@ EOF
+ }
+ 
+ check_sse42_intrinsics() {
+-    # Check whether compiler supports SSE4.2 CRC inline asm
+-    cat > $test.c << EOF
+-int main(void) {
+-    unsigned val = 0, h = 0;
+-    __asm__ __volatile__ ( "crc32 %1,%0" : "+r" (h) : "r" (val) );
+-    return (int) h;
+-}
+-EOF
+-    if try ${CC} ${CFLAGS} ${sse42flag} $test.c; then
+-        echo "Checking for SSE4.2 CRC inline assembly ... Yes." | tee -a configure.log
+-        HAVE_SSE42CRC_INLINE_ASM=1
+-    else
+-        echo "Checking for SSE4.2 CRC inline assembly ... No." | tee -a configure.log
+-        HAVE_SSE42CRC_INLINE_ASM=0
+-    fi
+-
+-    # Check whether compiler supports SSE4.2 CRC intrinsics
++    # Check whether compiler supports SSE4.2 intrinsics
+     cat > $test.c << EOF
++#include 
+ int main(void) {
+     unsigned crc = 0;
+     char c = 'c';
+-    crc = __builtin_ia32_crc32qi(crc, c);
++    crc = _mm_crc32_u32(crc, c);
+     (void)crc;
+     return 0;
+ }
+ EOF
+     if try ${CC} ${CFLAGS} ${sse42flag} $test.c; then
+-        echo "Checking for SSE4.2 CRC intrinsics ... Yes." | tee -a configure.log
+-        HAVE_SSE42CRC_INTRIN=1
++        echo "Checking for SSE4.2 intrinsics ... Yes." | tee -a configure.log
++        HAVE_SSE42_INTRIN=1
+     else
+-        echo "Checking for SSE4.2 CRC intrinsics ... No." | tee -a configure.log
+-        HAVE_SSE42CRC_INTRIN=0
++        echo "Checking for SSE4.2 intrinsics ... No." | tee -a configure.log
++        HAVE_SSE42_INTRIN=0
+     fi
+ }
+ 
+@@ -1606,15 +1591,9 @@ case "${ARCH}" in
+ 
+             check_sse42_intrinsics
+ 
+-            if test ${HAVE_SSE42CRC_INTRIN} -eq 1 || test ${HAVE_SSE42CRC_INLINE_ASM} -eq 1; then
++            if test ${HAVE_SSE42_INTRIN} -eq 1; then
+                 CFLAGS="${CFLAGS} -DX86_SSE42"
+                 SFLAGS="${SFLAGS} -DX86_SSE42"
+-
+-                if test ${HAVE_SSE42CRC_INTRIN} -eq 1; then
+-                  CFLAGS="${CFLAGS} -DX86_SSE42_CRC_INTRIN"
+-                  SFLAGS="${SFLAGS} -DX86_SSE42_CRC_INTRIN"
+-                fi
+-
+                 ARCH_STATIC_OBJS="${ARCH_STATIC_OBJS} adler32_sse42.o insert_string_sse42.o"
+                 ARCH_SHARED_OBJS="${ARCH_SHARED_OBJS} adler32_sse42.lo insert_string_sse42.lo"
+             fi
+diff --git a/win32/Makefile.msc b/win32/Makefile.msc
+index 9ed26f28..3035072b 100644
+--- a/win32/Makefile.msc
++++ b/win32/Makefile.msc
+@@ -31,7 +31,6 @@ WFLAGS  = \
+ 	-DX86_PCLMULQDQ_CRC \
+ 	-DX86_SSE2 \
+ 	-DX86_SSE42 \
+-	-DX86_SSE42_CRC_INTRIN \
+ 	-DX86_SSSE3 \
+ 	-DX86_AVX2
+ 
+-- 
+2.39.2
+
diff --git a/var/spack/repos/builtin/packages/zlib-ng/pr-1546.patch b/var/spack/repos/builtin/packages/zlib-ng/pr-1546.patch
new file mode 100644
index 00000000000000..8966cdd36bf439
--- /dev/null
+++ b/var/spack/repos/builtin/packages/zlib-ng/pr-1546.patch
@@ -0,0 +1,182 @@
+From 101bc10dcef8d2cb45a51534180c7134001b07a9 Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels 
+Date: Mon, 24 Jul 2023 09:37:57 +0200
+Subject: [PATCH 1/3] Relative paths CMAKE_INSTALL_*, absolute paths
+ CMAKE_INSTALL_FULL_*; use @rpath/libname as install name on macOS
+
+---
+ CMakeLists.txt   | 46 ++--------------------------------------------
+ configure        |  7 +------
+ test/pkgcheck.sh |  2 +-
+ 3 files changed, 4 insertions(+), 51 deletions(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 003e66db7..46b85bc48 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -951,36 +951,8 @@ if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
+     endif()
+ endif()
+ 
+-# Refer to prefix symbolically to ease relocation by end user,
+-# as Makefile-generated .pc file does.
+-string(FIND "${CMAKE_INSTALL_INCLUDEDIR}" "${CMAKE_INSTALL_PREFIX}/" INCLUDEDIR_POS)
+-string(FIND "${CMAKE_INSTALL_LIBDIR}" "${CMAKE_INSTALL_PREFIX}/" LIBDIR_POS)
+-string(LENGTH "${CMAKE_INSTALL_PREFIX}/" INSTALL_PREFIX_LEN)
+-
+-if(NOT IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}")
+-  set(PC_INC_INSTALL_DIR "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}")
+-elseif(INCLUDEDIR_POS EQUAL 0)
+-  string(SUBSTRING "${CMAKE_INSTALL_INCLUDEDIR}" "${INSTALL_PREFIX_LEN}" "-1" INCLUDEDIR_RELATIVE)
+-  set(PC_INC_INSTALL_DIR "\${prefix}/${INCLUDEDIR_RELATIVE}")
+-else()
+-  set(PC_INC_INSTALL_DIR "${CMAKE_INSTALL_INCLUDEDIR}")
+-endif()
+-
+-if(APPLE)
+-  option(WITH_RPATH "Enable RPATH for shared library" OFF)
+-endif()
+-if(NOT IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}")
+-  if(APPLE AND WITH_RPATH)
+-    set(PC_LIB_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}")
+-  else()
+-    set(PC_LIB_INSTALL_DIR "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}")
+-  endif()
+-elseif(LIBDIR_POS EQUAL 0)
+-  string(SUBSTRING "${CMAKE_INSTALL_LIBDIR}" "${INSTALL_PREFIX_LEN}" "-1" LIBDIR_RELATIVE)
+-  set(PC_LIB_INSTALL_DIR "\${exec_prefix}/${LIBDIR_RELATIVE}")
+-else()
+-  set(PC_LIB_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}")
+-endif()
++set(PC_INC_INSTALL_DIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
++set(PC_LIB_INSTALL_DIR "${CMAKE_INSTALL_FULL_LIBDIR}")
+ 
+ #============================================================================
+ # zlib
+@@ -1142,11 +1114,6 @@ if(NOT DEFINED BUILD_SHARED_LIBS OR BUILD_SHARED_LIBS)
+         if(NOT APPLE)
+             set_target_properties(zlib PROPERTIES LINK_FLAGS
+                 "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib${SUFFIX}.map\"")
+-        elseif(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}" OR NOT WITH_RPATH)
+-            # Match configure/make's behavior (i.e. don't use @rpath on mac when using absolute path).
+-            set_target_properties(zlib PROPERTIES INSTALL_NAME_DIR "@rpath/${CMAKE_INSTALL_FULL_LIBDIR}")
+-        else()
+-            set_target_properties(zlib PROPERTIES INSTALL_NAME_DIR "@rpath/${CMAKE_INSTALL_LIBDIR}")
+         endif()
+     endif()
+     if(MSYS)
+@@ -1183,11 +1150,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zlib${SUFFIX}.h.in
+ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gzread.c.in
+     ${CMAKE_CURRENT_BINARY_DIR}/gzread.c @ONLY)
+ 
+-# Fix install directory after generating zlib.pc/zlib-ng.pc
+-if (NOT IS_ABSOLUTE CMAKE_INSTALL_LIBDIR AND WITH_RPATH)
+-    set(CMAKE_INSTALL_LIBDIR "/${CMAKE_INSTALL_LIBDIR}")
+-endif()
+-
+ if (NOT ZLIB_SYMBOL_PREFIX STREQUAL "")
+     add_feature_info(ZLIB_SYMBOL_PREFIX ON "Publicly exported symbols have a custom prefix")
+     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zlib_name_mangling${SUFFIX}.h.in
+@@ -1286,8 +1248,4 @@ endif()
+ 
+ add_feature_info(INSTALL_UTILS INSTALL_UTILS "Copy minigzip and minideflate during install")
+ 
+-if(APPLE)
+-    add_feature_info(WITH_RPATH WITH_RPATH "Enable RPATH for shared library")
+-endif()
+-
+ FEATURE_SUMMARY(WHAT ALL INCLUDE_QUIET_PACKAGES)
+diff --git a/configure b/configure
+index 29a64badf..6881f7af5 100755
+--- a/configure
++++ b/configure
+@@ -513,12 +513,7 @@ if test "$gcc" -eq 1 && ($cc $CFLAGS -c $test.c) >> configure.log 2>&1; then
+              SHAREDLIBM=${LIBNAME}.$VER1$shared_ext
+              SHAREDTARGET=$SHAREDLIBV
+              LDSHARED=${LDSHARED-"$cc"}
+-             case ${libdir} in
+-             /*)
+-                 LDSHAREDFLAGS="-dynamiclib -install_name ${libdir}/${SHAREDLIBM} -compatibility_version ${VER1} -current_version ${VER3}" ;;
+-             *)
+-                 LDSHAREDFLAGS="-dynamiclib -install_name @rpath/${libdir}/${SHAREDLIBM} -compatibility_version ${VER1} -current_version ${VER3}" ;;
+-             esac
++             LDSHAREDFLAGS="-dynamiclib -install_name @rpath/${SHAREDLIBM} -compatibility_version ${VER1} -current_version ${VER3}"
+              if libtool -V 2>&1 | grep Apple > /dev/null; then
+                  AR="libtool"
+              else
+diff --git a/test/pkgcheck.sh b/test/pkgcheck.sh
+index 629f98ade..6641b300a 100644
+--- a/test/pkgcheck.sh
++++ b/test/pkgcheck.sh
+@@ -81,7 +81,7 @@ Darwin)
+   sysctl -n machdep.cpu.features
+   sysctl -n machdep.cpu.leaf7_features
+   sysctl -n machdep.cpu.extfeatures
+-  CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib -DPKGCONFIG_INSTALL_DIR=/lib/pkgconfig -DWITH_RPATH=on ${CMAKE_ARGS}"
++  CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
+   CONFIGURE_ARGS="--libdir=lib ${CONFIGURE_ARGS}"
+   ;;
+ *)
+
+From c6fbd1459eda454402c3450c670b7e538233b8d4 Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels 
+Date: Mon, 24 Jul 2023 11:37:31 +0200
+Subject: [PATCH 2/3] relocatable pc files by default, while allowing
+ discouraged absolute paths for CMAKE_INSTALL_[LIB|INCLUDE]DIR
+
+---
+ CMakeLists.txt | 15 +++++++++++++--
+ 1 file changed, 13 insertions(+), 2 deletions(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 46b85bc48..1894d10d5 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -951,8 +951,19 @@ if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
+     endif()
+ endif()
+ 
+-set(PC_INC_INSTALL_DIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
+-set(PC_LIB_INSTALL_DIR "${CMAKE_INSTALL_FULL_LIBDIR}")
++# The user is allowed (but discouraged) to set absolute CMAKE_INSTALL_*DIR paths.
++# If they do, we copy these non-relocatable paths into the pkg-config file.
++if(IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}")
++    set(PC_INC_INSTALL_DIR "${CMAKE_INSTALL_INCLUDEDIR}")
++else()
++    set(PC_INC_INSTALL_DIR "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}")
++endif()
++
++if(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}")
++    set(PC_LIB_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}")
++else()
++    set(PC_LIB_INSTALL_DIR "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}")
++endif()
+ 
+ #============================================================================
+ # zlib
+
+From 0bae8f7d8379e5863b7c9328f0d71a87a33ca02f Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels 
+Date: Mon, 24 Jul 2023 10:20:32 +0200
+Subject: [PATCH 3/3] Remove relative configure --lib
+
+---
+ test/pkgcheck.sh | 5 -----
+ 1 file changed, 5 deletions(-)
+
+diff --git a/test/pkgcheck.sh b/test/pkgcheck.sh
+index 6641b300a..94a67a191 100644
+--- a/test/pkgcheck.sh
++++ b/test/pkgcheck.sh
+@@ -81,11 +81,6 @@ Darwin)
+   sysctl -n machdep.cpu.features
+   sysctl -n machdep.cpu.leaf7_features
+   sysctl -n machdep.cpu.extfeatures
+-  CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
+-  CONFIGURE_ARGS="--libdir=lib ${CONFIGURE_ARGS}"
+-  ;;
+-*)
+-  CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
+   ;;
+ esac
+ 
diff --git a/var/spack/repos/builtin/packages/zlib-ng/pr-1561.patch b/var/spack/repos/builtin/packages/zlib-ng/pr-1561.patch
new file mode 100644
index 00000000000000..f87aea684bd916
--- /dev/null
+++ b/var/spack/repos/builtin/packages/zlib-ng/pr-1561.patch
@@ -0,0 +1,210 @@
+From f6fb1d350a7b8210cc9c45ed502b3cc34e4dac32 Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels 
+Date: Mon, 21 Aug 2023 10:13:10 +0200
+Subject: [PATCH 2/3] PR #1561
+
+---
+ configure | 50 +++++++++++++++++++++++++-------------------------
+ 1 file changed, 25 insertions(+), 25 deletions(-)
+
+diff --git a/configure b/configure
+index 6b4e7fff..fc78a135 100755
+--- a/configure
++++ b/configure
+@@ -1,4 +1,4 @@
+-#!/usr/bin/env bash
++#!/bin/sh
+ # configure script for zlib.
+ #
+ # Normally configure builds both a static and a shared library.
+@@ -230,13 +230,13 @@ test=ztest$$
+ # put arguments in log, also put test file in log if used in arguments
+ show()
+ {
+-  case "$*" in
++  case "$@" in
+     *$test.c*)
+       echo "=== $test.c ===" >> configure.log
+       cat $test.c >> configure.log
+       echo "===" >> configure.log;;
+   esac
+-  echo $* >> configure.log
++  echo "$@" >> configure.log
+ }
+ 
+ # check for gcc vs. cc and set compile and link flags based on the system identified by uname
+@@ -246,7 +246,7 @@ int main() {return getchar();}
+ EOF
+ 
+ cc=${CC-${CROSS_PREFIX}gcc}
+-echo -n "Checking for compiler... " | tee -a configure.log
++printf "Checking for compiler... " | tee -a configure.log
+ case "$cc" in
+   *gcc*) gcc=1 ;;
+   *clang*) gcc=1 ;;
+@@ -401,7 +401,7 @@ if test "$gcc" -eq 1 && ($cc $CFLAGS -c $test.c) >> configure.log 2>&1; then
+     SFLAGS="${SFLAGS} -DNDEBUG"
+   fi
+   if test -z "$uname"; then
+-    uname=$((uname -s || echo unknown) 2>/dev/null)
++    uname=$( (uname -s || echo unknown) 2>/dev/null)
+   fi
+   case "$uname" in
+   Linux* | linux* | GNU | GNU/* | solaris*)
+@@ -483,7 +483,7 @@ if test "$gcc" -eq 1 && ($cc $CFLAGS -c $test.c) >> configure.log 2>&1; then
+         fi
+         RC="${CROSS_PREFIX}windres"
+         RCFLAGS="-I ${BUILDDIR}"
+-        if [ "$CC" == "mingw32-gcc" ]; then
++        if [ "$CC" = "mingw32-gcc" ]; then
+           case $ARCH in
+           i386 | i486 | i586 | i686) RCFLAGS="${RCFLAGS} -F pe-i386";;
+           esac;
+@@ -498,7 +498,7 @@ if test "$gcc" -eq 1 && ($cc $CFLAGS -c $test.c) >> configure.log 2>&1; then
+   HP-UX*)
+          LDSHARED=${LDSHARED-"$cc"}
+          LDSHAREDFLAGS="-shared"
+-         case $((uname -m || echo unknown) 2>/dev/null) in
++         case $( (uname -m || echo unknown) 2>/dev/null) in
+          ia64)
+                  shared_ext='.so'
+                  SHAREDLIB='${LIBNAME}.so' ;;
+@@ -539,14 +539,14 @@ else
+   gcc=0
+   echo "$CC" | tee -a configure.log
+   if test -z "$uname"; then
+-    uname=$((uname -sr || echo unknown) 2>/dev/null)
++    uname=$( (uname -sr || echo unknown) 2>/dev/null)
+   fi
+   case "$uname" in
+   HP-UX*)    SFLAGS=${CFLAGS-"-O +z"}
+              CFLAGS=${CFLAGS-"-O"}
+              LDSHARED=${LDSHARED-"ld"}
+              LDSHAREDFLAGS="-b"
+-         case $((uname -m || echo unknown) 2>/dev/null) in
++         case $( (uname -m || echo unknown) 2>/dev/null) in
+          ia64)
+              shared_ext='.so'
+              SHAREDLIB='${LIBNAME}.so' ;;
+@@ -591,15 +591,15 @@ EOF
+ if ($CC -c $CFLAGS $test.c) 2>/dev/null; then
+   try()
+   {
+-    show $*
+-    test "$(\( $* \) 2>&1 | tee -a configure.log)" = ""
++    show "$@"
++    test "$( ("$@") 2>&1 | tee -a configure.log)" = ""
+   }
+   echo - using any output from compiler to indicate an error >> configure.log
+ else
+   try()
+   {
+-    show $*
+-    ( $* ) >> configure.log 2>&1
++    show "$@"
++    ( "$@" ) >> configure.log 2>&1
+     ret=$?
+     if test $ret -ne 0; then
+       echo "(exit code $ret)" >> configure.log
+@@ -627,7 +627,7 @@ extern int getchar();
+ int hello() {return getchar();}
+ EOF
+ if test $shared -eq 1; then
+-  echo -n "Checking for shared library support... " | tee -a configure.log
++  printf "Checking for shared library support... " | tee -a configure.log
+   # we must test in two steps (cc then ld), required at least on SunOS 4.x
+   if try $CC -w -c $SFLAGS $test.c &&
+      try $LDSHARED $LDSHAREDFLAGS $LDFLAGS -o $test$shared_ext $test.o $LDSHAREDLIBC; then
+@@ -784,7 +784,7 @@ fi
+ # Rename @ZLIB_SYMBOL_PREFIX@ to $symbol_prefix in gzread.c, zlib.h and zlib_name_mangling.h
+ sed < $SRCDIR/gzread.c.in "s/@ZLIB_SYMBOL_PREFIX@/$symbol_prefix/g" > gzread.c
+ sed < $SRCDIR/zlib${SUFFIX}.h.in "s/@ZLIB_SYMBOL_PREFIX@/$symbol_prefix/g" > zlib${SUFFIX}.h
+-if [[ ! -z $symbol_prefix ]]; then
++if [ ! -z "$symbol_prefix" ]; then
+   sed < $SRCDIR/zlib_name_mangling${SUFFIX}.h.in "s/@ZLIB_SYMBOL_PREFIX@/$symbol_prefix/g" > zlib_name_mangling${SUFFIX}.h
+ else
+   # symbol_prefix is not set, copy the empty mangling header
+@@ -814,7 +814,7 @@ fi
+ echo >> configure.log
+ 
+ # check for ptrdiff_t and save result in zconf.h
+-echo -n "Checking for ptrdiff_t... " | tee -a configure.log
++printf "Checking for ptrdiff_t... " | tee -a configure.log
+ cat > $test.c <
+ int fun(ptrdiff_t *a) { (void)a; return 0; }
+@@ -826,7 +826,7 @@ else
+     sed < zconf${SUFFIX}.h "/^#ifdef NEED_PTRDIFF_T.* may be/s/def NEED_PTRDIFF_T\(.*\) may be/ 1\1 was/" > zconf${SUFFIX}.temp.h
+     mv zconf${SUFFIX}.temp.h zconf${SUFFIX}.h
+ 
+-    echo -n "Checking for sizeof(void *)... " | tee -a configure.log
++    printf "Checking for sizeof(void *)... " | tee -a configure.log
+     cat > $test.c <
+ #define COMPILE_TIME_ASSERT(pred) struct s { int x: (pred) ? 1 : -1; }
+@@ -864,7 +864,7 @@ if test $compat -eq 1; then
+   esac
+ fi
+ 
+-if [[ ! -z $DEFFILE ]]; then
++if [ ! -z "$DEFFILE" ]; then
+   mkdir -p win32
+   sed < $SRCDIR/$DEFFILE.in "s/@ZLIB_SYMBOL_PREFIX@/$symbol_prefix/g" > $DEFFILE
+ fi
+@@ -1476,14 +1476,14 @@ EOF
+ 
+ check_vgfma_intrinsics() {
+     # Check whether "VECTOR GALOIS FIELD MULTIPLY SUM AND ACCUMULATE" intrinsic is available
+-    echo -n "Checking for -mzarch... " | tee -a configure.log
++    printf "Checking for -mzarch... " | tee -a configure.log
+     if try $CC -x c -c /dev/null -o /dev/null -mzarch; then
+         echo Yes. | tee -a configure.log
+         vgfmaflag="${vgfmaflag} -mzarch"
+     else
+         echo No. | tee -a configure.log
+     fi
+-    echo -n "Checking for -fzvector... " | tee -a configure.log
++    printf "Checking for -fzvector... " | tee -a configure.log
+     if try $CC -x c -c /dev/null -o /dev/null -fzvector; then
+         echo Yes. | tee -a configure.log
+         vgfmaflag="${vgfmaflag} -fzvector"
+@@ -1500,7 +1500,7 @@ int main(void) {
+     return c[0];
+ }
+ EOF
+-    echo -n "Checking for VGFMA support... " | tee -a configure.log
++    printf "Checking for VGFMA support... " | tee -a configure.log
+     if try $CC -c $CFLAGS $vgfmaflag $test.c; then
+         HAVE_VGFMA_INTRIN=1
+         echo "Yes." | tee -a configure.log
+@@ -2156,11 +2156,11 @@ for file in $SRCDIR/*.c $SRCDIR/test/*.c $SRCDIR/test/fuzz/*.c $SRCDIR/$ARCHDIR/
+                    # Check that the include file exists in the current dir,
+                    # otherwise it may be one of the system include header.
+                    if test -e $SRCDIR/$i; then
+-                       echo -n " \$(SRCDIR)/$i"
++                       printf " \$(SRCDIR)/$i"
+                    fi
+                    # We also need to check whether the include file is in the ARCHDIR.
+                    if test -e $SRCDIR/$ARCHDIR/$i; then
+-                       echo -n " \$(SRCDIR)/$ARCHDIR/$i"
++                       printf " \$(SRCDIR)/$ARCHDIR/$i"
+                    fi
+                done)
+     obj=$(basename $(echo $file | sed -e 's/\.c/\.o/g' -e 's#^\./##g'))
+@@ -2233,11 +2233,11 @@ for file in $SRCDIR/$ARCHDIR/*.c; do
+                    # Check that the include file exists in the current dir,
+                    # otherwise it may be one of the system include header.
+                    if test -e $SRCDIR/$i; then
+-                       echo -n " \$(SRCTOP)/$i"
++                       printf " \$(SRCTOP)/$i"
+                    fi
+                    # We also need to check whether the include file is in the ARCHDIR.
+                    if test -e $SRCDIR/$ARCHDIR/$i; then
+-                       echo -n " \$(SRCDIR)/$i"
++                       printf " \$(SRCDIR)/$i"
+                    fi
+                done)
+     obj=$(basename $(echo $file | sed -e 's/\.c/\.o/g' -e 's#^\./##g'))
+-- 
+2.39.2
+
diff --git a/var/spack/repos/builtin/packages/zlib-ng/pr-1562.patch b/var/spack/repos/builtin/packages/zlib-ng/pr-1562.patch
new file mode 100644
index 00000000000000..70806e900c36c5
--- /dev/null
+++ b/var/spack/repos/builtin/packages/zlib-ng/pr-1562.patch
@@ -0,0 +1,432 @@
+From 13df84c54aaf06cc7aeb1813ef60b17591d29ea3 Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels 
+Date: Mon, 21 Aug 2023 11:10:29 +0200
+Subject: [PATCH 3/3] PR #1562
+
+---
+ cmake/detect-intrinsics.cmake | 118 ++++++++++++----------------------
+ configure                     | 115 ++++++++++-----------------------
+ 2 files changed, 73 insertions(+), 160 deletions(-)
+
+diff --git a/cmake/detect-intrinsics.cmake b/cmake/detect-intrinsics.cmake
+index 52c54dc8..d476093f 100644
+--- a/cmake/detect-intrinsics.cmake
++++ b/cmake/detect-intrinsics.cmake
+@@ -62,28 +62,19 @@ macro(check_avx512_intrinsics)
+     set(CMAKE_REQUIRED_FLAGS "${AVX512FLAG} ${NATIVEFLAG}")
+     check_c_source_compile_or_run(
+         "#include 
+-        int main(void) {
+-            __m512i x = _mm512_set1_epi8(2);
+-            const __m512i y = _mm512_set_epi32(0x1020304, 0x5060708, 0x90a0b0c, 0xd0e0f10,
+-                                               0x11121314, 0x15161718, 0x191a1b1c, 0x1d1e1f20,
+-                                               0x21222324, 0x25262728, 0x292a2b2c, 0x2d2e2f30,
+-                                               0x31323334, 0x35363738, 0x393a3b3c, 0x3d3e3f40);
+-            x = _mm512_sub_epi8(x, y);
+-            (void)x;
+-            return 0;
+-        }"
++        __m512i f(__m512i y) {
++          __m512i x = _mm512_set1_epi8(2);
++          return _mm512_sub_epi8(x, y);
++        }
++        int main(void) { return 0; }"
+         HAVE_AVX512_INTRIN
+     )
+ 
+     # Evidently both GCC and clang were late to implementing these
+     check_c_source_compile_or_run(
+         "#include 
+-        int main(void) {
+-            __mmask16 a = 0xFF;
+-            a = _knot_mask16(a);
+-            (void)a;
+-            return 0;
+-        }"
++        __mmask16 f(__mmask16 x) { return _knot_mask16(x); }
++        int main(void) { return 0; }"
+         HAVE_MASK_INTRIN
+     )
+     set(CMAKE_REQUIRED_FLAGS)
+@@ -117,17 +108,11 @@ macro(check_avx512vnni_intrinsics)
+     set(CMAKE_REQUIRED_FLAGS "${AVX512VNNIFLAG} ${NATIVEFLAG}")
+     check_c_source_compile_or_run(
+         "#include 
+-        int main(void) {
+-            __m512i x = _mm512_set1_epi8(2);
+-            const __m512i y = _mm512_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+-                                              20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37,
+-                                              38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55,
+-                                              56, 57, 58, 59, 60, 61, 62, 63, 64);
++        __m512i f(__m512i x, __m512i y) {
+             __m512i z = _mm512_setzero_epi32();
+-            z = _mm512_dpbusd_epi32(z, x, y);
+-            (void)z;
+-            return 0;
+-        }"
++            return _mm512_dpbusd_epi32(z, x, y);
++        }
++        int main(void) { return 0; }"
+         HAVE_AVX512VNNI_INTRIN
+     )
+     set(CMAKE_REQUIRED_FLAGS)
+@@ -151,13 +136,11 @@ macro(check_avx2_intrinsics)
+     set(CMAKE_REQUIRED_FLAGS "${AVX2FLAG} ${NATIVEFLAG}")
+     check_c_source_compile_or_run(
+         "#include 
+-        int main(void) {
+-            __m256i x = _mm256_set1_epi16(2);
++        __m256i f(__m256i x) {
+             const __m256i y = _mm256_set1_epi16(1);
+-            x = _mm256_subs_epu16(x, y);
+-            (void)x;
+-            return 0;
+-        }"
++            return _mm256_subs_epu16(x, y);
++        }
++        int main(void) { return 0; }"
+         HAVE_AVX2_INTRIN
+     )
+     set(CMAKE_REQUIRED_FLAGS)
+@@ -204,12 +187,8 @@ macro(check_neon_ld4_intrinsics)
+         #else
+         #  include 
+         #endif
+-        int main(void) {
+-            int stack_var[16];
+-            int32x4x4_t v = vld1q_s32_x4(stack_var);
+-            (void)v;
+-            return 0;
+-        }"
++        int32x4x4_t f(int var[16]) { return vld1q_s32_x4(var); }
++        int main(void) { return 0; }"
+         NEON_HAS_LD4)
+     set(CMAKE_REQUIRED_FLAGS)
+ endmacro()
+@@ -226,13 +205,9 @@ macro(check_pclmulqdq_intrinsics)
+         set(CMAKE_REQUIRED_FLAGS "${PCLMULFLAG} ${NATIVEFLAG}")
+         check_c_source_compile_or_run(
+             "#include 
+-            int main(void) {
+-                __m128i a = _mm_setzero_si128();
+-                __m128i b = _mm_setzero_si128();
+-                __m128i c = _mm_clmulepi64_si128(a, b, 0x10);
+-                (void)c;
+-                return 0;
+-            }"
++            #include 
++            __m128i f(__m128i a, __m128i b) { return _mm_clmulepi64_si128(a, b, 0x10); }
++            int main(void) { return 0; }"
+             HAVE_PCLMULQDQ_INTRIN
+         )
+         set(CMAKE_REQUIRED_FLAGS)
+@@ -252,13 +227,12 @@ macro(check_vpclmulqdq_intrinsics)
+         set(CMAKE_REQUIRED_FLAGS "${VPCLMULFLAG} ${NATIVEFLAG}")
+         check_c_source_compile_or_run(
+             "#include 
+-            int main(void) {
+-                __m512i a = _mm512_setzero_si512();
++            #include 
++            __m512i f(__m512i a) {
+                 __m512i b = _mm512_setzero_si512();
+-                __m512i c = _mm512_clmulepi64_epi128(a, b, 0x10);
+-                (void)c;
+-                return 0;
+-            }"
++                return _mm512_clmulepi64_epi128(a, b, 0x10);
++            }
++            int main(void) { return 0; }"
+             HAVE_VPCLMULQDQ_INTRIN
+         )
+         set(CMAKE_REQUIRED_FLAGS)
+@@ -431,11 +405,8 @@ macro(check_sse2_intrinsics)
+     set(CMAKE_REQUIRED_FLAGS "${SSE2FLAG} ${NATIVEFLAG}")
+     check_c_source_compile_or_run(
+         "#include 
+-        int main(void) {
+-            __m128i zero = _mm_setzero_si128();
+-            (void)zero;
+-            return 0;
+-        }"
++        __m128i f(__m128i x, __m128i y) { return _mm_sad_epu8(x, y); }
++        int main(void) { return 0; }"
+         HAVE_SSE2_INTRIN
+     )
+     set(CMAKE_REQUIRED_FLAGS)
+@@ -457,14 +428,11 @@ macro(check_ssse3_intrinsics)
+     set(CMAKE_REQUIRED_FLAGS "${SSSE3FLAG} ${NATIVEFLAG}")
+     check_c_source_compile_or_run(
+         "#include 
+-        int main(void) {
+-            __m128i u, v, w;
+-            u = _mm_set1_epi32(1);
+-            v = _mm_set1_epi32(2);
+-            w = _mm_hadd_epi32(u, v);
+-            (void)w;
+-            return 0;
+-        }"
++        __m128i f(__m128i u) {
++          __m128i v = _mm_set1_epi32(1);
++          return _mm_hadd_epi32(u, v);
++        }
++        int main(void) { return 0; }"
+         HAVE_SSSE3_INTRIN
+     )
+ endmacro()
+@@ -485,13 +453,8 @@ macro(check_sse42_intrinsics)
+     set(CMAKE_REQUIRED_FLAGS "${SSE42FLAG} ${NATIVEFLAG}")
+     check_c_source_compile_or_run(
+         "#include 
+-        int main(void) {
+-            unsigned crc = 0;
+-            char c = 'c';
+-            crc = _mm_crc32_u32(crc, c);
+-            (void)crc;
+-            return 0;
+-        }"
++        unsigned int f(unsigned int a, unsigned int b) { return _mm_crc32_u32(a, b); }
++        int main(void) { return 0; }"
+         HAVE_SSE42_INTRIN
+     )
+     set(CMAKE_REQUIRED_FLAGS)
+@@ -529,13 +492,12 @@ macro(check_xsave_intrinsics)
+     set(CMAKE_REQUIRED_FLAGS "${XSAVEFLAG} ${NATIVEFLAG}")
+     check_c_source_compiles(
+         "#ifdef _WIN32
+-         #  include 
+-         #else
+-         #  include 
+-         #endif
+-         int main(void) {
+-             return _xgetbv(0);
+-         }"
++        #  include 
++        #else
++        #  include 
++        #endif
++        unsigned int f(unsigned int a) { return _xgetbv(a); }
++        int main(void) { return 0; }"
+         HAVE_XSAVE_INTRIN FAIL_REGEX "not supported")
+     set(CMAKE_REQUIRED_FLAGS)
+ endmacro()
+diff --git a/configure b/configure
+index fc78a135..e5a1965f 100755
+--- a/configure
++++ b/configure
+@@ -1023,12 +1023,8 @@ fi
+ 
+ # Check for __builtin_ctz() support in compiler
+ cat > $test.c << EOF
+-int main(void) {
+-    unsigned int zero = 0;
+-    long test = __builtin_ctz(zero);
+-    (void)test;
+-    return 0;
+-}
++long f(unsigned int x) { return __builtin_ctz(x); }
++int main(void) { return 0; }
+ EOF
+ if try ${CC} ${CFLAGS} $test.c $LDSHAREDLIBC; then
+     echo "Checking for __builtin_ctz ... Yes." | tee -a configure.log
+@@ -1040,12 +1036,8 @@ fi
+ 
+ # Check for __builtin_ctzll() support in compiler
+ cat > $test.c << EOF
+-int main(void) {
+-    unsigned long long zero = 0;
+-    long test = __builtin_ctzll(zero);
+-    (void)test;
+-    return 0;
+-}
++long f(unsigned long long x) { return __builtin_ctzll(x); }
++int main(void) { return 0; }
+ EOF
+ if try ${CC} ${CFLAGS} $test.c $LDSHAREDLIBC; then
+     echo "Checking for __builtin_ctzll ... Yes." | tee -a configure.log
+@@ -1059,13 +1051,11 @@ check_avx2_intrinsics() {
+     # Check whether compiler supports AVX2 intrinsics
+     cat > $test.c << EOF
+ #include 
+-int main(void) {
+-    __m256i x = _mm256_set1_epi16(2);
++__m256i f(__m256i x) {
+     const __m256i y = _mm256_set1_epi16(1);
+-    x = _mm256_subs_epu16(x, y);
+-    (void)x;
+-    return 0;
++    return _mm256_subs_epu16(x, y);
+ }
++int main(void) { return 0; }
+ EOF
+     if try ${CC} ${CFLAGS} ${avx2flag} $test.c; then
+         echo "Checking for AVX2 intrinsics ... Yes." | tee -a configure.log
+@@ -1080,16 +1070,11 @@ check_avx512_intrinsics() {
+     # Check whether compiler supports AVX512 intrinsics
+     cat > $test.c << EOF
+ #include 
+-int main(void) {
+-    __m512i x = _mm512_set1_epi8(2);
+-    const __m512i y = _mm512_set_epi32(0x1020304, 0x5060708, 0x90a0b0c, 0xd0e0f10,
+-                                       0x11121314, 0x15161718, 0x191a1b1c, 0x1d1e1f20,
+-                                       0x21222324, 0x25262728, 0x292a2b2c, 0x2d2e2f30,
+-                                       0x31323334, 0x35363738, 0x393a3b3c, 0x3d3e3f40);
+-    x = _mm512_sub_epi8(x, y);
+-    (void)x;
+-    return 0;
++__m512i f(__m512i y) {
++  __m512i x = _mm512_set1_epi8(2);
++  return _mm512_sub_epi8(x, y);
+ }
++int main(void) { return 0; }
+ EOF
+     if try ${CC} ${CFLAGS} ${avx512flag} $test.c; then
+         echo "Checking for AVX512 intrinsics ... Yes." | tee -a configure.log
+@@ -1133,17 +1118,11 @@ check_avx512vnni_intrinsics() {
+     # Check whether compiler supports AVX512-VNNI intrinsics
+     cat > $test.c << EOF
+ #include 
+-int main(void) {
+-    __m512i x = _mm512_set1_epi8(2);
+-    const __m512i y = _mm512_set_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+-                                      20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37,
+-                                      38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55,
+-                                      56, 57, 58, 59, 60, 61, 62, 63, 64);
++__m512i f(__m512i x, __m512i y) {
+     __m512i z = _mm512_setzero_epi32();
+-    z = _mm512_dpbusd_epi32(z, x, y);
+-    (void)z;
+-    return 0;
++    return _mm512_dpbusd_epi32(z, x, y);
+ }
++int main(void) { return 0; }
+ EOF
+     if try ${CC} ${CFLAGS} ${avx512vnniflag} $test.c; then
+         echo "Checking for AVX512VNNI intrinsics ... Yes." | tee -a configure.log
+@@ -1158,12 +1137,8 @@ check_mask_intrinsics() {
+     # Check whether compiler supports AVX512 k-mask intrinsics
+     cat > $test.c << EOF
+ #include 
+-int main(void) {
+-    __mmask16 a = 0xFF;
+-    a = _knot_mask16(a);
+-    (void)a;
+-    return 0;
+-}
++__mmask16 f(__mmask16 x) { return _knot_mask16(x); }
++int main(void) { return 0; }
+ EOF
+     if try ${CC} ${CFLAGS} ${avx512flag} $test.c; then
+         echo "Checking for AVX512 k-mask intrinsics ... Yes." | tee -a configure.log
+@@ -1230,12 +1205,8 @@ check_neon_ld4_intrinsics() {
+ #else
+ #  include 
+ #endif
+-int main(void) {
+-    int stack_var[16];
+-    int32x4x4_t v = vld1q_s32_x4(stack_var);
+-    (void)v;
+-    return 0;
+-}
++int32x4x4_t f(int var[16]) { return vld1q_s32_x4(var); }
++int main(void) { return 0; }
+ EOF
+     if try $CC -c $CFLAGS $neonflag $test.c; then
+         NEON_HAS_LD4=1
+@@ -1251,13 +1222,8 @@ check_pclmulqdq_intrinsics() {
+     cat > $test.c << EOF
+ #include 
+ #include 
+-int main(void) {
+-    __m128i a = _mm_setzero_si128();
+-    __m128i b = _mm_setzero_si128();
+-    __m128i c = _mm_clmulepi64_si128(a, b, 0x10);
+-    (void)c;
+-    return 0;
+-}
++__m128i f(__m128i a, __m128i b) { return _mm_clmulepi64_si128(a, b, 0x10); }
++int main(void) { return 0; }
+ EOF
+     if try ${CC} ${CFLAGS} ${pclmulflag} $test.c; then
+         echo "Checking for PCLMULQDQ intrinsics ... Yes." | tee -a configure.log
+@@ -1273,13 +1239,11 @@ check_vpclmulqdq_intrinsics() {
+     cat > $test.c << EOF
+ #include 
+ #include 
+-int main(void) {
+-    __m512i a = _mm512_setzero_si512();
++__m512i f(__m512i a) {
+     __m512i b = _mm512_setzero_si512();
+-    __m512i c = _mm512_clmulepi64_epi128(a, b, 0x10);
+-    (void)c;
+-    return 0;
++    return _mm512_clmulepi64_epi128(a, b, 0x10);
+ }
++int main(void) { return 0; }
+ EOF
+     if try ${CC} ${CFLAGS} ${vpclmulflag} $test.c; then
+         echo "Checking for VPCLMULQDQ intrinsics ... Yes." | tee -a configure.log
+@@ -1298,9 +1262,8 @@ check_xsave_intrinsics() {
+ #else
+ #  include 
+ #endif
+-int main(void) {
+-    return _xgetbv(0);
+-}
++unsigned int f(unsigned int a) { return _xgetbv(a); }
++int main(void) { return 0; }
+ EOF
+     if try ${CC} ${CFLAGS} ${xsaveflag} $test.c; then
+         echo "Checking for XSAVE intrinsics ... Yes." | tee -a configure.log
+@@ -1415,11 +1378,8 @@ check_sse2_intrinsics() {
+     # Check whether compiler supports SSE2 intrinsics
+     cat > $test.c << EOF
+ #include 
+-int main(void) {
+-    __m128i zero = _mm_setzero_si128();
+-    (void)zero;
+-    return 0;
+-}
++__m128i f(__m128i x, __m128i y) { return _mm_sad_epu8(x, y); }
++int main(void) { return 0; }
+ EOF
+     if try ${CC} ${CFLAGS} ${sse2flag} $test.c; then
+         echo "Checking for SSE2 intrinsics ... Yes." | tee -a configure.log
+@@ -1434,13 +1394,8 @@ check_sse42_intrinsics() {
+     # Check whether compiler supports SSE4.2 intrinsics
+     cat > $test.c << EOF
+ #include 
+-int main(void) {
+-    unsigned crc = 0;
+-    char c = 'c';
+-    crc = _mm_crc32_u32(crc, c);
+-    (void)crc;
+-    return 0;
+-}
++unsigned int f(unsigned int a, unsigned int b) { return _mm_crc32_u32(a, b); }
++int main(void) { return 0; }
+ EOF
+     if try ${CC} ${CFLAGS} ${sse42flag} $test.c; then
+         echo "Checking for SSE4.2 intrinsics ... Yes." | tee -a configure.log
+@@ -1455,15 +1410,11 @@ check_ssse3_intrinsics() {
+     # Check whether compiler supports SSSE3 intrinsics
+     cat > $test.c << EOF
+ #include 
+-int main(void)
+-{
+-    __m128i u, v, w;
+-    u = _mm_set1_epi32(1);
+-    v = _mm_set1_epi32(2);
+-    w = _mm_hadd_epi32(u, v);
+-    (void)w;
+-    return 0;
++__m128i f(__m128i u) {
++  __m128i v = _mm_set1_epi32(1);
++  return _mm_hadd_epi32(u, v);
+ }
++int main(void) { return 0; }
+ EOF
+     if try ${CC} ${CFLAGS} ${ssse3flag} $test.c; then
+         echo "Checking for SSSE3 intrinsics ... Yes." | tee -a configure.log
+-- 
+2.39.2
+
diff --git a/var/spack/repos/builtin/packages/zlib/package.py b/var/spack/repos/builtin/packages/zlib/package.py
index f77e3e0e3d2c56..144e3b0ec610bf 100644
--- a/var/spack/repos/builtin/packages/zlib/package.py
+++ b/var/spack/repos/builtin/packages/zlib/package.py
@@ -22,7 +22,9 @@ class Zlib(MakefilePackage, Package):
     homepage = "https://zlib.net"
     # URL must remain http:// so Spack can bootstrap curl
     url = "http://zlib.net/fossils/zlib-1.2.11.tar.gz"
+    git = "https://github.com/madler/zlib.git"
 
+    version("1.3", sha256="ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e")
     version("1.2.13", sha256="b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30")
     version(
         "1.2.12",
@@ -56,6 +58,10 @@ class Zlib(MakefilePackage, Package):
     patch("w_patch.patch", when="@1.2.11%cce")
     patch("configure-cc.patch", when="@1.2.12")
 
+    provides("zlib-api")
+
+    license("Zlib")
+
     @property
     def libs(self):
         shared = "+shared" in self.spec
diff --git a/var/spack/repos/builtin/packages/zoltan/package.py b/var/spack/repos/builtin/packages/zoltan/package.py
index 36ceca3a9e7fe3..87f8b6bd7be9e4 100644
--- a/var/spack/repos/builtin/packages/zoltan/package.py
+++ b/var/spack/repos/builtin/packages/zoltan/package.py
@@ -92,6 +92,9 @@ def configure_args(self):
         config_cflags = ["-O0" if "+debug" in spec else "-O3", "-g" if "+debug" in spec else ""]
 
         config_ldflags = []
+        config_libs = []
+        config_incdirs = []
+
         # PGI runtime libraries
         if "%pgi" in spec:
             config_ldflags.append("-pgf90libs")
@@ -102,9 +105,12 @@ def configure_args(self):
             config_args.extend(["RANLIB=echo", "--with-ar=$(CXX) -shared $(LDFLAGS) -o"])
             config_cflags.append(self.compiler.cc_pic_flag)
             if spec.satisfies("%gcc"):
-                config_args.append("--with-libs=-lgfortran")
+                config_libs.append("-lgfortran")
+                # Although adding to config_libs _should_ suffice, it does not
+                # Add to ldflags as well
+                config_ldflags.append("-lgfortran")
             if spec.satisfies("%intel"):
-                config_args.append("--with-libs=-lifcore")
+                config_libs.append("-lifcore")
 
         if "+int64" in spec:
             config_args.append("--with-id-type=ulong")
@@ -116,10 +122,16 @@ def configure_args(self):
                     "--with-parmetis",
                     "--with-parmetis-libdir={0}".format(parmetis_prefix.lib),
                     "--with-parmetis-incdir={0}".format(parmetis_prefix.include),
-                    "--with-incdirs=-I{0}".format(spec["metis"].prefix.include),
-                    "--with-ldflags=-L{0}".format(spec["metis"].prefix.lib),
                 ]
             )
+            config_ldflags.append("-L{0}".format(spec["metis"].prefix.lib))
+            config_incdirs.append("-I{0}".format(spec["metis"].prefix.include))
+            config_libs.append("-lparmetis")
+            config_libs.append("-lmetis")
+            # Although appending to config_libs _should_ suffice, it does not
+            # Add them to ldflags as well
+            config_ldflags.append("-lparmetis")
+            config_ldflags.append("-lmetis")
             if "+int64" in spec["metis"]:
                 config_args.append("--with-id-type=ulong")
             else:
@@ -143,19 +155,26 @@ def configure_args(self):
                 config_args.extend(["FC={0}".format(spec["mpi"].mpifc)])
 
         config_fcflags = config_cflags[:]
+        config_cxxflags = config_cflags[:]
+
         if spec.satisfies("%gcc@10:+fortran"):
             config_fcflags.append("-fallow-argument-mismatch")
+
         # NOTE: Early versions of Zoltan come packaged with a few embedded
         # library packages (e.g. ParMETIS, Scotch), which messes with Spack's
         # ability to descend directly into the package's source directory.
-        config_args.extend(
-            [
-                "--with-cflags={0}".format(" ".join(config_cflags)),
-                "--with-cxxflags={0}".format(" ".join(config_cflags)),
-                "--with-fcflags={0}".format(" ".join(config_fcflags)),
-                "--with-ldflags={0}".format(" ".join(config_ldflags)),
-            ]
-        )
+        if config_cflags:
+            config_args.append("--with-cflags={0}".format(" ".join(config_cflags)))
+        if config_cxxflags:
+            config_args.append("--with-cxxflags={0}".format(" ".join(config_cxxflags)))
+        if config_fcflags:
+            config_args.append("--with-fcflags={0}".format(" ".join(config_fcflags)))
+        if config_ldflags:
+            config_args.append("--with-ldflags={0}".format(" ".join(config_ldflags)))
+        if config_libs:
+            config_args.append("--with-libs={0}".format(" ".join(config_libs)))
+        if config_incdirs:
+            config_args.append("--with-incdirs={0}".format(" ".join(config_incdirs)))
         return config_args
 
     # NOTE: Unfortunately, Zoltan doesn't provide any configuration
diff --git a/var/spack/repos/builtin/packages/zstd/package.py b/var/spack/repos/builtin/packages/zstd/package.py
index 8c0cbd8bfe1c51..d0612985e94268 100644
--- a/var/spack/repos/builtin/packages/zstd/package.py
+++ b/var/spack/repos/builtin/packages/zstd/package.py
@@ -52,8 +52,9 @@ class Zstd(CMakePackage, MakefilePackage):
         values=any_combination_of("zlib", "lz4", "lzma"),
         description="Enable support for additional compression methods in programs",
     )
+    variant("pic", default=True, description="Enable position-independent code (PIC)")
 
-    depends_on("zlib", when="compression=zlib")
+    depends_on("zlib-api", when="compression=zlib")
     depends_on("lz4", when="compression=lz4")
     depends_on("xz", when="compression=lzma")
 
@@ -61,6 +62,8 @@ class Zstd(CMakePackage, MakefilePackage):
     # (last tested: nvhpc@22.3)
     conflicts("+programs %nvhpc")
 
+    conflicts("~pic libs=shared")
+
     build_system("cmake", "makefile", default="makefile")
 
 
@@ -77,6 +80,7 @@ def cmake_args(self):
             [
                 self.define("ZSTD_BUILD_STATIC", self.spec.satisfies("libs=static")),
                 self.define("ZSTD_BUILD_SHARED", self.spec.satisfies("libs=shared")),
+                self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
             ]
         )
         if "compression=zlib" in spec:
diff --git a/var/spack/repos/builtin/packages/zstr/package.py b/var/spack/repos/builtin/packages/zstr/package.py
index 64ab609126114b..6750d7bbb568eb 100644
--- a/var/spack/repos/builtin/packages/zstr/package.py
+++ b/var/spack/repos/builtin/packages/zstr/package.py
@@ -22,7 +22,7 @@ class Zstr(Package):
     version("1.0.1", sha256="e17e67e00ede182504b3165cebd802420770541465d4ba41df1a15bf4c2a63b7")
     version("1.0.0", sha256="9f4fa8cb0d2cbba03dfe67900c48b6e75c8380d9263a0ac71d795f11e0224b96")
 
-    depends_on("zlib")
+    depends_on("zlib-api")
 
     def install(self, spec, prefix):
         """Make the install targets - Note that this package
diff --git a/var/spack/repos/builtin/packages/zziplib/package.py b/var/spack/repos/builtin/packages/zziplib/package.py
index 9559e2b13228cd..08a00cdb2d280e 100644
--- a/var/spack/repos/builtin/packages/zziplib/package.py
+++ b/var/spack/repos/builtin/packages/zziplib/package.py
@@ -31,7 +31,7 @@ class Zziplib(AutotoolsPackage, CMakePackage):
 
     depends_on("python@3.5:", type="build", when="@0.13.72:")
     depends_on("python", type="build")
-    depends_on("zlib")
+    depends_on("zlib-api")
     # see zzip/CMakeLists.txt
     depends_on("coreutils", type="build", when="@0.13.72:")
     depends_on("pkgconfig", type="build", when="@0.13.72:")
diff --git a/var/spack/repos/duplicates.test/packages/cycle-a/package.py b/var/spack/repos/duplicates.test/packages/cycle-a/package.py
new file mode 100644
index 00000000000000..207f5aac447937
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/cycle-a/package.py
@@ -0,0 +1,17 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class CycleA(Package):
+    """Package that would lead to cycles if default variant values are used"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    version("2.0", md5="0123456789abcdef0123456789abcdef")
+
+    variant("cycle", default=True, description="activate cycles")
+    depends_on("cycle-b", when="+cycle")
diff --git a/var/spack/repos/duplicates.test/packages/cycle-b/package.py b/var/spack/repos/duplicates.test/packages/cycle-b/package.py
new file mode 100644
index 00000000000000..ef190a5649139f
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/cycle-b/package.py
@@ -0,0 +1,17 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class CycleB(Package):
+    """Package that would lead to cycles if default variant values are used"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    version("2.0", md5="0123456789abcdef0123456789abcdef")
+
+    variant("cycle", default=True, description="activate cycles")
+    depends_on("cycle-a", when="+cycle")
diff --git a/var/spack/repos/duplicates.test/packages/gmake/package.py b/var/spack/repos/duplicates.test/packages/gmake/package.py
new file mode 100644
index 00000000000000..64eed7ef67f66e
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/gmake/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class Gmake(Package):
+    """Simple build tool, with different versions"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    tags = ["build-tools"]
+
+    version("4.1", md5="0123456789abcdef0123456789abcdef")
+    version("4.0", md5="0123456789abcdef0123456789abcdef")
+    version("3.0", md5="0123456789abcdef0123456789abcdef")
+    version("2.0", md5="0123456789abcdef0123456789abcdef")
diff --git a/var/spack/repos/duplicates.test/packages/hdf5/package.py b/var/spack/repos/duplicates.test/packages/hdf5/package.py
new file mode 100644
index 00000000000000..98082cbbf95439
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/hdf5/package.py
@@ -0,0 +1,17 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class Hdf5(Package):
+    """Requires gmake at a version that doesn't match that of its dependency"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+
+    depends_on("pinned-gmake", type="link")
+    depends_on("gmake@4", type="build")
diff --git a/var/spack/repos/duplicates.test/packages/pinned-gmake/package.py b/var/spack/repos/duplicates.test/packages/pinned-gmake/package.py
new file mode 100644
index 00000000000000..71e90b1a824c65
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/pinned-gmake/package.py
@@ -0,0 +1,16 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class PinnedGmake(Package):
+    """Software that requires gmake at a specific version"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+
+    depends_on("gmake@3", type="build")
diff --git a/var/spack/repos/duplicates.test/packages/pkg-config/package.py b/var/spack/repos/duplicates.test/packages/pkg-config/package.py
new file mode 100644
index 00000000000000..eb7b84b88fc87c
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/pkg-config/package.py
@@ -0,0 +1,16 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class PkgConfig(Package):
+    """A package providing a virtual, which is frequently used as a pure build dependency."""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    version("1.0.0", md5="0123456789abcdef0123456789abcdef")
+
+    provides("pkgconfig")
diff --git a/var/spack/repos/duplicates.test/packages/py-floating/package.py b/var/spack/repos/duplicates.test/packages/py-floating/package.py
new file mode 100644
index 00000000000000..2921b617bd76ad
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/py-floating/package.py
@@ -0,0 +1,26 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class PyFloating(Package):
+    """An extension that depends on:
+    - py-setuptools without further constraints
+    - py-shapely, which depends on py-setuptools@=60
+    - py-numpy, which depends on py-setuptools@=59
+
+    We need to ensure that by default the root node gets the best version
+    of setuptools it could.
+    """
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    version("1.25.0", md5="0123456789abcdef0123456789abcdef")
+
+    extends("python")
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-shapely", type=("build", "run"))
+    depends_on("py-setuptools", type="build")
diff --git a/var/spack/repos/duplicates.test/packages/py-numpy/package.py b/var/spack/repos/duplicates.test/packages/py-numpy/package.py
new file mode 100644
index 00000000000000..2af50e4910b48c
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/py-numpy/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class PyNumpy(Package):
+    """An extension that depends on pinned build dependencies"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    tags = ["build-tools"]
+
+    version("1.25.0", md5="0123456789abcdef0123456789abcdef")
+
+    extends("python")
+    depends_on("py-setuptools@=59", type=("build", "run"))
+    depends_on("gmake@4.1", type="build")
diff --git a/var/spack/repos/duplicates.test/packages/py-setuptools/package.py b/var/spack/repos/duplicates.test/packages/py-setuptools/package.py
new file mode 100644
index 00000000000000..eaf78506975817
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/py-setuptools/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class PySetuptools(Package):
+    """Build tool for an extendable package"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    tags = ["build-tools"]
+
+    extends("python")
+
+    version("60", md5="0123456789abcdef0123456789abcdef")
+    version("59", md5="0123456789abcdef0123456789abcdef")
diff --git a/var/spack/repos/duplicates.test/packages/py-shapely/package.py b/var/spack/repos/duplicates.test/packages/py-shapely/package.py
new file mode 100644
index 00000000000000..b471bbda5ea7b4
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/py-shapely/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class PyShapely(Package):
+    """An extension that depends on pinned build dependencies"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    version("1.25.0", md5="0123456789abcdef0123456789abcdef")
+
+    extends("python")
+    depends_on("py-numpy", type=("build", "link", "run"))
+    depends_on("py-setuptools@=60", type="build")
diff --git a/var/spack/repos/duplicates.test/packages/python/package.py b/var/spack/repos/duplicates.test/packages/python/package.py
new file mode 100644
index 00000000000000..0c4db4781e9179
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/python/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class Python(Package):
+    """A package that can be extended"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    version("3.11.2", md5="0123456789abcdef0123456789abcdef")
+
+    extendable = True
+
+    depends_on("gmake@3", type="build")
diff --git a/var/spack/repos/duplicates.test/packages/virtual-build/package.py b/var/spack/repos/duplicates.test/packages/virtual-build/package.py
new file mode 100644
index 00000000000000..17fc60955d9b4f
--- /dev/null
+++ b/var/spack/repos/duplicates.test/packages/virtual-build/package.py
@@ -0,0 +1,16 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class VirtualBuild(Package):
+    """A package that has a pure build virtual dependency"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/tdep-1.0.tar.gz"
+
+    version("1.0.0", md5="0123456789abcdef0123456789abcdef")
+
+    depends_on("pkgconfig", type="build")
diff --git a/var/spack/repos/duplicates.test/repo.yaml b/var/spack/repos/duplicates.test/repo.yaml
new file mode 100644
index 00000000000000..fd021c858cce94
--- /dev/null
+++ b/var/spack/repos/duplicates.test/repo.yaml
@@ -0,0 +1,2 @@
+repo:
+  namespace: duplicates.test
diff --git a/var/spack/repos/edges.test/packages/blas-only-client/package.py b/var/spack/repos/edges.test/packages/blas-only-client/package.py
new file mode 100644
index 00000000000000..9e9652a752f44a
--- /dev/null
+++ b/var/spack/repos/edges.test/packages/blas-only-client/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class BlasOnlyClient(Package):
+    """This package depends on the 'blas' virtual only, but should be able to use also provider
+    that provide e.g. 'blas' together with 'lapack'.
+    """
+
+    homepage = "http://www.openblas.net"
+    url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz"
+
+    version("0.2.16", md5="b1190f3d3471685f17cfd1ec1d252ac9")
+
+    depends_on("blas")
diff --git a/var/spack/repos/edges.test/packages/conditional-edge/package.py b/var/spack/repos/edges.test/packages/conditional-edge/package.py
new file mode 100644
index 00000000000000..964596fcc14a7e
--- /dev/null
+++ b/var/spack/repos/edges.test/packages/conditional-edge/package.py
@@ -0,0 +1,24 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class ConditionalEdge(Package):
+    """This package has a variant that triggers a condition only if a required dependency is
+    providing a virtual.
+    """
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/a-1.0.tar.gz"
+
+    version("2.0", md5="abcdef0123456789abcdef0123456789")
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+
+    variant("foo", default=False, description="Just a regular foo")
+
+    # zlib is a real package, providing zlib-api
+    depends_on("zlib")
+    depends_on("zlib-api", when="+foo")
+    depends_on("zlib@1.0", when="^[virtuals=zlib-api] zlib")
diff --git a/var/spack/repos/edges.test/packages/openblas/package.py b/var/spack/repos/edges.test/packages/openblas/package.py
new file mode 100644
index 00000000000000..d162e069b0bae0
--- /dev/null
+++ b/var/spack/repos/edges.test/packages/openblas/package.py
@@ -0,0 +1,22 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Openblas(Package):
+    """This package provides two virtuals together, so if one is chosen the other
+    must be used too if needed.
+    """
+
+    homepage = "http://www.openblas.net"
+    url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz"
+
+    version("0.2.16", md5="b1190f3d3471685f17cfd1ec1d252ac9")
+    version("0.2.15", md5="b1190f3d3471685f17cfd1ec1d252ac9")
+    version("0.2.14", md5="b1190f3d3471685f17cfd1ec1d252ac9")
+    version("0.2.13", md5="b1190f3d3471685f17cfd1ec1d252ac9")
+
+    provides("blas", "lapack")
diff --git a/var/spack/repos/edges.test/packages/zlib/package.py b/var/spack/repos/edges.test/packages/zlib/package.py
new file mode 100644
index 00000000000000..66dfc4f58bb94b
--- /dev/null
+++ b/var/spack/repos/edges.test/packages/zlib/package.py
@@ -0,0 +1,19 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.package import *
+
+
+class Zlib(Package):
+    """This package has a variant that triggers a condition only if a required dependency is
+    providing a virtual.
+    """
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/a-1.0.tar.gz"
+
+    version("2.0", md5="abcdef0123456789abcdef0123456789")
+    version("1.0", md5="0123456789abcdef0123456789abcdef")
+
+    provides("zlib-api")
diff --git a/var/spack/repos/edges.test/repo.yaml b/var/spack/repos/edges.test/repo.yaml
new file mode 100644
index 00000000000000..86df79affe294a
--- /dev/null
+++ b/var/spack/repos/edges.test/repo.yaml
@@ -0,0 +1,2 @@
+repo:
+  namespace: edges.test
diff --git a/var/spack/repos/jcsda-emc-bundles/packages/jedi-base-env/package.py b/var/spack/repos/jcsda-emc-bundles/packages/jedi-base-env/package.py
index 5b015601162501..2e8b1b291adeea 100644
--- a/var/spack/repos/jcsda-emc-bundles/packages/jedi-base-env/package.py
+++ b/var/spack/repos/jcsda-emc-bundles/packages/jedi-base-env/package.py
@@ -28,7 +28,7 @@ class JediBaseEnv(BundlePackage):
     depends_on("boost", type="run")
     depends_on("bufr", type="run")
     # Force users to load manually
-    #depends_on("crtm@v2.4.1-jedi", type="run")
+    # depends_on("crtm@v2.4.1-jedi", type="run")
     depends_on("ecbuild", type="run")
     depends_on("eccodes", type="run")
     depends_on("eckit", type="run")
diff --git a/var/spack/repos/jcsda-emc-bundles/packages/ufs-pyenv/package.py b/var/spack/repos/jcsda-emc-bundles/packages/ufs-pyenv/package.py
index 8cccd7547acb15..3540c480934401 100644
--- a/var/spack/repos/jcsda-emc-bundles/packages/ufs-pyenv/package.py
+++ b/var/spack/repos/jcsda-emc-bundles/packages/ufs-pyenv/package.py
@@ -20,7 +20,7 @@ class UfsPyenv(BundlePackage):
 
     depends_on("py-cython")
     depends_on("py-cftime")
-    #depends_on("py-h5py")
+    # depends_on("py-h5py")
     depends_on("py-numpy")
     depends_on("py-pandas")
     depends_on("py-python-dateutil")
diff --git a/var/spack/repos/jcsda-emc-bundles/packages/ufs-weather-model-env/package.py b/var/spack/repos/jcsda-emc-bundles/packages/ufs-weather-model-env/package.py
index 2cfa0eba0bfc8a..4edb08674eb666 100644
--- a/var/spack/repos/jcsda-emc-bundles/packages/ufs-weather-model-env/package.py
+++ b/var/spack/repos/jcsda-emc-bundles/packages/ufs-weather-model-env/package.py
@@ -26,7 +26,7 @@ class UfsWeatherModelEnv(BundlePackage):
     depends_on("base-env", type="run")
     depends_on("ufs-pyenv", type="run", when="+python")
 
-    depends_on("fms@2023.02.01", type="run")
+    depends_on("fms@2023.03", type="run")
     depends_on("bacio", type="run")
     depends_on("crtm@2.4.0", type="run")
     depends_on("g2", type="run")
diff --git a/var/spack/repos/jcsda-emc/packages/ewok/package.py b/var/spack/repos/jcsda-emc/packages/ewok/package.py
index 3e252c689bffc4..7f7bca1380d9b0 100644
--- a/var/spack/repos/jcsda-emc/packages/ewok/package.py
+++ b/var/spack/repos/jcsda-emc/packages/ewok/package.py
@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from spack import *
+from spack.package import *
 
 
 class Ewok(PythonPackage):
@@ -13,16 +13,16 @@ class Ewok(PythonPackage):
     git = "https://github.com/JCSDA/ewok.git"
     url = "https://github.com/JCSDA/ewok/archive/refs/tags/1.0.0.tar.gz"
 
-    maintainers('climbfuji', 'ericlingerfelt')
+    maintainers("climbfuji", "ericlingerfelt")
 
-    version('develop', branch='develop', no_cache=True)
-    version('0.1.0', commit='2f88d2ea99bc88a16154520f69f8d70d4aedafae', preferred=True)
-    version('0.0.1', commit='69fff0f460fdb639db4fd38574dee8262b8a1f84')
+    version("develop", branch="develop", no_cache=True)
+    version("0.1.0", commit="2f88d2ea99bc88a16154520f69f8d70d4aedafae", preferred=True)
+    version("0.0.1", commit="69fff0f460fdb639db4fd38574dee8262b8a1f84")
 
-    depends_on('python@3.7:',         type=('build', 'run'))
-    depends_on('py-pyyaml',           type=('build', 'run'))
-    depends_on('py-jinja2',           type=('build', 'run'))
-    depends_on('py-ruamel-yaml',      type=('build', 'run'))
-    depends_on('py-ruamel-yaml-clib', type=('build', 'run'))
-    depends_on('py-shapely@1.8.0',    type=('build', 'run'))
-    depends_on('py-cartopy+plotting', type=('build', 'run'))
+    depends_on("python@3.7:", type=("build", "run"))
+    depends_on("py-pyyaml", type=("build", "run"))
+    depends_on("py-jinja2", type=("build", "run"))
+    depends_on("py-ruamel-yaml", type=("build", "run"))
+    depends_on("py-ruamel-yaml-clib", type=("build", "run"))
+    depends_on("py-shapely@1.8.0", type=("build", "run"))
+    depends_on("py-cartopy+plotting", type=("build", "run"))
diff --git a/var/spack/repos/jcsda-emc/packages/r2d2/package.py b/var/spack/repos/jcsda-emc/packages/r2d2/package.py
index 355242e590b0df..268920e17b3b8b 100644
--- a/var/spack/repos/jcsda-emc/packages/r2d2/package.py
+++ b/var/spack/repos/jcsda-emc/packages/r2d2/package.py
@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from spack import *
+from spack.package import *
 
 
 class R2d2(PythonPackage):
@@ -13,12 +13,12 @@ class R2d2(PythonPackage):
     git = "https://github.com/JCSDA/r2d2.git"
     url = "https://github.com/JCSDA/r2d2/archive/refs/tags/1.0.0.tar.gz"
 
-    maintainers('climbfuji', 'ericlingerfelt')
+    maintainers("climbfuji", "ericlingerfelt")
 
-    version('develop', branch='develop', no_cache=True)
-    version('1.0.0', commit='74236aba6817e8ca7880f8109268aae1048ccdbe', preferred=True)
-    version('0.0.1', commit='011990d36c9c651593e5e158b5ad7ef07aee16dc')
+    version("develop", branch="develop", no_cache=True)
+    version("1.0.0", commit="74236aba6817e8ca7880f8109268aae1048ccdbe", preferred=True)
+    version("0.0.1", commit="011990d36c9c651593e5e158b5ad7ef07aee16dc")
 
-    depends_on('python@3.7:', type=('build', 'run'))
-    depends_on('py-pyyaml@5.3.1:', type=('build', 'run'))
-    depends_on('py-boto3', type=('build', 'run'))
+    depends_on("python@3.7:", type=("build", "run"))
+    depends_on("py-pyyaml@5.3.1:", type=("build", "run"))
+    depends_on("py-boto3", type=("build", "run"))
diff --git a/var/spack/repos/jcsda-emc/packages/shumlib/package.py b/var/spack/repos/jcsda-emc/packages/shumlib/package.py
index 42a90bb2bf46df..df89f717beb7d6 100644
--- a/var/spack/repos/jcsda-emc/packages/shumlib/package.py
+++ b/var/spack/repos/jcsda-emc/packages/shumlib/package.py
@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-from spack import *
+from spack.package import *
 
 
 class Shumlib(MakefilePackage):
@@ -19,10 +19,10 @@ class Shumlib(MakefilePackage):
     git = "https://github.com/climbfuji/shumlib.git"
     url = "https://github.com/metomi/shumlib/archive/refs/tags/2021.10.1.zip"
 
-    maintainers('matthewrmshin', 'climbfuji')
+    maintainers("matthewrmshin", "climbfuji")
 
-    version('macos_clang_linux_intel_port', commit='84770606669463a54b51f9b8ed65a1d31f105fe9')
-    version('macos_clang_port', commit='e5e5c9f23ce2656aacd75a884c26b01a5380752e')
+    version("macos_clang_linux_intel_port", commit="84770606669463a54b51f9b8ed65a1d31f105fe9")
+    version("macos_clang_port", commit="e5e5c9f23ce2656aacd75a884c26b01a5380752e")
     # version('2021.10.1', commit='545874fba961deadf4b2758926be7c26f4c8dcb9')
     # version('2021.07.1', commit='a4ea525ad3bf04684ef39b0241991a350e2b7241')
     # version('2021.03.1', commit='58f599ce9cfb4bd47197125548a44039695fa7f1')
@@ -31,22 +31,22 @@ class Shumlib(MakefilePackage):
     depends_on("patchelf", type="build", when="platform=linux")
 
     def edit(self, spec, prefix):
-        env['LIBDIR_OUT'] = os.path.join(self.build_directory, 'spack-build')
+        env["LIBDIR_OUT"] = os.path.join(self.build_directory, "spack-build")
         # env['LIBDIR_ROOT'] = self.build_directory
 
     def build(self, spec, prefix):
-        if spec.satisfies('%clang') or spec.satisfies('%apple-clang'):
-            os.system('make -f make/vm-x86-gfortran-clang.mk')
-        elif spec.satisfies('%gcc'):
-            os.system('make -f make/vm-x86-gfortran-gcc.mk')
-        elif spec.satisfies('%intel'):
-            os.system('make -f make/vm-x86-ifort-icc.mk')
+        if spec.satisfies("%clang") or spec.satisfies("%apple-clang"):
+            os.system("make -f make/vm-x86-gfortran-clang.mk")
+        elif spec.satisfies("%gcc"):
+            os.system("make -f make/vm-x86-gfortran-gcc.mk")
+        elif spec.satisfies("%intel"):
+            os.system("make -f make/vm-x86-ifort-icc.mk")
         else:
-            raise InstallError('No shumlib make config for this compiler')
+            raise InstallError("No shumlib make config for this compiler")
 
     def install(self, spec, prefix):
-        install_tree(os.path.join(os.getenv('LIBDIR_OUT'), 'include'), prefix.include)
-        install_tree(os.path.join(os.getenv('LIBDIR_OUT'), 'lib'), prefix.lib)
+        install_tree(os.path.join(os.getenv("LIBDIR_OUT"), "include"), prefix.include)
+        install_tree(os.path.join(os.getenv("LIBDIR_OUT"), "lib"), prefix.lib)
 
     @run_after("install")
     def lib_path_fix(self):
@@ -57,17 +57,17 @@ def lib_path_fix(self):
         # without path, LD_LIBRARY_PATH needs to be set to find them.
         if self.spec.satisfies("platform=linux"):
             patchelf = which("patchelf")
-            libdirs = ['lib', 'lib64']
+            libdirs = ["lib", "lib64"]
             for libdir in libdirs:
                 libpath = os.path.join(self.prefix, libdir)
                 if not os.path.isdir(libpath):
                     continue
                 allfiles = os.listdir(libpath)
                 for filename in allfiles:
-                    if filename.startswith("lib") and filename.endswith('.so'):
+                    if filename.startswith("lib") and filename.endswith(".so"):
                         filepath = os.path.join(libpath, filename)
-                        ldd_output = subprocess.check_output(['ldd', filepath])
-                        ldd_output = ldd_output.decode("utf-8").split('\n')
+                        ldd_output = subprocess.check_output(["ldd", filepath])
+                        ldd_output = ldd_output.decode("utf-8").split("\n")
                         for line in ldd_output:
                             if self.build_directory in line:
                                 so_name_old = line.strip().split()[0]
@@ -75,12 +75,8 @@ def lib_path_fix(self):
                                 # part of the string, at this stage (post install)
                                 # the original build path still exists.
                                 if not os.path.isfile(so_name_old):
-                                    raise Exception("{} does not exist!".format(
-                                        so_name_old))
-                                so_name_new = os.path.join(self.prefix, libdir,
-                                    os.path.basename(so_name_old))
-                                patchelf_output = patchelf(
-                                    "--replace-needed",
-                                    so_name_old,
-                                    so_name_new,
-                                    filepath)
+                                    raise Exception("{} does not exist!".format(so_name_old))
+                                so_name_new = os.path.join(
+                                    self.prefix, libdir, os.path.basename(so_name_old)
+                                )
+                                patchelf("--replace-needed", so_name_old, so_name_new, filepath)
diff --git a/var/spack/repos/jcsda-emc/packages/solo/package.py b/var/spack/repos/jcsda-emc/packages/solo/package.py
index f0040622b6af31..41c99e6a83c5f2 100644
--- a/var/spack/repos/jcsda-emc/packages/solo/package.py
+++ b/var/spack/repos/jcsda-emc/packages/solo/package.py
@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from spack import *
+from spack.package import *
 
 
 class Solo(PythonPackage):
@@ -14,10 +14,10 @@ class Solo(PythonPackage):
     git = "https://github.com/JCSDA/solo.git"
     url = "https://github.com/JCSDA/solo/archive/refs/tags/1.0.0.tar.gz"
 
-    maintainers('climbfuji', 'ericlingerfelt')
+    maintainers("climbfuji", "ericlingerfelt")
 
-    version('develop', branch='develop', no_cache=True)
-    version('1.0.0', commit='4231f2f917225704cc53b5238a51853a1beeb9b0', preferred=True)
+    version("develop", branch="develop", no_cache=True)
+    version("1.0.0", commit="4231f2f917225704cc53b5238a51853a1beeb9b0", preferred=True)
 
-    depends_on('python@3.7:', type=('build', 'run'))
-    depends_on('py-pyyaml@5.3.1:', type=('build', 'run'))
+    depends_on("python@3.7:", type=("build", "run"))
+    depends_on("py-pyyaml@5.3.1:", type=("build", "run"))
diff --git a/var/spack/repos/tutorial/packages/hdf5/package.py b/var/spack/repos/tutorial/packages/hdf5/package.py
index 0fceeef891d47f..94335d4ae40c52 100644
--- a/var/spack/repos/tutorial/packages/hdf5/package.py
+++ b/var/spack/repos/tutorial/packages/hdf5/package.py
@@ -104,7 +104,7 @@ class Hdf5(CMakePackage):
     if sys.platform != "darwin":
         depends_on("numactl", when="+mpi+fortran")
     depends_on("szip", when="+szip")
-    depends_on("zlib@1.1.2:")
+    depends_on("zlib-api")
 
     # The compiler wrappers (h5cc, h5fc, etc.) run 'pkg-config'.
     depends_on("pkgconfig", type="run")