diff --git a/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/lib/jcsda-emc/spack-stack/stack/meta_modules.py index f5cc260ef9b4f2..6a8061e8193588 100755 --- a/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -137,6 +137,8 @@ def get_matched_dict(root_dir, candidate_list, sub_candidate_list=None): matched_dict[matched_name] = [matched_version] else: matched_dict[matched_name].append(matched_version) + # Dedupe + matched_dict[matched_name] = list(set(matched_dict[matched_name])) # MPI providers or example depend on compilers else: if matched_name not in matched_dict.keys(): @@ -302,11 +304,7 @@ def setup_meta_modules(): # Then, check for mpi providers - recursively for compilers mpi_dict = get_matched_dict(module_dir, mpi_candidate_list, compiler_candidate_list) if not mpi_dict: - user_input = input( - "No matching MPI providers found, proceed without creating MPI module hierarchy? (yes/no): " - ) - if not user_input.lower() in ["yes", "y"]: - raise Exception("No matching MPI providers found") + logging.warn("No matching MPI providers found, skipping MPI module hierarchy") else: logging.info(" ... stack mpi providers: '{}'".format(mpi_dict)) @@ -697,83 +695,47 @@ def setup_meta_modules(): f.write(module_content) logging.info(" ... writing {}".format(mpi_module_file)) - try: - del package_name - except: - pass # Create python modules. Need to accommodate both external # Python distributions and spack-built Python distributions. - # If there is no package config info for Python, then we are - # using a spack-built Python without knowing the version - not supported. - if not "python" in package_config.keys(): - raise Exception( - """No information on Python in package config. For external -Python distributions, specify complete specifications. For spack-built -Python, list the correct version in the package config""" - ) - else: - package_name = "python" - if "externals" in package_config[package_name].keys(): - # Loop through all external specs and find the - # latest version, this is what spack is using - spack_python_build = False - python_version = None - for i in range(len(package_config[package_name]["externals"])): - (python_name, python_version_test) = get_name_and_version_from_spec( - package_config[package_name]["externals"][i]["spec"] - ) - if not python_version or versiontuple(python_version_test) > versiontuple( - python_version - ): - python_version = python_version_test - python_package_config = package_config[python_name]["externals"][i] - logging.debug(" ... using external python version {}".format(python_version)) - else: - spack_python_build = True - python_name = "python" - python_version = None - python_package_config = None - # Loop through versions, pick the most-recent - for python_version_test in package_config[package_name]["version"]: - if not python_version or versiontuple(python_version_test) > versiontuple( - python_version - ): - python_version = python_version_test - logging.debug(" ... using spack-built python version {}".format(python_version)) - # Check that the Python version we determined is indeed what's installed - python_candidate_list = ["python@{}".format(python_version)] - for compiler_name in compiler_dict.keys(): - for compiler_version in compiler_dict[compiler_name]: - compiler_install_dir = os.path.join( - install_dir, compiler_name, compiler_version - ) - python_dict = get_matched_dict(compiler_install_dir, python_candidate_list) - logging.info(" ... stack python providers: '{}'".format(python_dict)) - if not python_dict: - user_input = input( - "No matching Python version found found, proceed without creating Python modules? (yes/no): " - ) - if not user_input.lower() in ["yes", "y"]: - raise Exception( - """"No matching Python version found. Make sure that the -Python version in the package config matches what spack installed.""" - ) - else: - logging.info( - "Metamodule generation completed successfully in {}".format( - meta_module_dir - ) - ) - return + python_dict = {} + python_name = "python" + + # First, collect all Python packages in the environment + for spec in ev.installed_specs(): + if spec.name == python_name: + compiler_name = spec.compiler.name + compiler_version = str(spec.compiler.version) + if not compiler_name in python_dict.keys(): + python_dict[compiler_name] = {} + if compiler_version in python_dict[compiler_name].keys(): + raise Exception("Duplicate Python packages for compiler {spec.compiler}") + python_dict[compiler_name][compiler_version] = spec for compiler_name in compiler_dict.keys(): for compiler_version in compiler_dict[compiler_name]: + if ( + not compiler_name in python_dict.keys() + or not compiler_version in python_dict[compiler_name].keys() + ): + logging.warn( + "No Python version found for compiler {compiler_name}@{compiler_version}, skipping Python modules" + ) + continue + spec = python_dict[compiler_name][compiler_version] + python_version = str(spec.version) logging.info( " ... configuring stack python interpreter {}@{} for compiler {}@{}".format( python_name, python_version, compiler_name, compiler_version ) ) + if spec.external: + logging.debug(f" ... using external python version {python_version}") + prefix = spec.external_path + modules = spec.external_modules + else: + logging.debug(f" ... using spack-built python version {python_version}") + # Path and name for lua module file python_module_dir = os.path.join( module_dir, compiler_name, compiler_version, "stack-" + python_name @@ -783,12 +745,12 @@ def setup_meta_modules(): ) substitutes = SUBSTITUTES_TEMPLATE.copy() - # - if spack_python_build: + # Spack-built Python vs external Python + if not spec.external: module = "python/{}".format(python_version) # Load spack python module substitutes["MODULELOADS"] += module_load_command(module_choice, module) - elif "modules" in python_package_config.keys(): + elif modules: # Existing non-spack modules to load for module in python_package_config["modules"]: substitutes["MODULELOADS"] += module_load_command(module_choice, module) @@ -798,14 +760,12 @@ def setup_meta_modules(): logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) # python_name_ROOT - replace "-" in python_name with "_" for environment variables - if "prefix" in python_package_config.keys(): - prefix = python_package_config["prefix"] + if prefix: substitutes["PYTHONROOT"] = setenv_command( module_choice, python_name.replace("-", "_") + "_ROOT", prefix ) logging.debug(" ... ... PYTHONROOT: {}".format(substitutes["PYTHONROOT"])) - elif "prefix" in python_package_config.keys(): - prefix = python_package_config["prefix"] + elif prefix: # PATH bindir = os.path.join(prefix, "bin") if os.path.isdir(bindir): diff --git a/var/spack/repos/builtin/packages/py-cylc-flow/package.py b/var/spack/repos/builtin/packages/py-cylc-flow/package.py index bed0abd1b19788..de70e1de50cc26 100644 --- a/var/spack/repos/builtin/packages/py-cylc-flow/package.py +++ b/var/spack/repos/builtin/packages/py-cylc-flow/package.py @@ -12,8 +12,9 @@ class PyCylcFlow(PythonPackage): homepage = "https://cylc.org" pypi = "cylc-flow/cylc-flow-8.1.4.tar.gz" - maintainers("LydDeb") + maintainers("LydDeb", "climbfuji") + version("8.2.3", sha256="dd5bea9e4b8dad00edd9c3459a38fb778e5a073da58ad2725bc9b84ad718e073") version("8.2.0", sha256="cbe35e0d72d1ca36f28a4cebe9b9040a3445a74253bc94051a3c906cf179ded0") version("8.1.4", sha256="d1835ac18f6f24f3115c56b2bc821185484e834a86b12fd0033ff7e4dc3c1f63") @@ -24,7 +25,8 @@ class PyCylcFlow(PythonPackage): depends_on("py-colorama@0.4:1", type=("build", "run")) depends_on("py-graphene@2.1:2", type=("build", "run")) depends_on("py-jinja2@3.0", type=("build", "run")) - depends_on("py-metomi-isodatetime@3.0", type=("build", "run")) + depends_on("py-metomi-isodatetime@3.0", type=("build", "run"), when="@:8.2.0") + depends_on("py-metomi-isodatetime@3:3.1", type=("build", "run"), when="@8.2.3:") depends_on("py-protobuf@4.21.2:4.21", type=("build", "run")) depends_on("py-psutil@5.6.0:", type=("build", "run")) depends_on("py-pyzmq@22:", type=("build", "run"), when="@8.2:") @@ -34,3 +36,6 @@ class PyCylcFlow(PythonPackage): depends_on("py-rx", type=("build", "run")) depends_on("py-promise", type=("build", "run")) depends_on("py-tomli@2:", type=("build", "run"), when="^python@:3.10") + + # Non-Python dependencies + depends_on("graphviz", type="run") diff --git a/var/spack/repos/builtin/packages/py-gevent/package.py b/var/spack/repos/builtin/packages/py-gevent/package.py index c41b815ef24320..7216c0c6d6dde2 100644 --- a/var/spack/repos/builtin/packages/py-gevent/package.py +++ b/var/spack/repos/builtin/packages/py-gevent/package.py @@ -39,7 +39,7 @@ class PyGevent(PythonPackage): conflicts("^py-cython@3:", when="@:20.5.0") # Deprecated compiler options. upstream PR: https://github.com/gevent/gevent/pull/1896 - patch("icc.patch", when="%intel") + patch("icc.patch", when="@:21.12.0 %intel") @run_before("install") def recythonize(self): diff --git a/var/spack/repos/builtin/packages/py-graphene-tornado/package.py b/var/spack/repos/builtin/packages/py-graphene-tornado/package.py index ba44045b4c6d79..d54b40676c9b1f 100644 --- a/var/spack/repos/builtin/packages/py-graphene-tornado/package.py +++ b/var/spack/repos/builtin/packages/py-graphene-tornado/package.py @@ -21,5 +21,8 @@ class PyGrapheneTornado(PythonPackage): depends_on("py-graphene@2.1:2", type=("build", "run")) depends_on("py-jinja2@2.10.1:", type=("build", "run")) depends_on("py-tornado@5.1.0:", type=("build", "run")) - depends_on("py-werkzeug@0.12.2", type=("build", "run")) + # py-werkzeug version requirements differ between setup.py (0.12.2) + # and requirements.txt (0.15.3); the latter seems to be correct, + # see also https://github.com/spack/spack/pull/41426 + depends_on("py-werkzeug@0.15.3", type=("build", "run")) depends_on("py-pytest", type=("build")) diff --git a/var/spack/repos/builtin/packages/py-metomi-isodatetime/package.py b/var/spack/repos/builtin/packages/py-metomi-isodatetime/package.py index feaa89161827dd..02d38147e22715 100644 --- a/var/spack/repos/builtin/packages/py-metomi-isodatetime/package.py +++ b/var/spack/repos/builtin/packages/py-metomi-isodatetime/package.py @@ -14,6 +14,11 @@ class PyMetomiIsodatetime(PythonPackage): maintainers("LydDeb") + version("3.1.0", sha256="2ec15eb9c323d5debd0678f33af99bc9a91aa0b534ee5f65f3487aed518ebf2d") version("3.0.0", sha256="2141e8aaa526ea7f7f1cb883e6c8ed83ffdab73269658d84d0624f63a6e1357e") depends_on("py-setuptools", type="build") + + def url_for_version(self, version): + url = "https://files.pythonhosted.org/packages/source/m/metomi-isodatetime/metomi-isodatetime-1!{}.tar.gz" + return url.format(version) diff --git a/var/spack/repos/jcsda-emc-bundles/packages/ewok-env/package.py b/var/spack/repos/jcsda-emc-bundles/packages/ewok-env/package.py index fc0b4a54a1b5c7..d0c82cbca9f742 100644 --- a/var/spack/repos/jcsda-emc-bundles/packages/ewok-env/package.py +++ b/var/spack/repos/jcsda-emc-bundles/packages/ewok-env/package.py @@ -17,6 +17,10 @@ class EwokEnv(BundlePackage): version("1.0.0") + # Variants for workflow engines + variant("ecflow", default=True, description="Use ecflow workflow engine") + variant("cylc", default=False, description="Use cylc workflow engine") + # Variants defining repositories that are not yet publicly available variant("solo", default=False, description="Build solo (general tools for Python programmers)") variant( @@ -41,7 +45,12 @@ class EwokEnv(BundlePackage): depends_on("py-ruamel-yaml-clib", type="run") # Workflow engines - depends_on("ecflow", type="run") + with when("+ecflow"): + depends_on("ecflow", type="run") + with when("+cylc"): + depends_on("py-cylc-flow", type="run") + depends_on("py-cylc-rose", type="run") + depends_on("py-cylc-uiserver", type="run") # R2D2 mysql backend depends_on("mysql", type="run")