From 961618ffabc60718668377763fc023f2422ec3d3 Mon Sep 17 00:00:00 2001 From: Tieqiong Zhang Date: Mon, 18 Aug 2025 01:18:06 -0500 Subject: [PATCH 1/8] cmi --- CHANGELOG.rst | 1 - CODE_OF_CONDUCT.rst => CODE-OF-CONDUCT.rst | 0 {doc => docs}/Makefile | 0 {doc => docs}/make.bat | 0 {doc => docs}/source/_static/wrap_table.css | 0 {doc => docs}/source/api/diffpy.cmi.rst | 0 {doc => docs}/source/conf.py | 0 {doc => docs}/source/data/Ni.cif | 0 {doc => docs}/source/data/Ni.gr | 0 {doc => docs}/source/getting-started.rst | 0 {doc => docs}/source/img/Ni_fit.png | Bin .../source/img/cmi_problem_types.png | Bin .../source/img/diffpycmi_screenshot.png | Bin {doc => docs}/source/img/makerecipe_flow.png | Bin {doc => docs}/source/img/pdfprimer.png | Bin {doc => docs}/source/index.rst | 0 {doc => docs}/source/license.rst | 0 {doc => docs}/source/release.rst | 0 {doc => docs}/source/snippets/data_params.rst | 0 .../source/snippets/diffpy_libraries.rst | 0 .../source/snippets/external_libraries.rst | 0 .../source/snippets/structure_params.rst | 0 {doc => docs}/source/tutorials/index.rst | 0 pyproject.toml | 6 +- requirements/build.txt | 0 requirements/conda.txt | 1 - requirements/packs/core.txt | 5 + requirements/{ => packs}/docs.txt | 0 requirements/packs/pdf.txt | 3 + requirements/packs/plotting.txt | 4 + requirements/packs/scripts/_pytest.sh | 61 ++ requirements/packs/scripts/diffpy-srreal.bat | 2 + requirements/packs/scripts/diffpy-srreal.sh | 2 + requirements/packs/scripts/tar_url.txt | 5 + requirements/{ => packs}/tests.txt | 3 + requirements/pip.txt | 1 - requirements/profiles/_tests.yml | 11 + requirements/profiles/all.yml | 9 + setup.py | 34 ++ src/diffpy/cmi/__init__.py | 14 +- src/diffpy/cmi/cli.py | 543 ++++++++++++++++++ src/diffpy/cmi/conda.py | 322 +++++++++++ src/diffpy/cmi/diffpy_cmi_app.py | 30 - src/diffpy/cmi/functions.py | 31 - src/diffpy/cmi/installer.py | 371 ++++++++++++ src/diffpy/cmi/log.py | 110 ++++ src/diffpy/cmi/packsmanager.py | 161 ++++++ src/diffpy/cmi/profilesmanager.py | 197 +++++++ tests/test_functions.py | 40 -- 49 files changed, 1859 insertions(+), 108 deletions(-) rename CODE_OF_CONDUCT.rst => CODE-OF-CONDUCT.rst (100%) rename {doc => docs}/Makefile (100%) rename {doc => docs}/make.bat (100%) rename {doc => docs}/source/_static/wrap_table.css (100%) rename {doc => docs}/source/api/diffpy.cmi.rst (100%) rename {doc => docs}/source/conf.py (100%) rename {doc => docs}/source/data/Ni.cif (100%) rename {doc => docs}/source/data/Ni.gr (100%) rename {doc => docs}/source/getting-started.rst (100%) rename {doc => docs}/source/img/Ni_fit.png (100%) rename {doc => docs}/source/img/cmi_problem_types.png (100%) rename {doc => docs}/source/img/diffpycmi_screenshot.png (100%) rename {doc => docs}/source/img/makerecipe_flow.png (100%) rename {doc => docs}/source/img/pdfprimer.png (100%) rename {doc => docs}/source/index.rst (100%) rename {doc => docs}/source/license.rst (100%) rename {doc => docs}/source/release.rst (100%) rename {doc => docs}/source/snippets/data_params.rst (100%) rename {doc => docs}/source/snippets/diffpy_libraries.rst (100%) rename {doc => docs}/source/snippets/external_libraries.rst (100%) rename {doc => docs}/source/snippets/structure_params.rst (100%) rename {doc => docs}/source/tutorials/index.rst (100%) delete mode 100644 requirements/build.txt delete mode 100644 requirements/conda.txt create mode 100644 requirements/packs/core.txt rename requirements/{ => packs}/docs.txt (100%) create mode 100644 requirements/packs/pdf.txt create mode 100644 requirements/packs/plotting.txt create mode 100644 requirements/packs/scripts/_pytest.sh create mode 100644 requirements/packs/scripts/diffpy-srreal.bat create mode 100644 requirements/packs/scripts/diffpy-srreal.sh create mode 100644 requirements/packs/scripts/tar_url.txt rename requirements/{ => packs}/tests.txt (63%) delete mode 100644 requirements/pip.txt create mode 100644 requirements/profiles/_tests.yml create mode 100644 requirements/profiles/all.yml create mode 100644 setup.py create mode 100644 src/diffpy/cmi/cli.py create mode 100644 src/diffpy/cmi/conda.py delete mode 100644 src/diffpy/cmi/diffpy_cmi_app.py delete mode 100644 src/diffpy/cmi/functions.py create mode 100644 src/diffpy/cmi/installer.py create mode 100644 src/diffpy/cmi/log.py create mode 100644 src/diffpy/cmi/packsmanager.py create mode 100644 src/diffpy/cmi/profilesmanager.py delete mode 100644 tests/test_functions.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a8d14ee..7ce36bd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,4 +15,3 @@ Release notes * Add long description to README. * Add light-weight documentation migrated from old diffpy-CMI documentation. * Add bulk Ni PDF fitting tutorial. - diff --git a/CODE_OF_CONDUCT.rst b/CODE-OF-CONDUCT.rst similarity index 100% rename from CODE_OF_CONDUCT.rst rename to CODE-OF-CONDUCT.rst diff --git a/doc/Makefile b/docs/Makefile similarity index 100% rename from doc/Makefile rename to docs/Makefile diff --git a/doc/make.bat b/docs/make.bat similarity index 100% rename from doc/make.bat rename to docs/make.bat diff --git a/doc/source/_static/wrap_table.css b/docs/source/_static/wrap_table.css similarity index 100% rename from doc/source/_static/wrap_table.css rename to docs/source/_static/wrap_table.css diff --git a/doc/source/api/diffpy.cmi.rst b/docs/source/api/diffpy.cmi.rst similarity index 100% rename from doc/source/api/diffpy.cmi.rst rename to docs/source/api/diffpy.cmi.rst diff --git a/doc/source/conf.py b/docs/source/conf.py similarity index 100% rename from doc/source/conf.py rename to docs/source/conf.py diff --git a/doc/source/data/Ni.cif b/docs/source/data/Ni.cif similarity index 100% rename from doc/source/data/Ni.cif rename to docs/source/data/Ni.cif diff --git a/doc/source/data/Ni.gr b/docs/source/data/Ni.gr similarity index 100% rename from doc/source/data/Ni.gr rename to docs/source/data/Ni.gr diff --git a/doc/source/getting-started.rst b/docs/source/getting-started.rst similarity index 100% rename from doc/source/getting-started.rst rename to docs/source/getting-started.rst diff --git a/doc/source/img/Ni_fit.png b/docs/source/img/Ni_fit.png similarity index 100% rename from doc/source/img/Ni_fit.png rename to docs/source/img/Ni_fit.png diff --git a/doc/source/img/cmi_problem_types.png b/docs/source/img/cmi_problem_types.png similarity index 100% rename from doc/source/img/cmi_problem_types.png rename to docs/source/img/cmi_problem_types.png diff --git a/doc/source/img/diffpycmi_screenshot.png b/docs/source/img/diffpycmi_screenshot.png similarity index 100% rename from doc/source/img/diffpycmi_screenshot.png rename to docs/source/img/diffpycmi_screenshot.png diff --git a/doc/source/img/makerecipe_flow.png b/docs/source/img/makerecipe_flow.png similarity index 100% rename from doc/source/img/makerecipe_flow.png rename to docs/source/img/makerecipe_flow.png diff --git a/doc/source/img/pdfprimer.png b/docs/source/img/pdfprimer.png similarity index 100% rename from doc/source/img/pdfprimer.png rename to docs/source/img/pdfprimer.png diff --git a/doc/source/index.rst b/docs/source/index.rst similarity index 100% rename from doc/source/index.rst rename to docs/source/index.rst diff --git a/doc/source/license.rst b/docs/source/license.rst similarity index 100% rename from doc/source/license.rst rename to docs/source/license.rst diff --git a/doc/source/release.rst b/docs/source/release.rst similarity index 100% rename from doc/source/release.rst rename to docs/source/release.rst diff --git a/doc/source/snippets/data_params.rst b/docs/source/snippets/data_params.rst similarity index 100% rename from doc/source/snippets/data_params.rst rename to docs/source/snippets/data_params.rst diff --git a/doc/source/snippets/diffpy_libraries.rst b/docs/source/snippets/diffpy_libraries.rst similarity index 100% rename from doc/source/snippets/diffpy_libraries.rst rename to docs/source/snippets/diffpy_libraries.rst diff --git a/doc/source/snippets/external_libraries.rst b/docs/source/snippets/external_libraries.rst similarity index 100% rename from doc/source/snippets/external_libraries.rst rename to docs/source/snippets/external_libraries.rst diff --git a/doc/source/snippets/structure_params.rst b/docs/source/snippets/structure_params.rst similarity index 100% rename from doc/source/snippets/structure_params.rst rename to docs/source/snippets/structure_params.rst diff --git a/doc/source/tutorials/index.rst b/docs/source/tutorials/index.rst similarity index 100% rename from doc/source/tutorials/index.rst rename to docs/source/tutorials/index.rst diff --git a/pyproject.toml b/pyproject.toml index fbe8be5..374a8b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,11 +49,11 @@ exclude = [] # exclude packages matching these glob patterns (empty by default) namespaces = false # to disable scanning PEP 420 namespaces (true by default) [project.scripts] -diffpy-cmi = "diffpy.cmi.diffpy_cmi_app:main" -cmi = "diffpy.cmi.diffpy_cmi_app:main" +# diffpy-cmi = "diffpy.cmi.apps.cmi:main" +cmi = "diffpy.cmi.cli:main" [tool.setuptools.dynamic] -dependencies = {file = ["requirements/pip.txt"]} +dependencies = {file = ["requirements/packs/core.txt"]} [tool.codespell] exclude-file = ".codespell/ignore_lines.txt" diff --git a/requirements/build.txt b/requirements/build.txt deleted file mode 100644 index e69de29..0000000 diff --git a/requirements/conda.txt b/requirements/conda.txt deleted file mode 100644 index 24ce15a..0000000 --- a/requirements/conda.txt +++ /dev/null @@ -1 +0,0 @@ -numpy diff --git a/requirements/packs/core.txt b/requirements/packs/core.txt new file mode 100644 index 0000000..36ca7ac --- /dev/null +++ b/requirements/packs/core.txt @@ -0,0 +1,5 @@ +packaging +PyYAML +diffpy.utils +diffpy.srfit +diffpy.structure diff --git a/requirements/docs.txt b/requirements/packs/docs.txt similarity index 100% rename from requirements/docs.txt rename to requirements/packs/docs.txt diff --git a/requirements/packs/pdf.txt b/requirements/packs/pdf.txt new file mode 100644 index 0000000..3aff7c0 --- /dev/null +++ b/requirements/packs/pdf.txt @@ -0,0 +1,3 @@ +diffpy-srreal.sh +diffpy-srreal.bat +pyobjcryst diff --git a/requirements/packs/plotting.txt b/requirements/packs/plotting.txt new file mode 100644 index 0000000..6b41443 --- /dev/null +++ b/requirements/packs/plotting.txt @@ -0,0 +1,4 @@ +ipywidgets +matplotlib +ipympl +py3dmol>=2.0.1 diff --git a/requirements/packs/scripts/_pytest.sh b/requirements/packs/scripts/_pytest.sh new file mode 100644 index 0000000..4f77876 --- /dev/null +++ b/requirements/packs/scripts/_pytest.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Usage: +# ./run-remote-tests.sh urls.txt +# ./run-remote-tests.sh https://host/a.tar.gz https://host/b.tgz +# From ChatGPT + +set -euo pipefail + +URLS=() +if (($# == 1)) && [ -f "$1" ]; then + URLS_FILE="$1" + case "$URLS_FILE" in /*) ;; *) URLS_FILE="$PWD/$URLS_FILE" ;; esac + while IFS= read -r line; do + [[ -z "${line// }" || "$line" =~ ^[[:space:]]*# ]] && continue + URLS+=("$line") + done < "$URLS_FILE" +else + URLS=("$@") +fi + +START_DIR="$PWD" +TMPROOT="$(mktemp -d -p "$START_DIR" ".tmp_remote_tests.XXXXXX")" +trap 'cd "$START_DIR" 2>/dev/null || true; rm -rf -- "$TMPROOT"' EXIT +cd "$TMPROOT" + +overall_ec=0 +i=0 +for url in "${URLS[@]}"; do + ((i++)) + echo -e "\n==> [$i] $url" + + tarball="$(mktemp -p "$TMPROOT" "dl_${i}.XXXXXX.tar.gz")" + curl -L --fail -o "$tarball" "$url" + + pkgdir="$TMPROOT/pkg_${i}" + mkdir -p "$pkgdir" + tar -xzf "$tarball" -C "$pkgdir" + + first_entry="$(tar -tzf "$tarball" | head -1 || true)" + top="${first_entry%%/*}" + if [ -n "$top" ] && [ -d "$pkgdir/$top" ]; then + projroot="$pkgdir/$top" + else + projroot="$pkgdir" + fi + + if [ -d "$projroot/src" ]; then + rm -rf -- "$projroot/src" + fi + + if [ -d "$projroot/tests" ]; then + ( cd "$projroot" && PYTHONPATH="$PWD:tests:${PYTHONPATH:-}" pytest ) || overall_ec=1 + else + ( cd "$projroot" && PYTHONPATH="$PWD:${PYTHONPATH:-}" pytest ) || overall_ec=1 + fi + + rm -f -- "$tarball" + rm -rf -- "$pkgdir" +done + +exit "$overall_ec" diff --git a/requirements/packs/scripts/diffpy-srreal.bat b/requirements/packs/scripts/diffpy-srreal.bat new file mode 100644 index 0000000..11852c6 --- /dev/null +++ b/requirements/packs/scripts/diffpy-srreal.bat @@ -0,0 +1,2 @@ +conda install -y -c conda-forge -q libdiffpy libboost-devel libobjcryst periodictable +pip install diffpy.srreal diff --git a/requirements/packs/scripts/diffpy-srreal.sh b/requirements/packs/scripts/diffpy-srreal.sh new file mode 100644 index 0000000..11852c6 --- /dev/null +++ b/requirements/packs/scripts/diffpy-srreal.sh @@ -0,0 +1,2 @@ +conda install -y -c conda-forge -q libdiffpy libboost-devel libobjcryst periodictable +pip install diffpy.srreal diff --git a/requirements/packs/scripts/tar_url.txt b/requirements/packs/scripts/tar_url.txt new file mode 100644 index 0000000..463189e --- /dev/null +++ b/requirements/packs/scripts/tar_url.txt @@ -0,0 +1,5 @@ +https://github.com/diffpy/diffpy.srreal/archive/refs/tags/1.4.0.tar.gz +https://github.com/diffpy/diffpy.srfit/archive/refs/tags/3.2.0.tar.gz +https://github.com/diffpy/pyobjcryst/archive/refs/tags/2025.1.0.tar.gz +https://github.com/diffpy/diffpy.structure/archive/refs/tags/3.3.1.tar.gz +https://github.com/diffpy/diffpy.utils/archive/refs/tags/3.6.1.tar.gz diff --git a/requirements/tests.txt b/requirements/packs/tests.txt similarity index 63% rename from requirements/tests.txt rename to requirements/packs/tests.txt index a727786..162f28c 100644 --- a/requirements/tests.txt +++ b/requirements/packs/tests.txt @@ -4,3 +4,6 @@ codecov coverage pytest-cov pytest-env +pytest-mock +freezegun +DeepDiff diff --git a/requirements/pip.txt b/requirements/pip.txt deleted file mode 100644 index 24ce15a..0000000 --- a/requirements/pip.txt +++ /dev/null @@ -1 +0,0 @@ -numpy diff --git a/requirements/profiles/_tests.yml b/requirements/profiles/_tests.yml new file mode 100644 index 0000000..f779737 --- /dev/null +++ b/requirements/profiles/_tests.yml @@ -0,0 +1,11 @@ +# script name must coincide with conda installation name +# script named "_*" can escape install presence check + +packs: + - core + - pdf + - plotting + - tests + +extras: + - _pytest.sh tar_url.txt diff --git a/requirements/profiles/all.yml b/requirements/profiles/all.yml new file mode 100644 index 0000000..13158ad --- /dev/null +++ b/requirements/profiles/all.yml @@ -0,0 +1,9 @@ +# all + +packs: + - core + - pdf + - plotting + +extras: + - ipykernel diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b1906ef --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +# setup.py +# +# DO NOT REMOVE setup.py. It is needed for +# including example files in the shipped package +import os +import shutil + +from setuptools import setup +from setuptools.command.build_py import build_py as _build_py + + +class build_py(_build_py): + def run(self): + super().run() + extra_dirs = [ + ("docs/build", os.path.join("diffpy", "cmi", "docs", "build")), + ( + "docs/examples", + os.path.join("diffpy", "cmi", "docs", "examples"), + ), + ("requirements", os.path.join("diffpy", "cmi", "requirements")), + ] + for src, rel_dst in extra_dirs: + if not os.path.exists(src): + print(f"Skipping missing directory: {src}") + continue + dst = os.path.join(self.build_lib, rel_dst) + shutil.rmtree(dst, ignore_errors=True) + shutil.copytree(src, dst) + + +setup( + cmdclass={"build_py": build_py}, +) diff --git a/src/diffpy/cmi/__init__.py b/src/diffpy/cmi/__init__.py index 158d6da..62108fd 100644 --- a/src/diffpy/cmi/__init__.py +++ b/src/diffpy/cmi/__init__.py @@ -15,10 +15,22 @@ """Complex modeling infrastructure: a modular framework for multi-modal modeling of scientific data.""" +from importlib.resources import as_file, files + + +def get_package_dir(): + resource = files(__name__) + return as_file(resource) + + +__all__ = [ + "__version__", + "get_package_dir", +] + # package version from diffpy.cmi.version import __version__ # noqa -# silence the pyflakes syntax checker assert __version__ or True # End of file diff --git a/src/diffpy/cmi/cli.py b/src/diffpy/cmi/cli.py new file mode 100644 index 0000000..2831081 --- /dev/null +++ b/src/diffpy/cmi/cli.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python +############################################################################## +# +# (c) 2025 The Trustees of Columbia University in the City of New York. +# All rights reserved. +# +# File coded by: Simon Billinge and members of the Billinge Group. +# +# See GitHub contributions for a more detailed list of contributors. +# https://github.com/diffpy/diffpy.cmi/graphs/contributors +# +# See LICENSE.rst for license information. +# +############################################################################## + +import argparse +from pathlib import Path +from shutil import copytree +from typing import List, Optional, Tuple + +from diffpy.cmi import __version__, get_package_dir +from diffpy.cmi.conda import env_info +from diffpy.cmi.log import plog, set_log_mode +from diffpy.cmi.packsmanager import PacksManager +from diffpy.cmi.profilesmanager import ProfilesManager + + +# Examples +def _installed_examples_dir() -> Path: + """Return the absolute path to the installed examples directory. + + Returns + ------- + pathlib.Path + Directory containing shipped examples. + + Raises + ------ + FileNotFoundError + If the examples directory cannot be located in the installation. + """ + with get_package_dir() as pkgdir: + pkg = Path(pkgdir).resolve() + for c in ( + pkg / "docs" / "examples", + pkg.parents[2] / "docs" / "examples", + ): + if c.is_dir(): + return c + raise FileNotFoundError( + "Could not locate requirements/packs. Check your installation." + ) + + +def list_examples() -> List[str]: + """List installed example names. + + Returns + ------- + list of str + Installed example directory names. + """ + root = _installed_examples_dir() + if not root.exists(): + return [] + return sorted([p.name for p in root.iterdir() if p.is_dir()]) + + +def copy_example(example: str) -> Path: + """Copy an example into the current working directory. + + Parameters + ---------- + example : str + Example directory name under the installed examples root. + + Returns + ------- + pathlib.Path + Destination path created under the current working directory. + + Raises + ------ + FileNotFoundError + If the example directory does not exist. + FileExistsError + If the destination directory already exists. + """ + src = _installed_examples_dir() / example + if not src.exists() or not src.is_dir(): + raise FileNotFoundError(f"Example not found: {example}") + dest = Path.cwd() / example + if dest.exists(): + raise FileExistsError(f"Destination {dest} already exists") + copytree(src, dest) + return dest + + +# Manual +def open_manual_and_exit() -> None: + """Open the installed manual or fall back to the online version, + then exit. + + Notes + ----- + This function terminates the process with ``SystemExit(0)``. + """ + import webbrowser + + v = __version__.split(".post")[0] + webdocbase = "https://www.diffpy.org/doc/cmi/" + v + with get_package_dir() as packagedir: + localpath = Path(packagedir) / "docs" / "build" / "html" / "index.html" + url = ( + localpath.resolve().as_uri() + if localpath.is_file() + else f"{webdocbase}/index.html" + ) + if not localpath.is_file(): + plog.info("Manual files not found, falling back to online version.") + plog.info("Opening manual at %s", url) + webbrowser.open(url) + raise SystemExit(0) + + +# Parser +def _build_parser() -> argparse.ArgumentParser: + """Build and return the top-level argument parser for the CLI. + + Returns + ------- + argparse.ArgumentParser + Configured parser with all subcommands and options. + """ + p = argparse.ArgumentParser( + prog="cmi", + description=( + "Welcome to diffpy-CMI, a complex modeling infrastructure " + "for multi-modal analysis of scientific data.\n\n" + ), + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p.add_argument( + "-v", "--verbose", action="store_true", help="Enable debug logging." + ) + p.add_argument( + "-V", "--version", action="version", version=f"%(prog)s {__version__}" + ) + p.add_argument( + "--manual", action="store_true", help="Open manual and exit." + ) + p.set_defaults(_parser=p) + + sub = p.add_subparsers(dest="cmd", metavar="") + + # example + p_example = sub.add_parser("example", help="List or copy an example") + p_example.set_defaults(_parser=p_example) + sub_ex = p_example.add_subparsers( + dest="example_cmd", metavar="" + ) + sub_ex.add_parser("list", help="List examples").set_defaults( + _parser=p_example + ) + p_example_copy = sub_ex.add_parser("copy", help="Copy an example to CWD") + p_example_copy.add_argument("name", metavar="EXAMPLE", help="Example name") + p_example_copy.set_defaults(_parser=p_example) + p_example.set_defaults(example_cmd=None) + + # pack + p_pack = sub.add_parser("pack", help="List packs or show a pack file") + p_pack.set_defaults(_parser=p_pack) + sub_pack = p_pack.add_subparsers(dest="pack_cmd", metavar="") + sub_pack.add_parser( + "list", help="List packs (Installed vs Available)" + ).set_defaults(_parser=p_pack) + p_pack_show = sub_pack.add_parser( + "show", help="Show a pack (by base name)" + ) + p_pack_show.add_argument("name", metavar="PACK", help="Pack base name") + p_pack_show.set_defaults(_parser=p_pack) + p_pack.set_defaults(pack_cmd=None) + + # profile + p_prof = sub.add_parser( + "profile", help="List profiles or show a profile file" + ) + p_prof.set_defaults(_parser=p_prof) + sub_prof = p_prof.add_subparsers(dest="profile_cmd", metavar="") + sub_prof.add_parser( + "list", help="List profiles (Installed vs Available)" + ).set_defaults(_parser=p_prof) + p_prof_show = sub_prof.add_parser( + "show", help="Show a profile (by base name)" + ) + p_prof_show.add_argument( + "name", metavar="PROFILE", help="Profile base name" + ) + p_prof_show.set_defaults(_parser=p_prof) + p_prof.set_defaults(profile_cmd=None) + + # install (multiple targets) + p_install = sub.add_parser("install", help="Install packs/profiles") + p_install.add_argument( + "targets", + nargs="*", + help="One or more targets: pack/profile base names \ + or absolute profile file/dir.", + ) + p_install.add_argument( + "-c", + "--channel", + dest="default_channel", + default="conda-forge", + help="Default conda channel for packages \ + without explicit per-line channel.", + ) + p_install.set_defaults(_parser=p_install) + + # env + sub.add_parser("env", help="Show basic conda environment info") + + return p + + +# Helpers +def _installed_pack_path(mgr: PacksManager, name: str) -> Path: + """Return the absolute path to an installed pack file. + + Parameters + ---------- + mgr : PacksManager + Packs manager instance. + name : str + Pack basename (without ``.txt``). + + Returns + ------- + pathlib.Path + Absolute path to the pack file. + + Raises + ------ + FileNotFoundError + If the pack cannot be found. + """ + path = mgr.packs_dir / f"{name}.txt" + if not path.is_file(): + raise FileNotFoundError(f"Pack not found: {name} ({path})") + return path + + +def _installed_profile_path(name: str) -> Path: + """Return the absolute path to an installed profile file by + basename. + + Parameters + ---------- + name : str + Profile basename (without extension). + + Returns + ------- + pathlib.Path + Absolute path to the profile file. + + Raises + ------ + FileNotFoundError + If the profile cannot be found under the installed profiles directory. + """ + base = ProfilesManager().profiles_dir + for cand in (base / f"{name}.yml", base / f"{name}.yaml"): + if cand.is_file(): + return cand + raise FileNotFoundError(f"Profile not found: {name} (under {base})") + + +def _resolve_target_for_install(s: str) -> Tuple[str, Path]: + """Return ('pack'|'profile', absolute path) for a single install + target. + + Delegates resolution to manager resolvers to keep rules centralized: + :meth:`PacksManager._resolve_pack_file` and + :meth:`ProfilesManager._resolve_profile_file`. + """ + mgr = PacksManager() + pm = ProfilesManager() + p = Path(s) + + if p.is_absolute(): + return "profile", pm._resolve_profile_file(p) + + pack_path = None + profile_path = None + try: + pack_path = mgr._resolve_pack_file(s) + except FileNotFoundError: + pass + try: + profile_path = pm._resolve_profile_file(s) + except FileNotFoundError: + pass + + if pack_path and profile_path: + raise ValueError( + f"Ambiguous install target '{s}': both a pack and a profile exist." + ) + if pack_path: + return "pack", pack_path + if profile_path: + return "profile", profile_path + + raise FileNotFoundError(f"No installed pack or profile named '{s}' found.") + + +def _cmd_example(ns: argparse.Namespace) -> int: + """Handle `cmi example` subcommands. + + Parameters + ---------- + ns : argparse.Namespace + Parsed arguments for the example subparser. + + Returns + ------- + int + Exit code (``0`` on success; non-zero on failure). + """ + if ns.example_cmd in (None, "copy"): + name = getattr(ns, "name", None) + if not name: + plog.error( + "Missing example name. Use `cmi example list` to see options." + ) + ns._parser.print_help() + return 1 + out = copy_example(name) + print(f"Example copied to: {out}") + return 0 + if ns.example_cmd == "list": + for g in list_examples(): + print(g) + return 0 + + plog.error("Unknown example subcommand.") + ns._parser.print_help() + return 2 + + +def _cmd_pack(ns: argparse.Namespace) -> int: + """Handle `cmi pack` subcommands. + + Parameters + ---------- + ns : argparse.Namespace + Parsed arguments for the pack subparser. + + Returns + ------- + int + Exit code (``0`` on success; non-zero on failure). + """ + mgr = PacksManager() + if ns.pack_cmd == "list": + names = mgr.available_packs() + installed, available = [], [] + for nm in names: + (installed if mgr.check_pack(nm) else available).append(nm) + + def dump(title: str, arr: List[str]) -> None: + print(title + ":") + if not arr: + print(" (none)") + else: + for n in arr: + print(f" - {n}") + + dump("Installed", installed) + dump("Available to install", available) + return 0 + + name = getattr(ns, "name", None) or getattr(ns, "pack_cmd", None) + if not name or name == "show": + plog.error("Usage: cmi pack (or: cmi pack show )") + ns._parser.print_help() + return 1 + + path = _installed_pack_path(mgr, name) + print(f"# pack: {name}\n# path: {path}\n") + print(path.read_text(encoding="utf-8")) + return 0 + + +def _cmd_profile(ns: argparse.Namespace) -> int: + """Handle `cmi profile` subcommands. + + Parameters + ---------- + ns : argparse.Namespace + Parsed arguments for the profile subparser. + + Returns + ------- + int + Exit code (``0`` on success; non-zero on failure). + """ + if ns.profile_cmd == "list": + pm = ProfilesManager() + names = pm.list_profiles() + installed, available = [], [] + for nm in names: + (installed if pm.check_profile(nm) else available).append(nm) + + def dump(title: str, arr: List[str]) -> None: + print(title + ":") + if not arr: + print(" (none)") + else: + for n in arr: + print(f" - {n}") + + dump("Installed", installed) + dump("Available to install", available) + return 0 + + name = getattr(ns, "name", None) or getattr(ns, "profile_cmd", None) + if not name or name == "show": + plog.error("Usage: cmi profile (or: cmi profile show )") + ns._parser.print_help() + return 1 + + path = _installed_profile_path(name) + print(f"# profile: {name}\n# path: {path}\n") + print(path.read_text(encoding="utf-8")) + return 0 + + +def _cmd_install(ns: argparse.Namespace) -> int: + """Handle `cmi install` subcommand for packs and profiles. + + Parameters + ---------- + ns : argparse.Namespace + Parsed arguments for the install subparser. + + Returns + ------- + int + Exit code (``0`` on success; non-zero on failure). + """ + if not getattr(ns, "targets", None): + plog.error( + "Missing install targets. " + "Provide pack/profile names or an absolute profile path." + ) + ns._parser.print_help() + return 1 + + rc = 0 + mgr = PacksManager() + pm = ProfilesManager() + for tgt in ns.targets: + try: + kind, path = _resolve_target_for_install(tgt) + if kind == "pack": + mgr.install_pack(path.stem) + else: + pm.install(path if path.is_absolute() else path.stem) + except (ValueError, FileNotFoundError) as e: + plog.error("%s", e) + ns._parser.print_help() + rc = 1 + except Exception as e: + plog.error("%s", e) + rc = 1 + return rc + + +def _cmd_env(_: argparse.Namespace) -> int: + """Print basic conda environment information. + + Parameters + ---------- + _ : argparse.Namespace + Unused parsed arguments placeholder. + + Returns + ------- + int + Always ``0``. + """ + info = env_info() + print("Conda environment:") + print(f" available : {info.available}") + print(f" mamba : {info.mamba}") + print(f" env_name : {info.env_name or '(unknown)'}") + print(f" prefix : {info.prefix or '(unknown)'}") + return 0 + + +def main(argv: Optional[List[str]] = None) -> int: + """Run the CMI CLI. + + Parameters + ---------- + argv : list of str, optional + Argument vector to parse. When ``None``, defaults to ``sys.argv[1:]``. + + Returns + ------- + int + Process exit code (``0`` success, ``1`` failure, ``2`` usage error). + """ + parser = _build_parser() + ns = parser.parse_args(argv) + + set_log_mode(ns.verbose) + + if ns.manual: + open_manual_and_exit() + + if ns.cmd is None: + parser.print_help() + return 2 + + if ns.cmd == "example": + return _cmd_example(ns) + if ns.cmd == "pack": + return _cmd_pack(ns) + if ns.cmd == "profile": + return _cmd_profile(ns) + if ns.cmd == "install": + return _cmd_install(ns) + if ns.cmd == "env": + return _cmd_env(ns) + + plog.error("Unknown command: %s", ns.cmd) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/diffpy/cmi/conda.py b/src/diffpy/cmi/conda.py new file mode 100644 index 0000000..516d8b7 --- /dev/null +++ b/src/diffpy/cmi/conda.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python +############################################################################## +# +# (c) 2025 The Trustees of Columbia University in the City of New York. +# All rights reserved. +# +# File coded by: Simon Billinge and members of the Billinge Group. +# +# See GitHub contributions for a more detailed list of contributors. +# https://github.com/diffpy/diffpy.cmi/graphs/contributors +# +# See LICENSE.rst for license information. +# +############################################################################## + +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Sequence, Tuple + +from diffpy.cmi.log import is_debug, plog + +__all__ = [ + "CondaEnvInfo", + "env_info", + "available", + "mamba_available", + "list_installed_names", + "install_specs", + "reset_cache", + "run", +] + +# Environment info + + +@dataclass +class CondaEnvInfo: + """Snapshot of conda/mamba availability and the active environment. + + Parameters + ---------- + available : bool + Whether ``conda`` is available on ``PATH``. + mamba : bool + Whether ``mamba`` is available on ``PATH``. + env_name : str or None + Name of the active environment, if known. + prefix : str or None + Filesystem prefix of the active environment, if known. + """ + + available: bool + mamba: bool + env_name: Optional[str] + prefix: Optional[str] + + +def run( + cmd: Sequence[str], + cwd: Optional[Path] = None, + *, + capture: Optional[bool] = None, +) -> Tuple[int, str]: + """Run a subprocess with sensible, mode-dependent defaults. + + The function does **not** decide logging policy. It simply runs the + command and returns the exit code and combined output. Visibility of + the child output follows CMI's current log mode: + + - In **debug** mode: stream output live by default. + - In **user** mode: capture output quietly by default. + + Parameters + ---------- + cmd : sequence of str + Command and arguments. + cwd : path-like, optional + Working directory for the child process. + capture : bool or None, optional + Override the default visibility. ``True`` forces capture; ``False`` + forces streaming; ``None`` selects the mode-dependent default. + + Returns + ------- + tuple of (int, str) + Exit code and captured output (empty string when streaming). + + Notes + ----- + When capturing in debug mode, the output is also echoed so developers + still see live progress. + """ + dbg = is_debug() + do_capture = (not dbg) if capture is None else bool(capture) + qcmd = " ".join(str(x) for x in cmd) + + try: + if do_capture: + cp = subprocess.run( + list(cmd), + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + check=False, + ) + rc = cp.returncode + out = (cp.stdout or "") + (cp.stderr or "") + # In debug, also echo captured output so devs see it live + if dbg: + if cp.stdout: + sys.stdout.write(cp.stdout) + if cp.stderr: + sys.stderr.write(cp.stderr) + if cp.stdout or cp.stderr: + sys.stdout.flush() + sys.stderr.flush() + else: + cp = subprocess.run( + list(cmd), + cwd=str(cwd) if cwd else None, + text=True, + check=False, + ) + rc = cp.returncode + out = "" + if rc != 0: + plog.debug("Command failed (%d): %s", rc, qcmd) + return rc, out + except FileNotFoundError as e: + plog.debug("Command not found: %s (%s)", cmd[0], e) + return 127, str(e) + + +def available() -> bool: + """Return whether ``conda`` is available on PATH. + + Returns + ------- + bool + ``True`` if the ``conda`` executable can be invoked. + """ + rc, _ = run(["conda", "--version"]) + return rc == 0 + + +def mamba_available() -> bool: + """Return whether ``mamba`` is available on PATH. + + Returns + ------- + bool + ``True`` if the ``mamba`` executable can be invoked. + """ + rc, _ = run(["mamba", "--version"]) + return rc == 0 + + +def env_info() -> CondaEnvInfo: + """Return availability and active-environment metadata. + + Returns + ------- + CondaEnvInfo + Structured information assembled from ``conda info --json`` and + availability probes. + """ + rc, out = run(["conda", "info", "--json"], capture=True) + if rc == 0: + try: + data = json.loads(out) if out else None + except Exception as e: + plog.debug("Failed to parse JSON: %s", e) + data = None + else: + data = None + env_name = None + prefix = None + if isinstance(data, dict): + env_name = data.get("active_prefix_name") + prefix = data.get("active_prefix") or data.get("default_prefix") + return CondaEnvInfo( + available=available(), + mamba=mamba_available(), + env_name=env_name, + prefix=prefix, + ) + + +_installed_names_cache: Optional[set[str]] = None + + +def reset_cache() -> None: + """Reset the internal cache of installed package names.""" + global _installed_names_cache + _installed_names_cache = None + + +def list_installed_names() -> List[str]: + """Return conda package names from ``conda list --json``. + + Results are cached for the current process to reduce repeated shell calls. + + Returns + ------- + list of str + Sorted unique package names. + """ + global _installed_names_cache + if _installed_names_cache is not None: + return sorted(_installed_names_cache) + + rc, out = run(["conda", "list", "--json"], capture=True) + names: set[str] = set() + if rc == 0: + try: + arr = json.loads(out) or [] + for rec in arr: + n = (rec or {}).get("name") + if isinstance(n, str) and n: + names.add(n) + except Exception as e: + plog.debug("conda list JSON parse failed: %s", e) + else: + plog.debug("conda list returned rc=%d", rc) + + _installed_names_cache = names + return sorted(names) + + +def _install_args(channel: Optional[str], default_channel: str) -> List[str]: + """Return common install arguments for conda/mamba. + + Parameters + ---------- + channel : str or None + Target channel for the batch, or ``None`` to use ``default_channel``. + default_channel : str + Channel used when ``channel`` is ``None``. + + Returns + ------- + list of str + Flattened list of CLI arguments. + """ + args: List[str] = ["-y"] + ch = (channel or default_channel).strip() + if ch: + args += ["-c", ch] + return args + + +def install_specs( + specs: Sequence[str], + *, + channel: Optional[str] = None, + default_channel: str = "conda-forge", +) -> Tuple[str, int, str]: + """Install a batch of specs, preferring mamba then conda. + + Parameters + ---------- + specs : sequence of str + Conda spec strings (e.g., ``"numpy>=1.24"``). + channel : str or None, optional + Preferred channel for this batch. + default_channel : str, optional + Channel used when ``channel`` is not given. + + Returns + ------- + tuple of (str, int, str) + The solver used (``"mamba"`` or ``"conda"``), the exit code, and the + captured output (empty when streaming). + + Notes + ----- + This function logs at ``INFO`` level when each batch starts, warns when a + mamba batch fails and a fallback is attempted, and leaves error decisions + to higher-level callers. + """ + specs = list(specs) + if not specs: + return "none", 0, "" + + # Try mamba first + if mamba_available(): + cmd = ( + ["mamba", "install"] + + _install_args(channel, default_channel) + + specs + ) + plog.info( + "mamba batch (%s): %s", channel or default_channel, " ".join(specs) + ) + rc, out = run(cmd) + if rc == 0: + reset_cache() + return "mamba", 0, out + plog.info( + "mamba batch failed for channel %s", channel or default_channel + ) + + # Fallback to conda + if available(): + cmd = ( + ["conda", "install"] + + _install_args(channel, default_channel) + + specs + ) + plog.info( + "conda batch (%s): %s", channel or default_channel, " ".join(specs) + ) + rc, out = run(cmd) + if rc == 0: + reset_cache() + return "conda", 0, out + return "conda", rc, out + + return "unavailable", 1, "Neither mamba nor conda is available." diff --git a/src/diffpy/cmi/diffpy_cmi_app.py b/src/diffpy/cmi/diffpy_cmi_app.py deleted file mode 100644 index 1a46163..0000000 --- a/src/diffpy/cmi/diffpy_cmi_app.py +++ /dev/null @@ -1,30 +0,0 @@ -import argparse - -from diffpy.cmi.version import __version__ - - -def main(): - parser = argparse.ArgumentParser( - prog="diffpy-cmi", - description=( - "Welcome to diffpy-CMI, a complex modeling infrastructure " - "for multi-modal analysis of scientific data.\n\n" - "Docs: https://www.diffpy.org/diffpy.cmi" - ), - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument( - "--version", - action="store_true", - help="Show the program's version number and exit", - ) - args = parser.parse_args() - if args.version: - print(f"diffpy.cmi {__version__}") - else: - # Default behavior when no arguments are given - parser.print_help() - - -if __name__ == "__main__": - main() diff --git a/src/diffpy/cmi/functions.py b/src/diffpy/cmi/functions.py deleted file mode 100644 index e7e2c8e..0000000 --- a/src/diffpy/cmi/functions.py +++ /dev/null @@ -1,31 +0,0 @@ -import numpy as np - - -def dot_product(a, b): - """Compute the dot product of two vectors of any size. - - Ensure that the inputs, a and b, are of the same size. - The supported types are "array_like" objects, which can - be converted to a NumPy array. Examples include lists and tuples. - - Parameters - ---------- - a : array_like - The first input vector. - b : array_like - The second input vector. - - Returns - ------- - float - The dot product of the two vectors. - - Examples - -------- - Compute the dot product of two lists: - >>> a = [1, 2, 3] - >>> b = [4, 5, 6] - >>> dot_product(a, b) - 32.0 - """ - return float(np.dot(a, b)) diff --git a/src/diffpy/cmi/installer.py b/src/diffpy/cmi/installer.py new file mode 100644 index 0000000..653b515 --- /dev/null +++ b/src/diffpy/cmi/installer.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python +############################################################################## +# +# (c) 2025 The Trustees of Columbia University in the City of New York. +# All rights reserved. +# +# File coded by: Simon Billinge and members of the Billinge Group. +# +# See GitHub contributions for a more detailed list of contributors. +# https://github.com/diffpy/diffpy.cmi/graphs/contributors +# +# See LICENSE.rst for license information. +# +############################################################################## + +import importlib.metadata as md +import os +import shlex +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +from packaging.requirements import InvalidRequirement +from packaging.requirements import Requirement as PkgRequirement + +from diffpy.cmi import conda +from diffpy.cmi.log import plog + +__all__ = [ + "ParsedReq", + "parse_requirement_line", + "presence_check", + "install_requirements", +] + + +# Types and parsing +@dataclass +class ParsedReq: + """Parsed representation of a single requirement line. + + A requirement line may declare a package or a script. Script lines set + :attr:`name` to the basename (stem) of the script so presence checking can + treat them like packages when appropriate. + + Parameters + ---------- + raw : str + Original, unmodified line (kept verbatim). + kind : {"script", "pkg", "skip"} + Classification of the line. + name : str or None, optional + Package name (for ``pkg``) or script basename (for ``script``). + spec : str or None, optional + Version specifier (e.g., ``">=1.2"``) for packages. + channel : str or None, optional + Optional channel (from ``channel::spec``) for packages. + """ + + raw: str + kind: str + name: Optional[str] = None + spec: Optional[str] = None + channel: Optional[str] = None + + +def parse_requirement_line(line: str) -> ParsedReq: + """Parse a single requirement definition. + + Supported forms + --------------- + - **Scripts:** a line whose first token ends with ``.sh`` or ``.bat``. + The script's basename becomes :attr:`ParsedReq.name`. + - **Packages:** an optional ``channel::`` prefix followed by a valid + PEP 508 requirement string (e.g., ``"numpy>=1.24"``). + + Parameters + ---------- + line : str + Raw line from a requirements file. + + Returns + ------- + ParsedReq + Structured representation of the requirement. + """ + s = (line or "").strip() + if not s or s.startswith("#"): + return ParsedReq(raw=line, kind="skip") + + # Script? + try: + first = shlex.split(s, posix=(os.name != "nt"))[0] + except Exception: + first = s.split()[0] if s.split() else s + if first.lower().endswith((".sh", ".bat")): + return ParsedReq(raw=line, kind="script", name=Path(first).stem) + + # Package + chan = None + body = s + if "::" in s: + c, rest = s.split("::", 1) + if c and all(ch not in c for ch in " <>="): + chan, body = c, rest + try: + req = PkgRequirement(body) + name = req.name + spec = str(req.specifier) or None + return ParsedReq( + raw=line, kind="pkg", name=name, spec=spec, channel=chan + ) + except InvalidRequirement: + plog.info("Skipping invalid requirement line: %s", line.rstrip()) + return ParsedReq(raw=line, kind="skip") + + +def _is_installed(name: str) -> bool: + """Return whether a package is importable or listed by conda. + + The check first tries Python package metadata and then falls back to + the names reported by ``conda list --json``. + + Parameters + ---------- + name : str + Distribution/package name to check. + + Returns + ------- + bool + ``True`` if present, ``False`` otherwise. + """ + try: + md.version(name) + return True + except md.PackageNotFoundError: + pass + except Exception as e: + plog.debug("Package %s not found in Python metadata: %s", name, e) + return name.lower() in {n.lower() for n in conda.list_installed_names()} + + +def presence_check( + reqs: List[ParsedReq], *, check_meta: Optional[bool] = True +) -> Tuple[bool, List[str]]: + """Check whether requirements appear satisfied. + + Semantics + --------- + - **Packages:** mark missing when :func:`_is_installed` is ``False``. + - **Scripts:** use the basename as a pseudo-package name. + * If the basename starts with ``"_"`` (meta-scripts), count them as + missing only when ``check_meta=True`` (profile checks). During + post-install verification ``check_meta=False`` so meta-scripts + are ignored (script exit code already enforced). + * Otherwise, treat like a package and require the basename to be present. + + Parameters + ---------- + reqs : list of ParsedReq + Parsed requirements to check. + check_meta : bool, optional + Whether meta-scripts should be considered missing. + Defaults to ``True``. + + Returns + ------- + tuple of (bool, list of str) + ``(ok, missing)`` where ``ok`` is ``True`` when all checks pass and + ``missing`` lists raw lines deemed missing. + """ + missing: List[str] = [] + for r in reqs: + if r.kind == "skip" or not r.name: + continue + elif r.kind == "script" and r.name[0] == "_": + if check_meta: + missing.append(r.raw.strip()) + continue + elif not _is_installed(r.name): + missing.append(r.raw.strip()) + return (len(missing) == 0), missing + + +def _script_supported_on_platform(ext: str) -> bool: + """Return whether scripts with the given extension are runnable + here. + + Parameters + ---------- + ext : str + File extension including the leading dot (e.g., ``".sh"``). + + Returns + ------- + bool + ``True`` if runnable on the current platform, ``False`` otherwise. + """ + ext = (ext or "").lower() + if os.name == "nt": + if ext == ".sh": + return shutil.which("bash") is not None + return ext == ".bat" + return ext == ".sh" + + +def _resolve_script_path(first_token: str, scripts_root: Path) -> Path: + """Resolve a script path under the configured scripts directory. + + Rules + ----- + 1) If ``first_token`` is an **absolute file path**, accept as-is. + 2) Otherwise treat it as a relative file under ``scripts_root``. + + Parameters + ---------- + first_token : str + First token from the script line (ends with ``.sh`` or ``.bat``). + scripts_root : path-like + Root directory containing available scripts. + + Returns + ------- + pathlib.Path + Absolute resolved path to the script. + + Raises + ------ + FileNotFoundError + If the path cannot be resolved by the above rules. + """ + p = Path(first_token) + if p.is_absolute(): + if p.is_file(): + return p.resolve() + raise FileNotFoundError(f"Script not found: {p}") + cand = scripts_root / first_token + if cand.is_file(): + return cand.resolve() + raise FileNotFoundError( + f"Script not found: {first_token}" + f" (expected under {scripts_root} or absolute file path)" + ) + + +def _script_exec_cmd(path: Path, args: List[str]) -> List[str]: + """Return the execution command for a script on this platform. + + Parameters + ---------- + path : path-like + Script path. + args : list of str + Arguments to pass to the script. + + Returns + ------- + list of str + Command vector to execute via :func:`conda.run`. + """ + ext = path.suffix.lower() + if os.name == "nt": + if ext == ".sh": + bash = shutil.which("bash") + if bash: + return [bash, str(path)] + args + return [str(path)] + args + if ext != ".bat": + return [str(path)] + args + return ["cmd", "/c", str(path)] + args + if ext == ".sh": + shell = shutil.which("bash") or shutil.which("sh") or "sh" + return [shell, str(path)] + args + return [str(path)] + args + + +# Install policy + + +def install_requirements( + reqs: List[ParsedReq], + *, + scripts_root: Path, + default_channel: str = "conda-forge", +) -> int: + """Install requirements, run scripts, and verify presence. + + Policy + ------ + 1) **Packages first**: batch per channel; solver failures are treated as + soft (final presence check decides). + 2) **Scripts next**: run native scripts only; a non-zero exit is a hard + failure. + 3) **Presence check**: verify packages and non-meta scripts. + + Parameters + ---------- + reqs : list of ParsedReq + Parsed requirements (from packs and profile extras). + scripts_root : path-like + Directory containing relative scripts used by recipes. + default_channel : str, optional + Default conda channel for batches. Defaults to ``"conda-forge"``. + + Returns + ------- + int + ``0`` on success, ``1`` on failure. + """ + # Batch packages by channel + by_channel: Dict[str, List[str]] = {} + for r in reqs: + if r.kind != "pkg" or not r.name: + continue + if _is_installed(r.name): + continue + chan = r.channel or default_channel + spec = f"{r.name}{r.spec or ''}" + lst = by_channel.setdefault(chan, []) + if spec not in lst: + lst.append(spec) + + for chan, specs in by_channel.items(): + solver, rc, _ = conda.install_specs( + specs, channel=chan, default_channel=default_channel + ) + if rc != 0: + plog.info( + "Batch install did not complete cleanly with %s " + "for channel '%s'. " + "Run in debug mode (-v) to see solver output.", + solver, + chan, + ) + conda.reset_cache() + + # Execute scripts + for r in reqs: + if r.kind != "script": + continue + first_token = shlex.split(r.raw.strip(), posix=(os.name != "nt"))[0] + ext = Path(first_token).suffix.lower() + if not _script_supported_on_platform(ext): + plog.info( + "Skipping non-native script on this platform: %s", first_token + ) + continue + + path = _resolve_script_path(first_token, scripts_root) + args = shlex.split(r.raw, posix=(os.name != "nt"))[1:] + cmd = _script_exec_cmd(path, args) + plog.info("Running script: %s", " ".join(cmd)) + rc, _ = conda.run(cmd, cwd=path.parent, capture=False) + if rc != 0: + plog.error("Script failed (exit %d): %s", rc, r.raw.strip()) + return 1 + conda.reset_cache() + + # Final presence check + ok, missing = presence_check(reqs, check_meta=False) + if not ok: + plog.error( + "Requirements not fully satisfied after install: %s", + ", ".join(missing), + ) + return 1 + + plog.info("Requirements installation complete.") + return 0 diff --git a/src/diffpy/cmi/log.py b/src/diffpy/cmi/log.py new file mode 100644 index 0000000..9eea488 --- /dev/null +++ b/src/diffpy/cmi/log.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +############################################################################## +# +# (c) 2025 The Trustees of Columbia University in the City of New York. +# All rights reserved. +# +# File coded by: Simon Billinge and members of the Billinge Group. +# +# See GitHub contributions for a more detailed list of contributors. +# https://github.com/diffpy/diffpy.cmi/graphs/contributors +# +# See LICENSE.rst for license information. +# +############################################################################## +"""Centralized logging utilities for the CMI package. + +This module exposes a single package logger :data:`plog` and helpers to +switch between a concise *user* mode and a verbose *debug* mode. + +Modes +----- +user + Only ``INFO`` and ``ERROR/CRITICAL`` records are shown. ``WARNING`` and + ``DEBUG`` are hidden. +debug + All levels are shown. + +Notes +----- +Use :func:`set_log_mode` in the CLI to toggle visibility. The logger itself +always emits at ``DEBUG`` level; a handler-side filter controls what is shown. +""" + +import logging + +__all__ = ["plog", "set_log_mode", "get_log_mode", "is_debug"] + +# Package logger +plog = logging.getLogger("diffpy.cmi") + + +class _AllowLevels(logging.Filter): + """Filter that allows only a chosen set of levels.""" + + def __init__(self, *levels: int): + super().__init__() + self._allowed = set(levels) + + def set_allowed(self, *levels: int) -> None: + """Set the allowed levels for this filter.""" + self._allowed = set(levels) + + def filter(self, record: logging.LogRecord) -> bool: + """Return True if the record's level is allowed.""" + return (not self._allowed) or (record.levelno in self._allowed) + + +# Global mode flag +_mode: str = "user" + +# Configure a default handler on first import +_handler = logging.StreamHandler() +_handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) +plog.setLevel( + logging.DEBUG +) # logger always emits; filter/handler decide visibility +plog.propagate = False +plog.handlers.clear() +plog.addHandler(_handler) + +# In user mode, suppress WARNING/DEBUG; in debug mode, show everything +_plog_filter = _AllowLevels(logging.INFO, logging.ERROR, logging.CRITICAL) +_handler.addFilter(_plog_filter) + + +def set_log_mode(mode: "str | bool" = "user") -> None: + """Set visible logging mode. + + Parameters + ---------- + mode : {"user","debug"} or bool, optional + - ``"user"`` (``False``): ``INFO`` and ``ERROR`` are visible + - ``"debug"`` (``True``): all levels visible. + """ + global _mode + if isinstance(mode, bool): + m = "debug" if mode else "user" + elif isinstance(mode, str): + m = (mode or "user").strip().lower() + else: + m = "user" + if m not in {"user", "debug"}: + m = "user" + if m == "debug": + _plog_filter.set_allowed() + _mode = "debug" + plog.debug("log mode set to debug") + else: + _plog_filter.set_allowed(logging.INFO, logging.ERROR, logging.CRITICAL) + _mode = "user" + + +def get_log_mode() -> str: + """Return ``"user"`` or ``"debug"``.""" + return _mode + + +def is_debug() -> bool: + """Return ``True`` when debug/verbose mode is active.""" + return _mode == "debug" diff --git a/src/diffpy/cmi/packsmanager.py b/src/diffpy/cmi/packsmanager.py new file mode 100644 index 0000000..58251de --- /dev/null +++ b/src/diffpy/cmi/packsmanager.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +############################################################################## +# +# (c) 2025 The Trustees of Columbia University in the City of New York. +# All rights reserved. +# +# File coded by: Simon Billinge and members of the Billinge Group. +# +# See GitHub contributions for a more detailed list of contributors. +# https://github.com/diffpy/diffpy.cmi/graphs/contributors +# +# See LICENSE.rst for license information. +# +############################################################################## + +from pathlib import Path +from typing import List, Union + +from diffpy.cmi import get_package_dir +from diffpy.cmi.installer import ( + ParsedReq, + install_requirements, + parse_requirement_line, + presence_check, +) +from diffpy.cmi.log import plog + +__all__ = ["PacksManager"] + + +def _installed_packs_dir() -> Path: + """Locate requirements/packs/ for the installed package.""" + with get_package_dir() as pkgdir: + pkg = Path(pkgdir).resolve() + for c in ( + pkg / "requirements" / "packs", + pkg.parents[2] / "requirements" / "packs", + ): + if c.is_dir(): + return c + raise FileNotFoundError( + "Could not locate requirements/packs. Check your installation." + ) + + +class PacksManager: + """Discovery, parsing, and installation for pack files. + + Attributes + ---------- + packs_dir : pathlib.Path + Absolute path to the installed packs directory. + Defaults to `requirements/packs` under the installed package. + """ + + def __init__(self) -> None: + self.packs_dir = _installed_packs_dir() + + def available_packs(self) -> List[str]: + """List all available packs. + + Returns + ------- + list of str + Pack basenames available under :attr:`packs_dir`. + """ + return sorted( + p.stem for p in self.packs_dir.glob("*.txt") if p.is_file() + ) + + def _resolve_pack_file(self, identifier: Union[str, Path]) -> Path: + """Resolve a pack identifier to an absolute .txt path. + + Rules + ----- + 1) Absolute path to a ``.txt`` file is NOT accepted. + 2) The identifier is treated as a basename that must exist + under :attr:`packs_dir`. + + Parameters + ---------- + identifier : str or path-like + Basename to resolve. + + Returns + ------- + pathlib.Path + Absolute path to the pack file. + + Raises + ------ + FileNotFoundError + If the pack cannot be found per the above rules. + """ + p = Path(identifier) + if p.is_absolute(): + raise FileNotFoundError( + f"Absolute pack paths are not supported: {p}.\ + Use a provided pack or \ + define extra requirements using a profile." + ) + cand = self.packs_dir / f"{p.name}.txt" + if cand.is_file(): + return cand.resolve() + raise FileNotFoundError(f"Pack not found: {identifier} ({cand})") + + def pack_requirements( + self, identifier: Union[str, Path] + ) -> List[ParsedReq]: + """Return parsed requirements for a pack. + + Parameters + ---------- + identifier : str or path-like + Installed pack name. + + Returns + ------- + list of ParsedReq + Parsed requirements from the pack file. + """ + path = self._resolve_pack_file(identifier) + lines: List[str] = [] + for ln in path.read_text(encoding="utf-8").splitlines(): + s = ln.strip() + if s and not s.startswith("#"): + lines.append(s) + return [parse_requirement_line(s) for s in lines] + + def check_pack(self, identifier: Union[str, Path]) -> bool: + """Return whether a pack is installed. + + Parameters + ---------- + identifier : str or path-like + Basename to the pack file. + + Returns + ------- + bool + ``True`` if the pack is installed, ``False`` otherwise. + """ + reqs = self.pack_requirements(identifier) + return presence_check(reqs)[0] + + def install_pack(self, identifier: str | Path) -> None: + """Install a pack and verify presence. + + Parameters + ---------- + identifier : str + Basename to the pack file. + """ + path = self._resolve_pack_file(identifier) + reqs = self.pack_requirements(path.stem) + scripts_root = self.packs_dir / "scripts" + plog.info("Installing pack: %s", path.stem) + if install_requirements(reqs, scripts_root=scripts_root) == 0: + plog.info("Pack '%s' installation complete.", path.stem) + else: + plog.error("Pack '%s' installation failed.", path.stem) diff --git a/src/diffpy/cmi/profilesmanager.py b/src/diffpy/cmi/profilesmanager.py new file mode 100644 index 0000000..bf4d5d0 --- /dev/null +++ b/src/diffpy/cmi/profilesmanager.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +############################################################################## +# +# (c) 2025 The Trustees of Columbia University in the City of New York. +# All rights reserved. +# +# File coded by: Simon Billinge and members of the Billinge Group. +# +# See GitHub contributions for a more detailed list of contributors. +# https://github.com/diffpy/diffpy.cmi/graphs/contributors +# +# See LICENSE.rst for license information. +# +############################################################################## + +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Union + +import yaml + +from diffpy.cmi.installer import ( + ParsedReq, + install_requirements, + parse_requirement_line, + presence_check, +) +from diffpy.cmi.log import plog +from diffpy.cmi.packsmanager import PacksManager + +__all__ = ["Profile", "ProfilesManager"] + + +@dataclass +class Profile: + """Container for a resolved profile. + + Parameters + ---------- + name : str + Profile name (defaults to the YAML stem). + packs : list of str + Pack basenames this profile depends on. + extras : list of str + Extra requirement lines (scripts or packages). + source : path-like + Absolute path to the YAML file that defined the profile. + """ + + name: str + packs: List[str] + extras: List[str] + source: Path + + +class ProfilesManager: + """Discovery, loading, checking and installation for profiles. + + Attributes + ---------- + packs_mgr : PacksManager, optional + The packs manager used for discovery and installation policy. + profiles_dir : pathlib.Path + Absolute path to the installed profiles directory. + Defaults to `requirements/profiles` under the installed package. + """ + + def __init__(self, packs_mgr: Optional[PacksManager] = None) -> None: + self.packs_mgr = packs_mgr or PacksManager() + self.profiles_dir = self.packs_mgr.packs_dir.parent / "profiles" + + # Resolution & loading + def _resolve_profile_file(self, identifier: Union[str, Path]) -> Path: + """Resolve a profile identifier to an absolute YAML path. + + Rules + ----- + 1) Absolute path to a ``.yml``/``.yaml`` file is accepted as-is. + 2) Otherwise treat ``identifier`` as a basename + under :attr:`profiles_dir`. + + Parameters + ---------- + identifier : str or path-like + Basename or absolute file to resolve. + + Returns + ------- + pathlib.Path + Absolute path to the profile YAML. + + Raises + ------ + FileNotFoundError + If the profile cannot be found per the above rules. + """ + p = Path(identifier) + if p.is_absolute(): + if p.is_file() and p.suffix.lower() in {".yml", ".yaml"}: + return p.resolve() + raise FileNotFoundError(f"Profile file not found: {p}") + + cand_y = self.profiles_dir / f"{p}.yml" + cand_ya = self.profiles_dir / f"{p}.yaml" + for c in (cand_y, cand_ya): + if c.is_file(): + return c.resolve() + raise FileNotFoundError( + f"No installed profile named '{identifier}' in {self.profiles_dir}" + ) + + def _profile_requirements(self, prof: Profile) -> List[ParsedReq]: + """Return parsed requirements for a profile. + + Parameters + ---------- + prof : Profile + Loaded profile. + + Returns + ------- + list of ParsedReq + Combined pack requirements and extras + with ``skip`` entries removed. + """ + reqs: List[ParsedReq] = [] + for pack_name in prof.packs: + reqs.extend(self.packs_mgr.pack_requirements(pack_name)) + reqs.extend(parse_requirement_line(x) for x in prof.extras) + return [r for r in reqs if r.kind != "skip"] + + def load(self, identifier: Union[str, Path]) -> Profile: + """Load a profile file into a :class:`Profile` object. + + Parameters + ---------- + identifier : str or path-like + Basename or absolute YAML path. + + Returns + ------- + Profile + Loaded profile with metadata. + """ + path = self._resolve_profile_file(identifier) + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + packs = list(data.get("packs") or []) + extras = list(data.get("extras") or []) + name = data.get("name") or path.stem + return Profile(name=name, packs=packs, extras=extras, source=path) + + def list_profiles(self) -> List[str]: + """Return available installed profiles by basename. + + Returns + ------- + list of str + Profile basenames available under :attr:`profiles_dir`. + """ + return sorted( + p.stem for p in self.profiles_dir.glob("*.yml") + ) + sorted(p.stem for p in self.profiles_dir.glob("*.yaml")) + + def check_profile(self, identifier: Union[str, Path]) -> bool: + """Return whether a profile appears installed on this system. + + Parameters + ---------- + identifier : str or path-like + Basename or absolute YAML path. + + Returns + ------- + bool + ``True`` if all packages and non-meta scripts appear present. + """ + prof = self.load(identifier) + reqs = self._profile_requirements(prof) + return presence_check(reqs)[0] + + def install(self, identifier: Union[str, Path]) -> None: + """Install a profile and verify presence. + + Parameters + ---------- + identifier : str or path-like + Basename or absolute YAML path. + """ + prof = self.load(identifier) + reqs = self._profile_requirements(prof) + scripts_root = self.packs_mgr.packs_dir / "scripts" + + plog.info("Installing profile: %s", prof.name) + if install_requirements(reqs, scripts_root=scripts_root) == 0: + plog.info("Profile '%s' installation complete.", prof.name) + else: + plog.error("Profile '%s' installation failed.", prof.name) diff --git a/tests/test_functions.py b/tests/test_functions.py deleted file mode 100644 index 3cbc345..0000000 --- a/tests/test_functions.py +++ /dev/null @@ -1,40 +0,0 @@ -import numpy as np -import pytest - -from diffpy.cmi import functions # noqa - - -def test_dot_product_2D_list(): - a = [1, 2] - b = [3, 4] - expected = 11.0 - actual = functions.dot_product(a, b) - assert actual == expected - - -def test_dot_product_3D_list(): - a = [1, 2, 3] - b = [4, 5, 6] - expected = 32.0 - actual = functions.dot_product(a, b) - assert actual == expected - - -@pytest.mark.parametrize( - "a, b, expected", - [ - # Test whether the dot product function works with 2D and 3D vectors - # C1: lists, expect correct float output - ([1, 2], [3, 4], 11.0), - ([1, 2, 3], [4, 5, 6], 32.0), - # C2: tuples, expect correct float output - ((1, 2), (3, 4), 11.0), - ((1, 2, 3), (4, 5, 6), 32.0), - # C3: numpy arrays, expect correct float output - (np.array([1, 2]), np.array([3, 4]), 11.0), - (np.array([1, 2, 3]), np.array([4, 5, 6]), 32.0), - ], -) -def test_dot_product(a, b, expected): - actual = functions.dot_product(a, b) - assert actual == expected From b9931702c5f66f25eb081c86ac96556be9e42282 Mon Sep 17 00:00:00 2001 From: Tieqiong Date: Mon, 18 Aug 2025 03:34:01 -0500 Subject: [PATCH 2/8] windows --- requirements/packs/scripts/_pytest.bat | 118 +++++++++++++++++++++++++ requirements/profiles/_tests.yml | 1 + src/diffpy/cmi/conda.py | 13 ++- src/diffpy/cmi/installer.py | 12 +-- 4 files changed, 134 insertions(+), 10 deletions(-) create mode 100644 requirements/packs/scripts/_pytest.bat diff --git a/requirements/packs/scripts/_pytest.bat b/requirements/packs/scripts/_pytest.bat new file mode 100644 index 0000000..b418bcb --- /dev/null +++ b/requirements/packs/scripts/_pytest.bat @@ -0,0 +1,118 @@ +@echo off +REM Usage: +REM _pytest.bat urls.txt +REM _pytest.bat https://host/a.tar.gz https://host/b.tgz +REM From ChatGPT + +setlocal EnableExtensions EnableDelayedExpansion + +REM ---- temp workspace under %TEMP% ---- +set "TMPROOT=%TEMP%\remote_tests_%RANDOM%%RANDOM%" +md "%TMPROOT%" || (echo Failed to create TMPROOT & exit /b 1) + +REM ---- resolve args -> URL list file ---- +if "%~2"=="" ( + if exist "%~1" ( + set "URLS_FILE=%~f1" + ) else ( + set "URLS_FILE=%TMPROOT%\urls.txt" + > "%URLS_FILE%" echo %~1 + ) +) else ( + set "URLS_FILE=%TMPROOT%\urls.txt" + (for %%U in (%*) do @echo %%~U) > "%URLS_FILE%" +) + +pushd "%TMPROOT%" >nul || (echo Failed to enter TMPROOT & exit /b 1) + +set /a i=0 +set /a overall_ec=0 + +REM read URL file line-by-line; hand off each line to a subroutine +for /f "usebackq delims=" %%L in ("%URLS_FILE%") do call :process_one "%%L" + +popd >nul +rmdir /s /q "%TMPROOT%" >nul 2>&1 +exit /b %overall_ec% + +REM ===================== subroutine ===================== +:process_one +setlocal EnableExtensions EnableDelayedExpansion + +REM grab the raw line and trim leading spaces +set "url=%~1" +if "%url%"=="" (endlocal & goto :eof) +:trim +if not "%url:~0,1%"==" " goto :trim_done +set "url=%url:~1%" +goto trim +:trim_done + +REM skip comments +if "%url:~0,1%"=="#" (endlocal & goto :eof) + +REM ----- do the work for this URL ----- +endlocal & set /a i+=1 & set "URL=%url%" +echo( +echo ==> [%i%] + +set "PKGDIR=%TMPROOT%\pkg_%i%" +md "%PKGDIR%" +pushd "%PKGDIR%" >nul || goto :after + +REM download archive into PKGDIR +curl -L --fail -o "archive.tar.gz" "%URL%" +if errorlevel 1 ( + echo curl failed + set /a overall_ec=1 + popd >nul & goto :after +) + +REM extract (try gzip flags, then plain) +tar -xzf "archive.tar.gz" >nul 2>&1 +if errorlevel 1 tar -xf "archive.tar.gz" >nul 2>&1 +if errorlevel 1 ( + echo tar extract failed + set /a overall_ec=1 + popd >nul & goto :after +) + +REM get first entry (try -tzf, then -tf) +set "FIRST=" +for /f "delims=" %%F in ('tar -tzf "archive.tar.gz" 2^>nul') do set "FIRST=%%F" & goto got_first +for /f "delims=" %%F in ('tar -tf "archive.tar.gz" 2^>nul') do set "FIRST=%%F" & goto got_first +:got_first + +REM choose project root (top dir if present) +set "PROJROOT=%CD%" +if defined FIRST for /f "tokens=1 delims=/" %%T in ("%FIRST%") do if exist ".\%%T\" set "PROJROOT=%CD%\%%T" + +REM mirror original: drop src\ if present +if exist "%PROJROOT%\src\" rmdir /s /q "%PROJROOT%\src" >nul 2>&1 + +REM run pytest from repo root (with tests on PYTHONPATH if exists) +pushd "%PROJROOT%" >nul +echo Running pytest in: "%CD%" +set "OLD_PYTHONPATH=%PYTHONPATH%" +if exist "tests\" ( + if defined OLD_PYTHONPATH ( + set "PYTHONPATH=%CD%;tests;%OLD_PYTHONPATH%" + ) else ( + set "PYTHONPATH=%CD%;tests" + ) +) else ( + if defined OLD_PYTHONPATH ( + set "PYTHONPATH=%CD%;%OLD_PYTHONPATH%" + ) else ( + set "PYTHONPATH=%CD%" + ) +) +pytest +if errorlevel 1 set /a overall_ec=1 +set "PYTHONPATH=%OLD_PYTHONPATH%" +popd >nul + +popd >nul +:after +rmdir /s /q "%PKGDIR%" >nul 2>&1 +goto :eof \ No newline at end of file diff --git a/requirements/profiles/_tests.yml b/requirements/profiles/_tests.yml index f779737..dd60e82 100644 --- a/requirements/profiles/_tests.yml +++ b/requirements/profiles/_tests.yml @@ -9,3 +9,4 @@ packs: extras: - _pytest.sh tar_url.txt + - _pytest.bat tar_url.txt diff --git a/src/diffpy/cmi/conda.py b/src/diffpy/cmi/conda.py index 516d8b7..ddcd5b1 100644 --- a/src/diffpy/cmi/conda.py +++ b/src/diffpy/cmi/conda.py @@ -16,6 +16,7 @@ import json import subprocess import sys +import os from dataclasses import dataclass from pathlib import Path from typing import List, Optional, Sequence, Tuple @@ -97,10 +98,18 @@ def run( do_capture = (not dbg) if capture is None else bool(capture) qcmd = " ".join(str(x) for x in cmd) + win = (os.name == "nt") + prog = str(cmd[0]).lower() if cmd else "" + needs_cmd = win and ( + prog.endswith("\\conda.bat") or prog.endswith("\\mamba.bat") + or prog in ("conda", "conda.bat", "mamba", "mamba.bat") + ) + argv = (["cmd", "/c"] + list(cmd)) if needs_cmd else list(cmd) + try: if do_capture: cp = subprocess.run( - list(cmd), + argv, cwd=str(cwd) if cwd else None, capture_output=True, text=True, @@ -119,7 +128,7 @@ def run( sys.stderr.flush() else: cp = subprocess.run( - list(cmd), + argv, cwd=str(cwd) if cwd else None, text=True, check=False, diff --git a/src/diffpy/cmi/installer.py b/src/diffpy/cmi/installer.py index 653b515..59faa11 100644 --- a/src/diffpy/cmi/installer.py +++ b/src/diffpy/cmi/installer.py @@ -200,8 +200,6 @@ def _script_supported_on_platform(ext: str) -> bool: """ ext = (ext or "").lower() if os.name == "nt": - if ext == ".sh": - return shutil.which("bash") is not None return ext == ".bat" return ext == ".sh" @@ -262,11 +260,6 @@ def _script_exec_cmd(path: Path, args: List[str]) -> List[str]: """ ext = path.suffix.lower() if os.name == "nt": - if ext == ".sh": - bash = shutil.which("bash") - if bash: - return [bash, str(path)] + args - return [str(path)] + args if ext != ".bat": return [str(path)] + args return ["cmd", "/c", str(path)] + args @@ -338,7 +331,10 @@ def install_requirements( # Execute scripts for r in reqs: - if r.kind != "script": + if r.kind != "script" or not r.name: + continue + if not r.name.startswith("_") and _is_installed(r.name): + plog.info("Skipping script (already satisfied): %s", r.raw.strip()) continue first_token = shlex.split(r.raw.strip(), posix=(os.name != "nt"))[0] ext = Path(first_token).suffix.lower() From 16cc1d0634682014963c4a32d32259c1806936a7 Mon Sep 17 00:00:00 2001 From: Tieqiong Zhang Date: Mon, 18 Aug 2025 03:40:38 -0500 Subject: [PATCH 3/8] pcmt --- requirements/packs/scripts/_pytest.bat | 2 +- src/diffpy/cmi/conda.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/requirements/packs/scripts/_pytest.bat b/requirements/packs/scripts/_pytest.bat index b418bcb..e09eec9 100644 --- a/requirements/packs/scripts/_pytest.bat +++ b/requirements/packs/scripts/_pytest.bat @@ -115,4 +115,4 @@ popd >nul popd >nul :after rmdir /s /q "%PKGDIR%" >nul 2>&1 -goto :eof \ No newline at end of file +goto :eof diff --git a/src/diffpy/cmi/conda.py b/src/diffpy/cmi/conda.py index ddcd5b1..ff9f505 100644 --- a/src/diffpy/cmi/conda.py +++ b/src/diffpy/cmi/conda.py @@ -14,9 +14,9 @@ ############################################################################## import json +import os import subprocess import sys -import os from dataclasses import dataclass from pathlib import Path from typing import List, Optional, Sequence, Tuple @@ -98,10 +98,11 @@ def run( do_capture = (not dbg) if capture is None else bool(capture) qcmd = " ".join(str(x) for x in cmd) - win = (os.name == "nt") + win = os.name == "nt" prog = str(cmd[0]).lower() if cmd else "" needs_cmd = win and ( - prog.endswith("\\conda.bat") or prog.endswith("\\mamba.bat") + prog.endswith("\\conda.bat") + or prog.endswith("\\mamba.bat") or prog in ("conda", "conda.bat", "mamba", "mamba.bat") ) argv = (["cmd", "/c"] + list(cmd)) if needs_cmd else list(cmd) From cd71934520b4e119fd6628cc7e19e79985437a59 Mon Sep 17 00:00:00 2001 From: Tieqiong Zhang Date: Mon, 18 Aug 2025 19:50:31 -0500 Subject: [PATCH 4/8] pr test workaround + add news --- .github/workflows/tests-on-pr.yml | 3 +++ news/cmi.rst | 28 +++++++++++++++++++++++++++ pyproject.toml | 1 - requirements/conda.txt | 6 ++++++ requirements/packs/scripts/_pytest.sh | 6 +++--- requirements/tests.txt | 10 ++++++++++ src/diffpy/cmi/cli.py | 2 +- src/diffpy/cmi/conda.py | 2 +- src/diffpy/cmi/installer.py | 2 +- src/diffpy/cmi/log.py | 2 +- src/diffpy/cmi/packsmanager.py | 2 +- src/diffpy/cmi/profilesmanager.py | 2 +- 12 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 news/cmi.rst create mode 100644 requirements/conda.txt create mode 100644 requirements/tests.txt diff --git a/.github/workflows/tests-on-pr.yml b/.github/workflows/tests-on-pr.yml index 86e5406..313555d 100644 --- a/.github/workflows/tests-on-pr.yml +++ b/.github/workflows/tests-on-pr.yml @@ -11,5 +11,8 @@ jobs: project: diffpy.cmi c_extension: false headless: false + run: | + echo "Running tests for diffpy.cmi" + cmi install _tests secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/news/cmi.rst b/news/cmi.rst new file mode 100644 index 0000000..ddeb682 --- /dev/null +++ b/news/cmi.rst @@ -0,0 +1,28 @@ +**Added:** + +* Add modules ``cli``, ``conda``, ``installer``, ``log``, ``packsmanager``, ``profilesmanager``. +* Add cmi cli commands for managing/installing profiles and packs; example and manual commands. +* Add `diffpy-srreal` script for script installation demonstration. +* Add `all.yml` for profile installation demonstration. +* Add `_tests.yml` profile for profile post-steps demonstration. + +**Changed:** + +* Update names to skpkg standard. +* Change requirements dir for packs and profiles management. + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +* diff --git a/pyproject.toml b/pyproject.toml index 374a8b0..55c71c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ exclude = [] # exclude packages matching these glob patterns (empty by default) namespaces = false # to disable scanning PEP 420 namespaces (true by default) [project.scripts] -# diffpy-cmi = "diffpy.cmi.apps.cmi:main" cmi = "diffpy.cmi.cli:main" [tool.setuptools.dynamic] diff --git a/requirements/conda.txt b/requirements/conda.txt new file mode 100644 index 0000000..e9bdbc1 --- /dev/null +++ b/requirements/conda.txt @@ -0,0 +1,6 @@ +# temporary before updating skpkg release-scripts +packaging +PyYAML +diffpy.utils +diffpy.srfit +diffpy.structure diff --git a/requirements/packs/scripts/_pytest.sh b/requirements/packs/scripts/_pytest.sh index 4f77876..fc641d6 100644 --- a/requirements/packs/scripts/_pytest.sh +++ b/requirements/packs/scripts/_pytest.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Usage: -# ./run-remote-tests.sh urls.txt -# ./run-remote-tests.sh https://host/a.tar.gz https://host/b.tgz +# ./_pytest.sh urls.txt +# ./_pytest.sh https://host/a.tar.gz https://host/b.tgz # From ChatGPT set -euo pipefail @@ -26,7 +26,7 @@ cd "$TMPROOT" overall_ec=0 i=0 for url in "${URLS[@]}"; do - ((i++)) + ((++i)) echo -e "\n==> [$i] $url" tarball="$(mktemp -p "$TMPROOT" "dl_${i}.XXXXXX.tar.gz")" diff --git a/requirements/tests.txt b/requirements/tests.txt new file mode 100644 index 0000000..fde0e31 --- /dev/null +++ b/requirements/tests.txt @@ -0,0 +1,10 @@ +# temporary before updating skpkg release-scripts +flake8 +pytest +codecov +coverage +pytest-cov +pytest-env +pytest-mock +freezegun +DeepDiff diff --git a/src/diffpy/cmi/cli.py b/src/diffpy/cmi/cli.py index 2831081..a740c90 100644 --- a/src/diffpy/cmi/cli.py +++ b/src/diffpy/cmi/cli.py @@ -4,7 +4,7 @@ # (c) 2025 The Trustees of Columbia University in the City of New York. # All rights reserved. # -# File coded by: Simon Billinge and members of the Billinge Group. +# File coded by: Tieqiong Zhang and members of the Billinge Group. # # See GitHub contributions for a more detailed list of contributors. # https://github.com/diffpy/diffpy.cmi/graphs/contributors diff --git a/src/diffpy/cmi/conda.py b/src/diffpy/cmi/conda.py index ff9f505..0b4cc78 100644 --- a/src/diffpy/cmi/conda.py +++ b/src/diffpy/cmi/conda.py @@ -4,7 +4,7 @@ # (c) 2025 The Trustees of Columbia University in the City of New York. # All rights reserved. # -# File coded by: Simon Billinge and members of the Billinge Group. +# File coded by: Tieqiong Zhang and members of the Billinge Group. # # See GitHub contributions for a more detailed list of contributors. # https://github.com/diffpy/diffpy.cmi/graphs/contributors diff --git a/src/diffpy/cmi/installer.py b/src/diffpy/cmi/installer.py index 59faa11..a626d8b 100644 --- a/src/diffpy/cmi/installer.py +++ b/src/diffpy/cmi/installer.py @@ -4,7 +4,7 @@ # (c) 2025 The Trustees of Columbia University in the City of New York. # All rights reserved. # -# File coded by: Simon Billinge and members of the Billinge Group. +# File coded by: Tieqiong Zhang and members of the Billinge Group. # # See GitHub contributions for a more detailed list of contributors. # https://github.com/diffpy/diffpy.cmi/graphs/contributors diff --git a/src/diffpy/cmi/log.py b/src/diffpy/cmi/log.py index 9eea488..ac91d6e 100644 --- a/src/diffpy/cmi/log.py +++ b/src/diffpy/cmi/log.py @@ -4,7 +4,7 @@ # (c) 2025 The Trustees of Columbia University in the City of New York. # All rights reserved. # -# File coded by: Simon Billinge and members of the Billinge Group. +# File coded by: Tieqiong Zhang and members of the Billinge Group. # # See GitHub contributions for a more detailed list of contributors. # https://github.com/diffpy/diffpy.cmi/graphs/contributors diff --git a/src/diffpy/cmi/packsmanager.py b/src/diffpy/cmi/packsmanager.py index 58251de..bd8b10b 100644 --- a/src/diffpy/cmi/packsmanager.py +++ b/src/diffpy/cmi/packsmanager.py @@ -4,7 +4,7 @@ # (c) 2025 The Trustees of Columbia University in the City of New York. # All rights reserved. # -# File coded by: Simon Billinge and members of the Billinge Group. +# File coded by: Tieqiong Zhang and members of the Billinge Group. # # See GitHub contributions for a more detailed list of contributors. # https://github.com/diffpy/diffpy.cmi/graphs/contributors diff --git a/src/diffpy/cmi/profilesmanager.py b/src/diffpy/cmi/profilesmanager.py index bf4d5d0..b004436 100644 --- a/src/diffpy/cmi/profilesmanager.py +++ b/src/diffpy/cmi/profilesmanager.py @@ -4,7 +4,7 @@ # (c) 2025 The Trustees of Columbia University in the City of New York. # All rights reserved. # -# File coded by: Simon Billinge and members of the Billinge Group. +# File coded by: Tieqiong Zhang and members of the Billinge Group. # # See GitHub contributions for a more detailed list of contributors. # https://github.com/diffpy/diffpy.cmi/graphs/contributors From fbb5e0e28c282905ee177c5e814e73ccebf1e2aa Mon Sep 17 00:00:00 2001 From: Tieqiong Zhang Date: Mon, 18 Aug 2025 20:10:24 -0500 Subject: [PATCH 5/8] add Getting Started in README --- README.rst | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 4c07f76..215b3a8 100644 --- a/README.rst +++ b/README.rst @@ -88,7 +88,40 @@ and run the following :: Getting Started --------------- -You may consult our `online documentation `_ for tutorials and API references. +Use the `cmi` command-line interface to install and manage modular optional dependencies, known as `packs`, +and to configure or execute user-defined workflows that combine multiple packs with optional post-installation steps, +known as `profiles`. To use `cmi`, you can run the following example commands: + +.. code-block:: bash + :caption: Show available commands and options + + cmi -h + +.. code-block:: bash + :caption: List installed and available packs and profiles + + cmi pack list + cmi profile list + +.. code-block:: bash + :caption: Show details of a specific pack or profile + + cmi pack show + cmi profile show + +.. code-block:: bash + :caption: Install a pack or profile (by name or path) + + cmi install + +.. code-block:: bash + :caption: List and get installed examples + + cmi example list + cmi example (copy) + +You may consult our `online documentation `_ for more information, +tutorials, and API references. Support and Contribute ---------------------- From 2bd2acf999e6e78a808cd7a3a2ec25f1a4e0576e Mon Sep 17 00:00:00 2001 From: Tieqiong Zhang Date: Mon, 18 Aug 2025 22:38:21 -0500 Subject: [PATCH 6/8] use diffpy.srreal from conda forge --- news/cmi.rst | 1 - requirements/packs/pdf.txt | 3 +-- requirements/packs/scripts/diffpy-srreal.bat | 2 -- requirements/packs/scripts/diffpy-srreal.sh | 2 -- 4 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 requirements/packs/scripts/diffpy-srreal.bat delete mode 100644 requirements/packs/scripts/diffpy-srreal.sh diff --git a/news/cmi.rst b/news/cmi.rst index ddeb682..1c02421 100644 --- a/news/cmi.rst +++ b/news/cmi.rst @@ -2,7 +2,6 @@ * Add modules ``cli``, ``conda``, ``installer``, ``log``, ``packsmanager``, ``profilesmanager``. * Add cmi cli commands for managing/installing profiles and packs; example and manual commands. -* Add `diffpy-srreal` script for script installation demonstration. * Add `all.yml` for profile installation demonstration. * Add `_tests.yml` profile for profile post-steps demonstration. diff --git a/requirements/packs/pdf.txt b/requirements/packs/pdf.txt index 3aff7c0..07b18a4 100644 --- a/requirements/packs/pdf.txt +++ b/requirements/packs/pdf.txt @@ -1,3 +1,2 @@ -diffpy-srreal.sh -diffpy-srreal.bat +diffpy.srreal pyobjcryst diff --git a/requirements/packs/scripts/diffpy-srreal.bat b/requirements/packs/scripts/diffpy-srreal.bat deleted file mode 100644 index 11852c6..0000000 --- a/requirements/packs/scripts/diffpy-srreal.bat +++ /dev/null @@ -1,2 +0,0 @@ -conda install -y -c conda-forge -q libdiffpy libboost-devel libobjcryst periodictable -pip install diffpy.srreal diff --git a/requirements/packs/scripts/diffpy-srreal.sh b/requirements/packs/scripts/diffpy-srreal.sh deleted file mode 100644 index 11852c6..0000000 --- a/requirements/packs/scripts/diffpy-srreal.sh +++ /dev/null @@ -1,2 +0,0 @@ -conda install -y -c conda-forge -q libdiffpy libboost-devel libobjcryst periodictable -pip install diffpy.srreal From b3bb807762516fe996c24dce3b5143a256e3a33c Mon Sep 17 00:00:00 2001 From: Tieqiong Zhang Date: Thu, 21 Aug 2025 17:08:37 -0500 Subject: [PATCH 7/8] add cmi test steps in matrix macos-13 shell script support Use personal diffpy.utils tests source temporaly properly pass exit code for install cmd --- .../matrix-and-codecov-on-merge-to-main.yml | 18 ++++++++++++++++ .github/workflows/tests-on-pr.yml | 17 ++++++++++++++- requirements/packs/scripts/_pytest.sh | 21 +++++++++++-------- requirements/packs/scripts/tar_url.txt | 2 +- requirements/profiles/_tests.yml | 1 - src/diffpy/cmi/cli.py | 12 +++++++---- src/diffpy/cmi/profilesmanager.py | 5 ++++- 7 files changed, 59 insertions(+), 17 deletions(-) diff --git a/.github/workflows/matrix-and-codecov-on-merge-to-main.yml b/.github/workflows/matrix-and-codecov-on-merge-to-main.yml index 2a27f6d..8d863f3 100644 --- a/.github/workflows/matrix-and-codecov-on-merge-to-main.yml +++ b/.github/workflows/matrix-and-codecov-on-merge-to-main.yml @@ -17,5 +17,23 @@ jobs: project: diffpy.cmi c_extension: false headless: false + run: | + set -Eeuo pipefail + echo "Test cmds" + cmi -h + cmi env + cmi pack list + cmi profile list + cmi install plotting + if [ "${RUNNER_OS}" != "Windows" ]; then + conda list | grep -i ipympl + else + source "$(cygpath -u "$CONDA")/etc/profile.d/conda.sh" + conda activate test + conda list | grep -i ipympl + fi + + echo "Running tests for diffpy.cmi dependencies" + cmi install _tests secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/tests-on-pr.yml b/.github/workflows/tests-on-pr.yml index 313555d..e8ef0fc 100644 --- a/.github/workflows/tests-on-pr.yml +++ b/.github/workflows/tests-on-pr.yml @@ -12,7 +12,22 @@ jobs: c_extension: false headless: false run: | - echo "Running tests for diffpy.cmi" + set -Eeuo pipefail + echo "Test cmds" + cmi -h + cmi env + cmi pack list + cmi profile list + cmi install plotting + if [ "${RUNNER_OS}" != "Windows" ]; then + conda list | grep -i ipympl + else + source "$(cygpath -u "$CONDA")/etc/profile.d/conda.sh" + conda activate test + conda list | grep -i ipympl + fi + + echo "Running tests for diffpy.cmi dependencies" cmi install _tests secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/requirements/packs/scripts/_pytest.sh b/requirements/packs/scripts/_pytest.sh index fc641d6..4fbe295 100644 --- a/requirements/packs/scripts/_pytest.sh +++ b/requirements/packs/scripts/_pytest.sh @@ -19,7 +19,7 @@ else fi START_DIR="$PWD" -TMPROOT="$(mktemp -d -p "$START_DIR" ".tmp_remote_tests.XXXXXX")" +TMPROOT="$(TMPDIR="$START_DIR" mktemp -d -t .tmp_remote_tests.XXXXXXXX)" trap 'cd "$START_DIR" 2>/dev/null || true; rm -rf -- "$TMPROOT"' EXIT cd "$TMPROOT" @@ -27,16 +27,21 @@ overall_ec=0 i=0 for url in "${URLS[@]}"; do ((++i)) - echo -e "\n==> [$i] $url" + printf '\n==> [%d] %s\n' "$i" "$url" - tarball="$(mktemp -p "$TMPROOT" "dl_${i}.XXXXXX.tar.gz")" + tfile="$(TMPDIR="$TMPROOT" mktemp -t "dl_${i}.XXXXXXXX")" + tarball="${tfile}.tar.gz" curl -L --fail -o "$tarball" "$url" pkgdir="$TMPROOT/pkg_${i}" mkdir -p "$pkgdir" - tar -xzf "$tarball" -C "$pkgdir" + tar -xzf "$tarball" -C "$pkgdir" 2>/dev/null || tar -xf "$tarball" -C "$pkgdir" + + first_entry="$(tar -tzf "$tarball" 2>/dev/null | head -1 || true)" + if [ -z "$first_entry" ]; then + first_entry="$(tar -tf "$tarball" 2>/dev/null | head -1 || true)" + fi - first_entry="$(tar -tzf "$tarball" | head -1 || true)" top="${first_entry%%/*}" if [ -n "$top" ] && [ -d "$pkgdir/$top" ]; then projroot="$pkgdir/$top" @@ -44,9 +49,7 @@ for url in "${URLS[@]}"; do projroot="$pkgdir" fi - if [ -d "$projroot/src" ]; then - rm -rf -- "$projroot/src" - fi + [ -d "$projroot/src" ] && rm -rf -- "$projroot/src" if [ -d "$projroot/tests" ]; then ( cd "$projroot" && PYTHONPATH="$PWD:tests:${PYTHONPATH:-}" pytest ) || overall_ec=1 @@ -54,7 +57,7 @@ for url in "${URLS[@]}"; do ( cd "$projroot" && PYTHONPATH="$PWD:${PYTHONPATH:-}" pytest ) || overall_ec=1 fi - rm -f -- "$tarball" + rm -f -- "$tarball" "$tfile" rm -rf -- "$pkgdir" done diff --git a/requirements/packs/scripts/tar_url.txt b/requirements/packs/scripts/tar_url.txt index 463189e..d477556 100644 --- a/requirements/packs/scripts/tar_url.txt +++ b/requirements/packs/scripts/tar_url.txt @@ -2,4 +2,4 @@ https://github.com/diffpy/diffpy.srreal/archive/refs/tags/1.4.0.tar.gz https://github.com/diffpy/diffpy.srfit/archive/refs/tags/3.2.0.tar.gz https://github.com/diffpy/pyobjcryst/archive/refs/tags/2025.1.0.tar.gz https://github.com/diffpy/diffpy.structure/archive/refs/tags/3.3.1.tar.gz -https://github.com/diffpy/diffpy.utils/archive/refs/tags/3.6.1.tar.gz +https://github.com/Tieqiong/diffpy.utils/archive/refs/tags/3.6.2-rc.0.tar.gz diff --git a/requirements/profiles/_tests.yml b/requirements/profiles/_tests.yml index dd60e82..b48aaf8 100644 --- a/requirements/profiles/_tests.yml +++ b/requirements/profiles/_tests.yml @@ -4,7 +4,6 @@ packs: - core - pdf - - plotting - tests extras: diff --git a/src/diffpy/cmi/cli.py b/src/diffpy/cmi/cli.py index a740c90..163593f 100644 --- a/src/diffpy/cmi/cli.py +++ b/src/diffpy/cmi/cli.py @@ -464,16 +464,20 @@ def _cmd_install(ns: argparse.Namespace) -> int: try: kind, path = _resolve_target_for_install(tgt) if kind == "pack": - mgr.install_pack(path.stem) + r = mgr.install_pack(path.stem) else: - pm.install(path if path.is_absolute() else path.stem) + r = pm.install(path if path.is_absolute() else path.stem) + if isinstance(r, bool): + if not r: rc = max(rc, 1) + elif isinstance(r, int): + rc = max(rc, r) except (ValueError, FileNotFoundError) as e: plog.error("%s", e) ns._parser.print_help() - rc = 1 + rc = max(rc, 1) except Exception as e: plog.error("%s", e) - rc = 1 + rc = max(rc, 1) return rc diff --git a/src/diffpy/cmi/profilesmanager.py b/src/diffpy/cmi/profilesmanager.py index b004436..533b239 100644 --- a/src/diffpy/cmi/profilesmanager.py +++ b/src/diffpy/cmi/profilesmanager.py @@ -191,7 +191,10 @@ def install(self, identifier: Union[str, Path]) -> None: scripts_root = self.packs_mgr.packs_dir / "scripts" plog.info("Installing profile: %s", prof.name) - if install_requirements(reqs, scripts_root=scripts_root) == 0: + exit_code = install_requirements(reqs, scripts_root=scripts_root) + if exit_code == 0: plog.info("Profile '%s' installation complete.", prof.name) else: plog.error("Profile '%s' installation failed.", prof.name) + + return exit_code From ac2b4c3dab368109fc1c8a63e6495d1fef53596c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Aug 2025 22:08:58 +0000 Subject: [PATCH 8/8] [pre-commit.ci] auto fixes from pre-commit hooks --- src/diffpy/cmi/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/diffpy/cmi/cli.py b/src/diffpy/cmi/cli.py index 163593f..50a793e 100644 --- a/src/diffpy/cmi/cli.py +++ b/src/diffpy/cmi/cli.py @@ -468,7 +468,8 @@ def _cmd_install(ns: argparse.Namespace) -> int: else: r = pm.install(path if path.is_absolute() else path.stem) if isinstance(r, bool): - if not r: rc = max(rc, 1) + if not r: + rc = max(rc, 1) elif isinstance(r, int): rc = max(rc, r) except (ValueError, FileNotFoundError) as e: