From f3356fd797394091af7ebf5ee37c389334e5d25b Mon Sep 17 00:00:00 2001 From: Bonelli Date: Mon, 9 Feb 2026 15:17:55 -0500 Subject: [PATCH 1/3] refactor(programs): multiple programs api improvements --- modflow_devtools/programs/__init__.py | 68 +++++--- modflow_devtools/programs/__main__.py | 16 +- modflow_devtools/programs/make_registry.py | 182 ++++++++++++++------- modflow_devtools/programs/programs.toml | 47 +++++- 4 files changed, 223 insertions(+), 90 deletions(-) diff --git a/modflow_devtools/programs/__init__.py b/modflow_devtools/programs/__init__.py index dd9fa28a..8bdf53a9 100644 --- a/modflow_devtools/programs/__init__.py +++ b/modflow_devtools/programs/__init__.py @@ -75,12 +75,14 @@ class ProgramRegistryDiscoveryError(Exception): pass -class ProgramBinary(BaseModel): - """Platform-specific binary information.""" +class ProgramDistribution(BaseModel): + """Distribution-specific information.""" + name: str = Field( + ..., description="Distribution name (e.g., linux, mac, macarm, win64, win64ext)" + ) asset: str = Field(..., description="Release asset filename") hash: str | None = Field(None, description="SHA256 hash") - exe: str = Field(..., description="Executable path within archive") model_config = {"arbitrary_types_allowed": True} @@ -92,8 +94,9 @@ class ProgramMetadata(BaseModel): description: str | None = Field(None, description="Program description") repo: str = Field(..., description="Source repository (owner/name)") license: str | None = Field(None, description="License identifier") - binaries: dict[str, ProgramBinary] = Field( - default_factory=dict, description="Platform-specific binaries (linux/mac/win64)" + exe: str = Field(..., description="Executable path within archive (e.g., bin/mf6)") + dists: list[ProgramDistribution] = Field( + default_factory=list, description="Available distributions" ) model_config = {"arbitrary_types_allowed": True} @@ -1239,15 +1242,19 @@ def install( if verbose: print(f"Detected platform: {platform}") - # 4. Get binary metadata - if platform not in program_meta.binaries: - available = ", ".join(program_meta.binaries.keys()) + # 4. Get distribution metadata + dist_meta = None + for dist in program_meta.dists: + if dist.name == platform: + dist_meta = dist + break + + if dist_meta is None: + available = ", ".join(d.name for d in program_meta.dists) raise ProgramInstallationError( - f"Binary not available for platform '{platform}'. Available platforms: {available}" + f"Distribution not available for platform '{platform}'. Available: {available}" ) - binary_meta = program_meta.binaries[platform] - # 5. Determine bindir if bindir is None: bindir_options = get_bindir_options(program) @@ -1270,13 +1277,16 @@ def install( if verbose: print(f"{program} {version} is already installed in {bindir}") # Return paths to existing executables - exe_name = Path(binary_meta.exe).name + exe_name = Path(program_meta.exe).name + # Add .exe extension on Windows + if platform.startswith("win") and not exe_name.endswith(".exe"): + exe_name += ".exe" return [bindir / exe_name] # 7. Download archive (if not cached) - asset_url = f"https://github.com/{program_meta.repo}/releases/download/{found_ref}/{binary_meta.asset}" + asset_url = f"https://github.com/{program_meta.repo}/releases/download/{found_ref}/{dist_meta.asset}" archive_dir = self.cache.get_archive_dir(program, version, platform) - archive_path = archive_dir / binary_meta.asset + archive_path = archive_dir / dist_meta.asset if verbose: print(f"Downloading archive from {asset_url}...") @@ -1284,14 +1294,18 @@ def install( download_archive( url=asset_url, dest=archive_path, - expected_hash=binary_meta.hash, + expected_hash=dist_meta.hash, force=force, verbose=verbose, ) # 8. Extract to binaries cache (if not already extracted) binary_dir = self.cache.get_binary_dir(program, version, platform) - exe_in_cache = binary_dir / binary_meta.exe + exe_path = program_meta.exe + # Add .exe extension on Windows for extraction path + if platform.startswith("win") and not exe_path.endswith(".exe"): + exe_path += ".exe" + exe_in_cache = binary_dir / exe_path if not exe_in_cache.exists() or force: if verbose: @@ -1300,12 +1314,15 @@ def install( extract_executables( archive=archive_path, dest_dir=binary_dir, - exe_path=binary_meta.exe, + exe_path=exe_path, verbose=verbose, ) # 9. Copy executables to bindir - exe_name = Path(binary_meta.exe).name + exe_name = Path(program_meta.exe).name + # Add .exe extension on Windows + if platform.startswith("win") and not exe_name.endswith(".exe"): + exe_name += ".exe" dest_exe = bindir / exe_name if verbose: @@ -1326,7 +1343,7 @@ def install( "repo": program_meta.repo, "tag": found_ref, "asset_url": asset_url, - "hash": binary_meta.hash or "", + "hash": dist_meta.hash or "", } installation = ProgramInstallation( version=version, @@ -1842,7 +1859,11 @@ def list_installed(program: str | None = None) -> dict[str, list[ProgramInstalla def _try_best_effort_sync(): - """Attempt to sync registries on first import (best-effort, fails silently).""" + """ + Attempt to sync registries (best-effort, fails silently). + + Called by consumer commands before accessing program registries. + """ global _SYNC_ATTEMPTED if _SYNC_ATTEMPTED: @@ -1861,10 +1882,6 @@ def _try_best_effort_sync(): _SYNC_ATTEMPTED = False """Track whether auto-sync has been attempted""" -# Attempt best-effort sync on import (unless disabled) -if not os.environ.get("MODFLOW_DEVTOOLS_NO_AUTO_SYNC"): - _try_best_effort_sync() - # ============================================================================ # Public API @@ -1875,8 +1892,8 @@ def _try_best_effort_sync(): "_DEFAULT_MANAGER", "DiscoveredProgramRegistry", "InstallationMetadata", - "ProgramBinary", "ProgramCache", + "ProgramDistribution", "ProgramInstallation", "ProgramInstallationError", "ProgramManager", @@ -1885,6 +1902,7 @@ def _try_best_effort_sync(): "ProgramRegistryDiscoveryError", "ProgramSourceConfig", "ProgramSourceRepo", + "_try_best_effort_sync", "download_archive", "extract_executables", "get_bindir_options", diff --git a/modflow_devtools/programs/__main__.py b/modflow_devtools/programs/__main__.py index 4ccff8cd..845ee03b 100644 --- a/modflow_devtools/programs/__main__.py +++ b/modflow_devtools/programs/__main__.py @@ -12,11 +12,13 @@ """ import argparse +import os import sys from . import ( _DEFAULT_CACHE, ProgramSourceConfig, + _try_best_effort_sync, get_executable, install_program, list_installed, @@ -68,6 +70,10 @@ def cmd_info(args): def cmd_list(args): """List command handler.""" + # Attempt auto-sync before listing (unless disabled) + if not os.environ.get("MODFLOW_DEVTOOLS_NO_AUTO_SYNC"): + _try_best_effort_sync() + cached = _DEFAULT_CACHE.list() if not cached: @@ -106,10 +112,10 @@ def cmd_list(args): # Show all programs in verbose mode for program_name, metadata in sorted(programs.items()): version = metadata.version - platforms = ( - ", ".join(metadata.binaries.keys()) if metadata.binaries else "none" + dist_names = ( + ", ".join(d.name for d in metadata.dists) if metadata.dists else "none" ) - print(f" - {program_name} ({version}) [{platforms}]") + print(f" - {program_name} ({version}) [{dist_names}]") else: print(" No programs") print() @@ -117,6 +123,10 @@ def cmd_list(args): def cmd_install(args): """Install command handler.""" + # Attempt auto-sync before installation (unless disabled) + if not os.environ.get("MODFLOW_DEVTOOLS_NO_AUTO_SYNC"): + _try_best_effort_sync() + try: paths = install_program( program=args.program, diff --git a/modflow_devtools/programs/make_registry.py b/modflow_devtools/programs/make_registry.py index 101d3265..dcb62e7e 100644 --- a/modflow_devtools/programs/make_registry.py +++ b/modflow_devtools/programs/make_registry.py @@ -69,50 +69,58 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - # Generate registry for MODFLOW 6 release + # Generate registry from local distribution files (for CI) python -m modflow_devtools.programs.make_registry \\ - --repo MODFLOW-ORG/modflow6 \\ - --tag 6.6.3 \\ + --dists *.zip \\ --programs mf6 zbud6 libmf6 mf5to6 \\ + --version 6.6.3 \\ + --repo MODFLOW-ORG/modflow6 \\ + --compute-hashes \\ --output programs.toml - # Generate with manual program metadata + # Generate registry from existing GitHub release (for testing) python -m modflow_devtools.programs.make_registry \\ --repo MODFLOW-ORG/modflow6 \\ --tag 6.6.3 \\ - --program mf6 --version 6.6.3 --description "MODFLOW 6" \\ + --programs mf6 zbud6 libmf6 mf5to6 \\ --output programs.toml - # Include hashes for verification + # With custom exe paths python -m modflow_devtools.programs.make_registry \\ - --repo MODFLOW-ORG/modflow6 \\ - --tag 6.6.3 \\ - --programs mf6 zbud6 \\ - --compute-hashes \\ - --output programs.toml + --dists *.zip \\ + --program mf6:bin/mf6 \\ + --program zbud6:bin/zbud6 \\ + --version 6.6.3 \\ + --repo MODFLOW-ORG/modflow6 """, ) parser.add_argument( "--repo", - required=True, + required=False, type=str, - help='Repository in "owner/name" format (e.g., MODFLOW-ORG/modflow6)', + help='Repository in "owner/name" format (e.g., MODFLOW-ORG/modflow6) [required]', ) parser.add_argument( "--tag", - required=True, + required=False, type=str, - help="Release tag (e.g., 6.6.3)", + help="Release tag (e.g., 6.6.3) [required when scanning GitHub release]", + ) + parser.add_argument( + "--dists", + type=str, + help="Glob pattern for local distribution files (e.g., *.zip) [for CI mode]", ) parser.add_argument( "--programs", nargs="+", required=True, - help="Program names to include in registry", + help="Program names to include (optionally with custom exe path: name:path)", ) parser.add_argument( "--version", - help="Program version (defaults to tag)", + required=False, + help="Program version [required when using --dists, defaults to --tag otherwise]", ) parser.add_argument( "--description", @@ -142,22 +150,69 @@ def main(): ) args = parser.parse_args() - # Get release assets - if args.verbose: - print(f"Fetching release assets for {args.repo}@{args.tag}...") + # Validate arguments + if args.dists: + # Local files mode + if not args.repo: + print("Error: --repo is required", file=sys.stderr) + sys.exit(1) + if not args.version: + print("Error: --version is required when using --dists", file=sys.stderr) + sys.exit(1) + else: + # GitHub release mode + if not args.repo or not args.tag: + print("Error: --repo and --tag are required when not using --dists", file=sys.stderr) + sys.exit(1) + + # Parse programs (support name:path syntax) + program_exes = {} + for prog_spec in args.programs: + if ":" in prog_spec: + name, exe = prog_spec.split(":", 1) + program_exes[name] = exe + else: + program_exes[prog_spec] = f"bin/{prog_spec}" # Default path + + # Get distribution files + if args.dists: + # Local files mode: scan for files matching pattern + + + dist_files = Path.glob(args.dists) + if not dist_files: + print(f"No files found matching pattern: {args.dists}", file=sys.stderr) + sys.exit(1) - try: - assets = get_release_assets(args.repo, args.tag) - except Exception as e: - print(f"Error fetching release assets: {e}", file=sys.stderr) - sys.exit(1) + if args.verbose: + print(f"Found {len(dist_files)} distribution file(s)") + + # Convert to asset-like structure + assets = [] + for file_path in dist_files: + assets.append( + { + "name": Path(file_path).name, + "local_path": file_path, + } + ) + else: + # GitHub release mode: fetch from GitHub API + if args.verbose: + print(f"Fetching release assets for {args.repo}@{args.tag}...") - if not assets: - print(f"No assets found for release {args.tag}", file=sys.stderr) - sys.exit(1) + try: + assets = get_release_assets(args.repo, args.tag) + except Exception as e: + print(f"Error fetching release assets: {e}", file=sys.stderr) + sys.exit(1) - if args.verbose: - print(f"Found {len(assets)} release assets") + if not assets: + print(f"No assets found for release {args.tag}", file=sys.stderr) + sys.exit(1) + + if args.verbose: + print(f"Found {len(assets)} release assets") # Build registry structure registry = { @@ -170,11 +225,14 @@ def main(): # Use tag as version if not specified version = args.version or args.tag - # Platform mappings for common names - platform_map = { + # Distribution name mappings for filenames + dist_map = { "linux": ["linux", "ubuntu"], "mac": ["mac", "osx", "darwin"], - "win64": ["win64", "win"], + "macarm": ["macarm", "mac_arm", "mac-arm"], + "win64": ["win64"], + "win64ext": ["win64ext"], + "win32": ["win32"], } temp_dir = None @@ -183,13 +241,14 @@ def main(): try: # Process each program - for program_name in args.programs: + for program_name in program_exes.keys(): if args.verbose: print(f"\nProcessing program: {program_name}") program_meta = { "version": version, "repo": args.repo, + "exe": program_exes[program_name], # Get exe path for this program } if args.description: @@ -197,60 +256,71 @@ def main(): if args.license: program_meta["license"] = args.license - # Find binaries for this program - binaries = {} + # Find distributions for this program + dists = [] for asset in assets: asset_name = asset["name"] - asset_url = asset["browser_download_url"] - # Try to match asset to platform + # Try to match asset to distribution name asset_lower = asset_name.lower() - matched_platform = None + matched_dist = None - for platform, keywords in platform_map.items(): + # Try to match distribution name from filename + # Check longest names first to match win64ext before win64 + for dist_name in sorted(dist_map.keys(), key=len, reverse=True): + keywords = dist_map[dist_name] if any(keyword in asset_lower for keyword in keywords): - matched_platform = platform + matched_dist = dist_name break - if not matched_platform: + if not matched_dist: if args.verbose: - print(f" Skipping asset (no platform match): {asset_name}") + print(f" Skipping asset (no distribution match): {asset_name}") continue if args.verbose: - print(f" Found {matched_platform} asset: {asset_name}") + print(f" Found {matched_dist} distribution: {asset_name}") - # Create binary entry - binary = { + # Create distribution entry + dist = { + "name": matched_dist, "asset": asset_name, - "exe": f"bin/{program_name}", # Default executable path } # Compute hash if requested if args.compute_hashes: if args.verbose: - print(" Downloading to compute hash...") + print(" Computing hash...") + + if args.dists: + # Local file - use local_path + asset_path = Path(asset["local_path"]) + else: + # GitHub release - download first + if args.verbose: + print(" Downloading to compute hash...") + asset_url = asset["browser_download_url"] + asset_path = temp_dir / asset_name + download_asset(asset_url, asset_path) - asset_path = temp_dir / asset_name - download_asset(asset_url, asset_path) hash_value = compute_sha256(asset_path) - binary["hash"] = f"sha256:{hash_value}" + dist["hash"] = f"sha256:{hash_value}" if args.verbose: print(f" SHA256: {hash_value}") - binaries[matched_platform] = binary + dists.append(dist) - if binaries: - program_meta["binaries"] = binaries + if dists: + program_meta["dists"] = dists registry["programs"][program_name] = program_meta if args.verbose: - print(f" Added {program_name} with {len(binaries)} platform(s)") + print(f" Added {program_name} with {len(dists)} distribution(s)") else: if args.verbose: - print(f" Warning: No binaries found for {program_name}") + print(f" Warning: No distributions found for {program_name}") # Write registry to file output_path = Path(args.output) diff --git a/modflow_devtools/programs/programs.toml b/modflow_devtools/programs/programs.toml index df9156d6..3f6b0cd6 100644 --- a/modflow_devtools/programs/programs.toml +++ b/modflow_devtools/programs/programs.toml @@ -7,22 +7,57 @@ # Program names are globally unique across all sources. [sources.modflow6] -repo = "MODFLOW-ORG/modflow6" -refs = ["6.6.3", "6.5.0"] +repo = "MODFLOW-USGS/modflow6" +refs = ["6.7.0", "6.6.3"] # Provides: mf6, zbud6, mf5to6, libmf6 -[sources.modpath7] -repo = "MODFLOW-ORG/modpath7" +[sources.gridgen] +repo = "MODFLOW-ORG/gridgen" +refs = ["v1.0.02"] +# Provides: gridgen + +[sources.triangle] +repo = "MODFLOW-ORG/triangle" +refs = ["v1.6"] +# Provides: triangle + +[sources.mfusg] +repo = "MODFLOW-ORG/mfusg" +refs = ["v1.5.00"] +# Provides: mfusg + +[sources.mfusgt] +repo = "MODFLOW-ORG/mfusgt" +refs = ["v2.6.0"] +# Provides: mfusg_gsi + +[sources.modpath-v7] +repo = "MODFLOW-ORG/modpath-v7" refs = ["7.2.001"] # Provides: mp7 [sources.mt3d-usgs] repo = "MODFLOW-ORG/mt3d-usgs" -refs = ["1.1.0"] +refs = ["v1.0.0"] # Provides: mt3dusgs +[sources.mt3dms] +repo = "MODFLOW-ORG/mt3dms" +refs = ["2.0"] +# Provides: mt3dms + +[sources.zonbud] +repo = "MODFLOW-ORG/zonbud" +refs = ["v3.01"] +# Provides: zonbud + +[sources.zonbudusg] +repo = "MODFLOW-ORG/zonbudusg" +refs = ["v1.01"] +# Provides: zonbudusg + [sources.executables] repo = "MODFLOW-ORG/executables" refs = ["latest"] # Consolidated repo for legacy programs -# Provides: mf2005, mf2000, mfnwt, mfusg, mt3dms, etc. +# Provides: mf2005, mf2000, mfnwt, etc. From 7fc45ffc89ce3222fee4f9e3ae79c8424e31a3f8 Mon Sep 17 00:00:00 2001 From: Bonelli Date: Mon, 9 Feb 2026 15:32:14 -0500 Subject: [PATCH 2/3] ruff --- modflow_devtools/programs/make_registry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modflow_devtools/programs/make_registry.py b/modflow_devtools/programs/make_registry.py index dcb62e7e..cf0693f8 100644 --- a/modflow_devtools/programs/make_registry.py +++ b/modflow_devtools/programs/make_registry.py @@ -178,7 +178,6 @@ def main(): if args.dists: # Local files mode: scan for files matching pattern - dist_files = Path.glob(args.dists) if not dist_files: print(f"No files found matching pattern: {args.dists}", file=sys.stderr) From 16ed63a6a45ee42bd22c945744527cdb86e125c9 Mon Sep 17 00:00:00 2001 From: Bonelli Date: Mon, 9 Feb 2026 16:10:30 -0500 Subject: [PATCH 3/3] fix tests --- autotest/test_programs.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/autotest/test_programs.py b/autotest/test_programs.py index 68cd0f6f..e143e602 100644 --- a/autotest/test_programs.py +++ b/autotest/test_programs.py @@ -32,6 +32,7 @@ def test_save_and_load_registry(self): "test-program": { "version": "1.0.0", "repo": "test/repo", + "exe": "bin/test-program", "binaries": {}, } }, @@ -272,6 +273,13 @@ def test_program_manager_list_installed_empty(self): # Use fresh cache cache = ProgramCache() cache.clear() + + # Also clear metadata directory to ensure no leftover installation data + if cache.metadata_dir.exists(): + import shutil + + shutil.rmtree(cache.metadata_dir) + manager = ProgramManager(cache=cache) installed = manager.list_installed()