perf: Performance optimization for deep dependency trees (#171)

.gitignore

@@ -2,6 +2,7 @@
 APPROACH.md
 next-steps.md
 apm-action-plan.md
+tests/benchmarks/BASELINE.md
 
 # Python
 __pycache__/

docs/cli-reference.md

@@ -122,6 +122,7 @@ apm install [PACKAGES...] [OPTIONS]
 - `--update` - Update dependencies to latest Git references  
 - `--force` - Overwrite locally-authored files on collision
 - `--dry-run` - Show what would be installed without installing
+- `--parallel-downloads INT` - Max concurrent package downloads (default: 4, 0 to disable)
 - `--verbose` - Show detailed installation information
 - `--trust-transitive-mcp` - Trust self-defined MCP servers from transitive packages (skip re-declaration requirement)
 

docs/dependencies.md

@@ -406,6 +406,14 @@ When both packages are installed, your project gains:
 4. **Build Dependency Graph**: Resolve transitive dependencies recursively
 5. **Check Conflicts**: Identify any circular dependencies or conflicts
 
+#### Resilient Downloads
+
+APM automatically retries failed HTTP requests with exponential backoff and jitter. Rate-limited responses (HTTP 429/503) are handled transparently, respecting `Retry-After` headers when provided. This ensures reliable installs even under heavy API usage or transient network issues.
+
+#### Parallel Downloads
+
+APM downloads packages in parallel using a thread pool, significantly reducing wall-clock time for large dependency trees. The concurrency level defaults to 4 and is configurable via `--parallel-downloads` (set to 0 to disable). For subdirectory packages in monorepos, APM attempts git sparse-checkout (git 2.25+) to download only the needed directory, falling back to a shallow clone if sparse-checkout is unavailable.
+
 ### File Processing and Content Merging
 
 APM uses instruction-level merging rather than file-level precedence. When local and dependency files contribute instructions with overlapping `applyTo` patterns:
@@ -527,7 +535,7 @@ The `deployed_files` field tracks exactly which files APM placed in your project
 ### How It Works
 
 1. **First install**: APM resolves dependencies, downloads packages, and writes `apm.lock`
-2. **Subsequent installs**: APM reads `apm.lock` and uses locked commits for exact reproducibility
+2. **Subsequent installs**: APM reads `apm.lock` and uses locked commits for exact reproducibility. If the local checkout already matches the locked commit SHA, the download is skipped entirely.
 3. **Updating**: Use `--update` to re-resolve dependencies and generate a fresh lockfile
 
 ### Version Control

docs/enhanced-primitive-discovery.md

@@ -186,6 +186,8 @@ The enhanced discovery system integrates with:
      - Keep existing primitive (higher priority)
      - Record conflict with losing source information
 
+Conflict detection uses O(1) name-indexed lookups, so performance remains constant regardless of collection size.
+
 ### Error Handling
 
 - Gracefully handles missing `apm_modules/` directory

pyproject.toml

@@ -70,7 +70,9 @@ warn_return_any = true
 warn_unused_configs = true
 
 [tool.pytest.ini_options]
+addopts = "-m 'not benchmark'"
 markers = [
     "integration: marks tests as integration tests that may require network access",
     "slow: marks tests as slow running tests",
+    "benchmark: marks performance benchmark tests (deselected by default, run with -m benchmark)",
 ]

src/apm_cli/cli.py

@@ -653,8 +653,15 @@ def _validate_package_exists(package):
     is_flag=True,
     help="Trust self-defined MCP servers from transitive packages (skip re-declaration requirement)",
 )
+@click.option(
+    "--parallel-downloads",
+    type=int,
+    default=4,
+    show_default=True,
+    help="Max concurrent package downloads (0 to disable parallelism)",
+)
 @click.pass_context
-def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbose, trust_transitive_mcp):
+def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbose, trust_transitive_mcp, parallel_downloads):
     """Install APM and MCP dependencies from apm.yml (like npm install).
 
     This command automatically detects AI runtimes from your apm.yml scripts and installs
@@ -753,7 +760,8 @@ def install(ctx, packages, runtime, exclude, only, update, dry_run, force, verbo
                 # Otherwise install all from apm.yml
                 only_pkgs = builtins.list(packages) if packages else None
                 apm_count, prompt_count, agent_count = _install_apm_dependencies(
-                    apm_package, update, verbose, only_pkgs, force=force
+                    apm_package, update, verbose, only_pkgs, force=force,
+                    parallel_downloads=parallel_downloads,
                 )
             except Exception as e:
                 _rich_error(f"Failed to install APM dependencies: {e}")
@@ -1554,6 +1562,7 @@ def _install_apm_dependencies(
     verbose: bool = False,
     only_packages: "builtins.list" = None,
     force: bool = False,
+    parallel_downloads: int = 4,
 ):
     """Install APM package dependencies.
 
@@ -1563,6 +1572,7 @@ def _install_apm_dependencies(
         verbose: Show detailed installation information
         only_packages: If provided, only install these specific packages (not all from apm.yml)
         force: Whether to overwrite locally-authored files on collision
+        parallel_downloads: Max concurrent downloads (0 disables parallelism)
     """
     if not APM_DEPS_AVAILABLE:
         raise RuntimeError("APM dependency system not available")
@@ -1765,7 +1775,75 @@ def download_callback(dep_ref, modules_dir):
         # downloader already created above for transitive resolution
         installed_count = 0
 
-        # Create progress display for downloads
+        # Phase 4 (#171): Parallel package downloads using ThreadPoolExecutor
+        # Pre-download all non-cached packages in parallel for wall-clock speedup.
+        # Results are stored and consumed by the sequential integration loop below.
+        from concurrent.futures import ThreadPoolExecutor, as_completed as _futures_completed
+
+        _pre_download_results = {}   # dep_key -> PackageInfo
+        _need_download = []
+        for _pd_ref in deps_to_install:
+            _pd_key = _pd_ref.get_unique_key()
+            _pd_path = (apm_modules_dir / _pd_ref.alias) if _pd_ref.alias else _pd_ref.get_install_path(apm_modules_dir)
+            # Skip if already downloaded during BFS resolution
+            if _pd_key in callback_downloaded:
+                continue
+            # Skip if lockfile SHA matches local HEAD (Phase 5 check)
+            if _pd_path.exists() and existing_lockfile and not update_refs:
+                _pd_locked = existing_lockfile.get_dependency(_pd_key)
+                if _pd_locked and _pd_locked.resolved_commit and _pd_locked.resolved_commit != "cached":
+                    try:
+                        from git import Repo as _PDGitRepo
+                        if _PDGitRepo(_pd_path).head.commit.hexsha == _pd_locked.resolved_commit:
+                            continue
+                    except Exception:
+                        pass
+            # Build download ref (use locked commit for reproducibility)
+            _pd_dlref = str(_pd_ref)
+            if existing_lockfile:
+                _pd_locked = existing_lockfile.get_dependency(_pd_key)
+                if _pd_locked and _pd_locked.resolved_commit and _pd_locked.resolved_commit != "cached":
+                    _pd_base = _pd_ref.repo_url
+                    if _pd_ref.virtual_path:
+                        _pd_base = f"{_pd_base}/{_pd_ref.virtual_path}"
+                    _pd_dlref = f"{_pd_base}#{_pd_locked.resolved_commit}"
+            _need_download.append((_pd_ref, _pd_path, _pd_dlref))
+
+        if _need_download and parallel_downloads > 0:
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[cyan]{task.description}[/cyan]"),
+                BarColumn(),
+                TaskProgressColumn(),
+                transient=True,
+            ) as _dl_progress:
+                _max_workers = min(parallel_downloads, len(_need_download))
+                with ThreadPoolExecutor(max_workers=_max_workers) as _executor:
+                    _futures = {}
+                    for _pd_ref, _pd_path, _pd_dlref in _need_download:
+                        _pd_disp = str(_pd_ref) if _pd_ref.is_virtual else _pd_ref.repo_url
+                        _pd_short = _pd_disp.split("/")[-1] if "/" in _pd_disp else _pd_disp
+                        _pd_tid = _dl_progress.add_task(description=f"Fetching {_pd_short}", total=None)
+                        _pd_fut = _executor.submit(
+                            downloader.download_package, _pd_dlref, _pd_path,
+                            progress_task_id=_pd_tid, progress_obj=_dl_progress,
+                        )
+                        _futures[_pd_fut] = (_pd_ref, _pd_tid, _pd_disp)
+                    for _pd_fut in _futures_completed(_futures):
+                        _pd_ref, _pd_tid, _pd_disp = _futures[_pd_fut]
+                        _pd_key = _pd_ref.get_unique_key()
+                        try:
+                            _pd_info = _pd_fut.result()
+                            _pre_download_results[_pd_key] = _pd_info
+                            _dl_progress.update(_pd_tid, visible=False)
+                            _dl_progress.refresh()
+                        except Exception:
+                            _dl_progress.remove_task(_pd_tid)
+                            # Silent: sequential loop below will retry and report errors
+
+        _pre_downloaded_keys = set(_pre_download_results.keys())
+
+        # Create progress display for sequential integration
         with Progress(
             SpinnerColumn(),
             TextColumn("[cyan]{task.description}[/cyan]"),
@@ -1791,7 +1869,7 @@ def download_callback(dep_ref, modules_dir):
                 from apm_cli.models.apm_package import GitReferenceType
 
                 resolved_ref = None
-                if dep_ref.reference:
+                if dep_ref.reference and dep_ref.get_unique_key() not in _pre_downloaded_keys:
                     try:
                         resolved_ref = downloader.resolve_git_reference(
                             f"{dep_ref.repo_url}@{dep_ref.reference}"
@@ -1806,8 +1884,20 @@ def download_callback(dep_ref, modules_dir):
                 ]
                 # Skip download if: already fetched by resolver callback, or cached tag/commit
                 already_resolved = dep_ref.get_unique_key() in callback_downloaded
+                # Phase 5 (#171): Also skip when lockfile SHA matches local HEAD
+                lockfile_match = False
+                if install_path.exists() and existing_lockfile and not update_refs:
+                    locked_dep = existing_lockfile.get_dependency(dep_ref.get_unique_key())
+                    if locked_dep and locked_dep.resolved_commit and locked_dep.resolved_commit != "cached":
+                        try:
+                            from git import Repo as GitRepo
+                            local_repo = GitRepo(install_path)
+                            if local_repo.head.commit.hexsha == locked_dep.resolved_commit:
+                                lockfile_match = True
+                        except Exception:
+                            pass  # Not a git repo or invalid — fall through to download
                 skip_download = install_path.exists() and (
-                    (is_cacheable and not update_refs) or already_resolved
+                    (is_cacheable and not update_refs) or already_resolved or lockfile_match
                 )
 
                 if skip_download:
@@ -2081,13 +2171,18 @@ def download_callback(dep_ref, modules_dir):
                                 base_ref = f"{base_ref}/{dep_ref.virtual_path}"
                             download_ref = f"{base_ref}#{locked_dep.resolved_commit}"
 
-                    # Download with live progress bar
-                    package_info = downloader.download_package(
-                        download_ref,
-                        install_path,
-                        progress_task_id=task_id,
-                        progress_obj=progress,
-                    )
+                    # Phase 4 (#171): Use pre-downloaded result if available
+                    _dep_key = dep_ref.get_unique_key()
+                    if _dep_key in _pre_download_results:
+                        package_info = _pre_download_results[_dep_key]
+                    else:
+                        # Fallback: sequential download (should rarely happen)
+                        package_info = downloader.download_package(
+                            download_ref,
+                            install_path,
+                            progress_task_id=task_id,
+                            progress_obj=progress,
+                        )
 
                     # CRITICAL: Hide progress BEFORE printing success message to avoid overlap
                     progress.update(task_id, visible=False)

src/apm_cli/compilation/constitution.py

@@ -2,10 +2,18 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
 from .constants import CONSTITUTION_RELATIVE_PATH
 
+# Module-level cache: resolved base_dir -> constitution content (#171)
+_constitution_cache: Dict[Path, Optional[str]] = {}
+
+
+def clear_constitution_cache() -> None:
+    """Clear the constitution read cache. Call in tests for isolation."""
+    _constitution_cache.clear()
+
 
 def find_constitution(base_dir: Path) -> Path:
     """Return path to constitution.md if present, else Path that does not exist.
@@ -19,15 +27,24 @@ def find_constitution(base_dir: Path) -> Path:
 def read_constitution(base_dir: Path) -> Optional[str]:
     """Read full constitution content if file exists.
 
+    Results are cached by resolved base_dir for the lifetime of the process.
+
     Args:
         base_dir: Repository root path.
     Returns:
         Full file text or None if absent.
     """
+    resolved = base_dir.resolve()
+    if resolved in _constitution_cache:
+        return _constitution_cache[resolved]
     path = find_constitution(base_dir)
     if not path.exists() or not path.is_file():
+        _constitution_cache[resolved] = None
         return None
     try:
-        return path.read_text(encoding="utf-8")
+        content = path.read_text(encoding="utf-8")
+        _constitution_cache[resolved] = content
+        return content
     except OSError:
+        _constitution_cache[resolved] = None
         return None

src/apm_cli/compilation/context_optimizer.py

@@ -121,6 +121,7 @@ def __init__(self, base_dir: str = ".", exclude_patterns: Optional[List[str]] =
         self._glob_cache: Dict[str, List[str]] = {}
         self._glob_set_cache: Dict[str, Set[Path]] = {}
         self._file_list_cache: Optional[List[Path]] = None
+        self._inheritance_cache: Dict[Path, List[Path]] = {}  # (#171)
         self._timing_enabled = False
         self._phase_timings: Dict[str, float] = {}
 
@@ -1234,6 +1235,10 @@ def _get_inheritance_chain(self, working_directory: Path) -> List[Path]:
         Returns:
             List[Path]: Inheritance chain (most specific to root).
         """
+        cached = self._inheritance_cache.get(working_directory)
+        if cached is not None:
+            return cached
+
         chain = []
         # Resolve the starting directory to ensure consistent path comparison
         try:
@@ -1261,6 +1266,7 @@ def _get_inheritance_chain(self, working_directory: Path) -> List[Path]:
             except (OSError, ValueError):
                 break
 
+        self._inheritance_cache[working_directory] = chain
         return chain
 
     def _is_child_directory(self, child: Path, parent: Path) -> bool:

src/apm_cli/config.py

@@ -2,11 +2,14 @@
 
 import os
 import json
+from typing import Optional
 
 
 CONFIG_DIR = os.path.expanduser("~/.apm")
 CONFIG_FILE = os.path.join(CONFIG_DIR, "config.json")
 
+_config_cache: Optional[dict] = None
+
 
 def ensure_config_exists():
     """Ensure the configuration directory and file exist."""
@@ -21,12 +24,24 @@ def ensure_config_exists():
 def get_config():
     """Get the current configuration.
 
+    Results are cached for the lifetime of the process.
+
     Returns:
         dict: Current configuration.
     """
+    global _config_cache
+    if _config_cache is not None:
+        return _config_cache
     ensure_config_exists()
     with open(CONFIG_FILE, "r") as f:
-        return json.load(f)
+        _config_cache = json.load(f)
+    return _config_cache
+
+
+def _invalidate_config_cache():
+    """Invalidate the config cache (called after writes)."""
+    global _config_cache
+    _config_cache = None
 
 
 def update_config(updates):
@@ -35,11 +50,13 @@ def update_config(updates):
     Args:
         updates (dict): Dictionary of configuration values to update.
     """
+    _invalidate_config_cache()
     config = get_config()
     config.update(updates)
 
     with open(CONFIG_FILE, "w") as f:
         json.dump(config, f, indent=2)
+    _invalidate_config_cache()
 
 
 def get_default_client():