diff --git a/cloudbuild_pr.yaml b/cloudbuild_pr.yaml index ef5aadd1..04f4c0ae 100644 --- a/cloudbuild_pr.yaml +++ b/cloudbuild_pr.yaml @@ -23,6 +23,9 @@ steps: echo "--- Building wheel ---" cp -r /workspace /tmp/build-39 && cd /tmp/build-39 pip install --upgrade pip setuptools wheel build -q + # Create a local-only tag so setuptools-scm generates a real version + # instead of 0.0.0.dev0. This tag is never pushed. + git tag 99.0.0.dev39 2>/dev/null || true python -m build -q ls -lh dist/ @@ -33,6 +36,27 @@ steps: pip show limacharlie limacharlie --version + echo "--- Verifying version consistency ---" + python -c " + from importlib.metadata import version as pkg_version + import limacharlie + + meta_ver = pkg_version('limacharlie') + attr_ver = limacharlie.__version__ + + print(f'importlib.metadata: {meta_ver}') + print(f'limacharlie.__version__: {attr_ver}') + + assert meta_ver == attr_ver, ( + f'Version mismatch: importlib.metadata={meta_ver!r} vs ' + f'__version__={attr_ver!r}' + ) + assert attr_ver != '0.0.0.dev0', ( + f'__version__ is the fallback 0.0.0.dev0 - _version.py import failed' + ) + print(f'Version consistency: OK ({meta_ver})') + " + echo "--- Verifying orjson (expect 3.10.x on Python 3.9) ---" python -c " import orjson @@ -61,6 +85,7 @@ steps: echo "--- Building wheel ---" cp -r /workspace /tmp/build-310 && cd /tmp/build-310 pip install --upgrade pip setuptools wheel build -q + git tag 99.0.0.dev310 2>/dev/null || true python -m build -q echo "--- Installing wheel ---" @@ -70,6 +95,17 @@ steps: pip show limacharlie limacharlie --version + echo "--- Verifying version consistency ---" + python -c " + from importlib.metadata import version as pkg_version + import limacharlie + meta_ver = pkg_version('limacharlie') + attr_ver = limacharlie.__version__ + assert meta_ver == attr_ver, f'Version mismatch: metadata={meta_ver!r} vs __version__={attr_ver!r}' + assert attr_ver != '0.0.0.dev0', '__version__ is fallback 0.0.0.dev0' + print(f'Version consistency: OK ({meta_ver})') + " + echo "--- Verifying orjson ---" python -c " from limacharlie.json_compat import backend_name, HAS_ORJSON @@ -94,6 +130,7 @@ steps: echo "--- Building wheel ---" cp -r /workspace /tmp/build-311 && cd /tmp/build-311 pip install --upgrade pip setuptools wheel build -q + git tag 99.0.0.dev311 2>/dev/null || true python -m build -q echo "--- Installing wheel ---" @@ -103,6 +140,17 @@ steps: pip show limacharlie limacharlie --version + echo "--- Verifying version consistency ---" + python -c " + from importlib.metadata import version as pkg_version + import limacharlie + meta_ver = pkg_version('limacharlie') + attr_ver = limacharlie.__version__ + assert meta_ver == attr_ver, f'Version mismatch: metadata={meta_ver!r} vs __version__={attr_ver!r}' + assert attr_ver != '0.0.0.dev0', '__version__ is fallback 0.0.0.dev0' + print(f'Version consistency: OK ({meta_ver})') + " + echo "--- Verifying orjson ---" python -c " from limacharlie.json_compat import backend_name, HAS_ORJSON @@ -127,6 +175,7 @@ steps: echo "--- Building wheel ---" cp -r /workspace /tmp/build-312 && cd /tmp/build-312 pip install --upgrade pip setuptools wheel build -q + git tag 99.0.0.dev312 2>/dev/null || true python -m build -q echo "--- Installing wheel ---" @@ -136,6 +185,17 @@ steps: pip show limacharlie limacharlie --version + echo "--- Verifying version consistency ---" + python -c " + from importlib.metadata import version as pkg_version + import limacharlie + meta_ver = pkg_version('limacharlie') + attr_ver = limacharlie.__version__ + assert meta_ver == attr_ver, f'Version mismatch: metadata={meta_ver!r} vs __version__={attr_ver!r}' + assert attr_ver != '0.0.0.dev0', '__version__ is fallback 0.0.0.dev0' + print(f'Version consistency: OK ({meta_ver})') + " + echo "--- Verifying orjson ---" python -c " from limacharlie.json_compat import backend_name, HAS_ORJSON @@ -160,6 +220,7 @@ steps: echo "--- Building wheel ---" cp -r /workspace /tmp/build-313 && cd /tmp/build-313 pip install --upgrade pip setuptools wheel build -q + git tag 99.0.0.dev313 2>/dev/null || true python -m build -q echo "--- Installing wheel ---" @@ -169,6 +230,17 @@ steps: pip show limacharlie limacharlie --version + echo "--- Verifying version consistency ---" + python -c " + from importlib.metadata import version as pkg_version + import limacharlie + meta_ver = pkg_version('limacharlie') + attr_ver = limacharlie.__version__ + assert meta_ver == attr_ver, f'Version mismatch: metadata={meta_ver!r} vs __version__={attr_ver!r}' + assert attr_ver != '0.0.0.dev0', '__version__ is fallback 0.0.0.dev0' + print(f'Version consistency: OK ({meta_ver})') + " + echo "--- Verifying orjson ---" python -c " from limacharlie.json_compat import backend_name, HAS_ORJSON @@ -193,6 +265,7 @@ steps: echo "--- Building wheel ---" cp -r /workspace /tmp/build-314 && cd /tmp/build-314 pip install --upgrade pip setuptools wheel build -q + git tag 99.0.0.dev314 2>/dev/null || true python -m build -q echo "--- Installing wheel ---" @@ -202,6 +275,17 @@ steps: pip show limacharlie limacharlie --version + echo "--- Verifying version consistency ---" + python -c " + from importlib.metadata import version as pkg_version + import limacharlie + meta_ver = pkg_version('limacharlie') + attr_ver = limacharlie.__version__ + assert meta_ver == attr_ver, f'Version mismatch: metadata={meta_ver!r} vs __version__={attr_ver!r}' + assert attr_ver != '0.0.0.dev0', '__version__ is fallback 0.0.0.dev0' + print(f'Version consistency: OK ({meta_ver})') + " + echo "--- Verifying orjson ---" python -c " from limacharlie.json_compat import backend_name, HAS_ORJSON @@ -226,6 +310,7 @@ steps: echo "--- Building sdist ---" cp -r /workspace /tmp/build-sdist && cd /tmp/build-sdist pip install --upgrade pip setuptools wheel build -q + git tag 99.0.0.dev999 2>/dev/null || true python -m build --sdist -q ls -lh dist/ @@ -236,6 +321,17 @@ steps: pip show limacharlie limacharlie --version + echo "--- Verifying version consistency ---" + python -c " + from importlib.metadata import version as pkg_version + import limacharlie + meta_ver = pkg_version('limacharlie') + attr_ver = limacharlie.__version__ + assert meta_ver == attr_ver, f'Version mismatch: metadata={meta_ver!r} vs __version__={attr_ver!r}' + assert attr_ver != '0.0.0.dev0', '__version__ is fallback 0.0.0.dev0' + print(f'Version consistency: OK ({meta_ver})') + " + echo "--- sdist check passed ---" waitFor: ["-"] @@ -370,6 +466,69 @@ steps: echo "--- Unit tests passed ---" waitFor: ["-"] +# --------------------------------------------------------------------------- +# Sanity check: latest stable release from PyPI. +# Installs the currently published version (not the PR branch) and verifies +# it works. Catches PyPI packaging issues, broken uploads, and dependency +# regressions in the published package. +# --------------------------------------------------------------------------- + +- id: "PyPI Stable Sanity Check" + name: 'python:3.11' + entrypoint: "bash" + args: + - '-c' + - | + set -e + echo "==========================================" + echo " PyPI Stable Sanity Check" + echo "==========================================" + + echo "--- Installing latest stable from PyPI ---" + pip install --upgrade pip -q + pip install limacharlie -q + INSTALLED_VERSION=$(python -c "from importlib.metadata import version; print(version('limacharlie'))") + echo "Installed version: $${INSTALLED_VERSION}" + + echo "--- Verifying package metadata ---" + pip show limacharlie + + echo "--- Verifying version is valid semver ---" + python -c " + import re + from importlib.metadata import version + v = version('limacharlie') + assert v, 'version is empty' + assert re.match(r'^\d+\.\d+\.\d+', v), f'version {v!r} does not look like semver' + print(f'Package version: {v} (OK)') + " + + echo "--- Verifying CLI entry point ---" + limacharlie --version + + echo "--- Verifying CLI --help ---" + limacharlie --help | head -20 + + echo "--- Verifying SDK core imports ---" + python -c " + from limacharlie.client import Client + from limacharlie.config import resolve_credentials, load_config + print('SDK core imports OK') + " + + echo "--- Verifying CLI command groups load ---" + for cmd in auth sensor org dr search tag hive event; do + echo -n " $${cmd} --help: " + limacharlie $${cmd} --help > /dev/null 2>&1 && echo 'OK' || echo 'FAILED' + done + + echo "--- Verifying completion generation ---" + limacharlie completion bash > /dev/null + echo "Bash completion: OK" + + echo "--- PyPI sanity check passed (version $${INSTALLED_VERSION}) ---" + waitFor: ["-"] + # --------------------------------------------------------------------------- # Integration tests and benchmarks (Python 3.14 only - tests API behavior, # not Python version compat). diff --git a/limacharlie/__init__.py b/limacharlie/__init__.py index efa2bdc7..dcfa93d0 100644 --- a/limacharlie/__init__.py +++ b/limacharlie/__init__.py @@ -1,6 +1,9 @@ """limacharlie SDK for limacharlie.io""" -from .client import __version__ +try: + from ._version import version as __version__ +except ImportError: + __version__ = "0.0.0.dev0" __author__ = "Refraction Point, Inc" __author_email__ = "info@refractionpoint.com" diff --git a/limacharlie/ai_help.py b/limacharlie/ai_help.py index e30770d4..23c2f01e 100644 --- a/limacharlie/ai_help.py +++ b/limacharlie/ai_help.py @@ -122,8 +122,11 @@ def _top_level_help(cli: click.Group) -> str: lines.append("## All Command Groups") lines.append("") - for name in sorted(cli.commands): - sub = cli.commands[name] + ctx = click.Context(cli) + for name in cli.list_commands(ctx): + sub = cli.get_command(ctx, name) + if sub is None: + continue short = (sub.get_short_help_str(limit=300) or "").strip() lines.append(f"- **{name}** — {short}") diff --git a/limacharlie/cli.py b/limacharlie/cli.py index 735cce3a..edd4827d 100644 --- a/limacharlie/cli.py +++ b/limacharlie/cli.py @@ -2,22 +2,30 @@ """Main CLI entry point for LimaCharlie v2. -Uses Click for the CLI framework. Global options (--oid, --output, --debug, etc.) -are passed via Click context to all subcommands. +Uses Click for the CLI framework with lazy command loading. Command +modules in ``limacharlie/commands/`` are only imported when actually +invoked (or when listing subcommands for help/completion). This keeps +CLI startup fast - importing this module loads only Click and the CLI +skeleton, not the full SDK or any command-specific dependencies. + +Global options (--oid, --output, --debug, etc.) are passed via Click +context to all subcommands and can appear anywhere on the command line. """ import importlib import os -import pkgutil import sys -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Callable import click -from .client import __version__ -from .ai_help import inject_ai_help -from .output import set_filter_expr, set_wide_mode +try: + from ._version import version as __version__ +except ImportError: + __version__ = "0.0.0.dev0" + +from .ai_help import inject_ai_help, _add_flag as _add_ai_help_flag def _make_debug_fn(enabled: bool) -> Callable[[str], None] | None: @@ -80,16 +88,157 @@ def _config_no_warnings() -> bool: return False -class _GlobalOptionsGroup(click.Group): - """Click Group that allows group-level options to appear anywhere on the command line. - - Normally Click only parses group options that appear before the subcommand - name. This subclass hoists recognized group options to the front of the - args list so that ``limacharlie org list --output json`` works the same as - ``limacharlie --output json org list``. +# Static mapping: Click command name -> (module_name, attribute_name). +# This allows resolving any command to its module without importing it, +# enabling truly lazy per-command loading. Generated from the current +# command modules; must be updated when adding/renaming commands. +# The regression test TestModuleMapping verifies this stays in sync. +_COMMAND_MODULE_MAP: dict[str, tuple[str, str]] = { + "ai": ("ai", "group"), + "api": ("api_cmd", "cmd"), + "api-key": ("api_key", "group"), + "arl": ("arl", "group"), + "artifact": ("artifact", "group"), + "audit": ("audit", "group"), + "auth": ("auth", "group"), + "billing": ("billing", "group"), + "case": ("case_cmd", "group"), + "cloud-adapter": ("cloud_sensor", "group"), + "completion": ("completion", "cmd"), + "detection": ("detection", "group"), + "download": ("download", "group"), + "dr": ("dr", "group"), + "endpoint-policy": ("endpoint_policy", "group"), + "event": ("event", "group"), + "exfil": ("exfil", "group"), + "extension": ("extension", "group"), + "external-adapter": ("adapter", "group"), + "fp": ("fp", "group"), + "group": ("group", "group"), + "help": ("help_cmd", "group"), + "hive": ("hive", "group"), + "ingestion-key": ("ingestion_key", "group"), + "installation-key": ("installation_key", "group"), + "integrity": ("integrity", "group"), + "ioc": ("ioc", "group"), + "job": ("job", "group"), + "logging": ("logging_cmd", "group"), + "lookup": ("lookup", "group"), + "note": ("note", "group"), + "org": ("org", "group"), + "output": ("output_cmd", "group"), + "payload": ("payload", "group"), + "playbook": ("playbook", "group"), + "replay": ("replay_cmd", "group"), + "schema": ("schema", "group"), + "search": ("search", "group"), + "secret": ("secret", "group"), + "sensor": ("sensor", "group"), + "sop": ("sop", "group"), + "spotcheck": ("spotcheck", "group"), + "stream": ("stream", "group"), + "sync": ("sync", "group"), + "tag": ("tag", "group"), + "task": ("task", "group"), + "user": ("user", "group"), + "usp": ("usp", "group"), + "yara": ("yara", "group"), +} + + +class _LazyCommandGroup(click.Group): + """Click Group with lazy command loading and global option hoisting. + + Combines two responsibilities: + + 1. **Lazy loading**: Command modules in ``limacharlie/commands/`` are + resolved via a static name map and only imported when a specific + command is invoked or its help is requested. This cuts CLI startup + from ~700ms to ~100ms for single-command invocations. + + 2. **Global option hoisting**: Group-level options (--oid, --output, etc.) + can appear anywhere on the command line, not just before the subcommand. + ``limacharlie org list --output json`` works the same as + ``limacharlie --output json org list``. + + The ``--ai-help`` flag is injected per-command on first access rather + than eagerly across the entire command tree. """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Track which commands have had --ai-help injected + self._ai_help_injected: set[str] = set() + + def _import_command(self, cmd_name: str) -> click.BaseCommand | None: + """Import a single command module and register its command. + + Uses the static _COMMAND_MODULE_MAP to find the right module + without scanning or importing other modules. + """ + entry = _COMMAND_MODULE_MAP.get(cmd_name) + if entry is None: + return None + + modname, attr_name = entry + module_path = f"limacharlie.commands.{modname}" + + try: + mod = importlib.import_module(module_path) + attr = getattr(mod, attr_name, None) + if isinstance(attr, click.BaseCommand): + self.add_command(attr) + self._inject_ai_help(attr.name, attr) + return attr + except Exception as e: + click.echo( + f"Warning: failed to load command module '{modname}': {e}", + err=True, + ) + if os.environ.get("LC_DEBUG"): + import traceback + traceback.print_exc() + return None + + def _inject_ai_help(self, cmd_name: str, cmd: click.BaseCommand) -> None: + """Inject --ai-help into a command and its descendants on first access.""" + if cmd_name in self._ai_help_injected: + return + self._ai_help_injected.add(cmd_name) + inject_ai_help(cmd) + + # -- Click Group overrides ------------------------------------------------ + + def list_commands(self, ctx: click.Context) -> list[str]: + """Return all command names. + + Returns the known command names from the static map merged with + any eagerly-added commands, without importing any modules. + """ + eager = set(self.commands.keys()) + lazy = set(_COMMAND_MODULE_MAP.keys()) + return sorted(eager | lazy) + + def get_command(self, ctx: click.Context, cmd_name: str) -> click.BaseCommand | None: + """Resolve a command by name, importing its module on first access.""" + # Already loaded? + if cmd_name in self.commands: + cmd = self.commands[cmd_name] + self._inject_ai_help(cmd_name, cmd) + return cmd + + # Lazy import + return self._import_command(cmd_name) + + # -- Global option hoisting ----------------------------------------------- + def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: + """Parse args with global option hoisting. + + Hoists recognized group options to the front of the args list so + that ``limacharlie org list --output json`` works the same as + ``limacharlie --output json org list``. + """ # Build a map of option strings defined on this group. opt_takes_value: dict[str, bool] = {} for param in self.params: @@ -160,8 +309,10 @@ def _find_shadowed_opts( else: i += 1 continue - # Non-option token — potential subcommand name. - sub = cmd.commands.get(arg) + # Non-option token - potential subcommand name. + # Use get_command() to trigger lazy loading for just this command. + ctx = click.Context(self) + sub = self.get_command(ctx, arg) if cmd is self else cmd.commands.get(arg) if sub is not None: cmd = sub for p in cmd.params: @@ -175,7 +326,7 @@ def _find_shadowed_opts( return shadowed -@click.group(cls=_GlobalOptionsGroup, context_settings={"help_option_names": ["-h", "--help"]}) +@click.group(cls=_LazyCommandGroup, context_settings={"help_option_names": ["-h", "--help"]}) @click.option("--oid", default=None, help="Organization ID (overrides env/config).") @click.option( "--output", "output_format", @@ -214,41 +365,17 @@ def cli(ctx: click.Context, oid: str | None, output_format: str | None, debug: b lc_ctx.filter_expr = filter_expr lc_ctx.profile = profile lc_ctx.environment = environment + # Lazy import: output pulls in jmespath, tabulate, yaml, csv (~14ms). + # Deferring to here avoids that cost for fast paths like --help, --version, + # and --ai-help that never render command output. + from .output import set_filter_expr, set_wide_mode set_wide_mode(wide) set_filter_expr(filter_expr) -def _auto_discover_commands() -> None: - """Auto-discover and register command modules from limacharlie/commands/.""" - commands_package = "limacharlie.commands" - try: - commands_mod = importlib.import_module(commands_package) - except ImportError: - return - - package_path = getattr(commands_mod, "__path__", None) - if package_path is None: - return - - for importer, modname, ispkg in pkgutil.iter_modules(package_path): - try: - mod = importlib.import_module(f"{commands_package}.{modname}") - # Each command module should have a 'group' or 'commands' attribute - # that is a Click group or list of Click commands - for attr_name in dir(mod): - attr = getattr(mod, attr_name) - if isinstance(attr, click.Command) and attr_name in ("group", "cmd"): - cli.add_command(attr) - except Exception as e: - if os.environ.get("LC_DEBUG"): - import traceback - click.echo(f"Warning: failed to load command module '{modname}': {e}", err=True) - traceback.print_exc() - - -# Auto-discover commands on import, then inject --ai-help everywhere. -_auto_discover_commands() -inject_ai_help(cli) +# Inject --ai-help on the root cli group itself (subcommands get it lazily +# via _LazyCommandGroup._inject_ai_help when first accessed). +_add_ai_help_flag(cli) def main() -> None: diff --git a/tests/microbenchmarks/test_cli_startup_microbenchmark.py b/tests/microbenchmarks/test_cli_startup_microbenchmark.py new file mode 100644 index 00000000..309c68b7 --- /dev/null +++ b/tests/microbenchmarks/test_cli_startup_microbenchmark.py @@ -0,0 +1,490 @@ +"""Microbenchmarks for CLI startup, command loading, and import overhead. + +Includes both in-process benchmarks (via CliRunner) and end-to-end +subprocess benchmarks that measure real-world latency including Python +interpreter startup. The subprocess tests represent what users actually +experience when running CLI commands. + +Run with: pytest tests/microbenchmarks/test_cli_startup_microbenchmark.py -v --benchmark-only +""" + +from __future__ import annotations + +import importlib +import os +import subprocess +import sys +import time + +import click +import click.testing +import pytest + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _fresh_import_cli(): + """Import limacharlie.cli from scratch by clearing cached modules. + + Returns the cli object and the time taken to import. + """ + to_remove = [k for k in sys.modules if k.startswith("limacharlie")] + saved = {} + for k in to_remove: + saved[k] = sys.modules.pop(k) + + try: + start = time.perf_counter() + mod = importlib.import_module("limacharlie.cli") + elapsed = time.perf_counter() - start + return mod.cli, elapsed + finally: + for k in list(sys.modules.keys()): + if k.startswith("limacharlie"): + del sys.modules[k] + sys.modules.update(saved) + + +def _invoke(cli_obj, args: list[str]) -> click.testing.Result: + """Invoke a CLI command via CliRunner.""" + runner = click.testing.CliRunner() + return runner.invoke(cli_obj, args, catch_exceptions=False) + + +def _subprocess_time(python_code: str) -> float: + """Run Python code in a subprocess and return wall-clock time in seconds. + + Uses the same Python interpreter and environment as the test process. + """ + start = time.perf_counter() + result = subprocess.run( + [sys.executable, "-c", python_code], + capture_output=True, + text=True, + timeout=30, + ) + elapsed = time.perf_counter() - start + if result.returncode != 0: + raise RuntimeError( + f"Subprocess failed (exit {result.returncode}):\n" + f"stdout: {result.stdout[:500]}\n" + f"stderr: {result.stderr[:500]}" + ) + return elapsed + + +# --------------------------------------------------------------------------- +# In-process import benchmarks +# --------------------------------------------------------------------------- + +class TestImportBenchmarks: + """Benchmark the cost of importing the CLI module (in-process).""" + + def test_full_cli_import(self, benchmark): + """Time to import limacharlie.cli. + + With lazy loading this imports only the CLI skeleton, not the + 49 command modules or the SDK. + """ + def do_import(): + cli_obj, _elapsed = _fresh_import_cli() + assert hasattr(cli_obj, "commands") + + benchmark(do_import) + + def test_cli_import_wall_time(self): + """Regression guard: import must complete within threshold.""" + _cli_obj, elapsed = _fresh_import_cli() + assert elapsed < 2.0, ( + f"CLI import took {elapsed:.3f}s, expected <2.0s" + ) + + def test_cli_import_does_not_load_output(self): + """Importing cli must not pull in limacharlie.output or its heavy deps. + + limacharlie.output is lazily imported inside the cli() callback + to avoid ~14ms of jmespath/tabulate/yaml/csv import overhead on + fast paths like --help, --version, and --ai-help. + """ + to_remove = [k for k in sys.modules if k.startswith("limacharlie")] + saved = {k: sys.modules.pop(k) for k in to_remove} + try: + importlib.import_module("limacharlie.cli") + assert "limacharlie.output" not in sys.modules, ( + "limacharlie.output imported at module level" + ) + for dep in ("jmespath", "tabulate", "yaml"): + assert dep not in sys.modules, ( + f"{dep} imported at module level via limacharlie.output" + ) + finally: + for k in list(sys.modules): + if k.startswith("limacharlie"): + del sys.modules[k] + sys.modules.update(saved) + + +# --------------------------------------------------------------------------- +# In-process help benchmarks +# --------------------------------------------------------------------------- + +class TestHelpBenchmarks: + """Benchmark help text generation (in-process).""" + + def test_top_level_help(self, benchmark): + """Time to generate top-level --help output.""" + from limacharlie.cli import cli + + def do_help(): + result = _invoke(cli, ["--help"]) + assert result.exit_code == 0 + + benchmark(do_help) + + def test_group_help(self, benchmark): + """Time to generate group-level --help (e.g. sensor --help).""" + from limacharlie.cli import cli + + def do_help(): + result = _invoke(cli, ["sensor", "--help"]) + assert result.exit_code == 0 + + benchmark(do_help) + + def test_subcommand_help(self, benchmark): + """Time to generate subcommand --help (e.g. sensor list --help).""" + from limacharlie.cli import cli + + def do_help(): + result = _invoke(cli, ["sensor", "list", "--help"]) + assert result.exit_code == 0 + + benchmark(do_help) + + def test_ai_help(self, benchmark): + """Time to generate --ai-help output. + + This triggers lazy loading of all commands to populate the + '## All Command Groups' section, so it measures the cost of + a full command tree materialization. + """ + from limacharlie.cli import cli + + def do_ai_help(): + result = _invoke(cli, ["--ai-help"]) + assert result.exit_code == 0 + assert "## All Command Groups" in result.output + + benchmark(do_ai_help) + + +# --------------------------------------------------------------------------- +# In-process completion benchmarks +# --------------------------------------------------------------------------- + +class TestCompletionBenchmarks: + """Benchmark completion script generation (in-process).""" + + def test_bash_completion(self, benchmark): + """Time to generate bash completion script.""" + from limacharlie.cli import cli + + def do_completion(): + result = _invoke(cli, ["completion", "bash"]) + assert result.exit_code == 0 + assert len(result.output) > 50 + + benchmark(do_completion) + + def test_zsh_completion(self, benchmark): + """Time to generate zsh completion script.""" + from limacharlie.cli import cli + + def do_completion(): + result = _invoke(cli, ["completion", "zsh"]) + assert result.exit_code == 0 + + benchmark(do_completion) + + +# --------------------------------------------------------------------------- +# In-process command resolution benchmarks +# --------------------------------------------------------------------------- + +class TestCommandResolutionBenchmarks: + """Benchmark command resolution (in-process).""" + + def test_resolve_leaf_command(self, benchmark): + """Time to resolve sensor list (lazy loads only sensor module).""" + from limacharlie.cli import cli + + def resolve(): + ctx = click.Context(cli) + sensor_cmd = cli.get_command(ctx, "sensor") + assert sensor_cmd is not None + if isinstance(sensor_cmd, click.Group): + sub_ctx = click.Context(sensor_cmd, parent=ctx) + list_cmd = sensor_cmd.get_command(sub_ctx, "list") + assert list_cmd is not None + + benchmark(resolve) + + def test_list_all_commands(self, benchmark): + """Time to enumerate all top-level command names. + + With lazy loading this returns names from the static map + without importing any modules. + """ + from limacharlie.cli import cli + + def list_cmds(): + ctx = click.Context(cli) + names = cli.list_commands(ctx) + assert len(names) >= 40 + + benchmark(list_cmds) + + +# --------------------------------------------------------------------------- +# Lazy import isolation tests +# --------------------------------------------------------------------------- + +class TestLazyImportIsolation: + """Verify that lazy loading defers heavy imports correctly.""" + + def test_help_does_not_import_requests(self): + """Top-level --help should not require importing requests.""" + from limacharlie.cli import cli + result = _invoke(cli, ["--help"]) + assert result.exit_code == 0 + + def test_sensor_list_help_does_not_need_auth(self): + """sensor list --help should not require auth credentials.""" + from limacharlie.cli import cli + result = _invoke(cli, ["sensor", "list", "--help"]) + assert result.exit_code == 0 + assert "--sid" in result.output or "--tag" in result.output + + +# --------------------------------------------------------------------------- +# End-to-end subprocess benchmarks +# --------------------------------------------------------------------------- + +class TestEndToEndSubprocess: + """End-to-end latency benchmarks using real subprocess invocations. + + These measure what users actually experience: Python startup + + module imports + command resolution + output generation. Each test + spawns a fresh Python process to avoid warm-cache effects. + """ + + def test_e2e_import_only(self, benchmark): + """Wall-clock time to import limacharlie.cli in a fresh process. + + This is the baseline cost every CLI invocation pays. + """ + def run(): + elapsed = _subprocess_time( + "from limacharlie.cli import cli" + ) + return elapsed + + result = benchmark(run) + + def test_e2e_version(self, benchmark): + """Wall-clock time for 'limacharlie --version'.""" + def run(): + elapsed = _subprocess_time( + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "CliRunner().invoke(cli, ['--version'])" + ) + return elapsed + + benchmark(run) + + def test_e2e_top_level_help(self, benchmark): + """Wall-clock time for 'limacharlie --help'. + + This lists all commands, triggering list_commands() which + with lazy loading only returns names from the static map. + """ + def run(): + elapsed = _subprocess_time( + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "CliRunner().invoke(cli, ['--help'])" + ) + return elapsed + + benchmark(run) + + def test_e2e_subcommand_help(self, benchmark): + """Wall-clock time for 'limacharlie sensor list --help'. + + With lazy loading, only imports the sensor module. + """ + def run(): + elapsed = _subprocess_time( + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "CliRunner().invoke(cli, ['sensor', 'list', '--help'])" + ) + return elapsed + + benchmark(run) + + def test_e2e_bash_completion(self, benchmark): + """Wall-clock time for 'limacharlie completion bash'. + + Generates the full bash completion script. + """ + def run(): + elapsed = _subprocess_time( + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "CliRunner().invoke(cli, ['completion', 'bash'])" + ) + return elapsed + + benchmark(run) + + def test_e2e_ai_help(self, benchmark): + """Wall-clock time for 'limacharlie --ai-help'. + + This loads all command modules to render the full command group + listing, so it measures the worst-case lazy loading overhead. + """ + def run(): + elapsed = _subprocess_time( + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "r = CliRunner().invoke(cli, ['--ai-help']); " + "assert r.exit_code == 0; " + "assert '## All Command Groups' in r.output" + ) + return elapsed + + benchmark(run) + + def test_e2e_unknown_command(self, benchmark): + """Wall-clock time for an unknown command (error path). + + Should be fast - lazy loading means no modules imported. + """ + def run(): + elapsed = _subprocess_time( + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "r = CliRunner().invoke(cli, ['nonexistent_xyz']); " + "assert r.exit_code != 0" + ) + return elapsed + + benchmark(run) + + +# --------------------------------------------------------------------------- +# End-to-end subprocess regression tests (non-benchmark) +# --------------------------------------------------------------------------- + +class TestEndToEndRegression: + """Verify end-to-end CLI behavior in fresh subprocesses. + + These are not benchmarks but correctness tests that ensure + the lazy loading works correctly from a clean Python process. + """ + + def test_e2e_help_output_complete(self): + """--help must list all expected command groups.""" + result = subprocess.run( + [sys.executable, "-c", + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "r = CliRunner().invoke(cli, ['--help']); " + "print(r.output)"], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0 + output = result.stdout + for cmd in ("sensor", "auth", "dr", "search", "hive", "tag"): + assert cmd in output, f"Missing {cmd!r} in --help output" + + def test_e2e_sensor_help_has_subcommands(self): + """sensor --help must list subcommands like list, get, delete.""" + result = subprocess.run( + [sys.executable, "-c", + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "r = CliRunner().invoke(cli, ['sensor', '--help']); " + "print(r.output)"], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0 + output = result.stdout + for sub in ("list", "get", "delete"): + assert sub in output, f"Missing {sub!r} in sensor --help" + + def test_e2e_version_matches(self): + """--version must output the package version.""" + result = subprocess.run( + [sys.executable, "-c", + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "r = CliRunner().invoke(cli, ['--version']); " + "print(r.output)"], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0 + import re + assert re.search(r"\d+\.\d+", result.stdout) + + def test_e2e_completion_bash_valid(self): + """completion bash must produce a non-trivial script.""" + result = subprocess.run( + [sys.executable, "-c", + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "r = CliRunner().invoke(cli, ['completion', 'bash']); " + "print(len(r.output))"], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0 + script_len = int(result.stdout.strip()) + assert script_len > 100, f"Bash completion script too short: {script_len}" + + def test_e2e_unknown_command_fails(self): + """Unknown command must exit non-zero.""" + result = subprocess.run( + [sys.executable, "-c", + "import sys; " + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "r = CliRunner().invoke(cli, ['nonexistent_xyz']); " + "sys.exit(r.exit_code)"], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode != 0 + + def test_e2e_ai_help_works(self): + """--ai-help must produce markdown with command groups from a fresh process.""" + result = subprocess.run( + [sys.executable, "-c", + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "r = CliRunner().invoke(cli, ['--ai-help']); " + "print(r.output)"], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0 + assert "LimaCharlie CLI" in result.stdout + assert "## All Command Groups" in result.stdout + # Verify command groups are actually listed (not empty) + group_lines = [ + l for l in result.stdout.splitlines() if l.startswith("- **") + ] + assert len(group_lines) >= 40, ( + f"Expected 40+ command groups, got {len(group_lines)}" + ) diff --git a/tests/unit/test_cli_command_map_lint.py b/tests/unit/test_cli_command_map_lint.py new file mode 100644 index 00000000..3353ef60 --- /dev/null +++ b/tests/unit/test_cli_command_map_lint.py @@ -0,0 +1,218 @@ +"""Lint tests for _COMMAND_MODULE_MAP in limacharlie.cli. + +Validates that the static command-to-module map used for lazy loading +is complete, correct, and in sync with the actual command modules on +disk. These tests fail immediately when a developer adds, removes, or +renames a command module without updating the map. + +This file is intentionally separate from the regression tests so that +the failure message clearly points to the map as the thing to fix. + +Run with: pytest tests/unit/test_cli_command_map_lint.py -v +""" + +from __future__ import annotations + +import importlib +import pkgutil + +import click +import pytest + +from limacharlie.cli import _COMMAND_MODULE_MAP + + +def _discover_command_modules() -> set[str]: + """Return all non-private module names in limacharlie/commands/.""" + commands_mod = importlib.import_module("limacharlie.commands") + return { + modname + for _importer, modname, _ispkg in pkgutil.iter_modules(commands_mod.__path__) + if not modname.startswith("_") + } + + +def _discover_actual_mapping() -> dict[str, tuple[str, str]]: + """Import every command module and build the ground-truth mapping. + + Returns: {click_command_name: (module_name, attr_name)} + """ + actual = {} + commands_mod = importlib.import_module("limacharlie.commands") + for _importer, modname, _ispkg in pkgutil.iter_modules(commands_mod.__path__): + if modname.startswith("_"): + continue + mod = importlib.import_module(f"limacharlie.commands.{modname}") + for attr_name in ("group", "cmd"): + attr = getattr(mod, attr_name, None) + if isinstance(attr, click.BaseCommand): + actual[attr.name] = (modname, attr_name) + break + return actual + + +class TestCommandModuleMapCompleteness: + """Verify _COMMAND_MODULE_MAP covers every command module on disk. + + If a new module is added to limacharlie/commands/ without a + corresponding entry in _COMMAND_MODULE_MAP, these tests fail with + a clear message telling the developer exactly what to add. + """ + + def test_no_missing_modules(self): + """Every command module on disk must have an entry in the map. + + This is the most important check - it catches the case where + a developer adds a new command module but forgets to update + _COMMAND_MODULE_MAP. + """ + on_disk = _discover_command_modules() + in_map = {modname for modname, _attr in _COMMAND_MODULE_MAP.values()} + + missing = on_disk - in_map + if missing: + # Import the missing modules to give a helpful error message + hints = [] + for modname in sorted(missing): + mod = importlib.import_module(f"limacharlie.commands.{modname}") + for attr_name in ("group", "cmd"): + attr = getattr(mod, attr_name, None) + if isinstance(attr, click.BaseCommand): + hints.append( + f' "{attr.name}": ("{modname}", "{attr_name}"),' + ) + break + else: + hints.append( + f" # {modname}: no 'group' or 'cmd' attribute found" + ) + + pytest.fail( + f"Command modules on disk but missing from _COMMAND_MODULE_MAP " + f"in limacharlie/cli.py:\n\n" + f" Missing modules: {sorted(missing)}\n\n" + f" Add these entries to _COMMAND_MODULE_MAP:\n" + + "\n".join(hints) + ) + + def test_no_stale_map_entries(self): + """Every entry in the map must correspond to a module on disk. + + Catches the case where a command module is deleted but its + map entry is left behind. + """ + on_disk = _discover_command_modules() + in_map = {modname for modname, _attr in _COMMAND_MODULE_MAP.values()} + + stale = in_map - on_disk + if stale: + # Find which map keys reference the stale modules + stale_keys = [ + cmd_name + for cmd_name, (modname, _) in _COMMAND_MODULE_MAP.items() + if modname in stale + ] + pytest.fail( + f"Stale entries in _COMMAND_MODULE_MAP reference modules " + f"that no longer exist on disk:\n\n" + f" Stale modules: {sorted(stale)}\n" + f" Remove these map entries: {sorted(stale_keys)}" + ) + + +class TestCommandModuleMapCorrectness: + """Verify every entry in _COMMAND_MODULE_MAP is correct.""" + + @pytest.mark.parametrize( + "cmd_name", + sorted(_COMMAND_MODULE_MAP.keys()), + ) + def test_map_entry_importable_and_correct(self, cmd_name: str): + """Each map entry must point to an importable module that exports + a Click command with the expected name. + + This catches: + - Typos in module names + - Wrong attribute name (group vs cmd) + - Command name mismatch after rename + """ + modname, attr_name = _COMMAND_MODULE_MAP[cmd_name] + module_path = f"limacharlie.commands.{modname}" + + # Must be importable + try: + mod = importlib.import_module(module_path) + except ImportError as e: + pytest.fail( + f"_COMMAND_MODULE_MAP[{cmd_name!r}] references " + f"module {module_path!r} which cannot be imported: {e}" + ) + + # Must have the expected attribute + attr = getattr(mod, attr_name, None) + assert attr is not None, ( + f"_COMMAND_MODULE_MAP[{cmd_name!r}] references " + f"{module_path}.{attr_name} which does not exist. " + f"Available attributes: {[a for a in dir(mod) if not a.startswith('_')]}" + ) + + # Must be a Click command + assert isinstance(attr, click.BaseCommand), ( + f"_COMMAND_MODULE_MAP[{cmd_name!r}] -> {module_path}.{attr_name} " + f"is {type(attr).__name__}, not a Click command" + ) + + # Command name must match + assert attr.name == cmd_name, ( + f"_COMMAND_MODULE_MAP[{cmd_name!r}] -> {module_path}.{attr_name} " + f"has name {attr.name!r}, expected {cmd_name!r}. " + f"Update the map key or the Click command name." + ) + + +class TestCommandModuleMapMatchesRuntime: + """Verify the static map exactly matches what auto-discovery would find. + + This is the ultimate consistency check: import every module (like + the old eager loading did) and verify the result is identical to + what the static map declares. + """ + + def test_map_matches_auto_discovery(self): + """The static map must produce the same result as importing + every module and inspecting its exports.""" + actual = _discover_actual_mapping() + + # Check both directions + map_names = set(_COMMAND_MODULE_MAP.keys()) + actual_names = set(actual.keys()) + + missing_from_map = actual_names - map_names + extra_in_map = map_names - actual_names + + errors = [] + if missing_from_map: + for name in sorted(missing_from_map): + modname, attr_name = actual[name] + errors.append( + f" Missing: \"{name}\": (\"{modname}\", \"{attr_name}\")," + ) + if extra_in_map: + for name in sorted(extra_in_map): + errors.append(f" Extra (remove): \"{name}\"") + + # Also check that matching entries agree on module/attr + for name in map_names & actual_names: + if _COMMAND_MODULE_MAP[name] != actual[name]: + errors.append( + f" Mismatch for {name!r}: " + f"map has {_COMMAND_MODULE_MAP[name]}, " + f"actual is {actual[name]}" + ) + + if errors: + pytest.fail( + "_COMMAND_MODULE_MAP does not match auto-discovered commands.\n" + "Update _COMMAND_MODULE_MAP in limacharlie/cli.py:\n\n" + + "\n".join(errors) + ) diff --git a/tests/unit/test_cli_lazy_loading_regression.py b/tests/unit/test_cli_lazy_loading_regression.py new file mode 100644 index 00000000..dc3dc25f --- /dev/null +++ b/tests/unit/test_cli_lazy_loading_regression.py @@ -0,0 +1,1125 @@ +"""Regression tests for CLI command loading, discovery, and help surface. + +Captures the full CLI contract - every command name, group structure, +help text, docstrings, --ai-help output, global option hoisting, and +explain registry entries - so that refactoring the command loading +mechanism (e.g. switching to lazy imports) cannot silently break or +lose any part of the CLI surface. + +Run with: pytest tests/unit/test_cli_lazy_loading_regression.py -v +""" + +from __future__ import annotations + +import importlib +import pkgutil +import re + +import click +import click.testing +import pytest + +from limacharlie.cli import cli, _LazyCommandGroup, LimaCharlieContext +from limacharlie.discovery import ( + PROFILES, + _EXPLAIN_REGISTRY, + get_explain, + list_profiles, + format_discovery, +) + +# Force full command loading so tests can inspect cli.commands directly. +# With lazy loading, list_commands() returns names from the static map +# but doesn't import modules. We must call get_command() for each name +# to populate cli.commands - this is what Click does when running --help +# or completion. +_ctx = click.Context(cli) +for _name in cli.list_commands(_ctx): + cli.get_command(_ctx, _name) +del _ctx, _name + + +# --------------------------------------------------------------------------- +# Snapshot of the expected CLI surface (captured from current eager loading) +# --------------------------------------------------------------------------- + +# Every top-level command/group that must be registered on cli. +EXPECTED_TOP_LEVEL_COMMANDS = frozenset({ + "ai", "api", "api-key", "arl", "artifact", "audit", "auth", "billing", + "case", "cloud-adapter", "completion", "detection", "download", "dr", + "endpoint-policy", "event", "exfil", "extension", "external-adapter", + "fp", "group", "help", "hive", "ingestion-key", "installation-key", + "integrity", "ioc", "job", "logging", "lookup", "note", "org", "output", + "payload", "playbook", "replay", "schema", "search", "secret", "sensor", + "sop", "spotcheck", "stream", "sync", "tag", "task", "user", "usp", + "yara", +}) + +# Module filename -> (attribute name, Click command name). +# This is the authoritative mapping that lazy loading must preserve. +EXPECTED_MODULE_MAP = { + "adapter": ("group", "external-adapter"), + "ai": ("group", "ai"), + "api_cmd": ("cmd", "api"), + "api_key": ("group", "api-key"), + "arl": ("group", "arl"), + "artifact": ("group", "artifact"), + "audit": ("group", "audit"), + "auth": ("group", "auth"), + "billing": ("group", "billing"), + "case_cmd": ("group", "case"), + "cloud_sensor": ("group", "cloud-adapter"), + "completion": ("cmd", "completion"), + "detection": ("group", "detection"), + "download": ("group", "download"), + "dr": ("group", "dr"), + "endpoint_policy": ("group", "endpoint-policy"), + "event": ("group", "event"), + "exfil": ("group", "exfil"), + "extension": ("group", "extension"), + "fp": ("group", "fp"), + "group": ("group", "group"), + "help_cmd": ("group", "help"), + "hive": ("group", "hive"), + "ingestion_key": ("group", "ingestion-key"), + "installation_key": ("group", "installation-key"), + "integrity": ("group", "integrity"), + "ioc": ("group", "ioc"), + "job": ("group", "job"), + "logging_cmd": ("group", "logging"), + "lookup": ("group", "lookup"), + "note": ("group", "note"), + "org": ("group", "org"), + "output_cmd": ("group", "output"), + "payload": ("group", "payload"), + "playbook": ("group", "playbook"), + "replay_cmd": ("group", "replay"), + "schema": ("group", "schema"), + "search": ("group", "search"), + "secret": ("group", "secret"), + "sensor": ("group", "sensor"), + "sop": ("group", "sop"), + "spotcheck": ("group", "spotcheck"), + "stream": ("group", "stream"), + "sync": ("group", "sync"), + "tag": ("group", "tag"), + "task": ("group", "task"), + "user": ("group", "user"), + "usp": ("group", "usp"), + "yara": ("group", "yara"), +} + +# Every (group, subcommand) pair that must exist. +EXPECTED_SUBCOMMANDS: dict[str, frozenset[str]] = { + "ai": frozenset({ + "generate-detection", "generate-playbook", "generate-query", + "generate-response", "generate-rule", "generate-selector", + "session", "start-session", "summarize-detection", "usage", + }), + "api-key": frozenset({"create", "delete", "list"}), + "arl": frozenset({"get"}), + "artifact": frozenset({"download", "list", "upload"}), + "audit": frozenset({"list"}), + "auth": frozenset({ + "get-token", "list-envs", "list-orgs", "login", "logout", + "signup", "test", "use-env", "use-org", "whoami", + }), + "billing": frozenset({"details", "invoice", "plans", "status"}), + "case": frozenset({ + "add-note", "artifact", "assignees", "bulk-update", + "config-get", "config-set", "create", "dashboard", "detection", + "entity", "export", "get", "list", "merge", "report", "tag", + "telemetry", "update", + }), + "cloud-adapter": frozenset({"delete", "disable", "enable", "get", "list", "set"}), + "detection": frozenset({"get", "list"}), + "download": frozenset({"adapter", "list", "sensor"}), + "dr": frozenset({ + "convert-rules", "delete", "export", "get", "import", + "list", "replay", "set", "test", "validate", + }), + "endpoint-policy": frozenset({"isolate", "rejoin", "seal", "status", "unseal"}), + "event": frozenset({ + "children", "get", "list", "overview", "retention", + "schema", "timeline", "types", + }), + "exfil": frozenset({"create-event", "create-watch", "delete", "list"}), + "extension": frozenset({ + "config-delete", "config-get", "config-list", "config-set", + "list", "list-available", "rekey", "request", "schema", + "subscribe", "unsubscribe", + }), + "external-adapter": frozenset({"delete", "disable", "enable", "get", "list", "set"}), + "fp": frozenset({"delete", "disable", "enable", "get", "list", "set"}), + "group": frozenset({ + "create", "delete", "get", "list", "logs", + "member-add", "member-remove", "org-add", "org-remove", + "owner-add", "owner-remove", "permissions-set", + }), + "help": frozenset({"cheatsheet", "discover", "topic"}), + "hive": frozenset({ + "delete", "disable", "enable", "export", "get", "import", + "list", "list-types", "rename", "set", "validate", + }), + "ingestion-key": frozenset({"create", "delete", "list"}), + "installation-key": frozenset({"create", "delete", "get", "list"}), + "integrity": frozenset({"create", "delete", "get", "list"}), + "ioc": frozenset({"batch-enrich", "batch-search", "enrich", "hosts", "search"}), + "job": frozenset({"delete", "get", "list", "wait"}), + "logging": frozenset({"create", "delete", "get", "list"}), + "lookup": frozenset({"delete", "disable", "enable", "get", "list", "set"}), + "note": frozenset({"delete", "disable", "enable", "get", "list", "set"}), + "org": frozenset({ + "check-name", "config-get", "config-set", "create", "delete", + "dismiss-error", "errors", "info", "list", "mitre", "quota", + "rename", "runtime-metadata", "schema", "stats", "urls", + }), + "output": frozenset({"create", "delete", "list"}), + "payload": frozenset({"delete", "download", "list", "upload"}), + "playbook": frozenset({"delete", "disable", "enable", "get", "list", "set"}), + "replay": frozenset({"run"}), + "schema": frozenset({"get", "list"}), + "search": frozenset({ + "checkpoint-show", "checkpoints", "estimate", "run", + "saved-create", "saved-delete", "saved-get", "saved-list", + "saved-run", "validate", + }), + "secret": frozenset({"delete", "disable", "enable", "get", "list", "set"}), + "sensor": frozenset({ + "delete", "dump", "export", "get", "list", "set-version", + "sweep", "upgrade", "wait-online", + }), + "sop": frozenset({"delete", "disable", "enable", "get", "list", "set"}), + "spotcheck": frozenset({"run"}), + "stream": frozenset({"audit", "detections", "events", "firehose"}), + "sync": frozenset({"pull", "push"}), + "tag": frozenset({"add", "find", "list", "mass-add", "mass-remove", "remove"}), + "task": frozenset({ + "reliable-delete", "reliable-list", "reliable-send", + "request", "send", + }), + "user": frozenset({"invite", "list", "permissions", "remove"}), + "usp": frozenset({"validate"}), + "yara": frozenset({ + "rule-add", "rule-delete", "rules-list", "scan", + "source-add", "source-delete", "source-get", "sources-list", + }), +} + +# Global options that must be present on the top-level cli group. +EXPECTED_GLOBAL_OPTIONS = frozenset({ + "oid", "output_format", "debug", "debug_full", "debug_curl", + "quiet", "wide", "no_warnings", "filter_expr", "profile", + "environment", +}) + + +# --------------------------------------------------------------------------- +# Helper +# --------------------------------------------------------------------------- + +def _invoke(args: list[str]) -> click.testing.Result: + """Invoke the real CLI and return the result.""" + runner = click.testing.CliRunner() + return runner.invoke(cli, args, catch_exceptions=False) + + +# --------------------------------------------------------------------------- +# Test: top-level command registration +# --------------------------------------------------------------------------- + +class TestCommandRegistration: + """Verify every expected command is registered on the CLI group.""" + + def test_all_expected_commands_registered(self): + """Every command in the snapshot must be present.""" + registered = set(cli.commands.keys()) + missing = EXPECTED_TOP_LEVEL_COMMANDS - registered + assert not missing, f"Missing commands: {sorted(missing)}" + + def test_no_unexpected_commands(self): + """No commands beyond the snapshot should appear (catches accidental duplication).""" + registered = set(cli.commands.keys()) + extra = registered - EXPECTED_TOP_LEVEL_COMMANDS + assert not extra, f"Unexpected commands: {sorted(extra)}" + + def test_command_count(self): + """Total command count must match exactly.""" + assert len(cli.commands) == len(EXPECTED_TOP_LEVEL_COMMANDS) + + +# --------------------------------------------------------------------------- +# Test: module-to-command mapping +# --------------------------------------------------------------------------- + +class TestModuleMapping: + """Verify the module filename to Click command name mapping. + + This is critical for lazy loading - the loader must know which module + to import for each command name without importing it first. + """ + + def test_all_modules_have_expected_export(self): + """Each command module must export either 'group' or 'cmd' + matching the expected mapping.""" + commands_mod = importlib.import_module("limacharlie.commands") + for modname, (attr_name, cmd_name) in EXPECTED_MODULE_MAP.items(): + mod = importlib.import_module(f"limacharlie.commands.{modname}") + attr = getattr(mod, attr_name, None) + assert attr is not None, ( + f"Module {modname} missing attribute '{attr_name}'" + ) + assert isinstance(attr, click.BaseCommand), ( + f"Module {modname}.{attr_name} is not a Click command" + ) + assert attr.name == cmd_name, ( + f"Module {modname}.{attr_name}.name = {attr.name!r}, " + f"expected {cmd_name!r}" + ) + + def test_no_undiscovered_command_modules(self): + """Every non-private .py file in commands/ must be in the mapping.""" + commands_mod = importlib.import_module("limacharlie.commands") + discovered = set() + for _importer, modname, _ispkg in pkgutil.iter_modules(commands_mod.__path__): + if not modname.startswith("_"): + discovered.add(modname) + mapped = set(EXPECTED_MODULE_MAP.keys()) + unmapped = discovered - mapped + assert not unmapped, ( + f"Command modules not in EXPECTED_MODULE_MAP: {sorted(unmapped)}" + ) + + def test_mapping_has_no_stale_entries(self): + """Every entry in the mapping must correspond to an actual module.""" + commands_mod = importlib.import_module("limacharlie.commands") + discovered = set() + for _importer, modname, _ispkg in pkgutil.iter_modules(commands_mod.__path__): + if not modname.startswith("_"): + discovered.add(modname) + mapped = set(EXPECTED_MODULE_MAP.keys()) + stale = mapped - discovered + assert not stale, ( + f"Stale entries in EXPECTED_MODULE_MAP: {sorted(stale)}" + ) + + +# --------------------------------------------------------------------------- +# Test: subcommand structure +# --------------------------------------------------------------------------- + +class TestSubcommandStructure: + """Verify every group has exactly the expected subcommands.""" + + @pytest.mark.parametrize( + "group_name", + sorted(EXPECTED_SUBCOMMANDS.keys()), + ) + def test_subcommands_match(self, group_name: str): + """Subcommands of each group must match the snapshot exactly.""" + cmd = cli.commands.get(group_name) + assert cmd is not None, f"Group {group_name!r} not registered" + assert isinstance(cmd, click.Group), ( + f"{group_name!r} is not a Group (is {type(cmd).__name__})" + ) + actual = set(cmd.commands.keys()) + expected = EXPECTED_SUBCOMMANDS[group_name] + missing = expected - actual + extra = actual - expected + assert not missing, ( + f"{group_name}: missing subcommands: {sorted(missing)}" + ) + assert not extra, ( + f"{group_name}: unexpected subcommands: {sorted(extra)}" + ) + + def test_standalone_commands_are_not_groups(self): + """'api' and 'completion' must be leaf commands, not groups.""" + for name in ("api", "completion"): + cmd = cli.commands.get(name) + assert cmd is not None, f"{name!r} not registered" + assert not isinstance(cmd, click.Group), ( + f"{name!r} should be a leaf Command, not a Group" + ) + + +# --------------------------------------------------------------------------- +# Test: global options +# --------------------------------------------------------------------------- + +class TestGlobalOptions: + """Verify global options are present and hoistable.""" + + def test_all_global_options_present(self): + """Every expected global option must be a parameter on cli.""" + param_names = {p.name for p in cli.params if isinstance(p, click.Option)} + missing = EXPECTED_GLOBAL_OPTIONS - param_names + assert not missing, f"Missing global options: {sorted(missing)}" + + def test_cli_uses_lazy_command_group(self): + """The CLI group class must support lazy loading and option hoisting.""" + assert isinstance(cli, _LazyCommandGroup), ( + f"cli is {type(cli).__name__}, expected _LazyCommandGroup" + ) + + def test_option_after_subcommand_works(self): + """Global options placed after the subcommand must be parsed.""" + result = _invoke(["auth", "whoami", "--help"]) + assert result.exit_code == 0 + # Verify the subcommand help renders correctly + assert "whoami" in result.output + + def test_global_options_in_help(self): + """Top-level --help must list all global options.""" + result = _invoke(["--help"]) + assert result.exit_code == 0 + for opt in ("--oid", "--output", "--debug", "--quiet", "--wide", + "--filter", "--profile", "--env", "--no-warnings"): + assert opt in result.output, f"{opt} missing from --help" + + +# --------------------------------------------------------------------------- +# Test: --help for every command +# --------------------------------------------------------------------------- + +class TestHelpOutput: + """Verify --help works for every registered command and group.""" + + def test_top_level_help(self): + result = _invoke(["--help"]) + assert result.exit_code == 0 + assert "LimaCharlie CLI" in result.output + + @pytest.mark.parametrize( + "group_name", + sorted(EXPECTED_TOP_LEVEL_COMMANDS), + ) + def test_group_help(self, group_name: str): + """--help must succeed for every top-level command/group.""" + result = _invoke([group_name, "--help"]) + assert result.exit_code == 0, ( + f"{group_name} --help failed with exit code {result.exit_code}: " + f"{result.output}" + ) + # Must contain the command name somewhere in the output + assert group_name in result.output + + @pytest.mark.parametrize( + "group_name,sub_name", + [ + (g, s) + for g in sorted(EXPECTED_SUBCOMMANDS) + for s in sorted(EXPECTED_SUBCOMMANDS[g]) + ], + ) + def test_subcommand_help(self, group_name: str, sub_name: str): + """--help must succeed for every subcommand.""" + result = _invoke([group_name, sub_name, "--help"]) + assert result.exit_code == 0, ( + f"{group_name} {sub_name} --help failed: {result.output}" + ) + + +# --------------------------------------------------------------------------- +# Test: docstrings +# --------------------------------------------------------------------------- + +class TestDocstrings: + """Verify all groups and commands have non-empty docstrings/help text.""" + + # Commands that currently lack docstrings (pre-existing). + _TOP_LEVEL_WITHOUT_HELP = {"api", "completion"} + + @pytest.mark.parametrize( + "group_name", + sorted(EXPECTED_TOP_LEVEL_COMMANDS), + ) + def test_top_level_has_help(self, group_name: str): + """Top-level commands with existing docstrings must keep them.""" + if group_name in self._TOP_LEVEL_WITHOUT_HELP: + pytest.skip(f"{group_name!r} has no docstring (pre-existing)") + cmd = cli.commands[group_name] + help_text = cmd.help or "" + assert help_text.strip(), ( + f"Command {group_name!r} has no help text/docstring" + ) + + def test_subcommands_with_help_count(self): + """Track how many subcommands have help text. + + Many subcommands currently lack docstrings. This test ensures + the count doesn't decrease (i.e., lazy loading doesn't lose + existing help text). Update the threshold as docstrings are added. + """ + with_help = 0 + total = 0 + for gname in EXPECTED_SUBCOMMANDS: + group = cli.commands[gname] + if not isinstance(group, click.Group): + continue + for sname in group.commands: + total += 1 + if (group.commands[sname].help or "").strip(): + with_help += 1 + # Current baseline: many subcommands have help text. + # This catches regressions where lazy loading loses help text. + assert with_help > 50, ( + f"Only {with_help}/{total} subcommands have help text, " + f"expected at least 50 (regression in help text loading?)" + ) + + +# --------------------------------------------------------------------------- +# Test: --ai-help injection +# --------------------------------------------------------------------------- + +class TestAiHelpInjection: + """Verify --ai-help is available on all commands after loading.""" + + def test_ai_help_on_root(self): + """Root CLI must have --ai-help.""" + names = {p.name for p in cli.params} + assert "ai_help" in names + + @pytest.mark.parametrize( + "group_name", + sorted(EXPECTED_TOP_LEVEL_COMMANDS), + ) + def test_ai_help_on_top_level(self, group_name: str): + """Every top-level command/group must have --ai-help.""" + cmd = cli.commands[group_name] + names = {p.name for p in cmd.params} + assert "ai_help" in names, ( + f"{group_name!r} missing --ai-help parameter" + ) + + @pytest.mark.parametrize( + "group_name,sub_name", + [ + (g, s) + for g in sorted(EXPECTED_SUBCOMMANDS) + for s in sorted(EXPECTED_SUBCOMMANDS[g]) + ], + ) + def test_ai_help_on_subcommand(self, group_name: str, sub_name: str): + """Every subcommand must have --ai-help.""" + group = cli.commands[group_name] + assert isinstance(group, click.Group) + cmd = group.commands[sub_name] + names = {p.name for p in cmd.params} + assert "ai_help" in names, ( + f"{group_name} {sub_name} missing --ai-help parameter" + ) + + +# --------------------------------------------------------------------------- +# Test: --ai-help output +# --------------------------------------------------------------------------- + +class TestAiHelpOutput: + """Verify --ai-help produces meaningful output at all levels.""" + + def test_top_level_ai_help(self): + result = _invoke(["--ai-help"]) + assert result.exit_code == 0 + assert "LimaCharlie CLI" in result.output + assert "Getting Started" in result.output + assert "Global Options" in result.output + + @pytest.mark.parametrize( + "group_name", + # Test a representative sample of groups for speed + ["sensor", "dr", "auth", "hive", "search", "case", "extension"], + ) + def test_group_ai_help(self, group_name: str): + result = _invoke([group_name, "--ai-help"]) + assert result.exit_code == 0 + assert group_name in result.output + # Must list subcommands + group = cli.commands[group_name] + if isinstance(group, click.Group): + for sub_name in list(group.commands.keys())[:3]: + assert sub_name in result.output + + @pytest.mark.parametrize( + "group_name,sub_name", + [ + ("sensor", "list"), + ("dr", "list"), + ("auth", "whoami"), + ("search", "run"), + ("tag", "add"), + ], + ) + def test_command_ai_help(self, group_name: str, sub_name: str): + result = _invoke([group_name, sub_name, "--ai-help"]) + assert result.exit_code == 0 + assert sub_name in result.output + # Should have Options section if command has options + cmd = cli.commands[group_name].commands[sub_name] + opts = [p for p in cmd.params if isinstance(p, click.Option) and p.name != "ai_help"] + if opts: + assert "Options" in result.output or "--" in result.output + + +# --------------------------------------------------------------------------- +# Test: explain registry +# --------------------------------------------------------------------------- + +class TestExplainRegistry: + """Verify the explain registry is populated for all commands.""" + + def test_explain_registry_has_entries(self): + """Registry must have substantial number of entries.""" + assert len(_EXPLAIN_REGISTRY) > 200, ( + f"Expected >200 explain entries, got {len(_EXPLAIN_REGISTRY)}" + ) + + def test_all_explain_values_are_nonempty_strings(self): + """Every explain entry must be a non-empty string.""" + for key, value in _EXPLAIN_REGISTRY.items(): + assert isinstance(value, str), ( + f"Explain entry {key!r} is {type(value).__name__}, not str" + ) + assert value.strip(), f"Explain entry {key!r} is empty" + + def test_explain_keys_use_dot_notation(self): + """All keys must use dot notation (e.g. 'sensor.list'). + + Single-segment keys (e.g. 'completion') are allowed for + standalone commands that are not part of a group. + """ + for key in _EXPLAIN_REGISTRY: + # Allow single-segment keys for standalone commands + assert isinstance(key, str) and len(key) > 0, ( + f"Explain key must be a non-empty string, got {key!r}" + ) + + @pytest.mark.parametrize( + "group_name", + sorted(EXPECTED_SUBCOMMANDS.keys()), + ) + def test_group_has_at_least_one_explain_entry(self, group_name: str): + """Each command group should have at least one explain entry. + + The key prefix uses the Click command name (e.g. 'endpoint-policy.isolate') + or may use alternate forms. We check that at least one key starts + with the group name. + """ + prefix = group_name + "." + # Some groups use underscores in explain keys + alt_prefix = group_name.replace("-", "_") + "." + matching = [ + k for k in _EXPLAIN_REGISTRY + if k.startswith(prefix) or k.startswith(alt_prefix) + ] + # Most groups should have explain entries; allow exceptions for + # very small groups that may not have been documented yet. + if group_name not in ("cloud-adapter",): + assert matching, ( + f"No explain entries found for group {group_name!r} " + f"(looked for prefix {prefix!r})" + ) + + +# --------------------------------------------------------------------------- +# Test: discovery profiles reference valid commands +# --------------------------------------------------------------------------- + +class TestDiscoveryProfiles: + """Verify discovery profiles reference commands that actually exist.""" + + def test_most_profile_commands_exist(self): + """Most command paths in profiles should correspond to real commands. + + Some profiles reference command aliases or planned commands + (e.g. 'rule list' instead of 'dr list', 'sensor online' which + doesn't exist yet). We verify the majority resolve correctly + and that lazy loading doesn't reduce the match rate. + """ + all_commands = set() + for name in cli.commands: + cmd = cli.commands[name] + if isinstance(cmd, click.Group): + for sub_name in cmd.commands: + all_commands.add(f"{name} {sub_name}") + else: + all_commands.add(name) + + total = 0 + found = 0 + for profile_name, profile in PROFILES.items(): + for cmd_path in profile["commands"]: + total += 1 + if cmd_path in all_commands: + found += 1 + + # Current baseline: most profile commands exist. + # Lazy loading must not reduce this ratio. + assert found > 100, ( + f"Only {found}/{total} profile commands resolve to real commands" + ) + + def test_format_discovery_works(self): + """format_discovery() must produce output without errors.""" + output = format_discovery() + assert "Command Discovery" in output + assert len(output) > 100 + + +# --------------------------------------------------------------------------- +# Test: completion command +# --------------------------------------------------------------------------- + +class TestCompletionCommand: + """Verify the completion command works for all supported shells.""" + + @pytest.mark.parametrize("shell", ["bash", "zsh", "fish"]) + def test_completion_generates_script(self, shell: str): + result = _invoke(["completion", shell]) + assert result.exit_code == 0 + assert len(result.output) > 50, ( + f"Completion script for {shell} is suspiciously short" + ) + + def test_completion_invalid_shell(self): + result = click.testing.CliRunner().invoke(cli, ["completion", "powershell"]) + assert result.exit_code != 0 + + +# --------------------------------------------------------------------------- +# Test: context propagation +# --------------------------------------------------------------------------- + +class TestContextPropagation: + """Verify the LimaCharlieContext dataclass is correctly populated.""" + + def test_context_object_type(self): + """Context object must be LimaCharlieContext.""" + captured = {} + + @cli.command("_test_ctx_probe", hidden=True) + @click.pass_context + def probe(ctx): + captured["obj"] = ctx.obj + + try: + result = _invoke(["_test_ctx_probe"]) + assert isinstance(captured.get("obj"), LimaCharlieContext) + finally: + cli.commands.pop("_test_ctx_probe", None) + + def test_debug_fn_off_by_default(self): + """debug_fn should be None when --debug is not passed.""" + ctx = LimaCharlieContext() + assert ctx.debug_fn is None + + def test_debug_fn_on_when_debug(self): + """debug_fn should be callable when debug=True.""" + ctx = LimaCharlieContext(debug=True) + assert ctx.debug_fn is not None + assert callable(ctx.debug_fn) + + def test_debug_verbose_combines_flags(self): + """debug_verbose should be True for --debug and --debug-full.""" + assert LimaCharlieContext(debug=True).debug_verbose is True + assert LimaCharlieContext(debug_full=True).debug_verbose is True + assert LimaCharlieContext(debug_curl=True).debug_verbose is False + assert LimaCharlieContext().debug_verbose is False + + +# --------------------------------------------------------------------------- +# Test: version +# --------------------------------------------------------------------------- + +class TestVersion: + def test_version_flag(self): + result = _invoke(["--version"]) + assert result.exit_code == 0 + # Should contain a version string (digits and dots) + assert re.search(r"\d+\.\d+", result.output) + + +# --------------------------------------------------------------------------- +# Test: unknown command error +# --------------------------------------------------------------------------- + +class TestErrorHandling: + def test_unknown_command_exits_nonzero(self): + runner = click.testing.CliRunner() + result = runner.invoke(cli, ["nonexistent_command_xyz"]) + assert result.exit_code != 0 + + def test_unknown_subcommand_exits_nonzero(self): + runner = click.testing.CliRunner() + result = runner.invoke(cli, ["sensor", "nonexistent_xyz"]) + assert result.exit_code != 0 + + +# --------------------------------------------------------------------------- +# Test: lazy loading specific behavior +# --------------------------------------------------------------------------- + +class TestLazyLoadingBehavior: + """Tests specific to the lazy command loading mechanism. + + These verify invariants that are only relevant when commands are + loaded on-demand rather than eagerly at import time. + """ + + def test_get_command_idempotent(self): + """Calling get_command twice returns the same object.""" + ctx = click.Context(cli) + cmd1 = cli.get_command(ctx, "sensor") + cmd2 = cli.get_command(ctx, "sensor") + assert cmd1 is cmd2 + + def test_get_command_unknown_returns_none(self): + """get_command for a non-existent command must return None.""" + ctx = click.Context(cli) + assert cli.get_command(ctx, "nonexistent_xyz_abc") is None + + def test_list_commands_sorted(self): + """list_commands must return a sorted list.""" + ctx = click.Context(cli) + names = cli.list_commands(ctx) + assert names == sorted(names) + + def test_list_commands_stable(self): + """list_commands called twice returns same result.""" + ctx = click.Context(cli) + first = cli.list_commands(ctx) + second = cli.list_commands(ctx) + assert first == second + + def test_static_map_matches_list_commands(self): + """_COMMAND_MODULE_MAP keys must match list_commands output.""" + from limacharlie.cli import _COMMAND_MODULE_MAP + ctx = click.Context(cli) + listed = set(cli.list_commands(ctx)) + mapped = set(_COMMAND_MODULE_MAP.keys()) + # list_commands may include eagerly-added commands (like the + # test probe), but all mapped commands must appear + missing = mapped - listed + assert not missing, f"Mapped but not listed: {sorted(missing)}" + + def test_static_map_all_importable(self): + """Every module in _COMMAND_MODULE_MAP must be importable and + must export the expected attribute.""" + from limacharlie.cli import _COMMAND_MODULE_MAP + for cmd_name, (modname, attr_name) in _COMMAND_MODULE_MAP.items(): + module_path = f"limacharlie.commands.{modname}" + mod = importlib.import_module(module_path) + attr = getattr(mod, attr_name, None) + assert attr is not None, ( + f"Map entry {cmd_name!r} -> {modname}.{attr_name} not found" + ) + assert isinstance(attr, click.BaseCommand), ( + f"Map entry {cmd_name!r} -> {modname}.{attr_name} is " + f"{type(attr).__name__}, not a Click command" + ) + assert attr.name == cmd_name, ( + f"Map entry {cmd_name!r} -> {modname}.{attr_name}.name = " + f"{attr.name!r}, expected {cmd_name!r}" + ) + + def test_loading_one_command_does_not_break_another(self): + """Loading 'tag' then 'sensor' must both work correctly.""" + ctx = click.Context(cli) + tag = cli.get_command(ctx, "tag") + sensor = cli.get_command(ctx, "sensor") + assert tag is not None + assert sensor is not None + assert tag.name == "tag" + assert sensor.name == "sensor" + # Both must have their subcommands + assert isinstance(tag, click.Group) + assert isinstance(sensor, click.Group) + assert "list" in tag.commands + assert "list" in sensor.commands + + def test_ai_help_not_double_injected(self): + """Accessing a command twice must not duplicate --ai-help.""" + ctx = click.Context(cli) + cli.get_command(ctx, "sensor") + cli.get_command(ctx, "sensor") + cmd = cli.commands["sensor"] + ai_params = [p for p in cmd.params if p.name == "ai_help"] + assert len(ai_params) == 1, ( + f"Expected 1 ai_help param, got {len(ai_params)}" + ) + + def test_ai_help_injected_on_subcommands(self): + """--ai-help must be present on subcommands after lazy loading.""" + ctx = click.Context(cli) + sensor = cli.get_command(ctx, "sensor") + assert isinstance(sensor, click.Group) + list_cmd = sensor.commands["list"] + names = {p.name for p in list_cmd.params} + assert "ai_help" in names + + def test_global_option_hoisting_with_lazy_commands(self): + """Global options after subcommand must be parsed correctly. + + This tests that option hoisting works when the subcommand + is lazily loaded during parse_args. + """ + result = _invoke(["sensor", "list", "--help", "--output", "json"]) + assert result.exit_code == 0 + + def test_global_option_hoisting_output_format(self): + """--output after subcommand must set the output format.""" + result = _invoke(["auth", "whoami", "--output", "json", "--help"]) + assert result.exit_code == 0 + + def test_shadowed_option_not_hoisted(self): + """Options defined by both the group and subcommand must not + be hoisted. e.g. auth login --oid should go to login, not cli.""" + # auth login has its own --oid option; it must not be hoisted + # to the global --oid. We verify by checking help shows --oid + # as an auth login option. + result = _invoke(["auth", "login", "--help"]) + assert result.exit_code == 0 + assert "--oid" in result.output + + def test_completion_works_with_lazy_loading(self): + """Completion must enumerate all commands without errors.""" + result = _invoke(["completion", "bash"]) + assert result.exit_code == 0 + # The completion script should reference limacharlie + assert "limacharlie" in result.output.lower() or "LIMACHARLIE" in result.output + + def test_help_topic_fallthrough_with_lazy_loading(self): + """help must work with lazily-loaded help command.""" + result = _invoke(["help", "auth"]) + assert result.exit_code == 0 + assert "Authentication" in result.output + + def test_hive_shortcut_commands_load_correctly(self): + """Hive shortcut commands (secret, fp, lookup, etc.) must load + correctly via lazy loading since they use a factory pattern.""" + ctx = click.Context(cli) + for name in ("secret", "fp", "lookup", "playbook", "sop", "note"): + cmd = cli.get_command(ctx, name) + assert cmd is not None, f"Hive shortcut {name!r} not loaded" + assert isinstance(cmd, click.Group), ( + f"{name!r} should be a Group" + ) + # All hive shortcuts have list, get, set, delete, enable, disable + for sub in ("list", "get", "set", "delete", "enable", "disable"): + assert sub in cmd.commands, ( + f"{name!r} missing subcommand {sub!r}" + ) + + def test_nonstandard_name_mappings(self): + """Commands where filename != command name must load correctly. + + These are the tricky cases that require the static map: + - adapter.py -> external-adapter + - cloud_sensor.py -> cloud-adapter + - api_cmd.py -> api + - case_cmd.py -> case + - help_cmd.py -> help + """ + ctx = click.Context(cli) + cases = { + "external-adapter": "adapter", + "cloud-adapter": "cloud_sensor", + "api": "api_cmd", + "case": "case_cmd", + "help": "help_cmd", + "output": "output_cmd", + "logging": "logging_cmd", + "replay": "replay_cmd", + } + for cmd_name, _module_name in cases.items(): + cmd = cli.get_command(ctx, cmd_name) + assert cmd is not None, ( + f"Non-standard mapping {cmd_name!r} failed to load" + ) + assert cmd.name == cmd_name + + def test_version_does_not_import_client(self): + """--version should use _version, not client.py. + + Verifies the __version__ import optimization is in place. + """ + from limacharlie.cli import __version__ + from limacharlie._version import version + assert __version__ == version + + +# --------------------------------------------------------------------------- +# Regression: --ai-help must list command groups with lazy loading +# --------------------------------------------------------------------------- + +class TestAiHelpLazyLoading: + """Verify --ai-help works correctly with lazy command loading. + + Regression tests for the bug where _top_level_help() iterated + cli.commands directly (empty with lazy loading) instead of using + list_commands() + get_command(). + """ + + def test_ai_help_lists_all_command_groups(self): + """--ai-help must list all command groups under '## All Command Groups'.""" + result = _invoke(["--ai-help"]) + assert result.exit_code == 0 + assert "## All Command Groups" in result.output + group_lines = [ + l for l in result.output.splitlines() if l.startswith("- **") + ] + assert len(group_lines) >= 40, ( + f"Expected 40+ command groups in --ai-help, got {len(group_lines)}" + ) + + def test_ai_help_groups_match_list_commands(self): + """Command groups in --ai-help must match list_commands().""" + ctx = click.Context(cli) + expected_names = set(cli.list_commands(ctx)) + + result = _invoke(["--ai-help"]) + assert result.exit_code == 0 + + # Extract group names from "- **name** - description" lines + listed_names = set() + for line in result.output.splitlines(): + if line.startswith("- **"): + name = line.split("**")[1] + listed_names.add(name) + + assert listed_names == expected_names, ( + f"Missing from --ai-help: {expected_names - listed_names}\n" + f"Extra in --ai-help: {listed_names - expected_names}" + ) + + +# --------------------------------------------------------------------------- +# Regression: broken command modules must warn on stderr +# --------------------------------------------------------------------------- + +class TestBrokenModuleWarning: + """Verify that broken command modules emit a warning.""" + + def test_broken_module_emits_warning(self): + """_import_command must emit a stderr warning for broken modules.""" + from limacharlie.cli import _COMMAND_MODULE_MAP, _LazyCommandGroup + + # Create a fresh group with a fake broken mapping + group = _LazyCommandGroup("test") + original_map = _COMMAND_MODULE_MAP.copy() + try: + _COMMAND_MODULE_MAP["__broken_test__"] = ("__nonexistent_module__", "cmd") + ctx = click.Context(group) + result = group._import_command("__broken_test__") + assert result is None + finally: + _COMMAND_MODULE_MAP.clear() + _COMMAND_MODULE_MAP.update(original_map) + + def test_broken_module_returns_none(self): + """_import_command must return None for non-existent modules.""" + from limacharlie.cli import _LazyCommandGroup + group = _LazyCommandGroup("test") + ctx = click.Context(group) + result = group._import_command("__definitely_not_a_command__") + assert result is None + + +# --------------------------------------------------------------------------- +# Lint: cli.py must not have unused imports +# --------------------------------------------------------------------------- + +class TestCliImportHygiene: + """Verify cli.py does not contain known dead imports.""" + + def test_no_pkgutil_import(self): + """cli.py must not import pkgutil (removed with eager discovery).""" + import inspect + from limacharlie import cli as cli_module + source = inspect.getsource(cli_module) + # Check that pkgutil is not imported at module level + assert "import pkgutil" not in source, ( + "cli.py still imports pkgutil which is unused after removing " + "_auto_discover_commands()" + ) + + def test_no_unused_field_import(self): + """cli.py must not import 'field' from dataclasses (unused).""" + import inspect + from limacharlie import cli as cli_module + source = inspect.getsource(cli_module) + assert "import dataclass, field" not in source, ( + "cli.py imports 'field' from dataclasses which is unused" + ) + + +# --------------------------------------------------------------------------- +# Regression: output module must be lazily imported in cli callback +# --------------------------------------------------------------------------- + +class TestOutputLazyImport: + """Verify limacharlie.output is not imported at cli.py module level. + + limacharlie.output pulls in jmespath, tabulate, yaml, csv (~14ms). + It should only be imported inside the cli() callback when a command + is actually invoked, not on bare import of limacharlie.cli. + """ + + def test_output_not_imported_at_module_level(self): + """Importing limacharlie.cli must not pull in limacharlie.output.""" + import subprocess + import sys + # Run in a fresh process to avoid module cache contamination + result = subprocess.run( + [sys.executable, "-c", + "import sys; " + "import limacharlie.cli; " + "mods = [k for k in sys.modules if k == 'limacharlie.output']; " + "print(len(mods))"], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0 + count = int(result.stdout.strip()) + assert count == 0, ( + "limacharlie.output was imported at module level - it should " + "be lazily imported inside the cli() callback" + ) + + def test_heavy_deps_not_imported_at_module_level(self): + """Importing limacharlie.cli must not pull in jmespath, tabulate, or yaml.""" + import subprocess + import sys + result = subprocess.run( + [sys.executable, "-c", + "import sys; " + "import limacharlie.cli; " + "heavy = [k for k in sys.modules if k in ('jmespath', 'tabulate', 'yaml')]; " + "print(','.join(heavy) if heavy else 'none')"], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0 + assert result.stdout.strip() == "none", ( + f"Heavy deps imported at module level: {result.stdout.strip()}. " + "These should be deferred via lazy import of limacharlie.output" + ) + + def test_output_imported_when_command_invoked(self): + """limacharlie.output must be available after a command runs.""" + import subprocess + import sys + result = subprocess.run( + [sys.executable, "-c", + "import sys; " + "from limacharlie.cli import cli; " + "from click.testing import CliRunner; " + "CliRunner().invoke(cli, ['auth', 'whoami', '--help']); " + "mods = [k for k in sys.modules if k == 'limacharlie.output']; " + "print(len(mods))"], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0 + count = int(result.stdout.strip()) + assert count == 1, ( + "limacharlie.output should be imported after a command invocation" + )