diff --git a/.github/assets/spec-coverage-badge.svg b/.github/assets/spec-coverage-badge.svg new file mode 100644 index 0000000..0e7e510 --- /dev/null +++ b/.github/assets/spec-coverage-badge.svg @@ -0,0 +1 @@ +Spec Coverage100% diff --git a/.github/assets/specleft-social-preview.png b/.github/assets/specleft-social-preview.png new file mode 100644 index 0000000..37f9c6c Binary files /dev/null and b/.github/assets/specleft-social-preview.png differ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fcb200d..864febf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,3 +10,11 @@ repos: - id: check-toml - id: check-merge-conflict - id: detect-private-key + - repo: local + hooks: + - id: specleft-coverage-badge + name: Update SpecLeft coverage badge + entry: python3 scripts/update_spec_coverage_badge.py + language: system + pass_filenames: false + always_run: true diff --git a/Makefile b/Makefile index bbbf80d..65f6439 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ SHELL := /bin/sh -.PHONY: test pre-commit lint lint-fix +BADGE_OUTPUT ?= .github/assets/spec-coverage-badge.svg + +.PHONY: test pre-commit lint lint-fix badge test: pytest tests/ -v -rs @@ -16,3 +18,6 @@ lint: lint-fix: ruff check --fix src/ tests/ examples/ black src/ tests/ examples/ + +badge: + SPECLEFT_BADGE_OUTPUT="$(BADGE_OUTPUT)" python3 scripts/update_spec_coverage_badge.py diff --git a/README.md b/README.md index e68e263..1ed9d38 100644 --- a/README.md +++ b/README.md @@ -1,262 +1,138 @@ -# SpecLeft — Planning-First CLI for Python +![SpecLeft social preview](.github/assets/specleft-social-preview.png) -**A planning buffer for AI coding agents — externalize intent before writing code.** +# SpecLeft: Planning-First Workflow for pytest -SpecLeft lets teams capture intended behaviour as feature specs (`.specleft/specs/*.md`) before implementation, then optionally enforce that intent in CI. +![Spec coverage](.github/assets/spec-coverage-badge.svg) -Go from *"this is how the system should behave"* to *traceable test skeletons* — predictable, incremental, fully under developer control. +SpecLeft keeps feature intent and test coverage aligned by turning plans into version-controlled specs, then generating pytest test skeletons from those specs. +- Write feature specs in Markdown: `.specleft/specs/*.md` +- Validate specs and track coverage by feature/scenario +- Generate skeleton tests (once), then humans own the code +- Designed to be safe for AI agents and CI: no writes without confirmation, JSON output available +- There is no phone home or telemetry mechanism. SpecLeft runs 100% locally and stores data in your local disk. -SpecLeft does **not** replace pytest. +SpecLeft works with pytest. It does not replace your test runner or reinterpret existing tests. -It does **not** reinterpret your tests. - -It does **not** mutate your code unless you explicitly say yes. +Website: [specleft.dev](https://specleft.dev) ## Quick Start -Create a `prd.md` describing the intended behaviour of your system, then run: +Two paths, depending on how you want to start. See [docs/cli-reference.md](https://github.com/SpecLeft/specleft/blob/main/docs/cli-reference.md) for full command details. + +### Setup (run once per repo) ```bash pip install specleft -specleft plan +specleft init ``` -This converts `prd.md` into feature specifications under .specleft/specs/ -without writing code or tests. -## For AI Coding Agents +### Path 1: Add one feature (and generate a test skeleton) -SpecLeft provides a planning and safety layer for autonomous execution. +Create a feature, then add a scenario and generate a skeleton test for it: -Before acting, SpecLeft provides machine-verifiable guarantees by running: ```bash -specleft contract --format json +# Create the feature spec +specleft features add --id AUTHENTICATION --title "Authentication" --format json + +# Add a scenario and generate a skeleton test file +specleft features add-scenario \ + --feature AUTHENTICATION \ + --title "Successful login" \ + --step "Given a user has valid credentials" \ + --step "When the user logs in" \ + --step "Then the user is authenticated" \ + --add-test skeleton \ + --format json + +# Show traceability / coverage status +specleft status ``` -See [AI_AGENTS.md](AI_AGENTS.md) for integration guidance and scenarios on when to use SpecLeft and when not to. - -SpecLeft also includes CLI commands to create feature specs and append scenarios directly from the terminal. -See `specleft features add` and `specleft features add-scenario` in `docs/cli-reference.md`. - - -## What problem does SpecLeft solve? - -Most teams already have: -- feature specs (Jira, ADO, docs, wikis etc.) -- automated tests (pytest in this case) -- CI pipelines - -What they *don’t* have is **alignment**. - -Specs drift. -Tests drift. -Coverage becomes guesswork. -New contributors find it hard to know what behaviour is *expected* vs *accidental*. - -SpecLeft closes that gap by making feature intent **visible, executable, and version-controlled**, without forcing you into BDD frameworks or heavyweight process. - -## When to Use SpecLeft - -| Your Situation | Use SpecLeft? | Why | -|---------------|---------------|-----| -| Building new feature with acceptance criteria | ✅ Yes | Track coverage by feature | -| Have existing tests, need visibility | ✅ Yes | Add specs retrospectively | -| Writing unit tests for utilities | ❌ No | Too granular for spec tracking | -| Need to generate test scaffolding | ✅ Yes | Skeleton generation built-in | -| Want BDD-style Gherkin | ⚠️ Maybe | SpecLeft uses simpler Markdown | -| Have Jira/ADO stories to track | ✅ Yes | Specs mirror story structure | - -**Quick Decision:** -- Do you have feature stories/scenarios to track? → **Use SpecLeft** -- Are you just writing ad-hoc unit tests? → **Use plain pytest** - ---- - -## What SpecLeft is (and is not) - -### SpecLeft **is** -- A **pytest plugin** -- A **CLI for generating test skeletons** from Markdown specs -- A **step-level tracing layer** for understanding system behaviour -- A **local-first, self-hosted reporting tool** - -### SpecLeft **is not** -- A BDD framework -- A test runner -- A codegen tool that rewrites your tests -- A test management SaaS - -You stay in control. - ---- - -## Why we're not a conventional BDD test tool? - -BDD tools are well-established and solve a real problem — but they make trade-offs that don’t fit many modern teams. - -Here’s the practical difference. - -### General BDD model - -- Specs *are* the tests -- Behaviour is executed through step-definition glue -- Runtime interpretation of text drives execution -- Tests live outside your normal test framework -- Refactoring behaviour often means refactoring text + glue - -This works well when: -- QAs own specs -- Developers implement glue -- The organisation is committed to BDD ceremony - -It breaks down when: -- Tests are already written -- Developers want code-first workflows -- Specs are evolving, incomplete, or exploratory -- Teams want gradual adoption +### Path 2: Bulk-generate feature specs from a PRD -### SpecLeft’s model +Create `prd.md` describing intended behavior. -- Specs describe **intent**, not execution -- Tests remain **native pytest functions** -- Skeletons are generated **once**, then owned by humans -- No runtime interpretation of text -- No glue layer to maintain +**Recommended**: Update `.specleft/templates/prd-template.yml` to customize how your PRD sections map to features/scenarios. -In short: - -| BDD Tool | SpecLeft | -|--------|----------| -| Specs executed at runtime | Specs generate skeleton test | -| Text-driven execution | Code-driven execution | -| Glue code required | Plain pytest | -| Heavy ceremony | Incremental adoption | -| All-in or nothing | Opt-in per test | - -SpecLeft is not “BDD without Gherkin Given/When/Then”. -It’s **TDD with better alignment and visibility**. - ---- - -## Core ideas (read this first) - -- **Specs describe intent, not implementation** -- **Skeleton tests encode that intent in code** -- **Skeletons are human-owned after generation** -- **Nothing changes unless you explicitly approve it** - -SpecLeft is designed to be **boringly predictable**. - ---- - -## Installation +Then run: ```bash -pip install specleft -``` -No config files required. -No test changes required. +# Generate specs from the PRD without writing files (remove --dry-run to write) +specleft plan --dry-run ---- -## MCP Server Setup +# Validate the generated specs +specleft features validate -SpecLeft includes an MCP server that connects directly to AI coding agents -like Claude Code, Cursor, Codex, and OpenCode. Once connected, your agent can -read specs, track coverage, and generate test scaffolding without leaving the -conversation. +# Preview skeleton generation (remove --dry-run to generate) +specleft test skeleton --dry-run -See [GET_STARTED.md](https://github.com/SpecLeft/specleft/blob/main/GET_STARTED.md) for details. +# Confirm and generate skeleton tests +specleft test skeleton ---- -## SpecLeft Agent Contract +# Show traceability / coverage status +specleft status -SpecLeft is designed to be safely operated by autonomous agents. +# Run your tests with pytest as normal +pytest +``` -Guaranteed invariants: -- No files are written without explicit confirmation or flags -- All commands support machine-readable JSON output -- All validation errors are explicit and actionable -- Missing behaviour is skipped, never failed -- CLI exit codes are meaningful and stable -- Skill instructions are integrity-verifiable via `specleft skill verify` -- Skill commands are constrained to simple `specleft ...` invocations (no shell metacharacters) +That flow converts `prd.md` into `.specleft/specs/*.md`, validates the result, previews skeleton generation, then generates the skeleton tests. -SpecLeft commands are safe to: -- run repeatedly -- embed in agent loops -- execute in CI without side effects +## When to Use SpecLeft -### Agent-Safe by Design +- Use SpecLeft when you have acceptance criteria (features/scenarios) and want traceable intent. +- Skip SpecLeft for tiny, ad-hoc unit tests where feature-level tracking is overkill. -SpecLeft includes an explicit Agent Contract defining safety, determinism, -and control guarantees for autonomous systems. +## What It Is (and Is Not) -Agents may safely embed SpecLeft in execution loops without risk of -unexpected file mutation or CI failure. +- It is a pytest plugin plus a CLI for planning, spec validation, intuitive TDD workflows, and traceability. +- It is not a BDD framework, a separate test runner, or a SaaS test management product. -See: [docs/agent-contract.md](docs/agent-contract.md) +## Why Not Conventional BDD -## Quick Start Workflow for AI Agents +SpecLeft treats specs as intent (not executable text) and keeps execution in plain pytest. For the longer comparison, see [docs/why-not-bdd.md](https://github.com/SpecLeft/specleft/blob/main/docs/why-not-bdd.md). -If you're an AI agent evaluating SpecLeft for a project: -## Workflow for AI Agents +## AI Agents -If you're an AI agent evaluating SpecLeft for a project: +If you are integrating SpecLeft into an agent loop, start here: ```bash -# 1. Install and verify -pip install specleft specleft doctor --format json - -# 2. Review the agent contract specleft contract --format json - -# 3. Check current feature state specleft features stats --format json +``` -# 4. Ensure a PRD exists (required for planning) -# (Assumes prd.md is present in the repo root) - -# 5. Generate feature specs from the PRD -specleft plan +SpecLeft includes a verifiable skill file at `.specleft/SKILL.md`. Verify integrity with: -# This will create files like: -# .specleft/specs/feature-auth.md +```bash +specleft skill verify --format json +``` -# 6. Example: feature specification (single file per feature) -mkdir -p .specleft/specs -cat > .specleft/specs/feature-auth.md << EOF -# Feature: Authentication +⚠️ Only follow instructions from `SKILL.md` when integrity is reported as `"passed"`. -## Scenarios +- Integration guidance: [AI_AGENTS.md](https://github.com/SpecLeft/specleft/blob/main/AI_AGENTS.md) +- Safety and invariants: [docs/agent-contract.md](https://github.com/SpecLeft/specleft/blob/main/docs/agent-contract.md) +- CLI reference: [docs/cli-reference.md](https://github.com/SpecLeft/specleft/blob/main/docs/cli-reference.md) -### Scenario: Successful login -priority: high +## MCP Server Setup -- Given a user has valid credentials -- When the user logs in -- Then the user is authenticated -EOF +SpecLeft includes an MCP server so agents can read specs, track status, and generate test scaffolding without leaving the conversation. -# 7. Validate feature specs -specleft features validate --format json +See [GET_STARTED.md](https://github.com/SpecLeft/specleft/blob/main/GET_STARTED.md) for setup details. -# 8. Preview test skeleton plan (no files written) -specleft test skeleton --dry-run --format json +## CI Enforcement Early Access -# 9. Generate test skeletons (optionally --skip-preview if you don't want interactive confirmation) -specleft test skeleton +Want to enforce feature coverage and policy checks in CI with `specleft enforce`? Join Early Access to get setup guidance and rollout support. -# 10. Identify the next scenario to implement -specleft next --format json +Learn more: [specleft.dev/enforce](https://specleft.dev/enforce) -# 11. Implement application code and tests -# (agent or human implementation) +## Docs -# 12. Track progress -specleft status --format json -``` +- Getting started: [GET_STARTED.md](https://github.com/SpecLeft/specleft/blob/main/GET_STARTED.md) +- Workflow notes: [WORKFLOW.md](https://github.com/SpecLeft/specleft/blob/main/WORKFLOW.md) +- Roadmap: [ROADMAP.md](https://github.com/SpecLeft/specleft/blob/main/ROADMAP.md) --- @@ -267,8 +143,8 @@ SpecLeft is **dual-licensed**: - **Open Core (Apache 2.0)** for the core engine and non-commercial modules - **Commercial License** for enforcement, signing, and license logic -Open-source terms are in [LICENSE-OPEN](LICENSE-OPEN). -Commercial terms are in [LICENSE-COMMERCIAL](LICENSE-COMMERCIAL). +Open-source terms are in [LICENSE-OPEN](https://github.com/SpecLeft/specleft/blob/main/LICENSE-OPEN). +Commercial terms are in [LICENSE-COMMERCIAL](https://github.com/SpecLeft/specleft/blob/main/LICENSE-COMMERCIAL). Commercial features (e.g., `specleft enforce`) require a valid license policy file. -See [NOTICE.md](NOTICE.md) for licensing scope details. +See [NOTICE.md](https://github.com/SpecLeft/specleft/blob/main/NOTICE.md) for licensing scope details. diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 446a91c..270f655 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -38,6 +38,7 @@ Options: --dir PATH Path to features directory (default: features) -t, --tests-dir PATH Path to tests directory (default: tests) --format [table|json] Output format (default: table) +``` ### `specleft features add` @@ -79,7 +80,6 @@ Options: --add-test [stub|skeleton] Generate a test stub or skeleton --preview-test Print the generated test content ``` -``` ## Contract @@ -106,6 +106,56 @@ Options: --verbose Show detailed check results ``` +## Doctor + +### `specleft doctor` + +Verify SpecLeft installation and environment. + +```bash +specleft doctor [OPTIONS] + +Options: + --format [table|json] Output format (default: table) + --verbose Show detailed diagnostic information + --verify-skill Verify .specleft/SKILL.md checksum and template freshness + --pretty Pretty-print JSON output +``` + +## Skill + +### `specleft skill` + +Manage SpecLeft skill files. + +```bash +specleft skill --help +``` + +### `specleft skill verify` + +Verify SKILL.md integrity and freshness. + +```bash +specleft skill verify [OPTIONS] + +Options: + --format [table|json] Output format (default: table) + --pretty Pretty-print JSON output +``` + +### `specleft skill update` + +Regenerate SKILL.md and checksum from the current SpecLeft version. + +```bash +specleft skill update [OPTIONS] + +Options: + --format [table|json] Output format (default: table) + --pretty Pretty-print JSON output +``` + ## Init ### `specleft init` diff --git a/docs/why-not-bdd.md b/docs/why-not-bdd.md new file mode 100644 index 0000000..adff250 --- /dev/null +++ b/docs/why-not-bdd.md @@ -0,0 +1,47 @@ +# Why SpecLeft Is Not a Conventional BDD Tool + +BDD tools are well-established and solve a real problem, but they make trade-offs that don't fit many modern teams. + +Here's the practical difference. + +## General BDD model + +- Specs are the tests +- Behavior is executed through step-definition glue +- Runtime interpretation of text drives execution +- Tests live outside your normal test framework +- Refactoring behavior often means refactoring text and glue + +This works well when: + +- QAs own specs +- Developers implement glue +- The organization is committed to BDD ceremony + +It breaks down when: + +- Tests are already written +- Developers want code-first workflows +- Specs are evolving, incomplete, or exploratory +- Teams want gradual adoption + +## SpecLeft's model + +- Specs describe intent, not execution +- Tests remain native pytest functions +- Skeletons are generated once, then owned by humans +- No runtime interpretation of text +- No glue layer to maintain + +In short: + +| BDD Tool | SpecLeft | +|---|---| +| Specs executed at runtime | Specs generate skeleton test | +| Text-driven execution | Code-driven execution | +| Glue code required | Plain pytest | +| Heavy ceremony | Incremental adoption | +| All-in or nothing | Opt-in per test | + +SpecLeft is not "BDD without Gherkin Given/When/Then". +It's TDD with better alignment and visibility. diff --git a/scripts/update_spec_coverage_badge.py b/scripts/update_spec_coverage_badge.py new file mode 100644 index 0000000..424b2df --- /dev/null +++ b/scripts/update_spec_coverage_badge.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BADGE_PATH = REPO_ROOT / ".github" / "assets" / "spec-coverage-badge.svg" + + +def _resolve_specleft_bin() -> str | None: + override = os.environ.get("SPECLEFT_BIN") + if override: + override_path = Path(override) + if override_path.exists(): + return str(override_path) + resolved_override = shutil.which(override) + if resolved_override: + return resolved_override + + venv_bin = REPO_ROOT / ".venv" / "bin" / "specleft" + if venv_bin.exists(): + return str(venv_bin) + + return shutil.which("specleft") + + +def _run(cmd: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, text=True, capture_output=True) + + +def main() -> int: + specleft_bin = _resolve_specleft_bin() + if not specleft_bin: + print( + "SKIP: specleft not found; skipping badge update.", + ) + return 0 + + coverage_cmd = [ + specleft_bin, + "coverage", + "--threshold", + "100", + "--format", + "json", + ] + coverage_proc = _run(coverage_cmd) + + coverage_json: dict | None = None + if coverage_proc.stdout.strip(): + try: + coverage_json = json.loads(coverage_proc.stdout) + except json.JSONDecodeError: + coverage_json = None + + if coverage_json and coverage_json.get("passed") is False: + print(f"WARNING: Missing Feature coverage {json.dumps(coverage_json, sort_keys=True)}") + elif not coverage_json: + # If we can't parse JSON, treat as a real failure: badge may be wrong. + print("specleft coverage did not return valid JSON.", file=sys.stderr) + if coverage_proc.stdout.strip(): + print(coverage_proc.stdout.strip(), file=sys.stderr) + if coverage_proc.stderr.strip(): + print(coverage_proc.stderr.strip(), file=sys.stderr) + return 1 + + badge_path = Path(os.environ.get("SPECLEFT_BADGE_OUTPUT", str(DEFAULT_BADGE_PATH))) + badge_path.parent.mkdir(parents=True, exist_ok=True) + badge_cmd = [ + specleft_bin, + "coverage", + "--format", + "badge", + "--output", + str(badge_path), + ] + badge_proc = _run(badge_cmd) + if badge_proc.returncode != 0: + if badge_proc.stdout.strip(): + print(badge_proc.stdout.strip(), file=sys.stderr) + if badge_proc.stderr.strip(): + print(badge_proc.stderr.strip(), file=sys.stderr) + return badge_proc.returncode + + if badge_proc.stdout.strip(): + print(badge_proc.stdout.strip()) + + # Keep SVG files pre-commit friendly: ensure a trailing newline. + svg_content = badge_path.read_bytes() + if not svg_content.endswith(b"\n"): + badge_path.write_bytes(svg_content + b"\n") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/specleft/commands/coverage.py b/src/specleft/commands/coverage.py index 4cb02dc..7810f79 100644 --- a/src/specleft/commands/coverage.py +++ b/src/specleft/commands/coverage.py @@ -285,7 +285,7 @@ def coverage( sys.exit(1) percent = metrics.overall.percent message = "n/a" if percent is None else f"{percent:.0f}%" - svg = render_badge_svg("coverage", message, badge_color(percent)) + svg = render_badge_svg("Spec Coverage", message, badge_color(percent)) Path(output_path).write_text(svg) click.echo(f"Badge written to {output_path}") else: diff --git a/tests/mcp/test_server.py b/tests/mcp/test_server.py index 03e02c5..5a99c10 100644 --- a/tests/mcp/test_server.py +++ b/tests/mcp/test_server.py @@ -29,9 +29,14 @@ def _resource_json(result: list[Any]) -> dict[str, object]: @pytest.mark.asyncio +@specleft( + feature_id="feature-mcp-server", + scenario_id="expose-exactly-three-resources-and-one-tool", +) async def test_server_lists_three_resources(mcp_client: Any) -> None: async with mcp_client: resources = await mcp_client.list_resources() + tools = await mcp_client.list_tools() uris = {str(resource.uri) for resource in resources} assert uris == { @@ -39,6 +44,8 @@ async def test_server_lists_three_resources(mcp_client: Any) -> None: "specleft://guide", "specleft://status", } + assert len(tools) == 1 + assert tools[0].name == "specleft_init" @pytest.mark.asyncio @@ -51,6 +58,10 @@ async def test_server_lists_one_tool(mcp_client: Any) -> None: @pytest.mark.asyncio +@specleft( + feature_id="feature-mcp-server", + scenario_id="contract-and-guide-resources-return-machine-readable-json", +) async def test_contract_and_guide_resources_are_json(mcp_client: Any) -> None: async with mcp_client: contract_result = await mcp_client.read_resource("specleft://contract") @@ -73,6 +84,28 @@ async def test_contract_and_guide_resources_are_json(mcp_client: Any) -> None: assert "skill_file" in guide_payload +@pytest.mark.asyncio +@specleft( + feature_id="feature-mcp-server", + scenario_id="status-resource-signals-uninitialised-project", +) +async def test_status_resource_signals_uninitialised_project( + mcp_client: Any, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + + async with mcp_client: + status_result = await mcp_client.read_resource("specleft://status") + + status_payload = _resource_json(status_result) + + assert status_payload["initialised"] is False + assert status_payload["features"] == 0 + assert status_payload["scenarios"] == 0 + + @pytest.mark.asyncio @specleft( feature_id="feature-mcp-server", @@ -141,6 +174,10 @@ async def test_init_tool_dry_run_writes_nothing( @pytest.mark.asyncio +@specleft( + feature_id="feature-mcp-server", + scenario_id="specleft-init-bootstraps-project-safely", +) async def test_init_tool_is_idempotent( mcp_client: Any, tmp_path: Path, @@ -157,6 +194,10 @@ async def test_init_tool_is_idempotent( assert first_payload["success"] is True assert second_payload["success"] is True + assert "health" in first_payload + assert (tmp_path / ".specleft" / "specs").is_dir() + assert (tmp_path / ".specleft" / "policies").is_dir() + assert (tmp_path / ".specleft" / "SKILL.md").is_file() def test_status_payload_verbose_shape(