diff --git a/.gitignore b/.gitignore index 96fab4f..6f85112 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,7 @@ coverage # Build Outputs .next/ out/ -build +/build dist @@ -36,3 +36,10 @@ yarn-error.log* # Misc .DS_Store *.pem +.gstack/ + +# Agent evals +evals/.tarballs/ +evals/results/ +evals/__agent_eval__/ +evals/**/__agent_eval__/ diff --git a/README.md b/README.md index 38ec227..dc5ebe9 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,34 @@ # leadtype -Shared docs tooling for any docs app: framework-neutral MDX-to-markdown conversion, LLM bundles, validation, and static search. +A docs pipeline. Write MDX once. Get a website for humans, an `llms.txt` for HTTP agents, an `AGENTS.md`-fronted bundle for offline coding agents, and a static search index — all from a single source. + +```mermaid +flowchart LR + src["docs/*.mdx"] + site_run["leadtype generate"] + bundle_run["leadtype generate --bundle"] + site_out["public/
llms.txt · llms-full/
docs/*.md · search-index.json"] + bundle_out["packages/<name>/
AGENTS.md · docs/*.md"] + humans["humans (browser)"] + http_agents["HTTP agents
(/llms.txt or
Accept: text/markdown)"] + search["search UI · AI answers"] + offline_agents["coding agents
(Claude Code, Codex, Cursor,
Copilot…) read
node_modules/<pkg>/AGENTS.md"] + src --> site_run + src --> bundle_run + site_run --> site_out + bundle_run --> bundle_out + site_out --> humans + site_out --> http_agents + site_out --> search + bundle_out --> offline_agents +``` + +leadtype is **not a docs website framework**. Bring your own UI — Next.js, TanStack Start, Astro, anything — and let leadtype handle conversion, validation, search, and the agent-facing outputs that website frameworks don't ship. -`leadtype` is split into focused public entry points: +## Choose your path -- `leadtype/remark`: remark plugins plus `defaultRemarkPlugins` -- `leadtype/convert`: MDX-to-markdown conversion APIs -- `leadtype/llm`: `llms.txt` and topic-scoped full-context generation -- `leadtype/search`: search runtime, content readers, guards, and rate limiter helpers -- `leadtype/search/node`: Node-only search index generation -- `leadtype/search/vercel`: Vercel AI Gateway / AI SDK answer streaming and bash tools -- `leadtype/search/tanstack`: TanStack AI answer streaming and bash tools -- `leadtype/search/cloudflare`: Cloudflare AI Gateway / Workers AI adapter helpers and bash tools -- `leadtype/lint`: docs validation and the `leadtype lint` CLI +- **[Build a docs site](https://docs.example.com/docs/build/connect-docs-site)** — wire leadtype into your build to convert MDX, index search, and serve markdown to agents. +- **[Bundle docs into your package](https://docs.example.com/docs/build/bundle-package-docs)** — ship `AGENTS.md` plus topic markdown inside the npm tarball so coding agents auto-discover them from `node_modules//AGENTS.md`. ## Install @@ -20,113 +36,53 @@ Shared docs tooling for any docs app: framework-neutral MDX-to-markdown conversi pnpm add leadtype ``` -## Basic Usage - -### Own MDX components in your app - -`leadtype` does not export prebuilt React, Vue, Nuxt, Svelte, or Astro components. Define the MDX component map in the docs app that renders your pages. - -## Live Example App +## 30-second example -The repo includes a canonical consumer demo at `apps/example`. - -- Renders real `.mdx` fixture files through app-owned `mdxComponents`. -- Uses TanStack Start for SSR and hydration coverage. -- Shows extracted `ExtractedTypeTable` output while keeping pipeline fixtures in the validation path. - -Local workflow: +For a hosted docs site: ```bash -bun install -bun run dev +npx leadtype generate --src . --out public --base-url https://docs.example.com ``` -Pipeline and browser checks: +For an npm-bundled doc set: ```bash -bun run --filter example pipeline:build -bun run --filter example pipeline:test -bun run --filter example test:e2e +npx leadtype generate --bundle --src . --out packages/my-package ``` -Validation layers: - -- Package unit tests in `packages/leadtype/src/**/*.test.ts*` cover framework-neutral conversion, search, linting, and generated docs behavior. -- Pipeline fixtures in `apps/example/scripts` and `apps/example/content` cover MDX conversion, LLM generation, and `ExtractedTypeTable`. -- The TanStack Start demo app in `apps/example/src` covers real browser rendering and hydration. - -## Where This Fits +The first produces `public/llms.txt`, `public/docs/llms-full/*.txt`, `public/docs/search-index.json`, and `public/docs/*.md`. The second produces `packages/my-package/AGENTS.md` and `packages/my-package/docs/*.md` — auto-discoverable by [25+ coding agents](https://agents.md) once the package is installed. -`leadtype` is not a hosted docs platform or a complete docs-site framework. Use tools such as Mintlify, Fumadocs, or Starlight when the primary job is shipping a polished docs website quickly. +## Documentation -Use this package when the primary job is shared docs infrastructure: MDX-to-markdown conversion, LLM bundles, linting, static search artifacts, answer helpers, and agent-facing docs output that can feed multiple apps and tools. +Full docs at [docs.example.com](https://docs.example.com/docs): -The pipeline entry points are framework-neutral. React, Vue, Nuxt, Svelte, Astro, and other stacks can use conversion, LLM, lint, and search APIs while owning their own runtime component rendering. +- [Quickstart](https://docs.example.com/docs/quickstart) +- [How it works](https://docs.example.com/docs/how-it-works) +- [Frontmatter](https://docs.example.com/docs/authoring/frontmatter) +- [CLI reference](https://docs.example.com/docs/reference/cli) +- [Methodology](https://docs.example.com/docs/methodology) — how leadtype differs from Fumadocs, Starlight, and Mintlify -## Wiring It Into An App +## Repo layout -In a c15t-style repo with a top-level `docs/` directory, wire `leadtype` into the docs app and docs scripts: +- `packages/leadtype/` — the npm package (CLI + library entry points). +- `apps/example/` — production docs site and reference template, on TanStack Start. +- `docs/` — the source MDX rendered by both this site and the package's bundled docs. -- The docs app owns `mdxComponents` if it renders MDX directly. -- A conversion script runs `convertAllMdx({ srcDir: process.cwd(), outDir: "public" })`. -- LLM and search scripts read the converted markdown under `public/docs/`. -- Product code does not import `leadtype` unless it also renders docs pages. - -### Convert MDX to markdown - -```ts -import { convertAllMdx } from "leadtype/convert"; -import { defaultRemarkPlugins, remarkInclude } from "leadtype/remark"; - -await convertAllMdx({ - srcDir: "content", - outDir: "public", - remarkPlugins: [remarkInclude, ...defaultRemarkPlugins], -}); -``` - -### Generate agent-facing docs bundles - -```ts -import { generateLLMFullContextFiles, generateLlmsTxt } from "leadtype/llm"; -``` - -Source MDX for the package's own docs lives at the repo root in `/docs` (with `meta.json`). Run the docs generator locally with: +## Local workflow ```bash -LEADTYPE_AGENT_BASE_URL=https://docs.example.com/leadtype bun run --filter leadtype docs:generate +bun install +bun run dev # build the package, run the pipeline, start the example app ``` -This converts `/docs/*.mdx` into `packages/leadtype/docs/` (markdown, `llms.txt`, `llms-full.txt`, `llms-full/`). The output folder is gitignored and produced fresh at build time; only the converted output ships in the published tarball — the `.mdx` source does not. - -### Generate a static search index +Pipeline checks: -```ts -import { generateDocsSearchFiles } from "leadtype/search/node"; - -await generateDocsSearchFiles({ - outDir: "public", - baseUrl: "https://docs.example.com", -}); +```bash +bun run --filter example pipeline:build +bun run --filter example pipeline:test +bun run --filter example test:e2e ``` -At runtime, query the generated JSON with `leadtype/search`. Add a provider entrypoint such as `leadtype/search/vercel` only when a user explicitly asks for a source-grounded answer. - -## Agent Docs - -The package ships a small, topic-scoped agent reference bundle in `docs/`: - -- `docs/llms.txt`: routing index -- `docs/components.md` -- `docs/convert.md` -- `docs/remark.md` -- `docs/llm.md` -- `docs/search.md` -- `docs/lint.md` - -Set `LEADTYPE_AGENT_BASE_URL` to the hosted docs base before generating publishable `llms*.txt` files. -For the example app generator, base URL precedence is `LEADTYPE_AGENT_BASE_URL`, then generic deployment `BASE_URL`, then `PORTLESS_URL`, then the local default. - -## Repo Skill +## License -This repo also includes a local agent skill at `.agents/skills/leadtype/SKILL.md`. It routes agents to the packaged `docs` bundle in `node_modules/leadtype/docs` and falls back to the local workspace copy at `packages/leadtype/docs/` when the package is not installed. +MIT. diff --git a/apps/example/package.json b/apps/example/package.json index d707cb5..17c0e26 100644 --- a/apps/example/package.json +++ b/apps/example/package.json @@ -4,9 +4,9 @@ "private": true, "type": "module", "scripts": { - "dev": "bun run --filter leadtype build && portless run vite dev", - "build": "bun run --filter leadtype build && vite build", - "preview": "portless run vite preview", + "dev": "bun run --filter leadtype build && bun run pipeline:convert && PORTLESS_PORT=1355 PORTLESS_HTTPS=0 portless run vite dev", + "build": "bun run --filter leadtype build && bun run pipeline:convert && vite build", + "preview": "PORTLESS_PORT=1355 PORTLESS_HTTPS=0 portless run vite preview", "check-types": "tsgo --noEmit", "test:e2e": "bun run --filter leadtype build && playwright test", "convert": "bun run pipeline:convert", @@ -28,15 +28,16 @@ "dependencies": { "@fontsource-variable/geist": "^5.2.8", "@fontsource-variable/geist-mono": "^5.2.7", - "leadtype": "workspace:*", "@mdx-js/react": "^3.1.1", "@radix-ui/react-separator": "^1.1.7", "@radix-ui/react-slot": "^1.2.3", + "@streamdown/mermaid": "^1.0.2", "@tanstack/react-router": "^1.167.4", "@tanstack/react-start": "^1.166.15", "ai": "^6.0.168", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", + "leadtype": "workspace:*", "nitro": "3.0.260415-beta", "react": "^19.0.0", "react-dom": "^19.0.0", diff --git a/apps/example/playwright.config.ts b/apps/example/playwright.config.ts index ee595aa..1de72be 100644 --- a/apps/example/playwright.config.ts +++ b/apps/example/playwright.config.ts @@ -4,6 +4,7 @@ import { defineConfig, devices } from "@playwright/test"; const isCI = Boolean(process.env.CI); const HTTPS_PROTOCOL = "https://"; const HTTP_PROTOCOL = "http://"; +const PORTLESS_HTTP_PORT = "1355"; const DEFAULT_BASE_URL = "http://localhost:3000"; function getExampleBaseUrl(): string { @@ -16,6 +17,11 @@ function getExampleBaseUrl(): string { try { portlessUrl = execFileSync("portless", ["get", "example"], { encoding: "utf8", + env: { + ...process.env, + PORTLESS_HTTPS: "0", + PORTLESS_PORT: PORTLESS_HTTP_PORT, + }, maxBuffer: 10 * 1024 * 1024, timeout: 5000, }).trim(); diff --git a/apps/example/src/components/docs-mdx/command-tabs.tsx b/apps/example/src/components/docs-mdx/command-tabs.tsx index a0ab2bb..2c4be40 100644 --- a/apps/example/src/components/docs-mdx/command-tabs.tsx +++ b/apps/example/src/components/docs-mdx/command-tabs.tsx @@ -86,7 +86,6 @@ export function CommandTabs({ role="tab" since we don't implement the full tabs keyboard pattern (roving tabindex, ArrowLeft/Right, associated tabpanel). */}
- Package manager {MANAGERS.map((manager) => (