Support Vercel Agent Readability spec end-to-end (sitemap, llms.txt, markdown mirrors)

apps/example/package.json

@@ -4,11 +4,11 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "dev": "bun run --filter leadtype build && bun run pipeline:convert && PORTLESS_PORT=1355 PORTLESS_HTTPS=0 portless run vite dev",
-    "build": "bun run --filter leadtype build && bun run pipeline:convert && vite build",
+    "dev": "bun run --filter leadtype build && bun run pipeline:build && PORTLESS_PORT=1355 PORTLESS_HTTPS=0 portless run vite dev",
+    "build": "bun run --filter leadtype build && bun run pipeline:build && vite build",
     "preview": "PORTLESS_PORT=1355 PORTLESS_HTTPS=0 portless run vite preview",
     "check-types": "tsgo --noEmit",
-    "test:e2e": "bun run --filter leadtype build && playwright test",
+    "test:e2e": "bun run --filter leadtype build && bun run pipeline:build && playwright test",
     "convert": "bun run pipeline:convert",
     "llm": "bun run pipeline:llm",
     "setup:real": "bun run pipeline:setup-real",

apps/example/scripts/llm-generate.ts

@@ -7,10 +7,11 @@
  * consumers.
  */
 
-import { mkdir, writeFile } from "node:fs/promises";
+import { mkdir, rm, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import {
+  generateAgentReadabilityArtifacts,
   generateLLMFullContextFiles,
   generateLlmsTxt,
   resolveDocsNavigation,
@@ -48,6 +49,16 @@ await generateLLMFullContextFiles({
   groups: docsConfig.groups,
 });
 
+const agentReadability = await generateAgentReadabilityArtifacts({
+  outDir,
+  baseUrl,
+  product: {
+    name: docsConfig.product.name,
+    summary: docsConfig.product.summary,
+  },
+  groups: docsConfig.groups,
+});
+
 // Build the runtime sidebar manifest. Doing this in the build pipeline keeps
 // the docs.config.ts as the single source of truth: the same call resolves
 // frontmatter membership for the LLM bundles AND for the in-app sidebar.
@@ -71,5 +82,22 @@ await writeFile(
   join(generatedDir, "docs-nav.json"),
   `${JSON.stringify(navigation, null, 2)}\n`
 );
+await writeFile(
+  join(generatedDir, "agent-readability.json"),
+  `${JSON.stringify(agentReadability.manifest, null, 2)}\n`
+);
+
+// Static copies would be served by Vite/nitro before the middleware runs,
+// so the live origin would never make it into <loc> / Sitemap:.
+await Promise.all(
+  [
+    join(outDir, "sitemap.xml"),
+    join(outDir, "sitemap.md"),
+    join(outDir, "robots.txt"),
+    join(outDir, "docs", "sitemap.xml"),
+    join(outDir, "docs", "sitemap.md"),
+    join(outDir, "docs", "robots.txt"),
+  ].map((file) => rm(file, { force: true }))
+);
 
-process.stdout.write("LLM files + nav manifest generated\n");
+process.stdout.write("LLM files + agent readability manifests generated\n");

apps/example/server/middleware/agent-readability.ts

@@ -0,0 +1,66 @@
+import {
+  createAgentMarkdownResponse,
+  createRobotsTxtResponse,
+  createSitemapMarkdownResponse,
+  createSitemapXmlResponse,
+} from "leadtype/llm/readability";
+import {
+  defineEventHandler,
+  getHeaders,
+  getMethod,
+  getRequestURL,
+} from "nitro/h3";
+import {
+  agentReadabilityManifest,
+  getRequestOrigin,
+  readMarkdownFile,
+} from "../utils/agent-readability";
+
+export default defineEventHandler(async (event) => {
+  const method = getMethod(event);
+  if (method !== "GET" && method !== "HEAD") {
+    return;
+  }
+
+  const url = getRequestURL(event);
+  const requestOrigin = getRequestOrigin(event);
+  const pathname = url.pathname;
+
+  switch (pathname) {
+    case "/sitemap.xml":
+    case "/docs/sitemap.xml":
+      return createSitemapXmlResponse({
+        manifest: agentReadabilityManifest,
+        requestOrigin,
+      });
+    case "/sitemap.md":
+    case "/docs/sitemap.md":
+      return createSitemapMarkdownResponse({
+        manifest: agentReadabilityManifest,
+        requestOrigin,
+      });
+    case "/robots.txt":
+      return createRobotsTxtResponse({
+        manifest: agentReadabilityManifest,
+        requestOrigin,
+      });
+    case "/docs/robots.txt":
+      return createRobotsTxtResponse({
+        manifest: agentReadabilityManifest,
+        requestOrigin,
+        sitemapUrlPath: "/docs/sitemap.xml",
+      });
+    default:
+      break;
+  }
+
+  const response = await createAgentMarkdownResponse({
+    urlPath: pathname,
+    method,
+    headers: getHeaders(event),
+    manifest: agentReadabilityManifest,
+    readMarkdownFile,
+    requestOrigin,
+  });
+  return response ?? undefined;
+});

apps/example/server/utils/agent-readability.ts

@@ -0,0 +1,61 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import type {
+  AgentReadabilityManifest,
+  MarkdownMirrorTarget,
+} from "leadtype/llm/readability";
+import {
+  getHeader,
+  getRequestProtocol,
+  getRequestURL,
+  type H3Event,
+} from "nitro/h3";
+import manifestJson from "../../src/generated/agent-readability.json" with {
+  type: "json",
+};
+
+export const agentReadabilityManifest: AgentReadabilityManifest = {
+  ...manifestJson,
+  version: 1,
+};
+
+export function getRequestOrigin(event: H3Event): string | undefined {
+  const forwardedHost = getHeader(event, "x-forwarded-host")
+    ?.split(",")[0]
+    ?.trim();
+  const forwardedProto = getHeader(event, "x-forwarded-proto")
+    ?.split(",")[0]
+    ?.trim();
+  if (forwardedHost) {
+    const protocol = forwardedProto || getRequestProtocol(event) || "http";
+    return `${protocol}://${forwardedHost}`;
+  }
+  const url = getRequestURL(event);
+  return url.origin;
+}
+
+function isMissingFileError(error: unknown): boolean {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    (error.code === "ENOENT" || error.code === "ENOTDIR")
+  );
+}
+
+export async function readMarkdownFile(
+  target: MarkdownMirrorTarget
+): Promise<string | null> {
+  try {
+    return await readFile(
+      join(process.cwd(), "public", target.filePath),
+      "utf8"
+    );
+  } catch (error) {
+    if (isMissingFileError(error)) {
+      return null;
+    }
+
+    throw error;
+  }
+}