From ca3d70589e0ea73e4f240699d01a3f607e4d5cd9 Mon Sep 17 00:00:00 2001 From: Snigdha Banda Date: Wed, 14 May 2025 01:11:34 -0400 Subject: [PATCH 1/6] add patronus --- .../providers/05-observability/patronus.mdx | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 content/providers/05-observability/patronus.mdx diff --git a/content/providers/05-observability/patronus.mdx b/content/providers/05-observability/patronus.mdx new file mode 100644 index 000000000000..b9cc1eab1186 --- /dev/null +++ b/content/providers/05-observability/patronus.mdx @@ -0,0 +1,135 @@ +--- +title: Patronus +description: Monitor, evaluate and debug your AI SDK application with Patronus +--- + +## Patronus Observability + +[Patronus](https://patronus.ai/) Patronus AI is the leading tool to score and optimize Generative AI applications. + +Patronus provides an end-to-end system to evaluate, monitor and improve performance of an LLM system, enabling developers to ship AI products safely and confidently. Learn more [here](https://docs.patronus.ai/docs). +When you build with the **Vercel AI SDK**, you can stream OpenTelemetry (OTEL) traces straight into Patronus and pair every generation with rich automatic evaluations. + +--- + +\## Setup + +\### 1. OpenTelemetry +Patronus exposes a fully‑managed OTEL endpoint. Configure an **OTLP exporter** to point at it, pass your API key, and you’re done—Patronus will automatically convert LLM spans into prompt/response records you can explore and evaluate. + +\#### Environment variables (recommended) + +```bash +# .env.local +OTEL_EXPORTER_OTLP_ENDPOINT=https://otel.patronus.ai/v1/traces +OTEL_EXPORTER_OTLP_HEADERS="x-api-key:" +``` + +\#### With `@vercel/otel` + +```ts title="instrumentation.ts" +import { registerOTel } from '@vercel/otel'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-node'; + +export function register() { + registerOTel({ + serviceName: 'next-app', + additionalSpanProcessors: [ + new BatchSpanProcessor( + new OTLPTraceExporter({ + url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT, + headers: { + 'x-api-key': process.env.PATRONUS_API_KEY!, + }, + }) + ), + ], + }); +} +``` + +*If you need gRPC instead of HTTP, swap the exporter for `@opentelemetry/exporter-trace-otlp-grpc` and use `https://otel.patronus.ai:4317`.* + +\### 2. Enable telemetry on individual calls +The AI SDK emits a span only when you opt in with `experimental_telemetry`: + +```ts +import { generateText } from 'ai'; +import { openai } from '@ai-sdk/openai'; + +const result = await generateText({ + model: openai('gpt-4o'), + prompt: 'Write a haiku about spring.', + experimental_telemetry: { + isEnabled: true, + functionId: 'spring-haiku', // span name + metadata: { + userId: 'user-123', // custom attrs surface in Patronus UI + }, + }, +}); +``` + +Every attribute inside `metadata` becomes an OTEL attribute and is indexed by Patronus for filtering. + +--- + +\## Example — tracing and automated evaluation + +```ts title="app/api/chat/route.ts" +import { trace } from '@opentelemetry/api'; +import { generateText } from 'ai'; +import { openai } from '@ai-sdk/openai'; + +export async function POST(req: Request) { + const body = await req.json(); + const tracer = trace.getTracer('next-app'); + + return await tracer.startActiveSpan('chat-evaluate', async (span) => { + try { + /* 1️⃣ generate answer */ + const answer = await generateText({ + model: openai('gpt-4o'), + prompt: body.prompt, + experimental_telemetry: { isEnabled: true, functionId: 'chat' }, + }); + + /* 2️⃣ run Patronus evaluation inside the same trace */ + await fetch('https://api.patronus.ai/v1/evaluate', { + method: 'POST', + headers: { + 'X-API-Key': process.env.PATRONUS_API_KEY!, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + evaluators: [{ evaluator: 'lynx', criteria: 'patronus:hallucination' }], + evaluated_model_input: body.prompt, + evaluated_model_output: answer.text, + trace_id: span.spanContext().traceId, + span_id: span.spanContext().spanId, + }), + }); + + return new Response(answer.text); + } finally { + span.end(); + } + }); +} +``` + +## *Result:* a single trace containing the root HTTP request, the LLM generation span, and your evaluation span—**all visible in Patronus** with the hallucination score attached. + +\## Once you've traced + +* If you're tracing an agent, Patronus's AI assistant Percival will assist with error analysis and prompt optimization. Learn more [here](https://docs.patronus.ai/docs/percival/percival) +* Get set up on production monitoring and alerting by viewing logs and traces on Patronus and configuring webhooks for alerting. Learn more [here](https://docs.patronus.ai/docs/real_time_monitoring/webhooks) + +--- + +\## Resources + +* **Patronus docs:** [https://docs.patronus.ai](https://docs.patronus.ai) +* **OpenTelemetry SDK (JS):** [https://opentelemetry.io/docs/instrumentation/js/](https://opentelemetry.io/docs/instrumentation/js/) +* **Vercel AI SDK:** [https://sdk.vercel.ai](https://sdk.vercel.ai) From 9bb4741e2a0c818acbb733b2021c057c21a5dd52 Mon Sep 17 00:00:00 2001 From: Snigdha Banda Date: Wed, 14 May 2025 01:16:28 -0400 Subject: [PATCH 2/6] formatting issues --- content/providers/05-observability/patronus.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/providers/05-observability/patronus.mdx b/content/providers/05-observability/patronus.mdx index b9cc1eab1186..a1bd6aee204a 100644 --- a/content/providers/05-observability/patronus.mdx +++ b/content/providers/05-observability/patronus.mdx @@ -5,7 +5,7 @@ description: Monitor, evaluate and debug your AI SDK application with Patronus ## Patronus Observability -[Patronus](https://patronus.ai/) Patronus AI is the leading tool to score and optimize Generative AI applications. +[Patronus AI](https://patronus.ai) is the leading tool to score and optimize Generative AI applications. Patronus provides an end-to-end system to evaluate, monitor and improve performance of an LLM system, enabling developers to ship AI products safely and confidently. Learn more [here](https://docs.patronus.ai/docs). When you build with the **Vercel AI SDK**, you can stream OpenTelemetry (OTEL) traces straight into Patronus and pair every generation with rich automatic evaluations. @@ -119,7 +119,7 @@ export async function POST(req: Request) { } ``` -## *Result:* a single trace containing the root HTTP request, the LLM generation span, and your evaluation span—**all visible in Patronus** with the hallucination score attached. +Result: a single trace containing the root HTTP request, the LLM generation span, and your evaluation span—**all visible in Patronus** with the hallucination score attached. \## Once you've traced From eecdd3e9e5ad01448687ecf7668b31fb0bcb6536 Mon Sep 17 00:00:00 2001 From: Snigdha Banda Date: Wed, 14 May 2025 01:17:53 -0400 Subject: [PATCH 3/6] more formatting --- content/providers/05-observability/patronus.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/providers/05-observability/patronus.mdx b/content/providers/05-observability/patronus.mdx index a1bd6aee204a..2ae273c9172d 100644 --- a/content/providers/05-observability/patronus.mdx +++ b/content/providers/05-observability/patronus.mdx @@ -119,7 +119,7 @@ export async function POST(req: Request) { } ``` -Result: a single trace containing the root HTTP request, the LLM generation span, and your evaluation span—**all visible in Patronus** with the hallucination score attached. +####Result: a single trace containing the root HTTP request, the LLM generation span, and your evaluation span—**all visible in Patronus** with the hallucination score attached. \## Once you've traced From 26e65ed29556c8a5e0749a70d544161d38378419 Mon Sep 17 00:00:00 2001 From: nicoalbanese Date: Wed, 14 May 2025 14:03:05 +0100 Subject: [PATCH 4/6] Formatting fixes and update intro --- .../providers/05-observability/patronus.mdx | 54 +++++++++---------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/content/providers/05-observability/patronus.mdx b/content/providers/05-observability/patronus.mdx index 2ae273c9172d..33a0d5a741d3 100644 --- a/content/providers/05-observability/patronus.mdx +++ b/content/providers/05-observability/patronus.mdx @@ -5,19 +5,17 @@ description: Monitor, evaluate and debug your AI SDK application with Patronus ## Patronus Observability -[Patronus AI](https://patronus.ai) is the leading tool to score and optimize Generative AI applications. +[Patronus AI](https://patronus.ai) provides an end-to-end system to evaluate, monitor and improve performance of an LLM system, enabling developers to ship AI products safely and confidently. Learn more [here](https://docs.patronus.ai/docs). -Patronus provides an end-to-end system to evaluate, monitor and improve performance of an LLM system, enabling developers to ship AI products safely and confidently. Learn more [here](https://docs.patronus.ai/docs). -When you build with the **Vercel AI SDK**, you can stream OpenTelemetry (OTEL) traces straight into Patronus and pair every generation with rich automatic evaluations. +When you build with the **AI SDK**, you can stream OpenTelemetry (OTEL) traces straight into Patronus and pair every generation with rich automatic evaluations. ---- +## Setup -\## Setup +### 1. OpenTelemetry -\### 1. OpenTelemetry -Patronus exposes a fully‑managed OTEL endpoint. Configure an **OTLP exporter** to point at it, pass your API key, and you’re done—Patronus will automatically convert LLM spans into prompt/response records you can explore and evaluate. +Patronus exposes a fully‑managed OTEL endpoint. Configure an **OTLP exporter** to point at it, pass your API key, and you’re done—Patronus will automatically convert LLM spans into prompt/response records you can explore and evaluate. -\#### Environment variables (recommended) +#### Environment variables (recommended) ```bash # .env.local @@ -25,7 +23,7 @@ OTEL_EXPORTER_OTLP_ENDPOINT=https://otel.patronus.ai/v1/traces OTEL_EXPORTER_OTLP_HEADERS="x-api-key:" ``` -\#### With `@vercel/otel` +#### With `@vercel/otel` ```ts title="instrumentation.ts" import { registerOTel } from '@vercel/otel'; @@ -42,17 +40,18 @@ export function register() { headers: { 'x-api-key': process.env.PATRONUS_API_KEY!, }, - }) + }), ), ], }); } ``` -*If you need gRPC instead of HTTP, swap the exporter for `@opentelemetry/exporter-trace-otlp-grpc` and use `https://otel.patronus.ai:4317`.* +_If you need gRPC instead of HTTP, swap the exporter for `@opentelemetry/exporter-trace-otlp-grpc` and use `https://otel.patronus.ai:4317`._ -\### 2. Enable telemetry on individual calls -The AI SDK emits a span only when you opt in with `experimental_telemetry`: +### 2. Enable telemetry on individual calls + +The AI SDK emits a span only when you opt in with `experimental_telemetry`: ```ts import { generateText } from 'ai'; @@ -63,9 +62,9 @@ const result = await generateText({ prompt: 'Write a haiku about spring.', experimental_telemetry: { isEnabled: true, - functionId: 'spring-haiku', // span name + functionId: 'spring-haiku', // span name metadata: { - userId: 'user-123', // custom attrs surface in Patronus UI + userId: 'user-123', // custom attrs surface in Patronus UI }, }, }); @@ -73,9 +72,7 @@ const result = await generateText({ Every attribute inside `metadata` becomes an OTEL attribute and is indexed by Patronus for filtering. ---- - -\## Example — tracing and automated evaluation +## Example — tracing and automated evaluation ```ts title="app/api/chat/route.ts" import { trace } from '@opentelemetry/api'; @@ -86,7 +83,7 @@ export async function POST(req: Request) { const body = await req.json(); const tracer = trace.getTracer('next-app'); - return await tracer.startActiveSpan('chat-evaluate', async (span) => { + return await tracer.startActiveSpan('chat-evaluate', async span => { try { /* 1️⃣ generate answer */ const answer = await generateText({ @@ -103,7 +100,9 @@ export async function POST(req: Request) { 'Content-Type': 'application/json', }, body: JSON.stringify({ - evaluators: [{ evaluator: 'lynx', criteria: 'patronus:hallucination' }], + evaluators: [ + { evaluator: 'lynx', criteria: 'patronus:hallucination' }, + ], evaluated_model_input: body.prompt, evaluated_model_output: answer.text, trace_id: span.spanContext().traceId, @@ -121,15 +120,12 @@ export async function POST(req: Request) { ####Result: a single trace containing the root HTTP request, the LLM generation span, and your evaluation span—**all visible in Patronus** with the hallucination score attached. -\## Once you've traced - -* If you're tracing an agent, Patronus's AI assistant Percival will assist with error analysis and prompt optimization. Learn more [here](https://docs.patronus.ai/docs/percival/percival) -* Get set up on production monitoring and alerting by viewing logs and traces on Patronus and configuring webhooks for alerting. Learn more [here](https://docs.patronus.ai/docs/real_time_monitoring/webhooks) +## Once you've traced ---- +- If you're tracing an agent, Patronus's AI assistant Percival will assist with error analysis and prompt optimization. Learn more [here](https://docs.patronus.ai/docs/percival/percival) +- Get set up on production monitoring and alerting by viewing logs and traces on Patronus and configuring webhooks for alerting. Learn more [here](https://docs.patronus.ai/docs/real_time_monitoring/webhooks) -\## Resources +## Resources -* **Patronus docs:** [https://docs.patronus.ai](https://docs.patronus.ai) -* **OpenTelemetry SDK (JS):** [https://opentelemetry.io/docs/instrumentation/js/](https://opentelemetry.io/docs/instrumentation/js/) -* **Vercel AI SDK:** [https://sdk.vercel.ai](https://sdk.vercel.ai) +- **Patronus docs:** [https://docs.patronus.ai](https://docs.patronus.ai) +- **OpenTelemetry SDK (JS):** [https://opentelemetry.io/docs/instrumentation/js/](https://opentelemetry.io/docs/instrumentation/js/) From 4e54f13922f5f9215eca732b773aaddf5e9af809 Mon Sep 17 00:00:00 2001 From: nicoalbanese Date: Wed, 14 May 2025 14:08:40 +0100 Subject: [PATCH 5/6] Update heading and misc formatting --- content/providers/05-observability/patronus.mdx | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/content/providers/05-observability/patronus.mdx b/content/providers/05-observability/patronus.mdx index 33a0d5a741d3..f2f671deef8c 100644 --- a/content/providers/05-observability/patronus.mdx +++ b/content/providers/05-observability/patronus.mdx @@ -3,11 +3,11 @@ title: Patronus description: Monitor, evaluate and debug your AI SDK application with Patronus --- -## Patronus Observability +# Patronus Observability [Patronus AI](https://patronus.ai) provides an end-to-end system to evaluate, monitor and improve performance of an LLM system, enabling developers to ship AI products safely and confidently. Learn more [here](https://docs.patronus.ai/docs). -When you build with the **AI SDK**, you can stream OpenTelemetry (OTEL) traces straight into Patronus and pair every generation with rich automatic evaluations. +When you build with the AI SDK, you can stream OpenTelemetry (OTEL) traces straight into Patronus and pair every generation with rich automatic evaluations. ## Setup @@ -47,7 +47,11 @@ export function register() { } ``` -_If you need gRPC instead of HTTP, swap the exporter for `@opentelemetry/exporter-trace-otlp-grpc` and use `https://otel.patronus.ai:4317`._ + + If you need gRPC instead of HTTP, swap the exporter for + `@opentelemetry/exporter-trace-otlp-grpc` and use + `https://otel.patronus.ai:4317`. + ### 2. Enable telemetry on individual calls @@ -118,7 +122,7 @@ export async function POST(req: Request) { } ``` -####Result: a single trace containing the root HTTP request, the LLM generation span, and your evaluation span—**all visible in Patronus** with the hallucination score attached. +Result: a single trace containing the root HTTP request, the LLM generation span, and your evaluation span—**all visible in Patronus** with the hallucination score attached. ## Once you've traced @@ -127,5 +131,5 @@ export async function POST(req: Request) { ## Resources -- **Patronus docs:** [https://docs.patronus.ai](https://docs.patronus.ai) -- **OpenTelemetry SDK (JS):** [https://opentelemetry.io/docs/instrumentation/js/](https://opentelemetry.io/docs/instrumentation/js/) +- [Patronus docs](https://docs.patronus.ai) +- [OpenTelemetry SDK (JS)](https://opentelemetry.io/docs/instrumentation/js/) From 4359e3add646e51f6258796dff734226bf69b7cb Mon Sep 17 00:00:00 2001 From: nicoalbanese Date: Wed, 14 May 2025 21:42:43 +0100 Subject: [PATCH 6/6] fix filename in codesnippets --- content/providers/05-observability/patronus.mdx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/content/providers/05-observability/patronus.mdx b/content/providers/05-observability/patronus.mdx index f2f671deef8c..6ff3f15be5d0 100644 --- a/content/providers/05-observability/patronus.mdx +++ b/content/providers/05-observability/patronus.mdx @@ -17,15 +17,14 @@ Patronus exposes a fully‑managed OTEL endpoint. Configure an **OTLP exporter** #### Environment variables (recommended) -```bash -# .env.local +```bash filename=".env.local" OTEL_EXPORTER_OTLP_ENDPOINT=https://otel.patronus.ai/v1/traces OTEL_EXPORTER_OTLP_HEADERS="x-api-key:" ``` #### With `@vercel/otel` -```ts title="instrumentation.ts" +```ts filename="instrumentation.ts" import { registerOTel } from '@vercel/otel'; import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-node'; @@ -78,7 +77,7 @@ Every attribute inside `metadata` becomes an OTEL attribute and is indexed by Pa ## Example — tracing and automated evaluation -```ts title="app/api/chat/route.ts" +```ts filename="app/api/chat/route.ts" import { trace } from '@opentelemetry/api'; import { generateText } from 'ai'; import { openai } from '@ai-sdk/openai';