From 1b5a170cd824015c5be05fb883345457ca08b3bc Mon Sep 17 00:00:00 2001 From: colegottdank Date: Mon, 31 Mar 2025 14:50:18 -0700 Subject: [PATCH 1/4] Add Helicone docs --- .../providers/05-observability/helicone.mdx | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 content/providers/05-observability/helicone.mdx diff --git a/content/providers/05-observability/helicone.mdx b/content/providers/05-observability/helicone.mdx new file mode 100644 index 000000000000..cd1a80107ada --- /dev/null +++ b/content/providers/05-observability/helicone.mdx @@ -0,0 +1,217 @@ +--- +title: Helicone +description: Monitor and optimize your AI SDK applications with minimal configuration using Helicone +--- + +# Helicone Observability + +[Helicone](https://helicone.ai) is an open-source LLM observability platform that helps you monitor, analyze, and optimize your AI applications through a proxy-based approach, requiring minimal setup and zero additional dependencies. + +## 30-Second Setup + +Setting up Helicone takes just three simple steps: + +1. Create a Helicone account at [helicone.ai](https://helicone.ai) +2. Set your API key as an environment variable: + ```bash filename=".env" + HELICONE_API_KEY=your-helicone-api-key + ``` +3. Update your model provider configuration to use Helicone's proxy: + ```javascript + import { createOpenAI } from "@ai-sdk/openai"; + + const openai = createOpenAI({ + baseURL: "https://oai.helicone.ai/v1", + headers: { + "Helicone-Auth": `Bearer ${process.env.HELICONE_API_KEY}`, + }, + }); + + // Use normally with AI SDK + const response = await generateText({ + model: openai("gpt-4o-mini"), + prompt: "Hello world", + }); + ``` + +That's it! Your requests are now being logged and monitored through Helicone. + +[→ Learn more about getting started with Helicone on Vercel AI](https://docs.helicone.ai/getting-started/integration-method/vercelai) + +## Why Proxy vs OTEL? + +While other observability solutions require complex OpenTelemetry instrumentation, Helicone uses a simpler proxy approach: + + + + ```javascript + const openai = createOpenAI({ + baseURL: "https://oai.helicone.ai/v1", + headers: { "Helicone-Auth": `Bearer ${process.env.HELICONE_API_KEY}` }, + }); + ``` + + + + ```javascript + // Install multiple packages + // @vercel/otel, @opentelemetry/sdk-node, @opentelemetry/auto-instrumentations-node, etc. + + // Create exporter + const exporter = new OtherProviderExporter({ + projectApiKey: process.env.API_KEY + }); + + // Setup SDK + const sdk = new NodeSDK({ + traceExporter: exporter, + instrumentations: [getNodeAutoInstrumentations()], + resource: new Resource({...}), + }); + + // Start SDK + sdk.start(); + + // Enable telemetry on each request + const response = await generateText({ + model: openai("gpt-4o-mini"), + prompt: "Hello world", + experimental_telemetry: { isEnabled: true } + }); + + // Shutdown SDK to flush traces + await sdk.shutdown(); + ``` + + + +**Benefits of Helicone's Proxy Approach:** +- **Zero Dependencies**: No additional packages needed +- **Works Everywhere**: Compatible with any JavaScript environment +- **No Code Changes**: Minimal changes to your existing code +- **Advanced Features**: Enables unique capabilities like caching and rate limiting + +[→ Learn more about Helicone's proxy approach](https://docs.helicone.ai/references/proxy-vs-async) + +## Core Observability Features + +### User Tracking + +Monitor how individual users interact with your AI application: + +```javascript +const response = await generateText({ + model: openai("gpt-4o-mini"), + prompt: "Hello world", + headers: { + "Helicone-User-Id": "user@example.com", + }, +}); +``` + +[→ Learn more about User Metrics](https://docs.helicone.ai/features/advanced-usage/user-metrics) + +### Custom Properties + +Add structured metadata to filter and analyze requests: + +```javascript +const response = await generateText({ + model: openai("gpt-4o-mini"), + prompt: "Translate this text to French", + headers: { + "Helicone-Property-Feature": "translation", + "Helicone-Property-Source": "mobile-app", + "Helicone-Property-Language": "French", + }, +}); +``` + +[→ Learn more about Custom Properties](https://docs.helicone.ai/features/advanced-usage/custom-properties) + +### Session Tracking + +Group related requests into coherent conversations: + +```javascript +const response = await generateText({ + model: openai("gpt-4o-mini"), + prompt: "Tell me more about that", + headers: { + "Helicone-Session-Id": "convo-123", + "Helicone-Session-Name": "Travel Planning", + "Helicone-Session-Path": "/chats/travel", + }, +}); +``` + +[→ Learn more about Sessions](https://docs.helicone.ai/features/sessions) + +## Unique Proxy-Powered Features + +**Request Caching** +Reduce costs by caching identical requests: + +```javascript +const response = await generateText({ + model: openai("gpt-4o-mini"), + prompt: "What is the capital of France?", + headers: { + "Helicone-Cache-Enabled": "true", + }, +}); +``` + +[→ Learn more about Caching](https://docs.helicone.ai/features/advanced-usage/caching) + +**Rate Limiting** +Control usage by adding a simple rate limit policy: + +```javascript +const response = await generateText({ + model: openai("gpt-4o-mini"), + prompt: "Generate creative content", + headers: { + // Allow 10,000 requests per hour + "Helicone-RateLimit-Policy": "10000;w=3600", + + // Optional: limit by user + "Helicone-User-Id": "user@example.com", + }, +}); +``` + +Format: `[quota];w=[time_window];u=[unit];s=[segment]` where: +- `quota`: Maximum requests allowed in the time window +- `w`: Time window in seconds (minimum 60s) +- `u`: Optional unit - "request" (default) or "cents" +- `s`: Optional segment - "user", custom property, or global (default) + +[→ Learn more about Rate Limiting](https://docs.helicone.ai/features/advanced-usage/custom-rate-limits) + +**LLM Security** +Protect against prompt injection, jailbreaking, and other LLM-specific threats: + +```javascript +const response = await generateText({ + model: openai("gpt-4o-mini"), + prompt: userInput, + headers: { + // Basic protection (Prompt Guard model) + "Helicone-LLM-Security-Enabled": "true", + + // Optional: Advanced protection (Llama Guard model) + "Helicone-LLM-Security-Advanced": "true", + }, +}); +``` + +Protects against multiple attack vectors in 8 languages with minimal latency. Advanced mode adds protection across 14 threat categories. + +[→ Learn more about LLM Security](https://docs.helicone.ai/features/advanced-usage/llm-security) + +## Resources + +- [Helicone Documentation](https://docs.helicone.ai) +- [GitHub Repository](https://github.com/Helicone/helicone) +- [Discord Community](https://discord.com/invite/2TkeWdXNPQ) \ No newline at end of file From 5543f3e89fc624570f9eea5c98712ddf4e39b966 Mon Sep 17 00:00:00 2001 From: colegottdank Date: Mon, 31 Mar 2025 14:51:36 -0700 Subject: [PATCH 2/4] Add Helicone to index.mdx --- content/providers/05-observability/index.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/content/providers/05-observability/index.mdx b/content/providers/05-observability/index.mdx index fa54df2979d7..2f846d3a748d 100644 --- a/content/providers/05-observability/index.mdx +++ b/content/providers/05-observability/index.mdx @@ -8,6 +8,7 @@ description: AI SDK Integration for monitoring and tracing LLM applications Several LLM observability providers offer integrations with the AI SDK telemetry data: - [Braintrust](/providers/observability/braintrust) +- [Helicone](/providers/observability/helicone) - [Traceloop](/providers/observability/traceloop) - [Langfuse](/providers/observability/langfuse) - [LangSmith](/providers/observability/langsmith) From 417c1c7ac3991e3f8ba4a9196248c9244f49f351 Mon Sep 17 00:00:00 2001 From: colegottdank Date: Wed, 2 Apr 2025 12:21:16 -0700 Subject: [PATCH 3/4] Adjust from feedback --- .../providers/05-observability/helicone.mdx | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/content/providers/05-observability/helicone.mdx b/content/providers/05-observability/helicone.mdx index cd1a80107ada..042e2dceabed 100644 --- a/content/providers/05-observability/helicone.mdx +++ b/content/providers/05-observability/helicone.mdx @@ -7,9 +7,9 @@ description: Monitor and optimize your AI SDK applications with minimal configur [Helicone](https://helicone.ai) is an open-source LLM observability platform that helps you monitor, analyze, and optimize your AI applications through a proxy-based approach, requiring minimal setup and zero additional dependencies. -## 30-Second Setup +## Setup -Setting up Helicone takes just three simple steps: +Setting up Helicone: 1. Create a Helicone account at [helicone.ai](https://helicone.ai) 2. Set your API key as an environment variable: @@ -36,11 +36,11 @@ Setting up Helicone takes just three simple steps: That's it! Your requests are now being logged and monitored through Helicone. -[→ Learn more about getting started with Helicone on Vercel AI](https://docs.helicone.ai/getting-started/integration-method/vercelai) +[→ Learn more about getting started with Helicone on AI SDK](https://docs.helicone.ai/getting-started/integration-method/vercelai) -## Why Proxy vs OTEL? +## Integration Approach -While other observability solutions require complex OpenTelemetry instrumentation, Helicone uses a simpler proxy approach: +While other observability solutions require OpenTelemetry instrumentation, Helicone uses a simple proxy approach: @@ -85,15 +85,15 @@ While other observability solutions require complex OpenTelemetry instrumentatio -**Benefits of Helicone's Proxy Approach:** -- **Zero Dependencies**: No additional packages needed -- **Works Everywhere**: Compatible with any JavaScript environment -- **No Code Changes**: Minimal changes to your existing code -- **Advanced Features**: Enables unique capabilities like caching and rate limiting +**Characteristics of Helicone's Proxy Approach:** +- No additional packages required +- Compatible with JavaScript environments +- Minimal code changes to existing implementations +- Supports features such as caching and rate limiting [→ Learn more about Helicone's proxy approach](https://docs.helicone.ai/references/proxy-vs-async) -## Core Observability Features +## Core Features ### User Tracking @@ -147,9 +147,10 @@ const response = await generateText({ [→ Learn more about Sessions](https://docs.helicone.ai/features/sessions) -## Unique Proxy-Powered Features +## Advanced Configuration + +### Request Caching -**Request Caching** Reduce costs by caching identical requests: ```javascript @@ -164,8 +165,9 @@ const response = await generateText({ [→ Learn more about Caching](https://docs.helicone.ai/features/advanced-usage/caching) -**Rate Limiting** -Control usage by adding a simple rate limit policy: +### Rate Limiting + +Control usage by adding a rate limit policy: ```javascript const response = await generateText({ @@ -189,7 +191,8 @@ Format: `[quota];w=[time_window];u=[unit];s=[segment]` where: [→ Learn more about Rate Limiting](https://docs.helicone.ai/features/advanced-usage/custom-rate-limits) -**LLM Security** +### LLM Security + Protect against prompt injection, jailbreaking, and other LLM-specific threats: ```javascript From a4996a410163ec7ba808ad6979da977a6e4d5ea8 Mon Sep 17 00:00:00 2001 From: colegottdank Date: Thu, 3 Apr 2025 11:07:08 -0700 Subject: [PATCH 4/4] Run prettier --- .../providers/05-observability/helicone.mdx | 69 ++++++++++--------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/content/providers/05-observability/helicone.mdx b/content/providers/05-observability/helicone.mdx index 042e2dceabed..cbf02779bfe0 100644 --- a/content/providers/05-observability/helicone.mdx +++ b/content/providers/05-observability/helicone.mdx @@ -17,20 +17,21 @@ Setting up Helicone: HELICONE_API_KEY=your-helicone-api-key ``` 3. Update your model provider configuration to use Helicone's proxy: + ```javascript - import { createOpenAI } from "@ai-sdk/openai"; - + import { createOpenAI } from '@ai-sdk/openai'; + const openai = createOpenAI({ - baseURL: "https://oai.helicone.ai/v1", + baseURL: 'https://oai.helicone.ai/v1', headers: { - "Helicone-Auth": `Bearer ${process.env.HELICONE_API_KEY}`, + 'Helicone-Auth': `Bearer ${process.env.HELICONE_API_KEY}`, }, }); - + // Use normally with AI SDK const response = await generateText({ - model: openai("gpt-4o-mini"), - prompt: "Hello world", + model: openai('gpt-4o-mini'), + prompt: 'Hello world', }); ``` @@ -86,6 +87,7 @@ While other observability solutions require OpenTelemetry instrumentation, Helic **Characteristics of Helicone's Proxy Approach:** + - No additional packages required - Compatible with JavaScript environments - Minimal code changes to existing implementations @@ -101,10 +103,10 @@ Monitor how individual users interact with your AI application: ```javascript const response = await generateText({ - model: openai("gpt-4o-mini"), - prompt: "Hello world", + model: openai('gpt-4o-mini'), + prompt: 'Hello world', headers: { - "Helicone-User-Id": "user@example.com", + 'Helicone-User-Id': 'user@example.com', }, }); ``` @@ -117,12 +119,12 @@ Add structured metadata to filter and analyze requests: ```javascript const response = await generateText({ - model: openai("gpt-4o-mini"), - prompt: "Translate this text to French", + model: openai('gpt-4o-mini'), + prompt: 'Translate this text to French', headers: { - "Helicone-Property-Feature": "translation", - "Helicone-Property-Source": "mobile-app", - "Helicone-Property-Language": "French", + 'Helicone-Property-Feature': 'translation', + 'Helicone-Property-Source': 'mobile-app', + 'Helicone-Property-Language': 'French', }, }); ``` @@ -135,12 +137,12 @@ Group related requests into coherent conversations: ```javascript const response = await generateText({ - model: openai("gpt-4o-mini"), - prompt: "Tell me more about that", + model: openai('gpt-4o-mini'), + prompt: 'Tell me more about that', headers: { - "Helicone-Session-Id": "convo-123", - "Helicone-Session-Name": "Travel Planning", - "Helicone-Session-Path": "/chats/travel", + 'Helicone-Session-Id': 'convo-123', + 'Helicone-Session-Name': 'Travel Planning', + 'Helicone-Session-Path': '/chats/travel', }, }); ``` @@ -155,10 +157,10 @@ Reduce costs by caching identical requests: ```javascript const response = await generateText({ - model: openai("gpt-4o-mini"), - prompt: "What is the capital of France?", + model: openai('gpt-4o-mini'), + prompt: 'What is the capital of France?', headers: { - "Helicone-Cache-Enabled": "true", + 'Helicone-Cache-Enabled': 'true', }, }); ``` @@ -171,19 +173,20 @@ Control usage by adding a rate limit policy: ```javascript const response = await generateText({ - model: openai("gpt-4o-mini"), - prompt: "Generate creative content", + model: openai('gpt-4o-mini'), + prompt: 'Generate creative content', headers: { // Allow 10,000 requests per hour - "Helicone-RateLimit-Policy": "10000;w=3600", - + 'Helicone-RateLimit-Policy': '10000;w=3600', + // Optional: limit by user - "Helicone-User-Id": "user@example.com", + 'Helicone-User-Id': 'user@example.com', }, }); ``` Format: `[quota];w=[time_window];u=[unit];s=[segment]` where: + - `quota`: Maximum requests allowed in the time window - `w`: Time window in seconds (minimum 60s) - `u`: Optional unit - "request" (default) or "cents" @@ -197,14 +200,14 @@ Protect against prompt injection, jailbreaking, and other LLM-specific threats: ```javascript const response = await generateText({ - model: openai("gpt-4o-mini"), + model: openai('gpt-4o-mini'), prompt: userInput, headers: { // Basic protection (Prompt Guard model) - "Helicone-LLM-Security-Enabled": "true", - + 'Helicone-LLM-Security-Enabled': 'true', + // Optional: Advanced protection (Llama Guard model) - "Helicone-LLM-Security-Advanced": "true", + 'Helicone-LLM-Security-Advanced': 'true', }, }); ``` @@ -217,4 +220,4 @@ Protects against multiple attack vectors in 8 languages with minimal latency. Ad - [Helicone Documentation](https://docs.helicone.ai) - [GitHub Repository](https://github.com/Helicone/helicone) -- [Discord Community](https://discord.com/invite/2TkeWdXNPQ) \ No newline at end of file +- [Discord Community](https://discord.com/invite/2TkeWdXNPQ)