diff --git a/.gitignore b/.gitignore index 0dc450b..4ca2595 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,5 @@ hs_err_pid* # Local environment .env -*.local \ No newline at end of file +*.local +.claude \ No newline at end of file diff --git a/FIXES_SUMMARY.md b/FIXES_SUMMARY.md deleted file mode 100644 index 6b9dbce..0000000 --- a/FIXES_SUMMARY.md +++ /dev/null @@ -1,70 +0,0 @@ -# Braintrust Java SDK Fixes Summary - -## Issues Fixed - -### 1. API Endpoint Paths (Critical) -**Problem**: Using plural endpoints like `/projects` and `/experiments` -**Solution**: Changed to singular with `/v1/` prefix: -- `/projects` → `/v1/project` -- `/experiments` → `/v1/experiment` -- `/datasets` → `/v1/dataset` - -### 2. Missing x-bt-parent Header (Critical) -**Problem**: Traces were not appearing in experiments because the `x-bt-parent` header was missing -**Solution**: Created custom `BraintrustSpanExporter` that dynamically adds the header based on span attributes: -- Format: `experiment_id:` or `project_id:` -- This header tells Braintrust which experiment/project the traces belong to - -### 3. Jackson Deserialization Errors -**Problem**: API responses included fields like `org_id` that weren't in our model classes -**Solution**: -- Added `orgId` field to Project record -- Configured ObjectMapper to ignore unknown fields with `FAIL_ON_UNKNOWN_PROPERTIES = false` - -### 4. Compilation Error in Example -**Problem**: `OpenTelemetry` interface doesn't have `getSdkTracerProvider()` method -**Solution**: Added instanceof check and cast to `OpenTelemetrySdk` when flushing spans - -### 5. Missing Span Data -**Problem**: Evaluation spans were not logging input/output data -**Solution**: Added `input` and `output` attributes to evaluation spans - -## Files Modified - -1. `/src/main/java/dev/braintrust/api/BraintrustApiClient.java` - - Updated all API endpoints to use `/v1/` prefix with singular names - - Added `orgId` field to Project record - - Configured Jackson to ignore unknown fields - -2. `/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java` (NEW) - - Custom exporter that dynamically sets `x-bt-parent` header - - Groups spans by parent and exports with appropriate headers - -3. `/src/main/java/dev/braintrust/trace/BraintrustTracing.java` - - Changed to use custom `BraintrustSpanExporter` instead of standard OTLP exporter - -4. `/src/main/java/dev/braintrust/eval/Evaluation.java` - - Added input/output attributes to evaluation spans - -5. `/examples/src/main/java/dev/braintrust/examples/SimpleExperimentWithRegistration.java` - - Fixed SDK type check for flushing spans - - Added proper shutdown sequence - -6. `/CLAUDE.md` (NEW) - - Documented all Braintrust API quirks and solutions - - Critical reference for future development - -## Key Learnings - -1. **404 as 403**: Braintrust returns 403 "Invalid key=value pair" errors when hitting non-existent endpoints -2. **x-bt-parent Required**: This header is critical for traces to appear in experiments -3. **Singular API Paths**: Unlike typical REST APIs, Braintrust uses singular resource names -4. **Dynamic Headers**: Standard OTLP exporters don't support dynamic headers, requiring custom implementation - -## Testing - -The SDK now compiles successfully. With a valid API key, the Go-style experiment example should: -1. Create a project and experiment via the REST API -2. Run evaluations that generate traced spans -3. Export spans with proper `x-bt-parent` header -4. Display results in the Braintrust dashboard under the correct experiment \ No newline at end of file diff --git a/build.gradle b/build.gradle index 0ac6555..d36255d 100644 --- a/build.gradle +++ b/build.gradle @@ -61,6 +61,10 @@ dependencies { compileOnly 'com.openai:openai-java:2.8.1' testImplementation 'com.openai:openai-java:2.8.1' implementation "io.opentelemetry.instrumentation:opentelemetry-openai-java-1.1:2.19.0-alpha" + + // Anthropic Instrumentation + compileOnly "com.anthropic:anthropic-java:2.8.1" + testImplementation "com.anthropic:anthropic-java:2.8.1" } /** @@ -281,12 +285,13 @@ task validateJavaVersion { // Task to test the jar by running it -task testJar(type: Exec) { +task testJar(type: JavaExec) { description = 'Test the jar by running it and fail build if non-zero exit code' group = 'verification' dependsOn jar - commandLine 'java', '-jar', jar.archiveFile.get().asFile.absolutePath + classpath = files(jar.archiveFile) + javaLauncher = javaToolchains.launcherFor(java.toolchain) doFirst { // println "Testing jar: ${jar.archiveFile.get().asFile.absolutePath}" diff --git a/examples/build.gradle b/examples/build.gradle index 5eecf08..8c98c46 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -20,6 +20,8 @@ dependencies { implementation "io.opentelemetry:opentelemetry-exporter-otlp:${otelVersion}" // to run OAI instrumentation examples implementation 'com.openai:openai-java:2.8.1' + // to run anthropic examples + implementation "com.anthropic:anthropic-java:2.8.1" } application { @@ -69,3 +71,17 @@ task runExperiment(type: JavaExec) { suspend = false } } + +task runAnthropicInstrumentation(type: JavaExec) { + group = 'Braintrust SDK Examples' + description = 'Run the Anthropic instrumentation example. NOTE: this requires ANTHROPIC_API_KEY to be exported and will make a small call to anthropic, using your tokens' + classpath = sourceSets.main.runtimeClasspath + mainClass = 'dev.braintrust.examples.AnthropicInstrumentationExample' + systemProperty 'org.slf4j.simpleLogger.log.dev.braintrust', braintrustLogLevel + debugOptions { + enabled = true + port = 5566 + server = true + suspend = false + } +} diff --git a/examples/src/main/java/dev/braintrust/examples/AnthropicInstrumentationExample.java b/examples/src/main/java/dev/braintrust/examples/AnthropicInstrumentationExample.java new file mode 100644 index 0000000..5bc3c9b --- /dev/null +++ b/examples/src/main/java/dev/braintrust/examples/AnthropicInstrumentationExample.java @@ -0,0 +1,59 @@ +package dev.braintrust.examples; + +import com.anthropic.client.AnthropicClient; +import com.anthropic.client.okhttp.AnthropicOkHttpClient; +import com.anthropic.models.messages.MessageCreateParams; +import com.anthropic.models.messages.Model; +import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.instrumentation.anthropic.BraintrustAnthropic; +import dev.braintrust.trace.BraintrustTracing; + +/** Basic OTel + Anthropic instrumentation example */ +public class AnthropicInstrumentationExample { + public static void main(String[] args) throws Exception { + if (null == System.getenv("ANTHROPIC_API_KEY")) { + System.err.println( + "\n" + + "WARNING envar ANTHROPIC_API_KEY not found. This example will likely" + + " fail.\n"); + } + + var braintrustConfig = BraintrustConfig.fromEnvironment(); + var openTelemetry = BraintrustTracing.of(braintrustConfig, true); + var tracer = BraintrustTracing.getTracer(openTelemetry); + + // Wrap Anthropic client with Braintrust instrumentation + AnthropicClient anthropicClient = + BraintrustAnthropic.wrap(openTelemetry, AnthropicOkHttpClient.fromEnv()); + + var rootSpan = tracer.spanBuilder("anthropic-java-instrumentation-example").startSpan(); + try (var ignored = rootSpan.makeCurrent()) { + Thread.sleep(70); // just to make span look interesting + + var request = + MessageCreateParams.builder() + .model(Model.CLAUDE_3_5_HAIKU_20241022) + .system("You are the world's greatest philosopher") + .addUserMessage("What's the meaning of life? Be very brief.") + .maxTokens(50) + .temperature(0.0) + .build(); + + var response = anthropicClient.messages().create(request); + System.out.println("~~~ GOT RESPONSE: " + response); + + Thread.sleep(30); // not required, just to show span duration + } finally { + rootSpan.end(); + } + + var url = + braintrustConfig.fetchProjectURI() + + "/logs?r=%s&s=%s" + .formatted( + rootSpan.getSpanContext().getTraceId(), + rootSpan.getSpanContext().getSpanId()); + + System.out.println("\n\n Example complete! View your data in Braintrust: " + url); + } +} diff --git a/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java b/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java index 8e16397..388d0fa 100644 --- a/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java +++ b/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java @@ -77,7 +77,7 @@ public static void main(String[] args) throws Exception { braintrustConfig.fetchProjectURI() + "/logs?r=%s&s=%s" .formatted( - span.getSpanContext().getSpanId(), + span.getSpanContext().getTraceId(), span.getSpanContext().getSpanId()); System.out.println("\n\n Example complete! View your data in Braintrust: " + url); } diff --git a/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java b/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java index aaddfbd..3d52c32 100644 --- a/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java +++ b/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java @@ -20,7 +20,7 @@ public static void main(String[] args) throws Exception { var tracer = BraintrustTracing.getTracer(openTelemetry); OpenAIClient openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv()); - var rootSpan = tracer.spanBuilder("java-braintrust-example").startSpan(); + var rootSpan = tracer.spanBuilder("openai-java-instrumentation-example").startSpan(); try (var ignored = rootSpan.makeCurrent()) { Thread.sleep(70); // Not required. This is just to make the span look interesting var request = @@ -41,7 +41,7 @@ public static void main(String[] args) throws Exception { braintrustConfig.fetchProjectURI() + "/logs?r=%s&s=%s" .formatted( - rootSpan.getSpanContext().getSpanId(), + rootSpan.getSpanContext().getTraceId(), rootSpan.getSpanContext().getSpanId()); System.out.println("\n\n Example complete! View your data in Braintrust: " + url); } diff --git a/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java b/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java index 857c4c0..b1a6c50 100644 --- a/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java +++ b/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java @@ -21,7 +21,7 @@ public static void main(String[] args) throws Exception { braintrustConfig.fetchProjectURI() + "/logs?r=%s&s=%s" .formatted( - span.getSpanContext().getSpanId(), + span.getSpanContext().getTraceId(), span.getSpanContext().getSpanId()); System.out.println("\n\n Example complete! View your data in Braintrust: " + url); } diff --git a/run-with-env.sh b/run-with-env.sh deleted file mode 100755 index 8bde6ae..0000000 --- a/run-with-env.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# Script to run Gradle tasks with environment variables from .env file - -cd "$(dirname "$0")" - -# Check if .env file exists -if [ -f .env ]; then - # Load environment variables from .env file - set -a - source .env - set +a - echo "Loaded environment variables from .env file" -else - echo "Warning: .env file not found!" - echo "Using fallback test API key for compilation only." - export BRAINTRUST_API_KEY="sk-test-1234567890abcdef1234567890abcdef" -fi - -# Check if we should use staging -if [ "$1" == "--staging" ]; then - export BRAINTRUST_API_URL="https://staging-api.braintrust.dev" - shift -fi - -# Run the gradle command passed as arguments -if [ -z "$1" ]; then - echo "Usage: ./run-with-env.sh [--staging] " - echo "Example: ./run-with-env.sh :examples:runOpenTelemetry" - echo "Example: ./run-with-env.sh --staging :examples:runTestEndpoints" - echo "" - if [ ! -f .env ]; then - echo "To use a real API key, create a .env file with:" - echo "BRAINTRUST_API_KEY=your-actual-api-key" - fi - exit 1 -fi - -if [ -n "$BRAINTRUST_API_KEY" ]; then - echo "Running with BRAINTRUST_API_KEY set (${BRAINTRUST_API_KEY:0:10}...)" -else - echo "Error: BRAINTRUST_API_KEY not set!" - exit 1 -fi - -if [ -n "$BRAINTRUST_API_URL" ]; then - echo "Using API URL: $BRAINTRUST_API_URL" -fi - -./gradlew "$@" \ No newline at end of file diff --git a/src/main/java/dev/braintrust/instrumentation/anthropic/BraintrustAnthropic.java b/src/main/java/dev/braintrust/instrumentation/anthropic/BraintrustAnthropic.java new file mode 100644 index 0000000..8c20a8a --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/anthropic/BraintrustAnthropic.java @@ -0,0 +1,14 @@ +package dev.braintrust.instrumentation.anthropic; + +import com.anthropic.client.AnthropicClient; +import dev.braintrust.instrumentation.anthropic.otel.AnthropicTelemetry; +import io.opentelemetry.api.OpenTelemetry; + +/** Braintrust Anthropic client instrumentation. */ +public final class BraintrustAnthropic { + + /** Instrument Anthropic client with braintrust traces */ + public static AnthropicClient wrap(OpenTelemetry otel, AnthropicClient client) { + return AnthropicTelemetry.builder(otel).setCaptureMessageContent(true).build().wrap(client); + } +} diff --git a/src/main/java/dev/braintrust/instrumentation/anthropic/otel/AnthropicTelemetry.java b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/AnthropicTelemetry.java new file mode 100644 index 0000000..3d91870 --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/AnthropicTelemetry.java @@ -0,0 +1,46 @@ +package dev.braintrust.instrumentation.anthropic.otel; + +import com.anthropic.client.AnthropicClient; +import com.anthropic.models.messages.Message; +import com.anthropic.models.messages.MessageCreateParams; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter; + +/** Entrypoint for instrumenting Anthropic clients. */ +public final class AnthropicTelemetry { + /** Returns a new {@link AnthropicTelemetry} configured with the given {@link OpenTelemetry}. */ + public static AnthropicTelemetry create(OpenTelemetry openTelemetry) { + return builder(openTelemetry).build(); + } + + /** + * Returns a new {@link AnthropicTelemetryBuilder} configured with the given {@link + * OpenTelemetry}. + */ + public static AnthropicTelemetryBuilder builder(OpenTelemetry openTelemetry) { + return new AnthropicTelemetryBuilder(openTelemetry); + } + + private final Instrumenter messageInstrumenter; + + private final Logger eventLogger; + + private final boolean captureMessageContent; + + AnthropicTelemetry( + Instrumenter messageInstrumenter, + Logger eventLogger, + boolean captureMessageContent) { + this.messageInstrumenter = messageInstrumenter; + this.eventLogger = eventLogger; + this.captureMessageContent = captureMessageContent; + } + + /** Wraps the provided AnthropicClient, enabling telemetry for it. */ + public AnthropicClient wrap(AnthropicClient client) { + return new InstrumentedAnthropicClient( + client, messageInstrumenter, eventLogger, captureMessageContent) + .createProxy(); + } +} diff --git a/src/main/java/dev/braintrust/instrumentation/anthropic/otel/AnthropicTelemetryBuilder.java b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/AnthropicTelemetryBuilder.java new file mode 100644 index 0000000..da68316 --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/AnthropicTelemetryBuilder.java @@ -0,0 +1,56 @@ +package dev.braintrust.instrumentation.anthropic.otel; + +import com.anthropic.models.messages.Message; +import com.anthropic.models.messages.MessageCreateParams; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiAttributesExtractor; +import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiClientMetrics; +import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiSpanNameExtractor; +import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter; +import io.opentelemetry.instrumentation.api.instrumenter.SpanKindExtractor; + +/** A builder of {@link AnthropicTelemetry}. */ +public final class AnthropicTelemetryBuilder { + static final String INSTRUMENTATION_NAME = "io.opentelemetry.anthropic-java-2.8"; + + private final OpenTelemetry openTelemetry; + + private boolean captureMessageContent; + + AnthropicTelemetryBuilder(OpenTelemetry openTelemetry) { + this.openTelemetry = openTelemetry; + } + + /** + * Sets whether emitted log events include full content of user and assistant messages. + * + *

Note that full content can have data privacy and size concerns and care should be taken + * when enabling this. + */ + @CanIgnoreReturnValue + public AnthropicTelemetryBuilder setCaptureMessageContent(boolean captureMessageContent) { + this.captureMessageContent = captureMessageContent; + return this; + } + + /** + * Returns a new {@link AnthropicTelemetry} with the settings of this {@link + * AnthropicTelemetryBuilder}. + */ + public AnthropicTelemetry build() { + Instrumenter messageInstrumenter = + Instrumenter.builder( + openTelemetry, + INSTRUMENTATION_NAME, + GenAiSpanNameExtractor.create(MessageAttributesGetter.INSTANCE)) + .addAttributesExtractor( + GenAiAttributesExtractor.create(MessageAttributesGetter.INSTANCE)) + .addOperationMetrics(GenAiClientMetrics.get()) + .buildInstrumenter(SpanKindExtractor.alwaysClient()); + + Logger eventLogger = openTelemetry.getLogsBridge().get(INSTRUMENTATION_NAME); + return new AnthropicTelemetry(messageInstrumenter, eventLogger, captureMessageContent); + } +} diff --git a/src/main/java/dev/braintrust/instrumentation/anthropic/otel/DelegatingInvocationHandler.java b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/DelegatingInvocationHandler.java new file mode 100644 index 0000000..8d045ee --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/DelegatingInvocationHandler.java @@ -0,0 +1,37 @@ +package dev.braintrust.instrumentation.anthropic.otel; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; + +abstract class DelegatingInvocationHandler> + implements InvocationHandler { + + private static final ClassLoader CLASS_LOADER = + DelegatingInvocationHandler.class.getClassLoader(); + + protected final T delegate; + + public DelegatingInvocationHandler(T delegate) { + this.delegate = delegate; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + try { + return method.invoke(delegate, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + + protected abstract Class getProxyType(); + + @SuppressWarnings("rawtypes") + public T createProxy() { + Class proxyType = getProxyType(); + Object proxy = Proxy.newProxyInstance(CLASS_LOADER, new Class[] {proxyType}, this); + return proxyType.cast(proxy); + } +} diff --git a/src/main/java/dev/braintrust/instrumentation/anthropic/otel/GenAiAttributes.java b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/GenAiAttributes.java new file mode 100644 index 0000000..305bb38 --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/GenAiAttributes.java @@ -0,0 +1,24 @@ +package dev.braintrust.instrumentation.anthropic.otel; + +import static io.opentelemetry.api.common.AttributeKey.stringKey; + +import io.opentelemetry.api.common.AttributeKey; + +// copied from GenAiIncubatingAttributes +final class GenAiAttributes { + static final AttributeKey GEN_AI_PROVIDER_NAME = stringKey("gen_ai.provider.name"); + + static final class GenAiOperationNameIncubatingValues { + static final String CHAT = "chat"; + + private GenAiOperationNameIncubatingValues() {} + } + + static final class GenAiProviderNameIncubatingValues { + static final String ANTHROPIC = "anthropic"; + + private GenAiProviderNameIncubatingValues() {} + } + + private GenAiAttributes() {} +} diff --git a/src/main/java/dev/braintrust/instrumentation/anthropic/otel/InstrumentedAnthropicClient.java b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/InstrumentedAnthropicClient.java new file mode 100644 index 0000000..dece35f --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/InstrumentedAnthropicClient.java @@ -0,0 +1,47 @@ +package dev.braintrust.instrumentation.anthropic.otel; + +import com.anthropic.client.AnthropicClient; +import com.anthropic.models.messages.Message; +import com.anthropic.models.messages.MessageCreateParams; +import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter; +import java.lang.reflect.Method; + +final class InstrumentedAnthropicClient + extends DelegatingInvocationHandler { + + private final Instrumenter messageInstrumenter; + private final Logger eventLogger; + private final boolean captureMessageContent; + + InstrumentedAnthropicClient( + AnthropicClient delegate, + Instrumenter messageInstrumenter, + Logger eventLogger, + boolean captureMessageContent) { + super(delegate); + this.messageInstrumenter = messageInstrumenter; + this.eventLogger = eventLogger; + this.captureMessageContent = captureMessageContent; + } + + @Override + protected Class getProxyType() { + return AnthropicClient.class; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + String methodName = method.getName(); + Class[] parameterTypes = method.getParameterTypes(); + if (methodName.equals("messages") && parameterTypes.length == 0) { + return new InstrumentedMessageService( + delegate.messages(), + messageInstrumenter, + eventLogger, + captureMessageContent) + .createProxy(); + } + return super.invoke(proxy, method, args); + } +} diff --git a/src/main/java/dev/braintrust/instrumentation/anthropic/otel/InstrumentedMessageService.java b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/InstrumentedMessageService.java new file mode 100644 index 0000000..9746783 --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/InstrumentedMessageService.java @@ -0,0 +1,111 @@ +package dev.braintrust.instrumentation.anthropic.otel; + +import com.anthropic.core.RequestOptions; +import com.anthropic.models.messages.Message; +import com.anthropic.models.messages.MessageCreateParams; +import com.anthropic.models.messages.MessageParam; +import com.anthropic.models.messages.TextBlockParam; +import com.anthropic.services.blocking.MessageService; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import lombok.SneakyThrows; + +final class InstrumentedMessageService + extends DelegatingInvocationHandler { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + + private final Instrumenter instrumenter; + private final Logger eventLogger; + private final boolean captureMessageContent; + + InstrumentedMessageService( + MessageService delegate, + Instrumenter instrumenter, + Logger eventLogger, + boolean captureMessageContent) { + super(delegate); + this.instrumenter = instrumenter; + this.eventLogger = eventLogger; + this.captureMessageContent = captureMessageContent; + } + + @Override + protected Class getProxyType() { + return MessageService.class; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + String methodName = method.getName(); + Class[] parameterTypes = method.getParameterTypes(); + + if (methodName.equals("create")) { + if (parameterTypes.length >= 1 && parameterTypes[0] == MessageCreateParams.class) { + if (parameterTypes.length == 1) { + return create((MessageCreateParams) args[0], RequestOptions.none()); + } else if (parameterTypes.length == 2 + && parameterTypes[1] == RequestOptions.class) { + return create((MessageCreateParams) args[0], (RequestOptions) args[1]); + } + } + } + + return super.invoke(proxy, method, args); + } + + @SneakyThrows + private Message create(MessageCreateParams inputMessage, RequestOptions requestOptions) { + Context parentContext = Context.current(); + if (!instrumenter.shouldStart(parentContext, inputMessage)) { + return delegate.create(inputMessage, requestOptions); + } + + Context context = instrumenter.start(parentContext, inputMessage); + Message outputMessage; + try (Scope ignored = context.makeCurrent()) { + List inputMessages = new ArrayList<>(inputMessage.messages()); + // Put system in the input message so the backend will pick it up in the LLM display + if (inputMessage.system().isPresent()) { + inputMessages.add( + 0, + MessageParam.builder() + .role(MessageParam.Role.of("system")) + .content(inputMessage.system().get().asString()) + .build()); + } + Span.current() + .setAttribute( + "braintrust.input_json", JSON_MAPPER.writeValueAsString(inputMessages)); + outputMessage = delegate.create(inputMessage, requestOptions); + Span.current() + .setAttribute( + "braintrust.output_json", + JSON_MAPPER.writeValueAsString(new Message[] {outputMessage})); + } catch (Throwable t) { + instrumenter.end(context, inputMessage, null, t); + throw t; + } + + instrumenter.end(context, inputMessage, outputMessage, null); + return outputMessage; + } + + private static String contentToString(MessageCreateParams.System content) { + if (content.isString()) { + return content.asString(); + } else if (content.isTextBlockParams()) { + return content.asTextBlockParams().stream() + .map(TextBlockParam::text) + .collect(Collectors.joining()); + } + return ""; + } +} diff --git a/src/main/java/dev/braintrust/instrumentation/anthropic/otel/MessageAttributesGetter.java b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/MessageAttributesGetter.java new file mode 100644 index 0000000..a347751 --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/anthropic/otel/MessageAttributesGetter.java @@ -0,0 +1,129 @@ +package dev.braintrust.instrumentation.anthropic.otel; + +import static java.util.Collections.emptyList; + +import com.anthropic.models.messages.Message; +import com.anthropic.models.messages.MessageCreateParams; +import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiAttributesGetter; +import java.util.List; +import org.jetbrains.annotations.Nullable; + +enum MessageAttributesGetter implements GenAiAttributesGetter { + INSTANCE; + + @Override + public String getOperationName(MessageCreateParams request) { + return GenAiAttributes.GenAiOperationNameIncubatingValues.CHAT; + } + + @Override + public String getSystem(MessageCreateParams request) { + return GenAiAttributes.GenAiProviderNameIncubatingValues.ANTHROPIC; + } + + @Override + public String getRequestModel(MessageCreateParams request) { + return request.model().asString(); + } + + @Nullable + @Override + public Long getRequestSeed(MessageCreateParams request) { + return null; + } + + @Nullable + @Override + public List getRequestEncodingFormats(MessageCreateParams request) { + return null; + } + + @Nullable + @Override + public Double getRequestFrequencyPenalty(MessageCreateParams request) { + return null; + } + + @Nullable + @Override + public Long getRequestMaxTokens(MessageCreateParams request) { + // maxTokens() returns a primitive long, so we convert to Long + long maxTokens = request.maxTokens(); + return maxTokens > 0 ? maxTokens : null; + } + + @Nullable + @Override + public Double getRequestPresencePenalty(MessageCreateParams request) { + return null; + } + + @Nullable + @Override + public List getRequestStopSequences(MessageCreateParams request) { + return request.stopSequences().orElse(null); + } + + @Nullable + @Override + public Double getRequestTemperature(MessageCreateParams request) { + return request.temperature().orElse(null); + } + + @Nullable + @Override + public Double getRequestTopK(MessageCreateParams request) { + return request.topK().map(Long::doubleValue).orElse(null); + } + + @Nullable + @Override + public Double getRequestTopP(MessageCreateParams request) { + return request.topP().orElse(null); + } + + @Override + public List getResponseFinishReasons( + MessageCreateParams request, @Nullable Message response) { + if (response == null) { + return emptyList(); + } + return response.stopReason().map(reason -> List.of(reason.asString())).orElse(emptyList()); + } + + @Override + @Nullable + public String getResponseId(MessageCreateParams request, @Nullable Message response) { + if (response == null) { + return null; + } + return response.id(); + } + + @Override + @Nullable + public String getResponseModel(MessageCreateParams request, @Nullable Message response) { + if (response == null) { + return null; + } + return response.model().asString(); + } + + @Override + @Nullable + public Long getUsageInputTokens(MessageCreateParams request, @Nullable Message response) { + if (response == null) { + return null; + } + return response.usage().inputTokens(); + } + + @Override + @Nullable + public Long getUsageOutputTokens(MessageCreateParams request, @Nullable Message response) { + if (response == null) { + return null; + } + return response.usage().outputTokens(); + } +} diff --git a/src/main/java/dev/braintrust/instrumentation/openai/otel/ChatCompletionEventsHelper.java b/src/main/java/dev/braintrust/instrumentation/openai/otel/ChatCompletionEventsHelper.java index 92b023a..0814c16 100644 --- a/src/main/java/dev/braintrust/instrumentation/openai/otel/ChatCompletionEventsHelper.java +++ b/src/main/java/dev/braintrust/instrumentation/openai/otel/ChatCompletionEventsHelper.java @@ -7,7 +7,6 @@ import static io.opentelemetry.api.common.AttributeKey.stringKey; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.chat.completions.ChatCompletion; import com.openai.models.chat.completions.ChatCompletionAssistantMessageParam; @@ -15,7 +14,6 @@ import com.openai.models.chat.completions.ChatCompletionCreateParams; import com.openai.models.chat.completions.ChatCompletionDeveloperMessageParam; import com.openai.models.chat.completions.ChatCompletionMessage; -import com.openai.models.chat.completions.ChatCompletionMessageParam; import com.openai.models.chat.completions.ChatCompletionMessageToolCall; import com.openai.models.chat.completions.ChatCompletionSystemMessageParam; import com.openai.models.chat.completions.ChatCompletionToolMessageParam; @@ -36,6 +34,7 @@ import java.util.Optional; import java.util.stream.Collectors; import javax.annotation.Nullable; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -45,80 +44,16 @@ final class ChatCompletionEventsHelper { private static final ObjectMapper JSON_MAPPER = new com.fasterxml.jackson.databind.ObjectMapper(); + @SneakyThrows public static void emitPromptLogEvents( Context context, Logger eventLogger, ChatCompletionCreateParams request, boolean captureMessageContent) { - for (ChatCompletionMessageParam msg : request.messages()) { - String eventType; - Map> body = new HashMap<>(); - if (msg.isSystem()) { - eventType = "gen_ai.system.message"; - if (captureMessageContent) { - var content = contentToString(msg.asSystem().content()); - Span.current().setAttribute("instructions", content); - body.put("content", Value.of(content)); - } - } else if (msg.isDeveloper()) { - eventType = "gen_ai.system.message"; - body.put("role", Value.of("developer")); - if (captureMessageContent) { - var content = contentToString(msg.asDeveloper().content()); - body.put("content", Value.of(content)); - } - } else if (msg.isUser()) { - eventType = "gen_ai.user.message"; - if (captureMessageContent) { - var content = contentToString(msg.asUser().content()); - try { - Span.current() - .setAttribute( - "braintrust.input_json", - JSON_MAPPER.writeValueAsString(content)); - } catch (JsonProcessingException e) { - log.error("Error mapping json", e); - } - body.put("content", Value.of(content)); - } - } else if (msg.isAssistant()) { - ChatCompletionAssistantMessageParam assistantMsg = msg.asAssistant(); - eventType = "gen_ai.assistant.message"; - if (captureMessageContent) { - assistantMsg - .content() - .ifPresent( - content -> - body.put( - "content", Value.of(contentToString(content)))); - } - assistantMsg - .toolCalls() - .ifPresent( - toolCalls -> { - List> toolCallsJson = - toolCalls.stream() - .map( - call -> - buildToolCallEventObject( - call, - captureMessageContent)) - .collect(Collectors.toList()); - body.put("tool_calls", Value.of(toolCallsJson)); - }); - } else if (msg.isTool()) { - ChatCompletionToolMessageParam toolMsg = msg.asTool(); - eventType = "gen_ai.tool.message"; - if (captureMessageContent) { - body.put("content", Value.of(contentToString(toolMsg.content()))); - } - body.put("id", Value.of(toolMsg.toolCallId())); - } else { - continue; - } - // newEvent(eventLogger, eventType).setContext(context).setBody(Value.of(body)).emit(); - } - // "gen_ai.input.messages" + Span.current() + .setAttribute( + "braintrust.input_json", + JSON_MAPPER.writeValueAsString(request.messages())); } private static String contentToString(ChatCompletionToolMessageParam.Content content) { @@ -192,16 +127,24 @@ private static String joinContentParts(List conte .collect(Collectors.joining()); } + @SneakyThrows public static void emitCompletionLogEvents( Context context, Logger eventLogger, ChatCompletion completion, boolean captureMessageContent) { - try { - var completionJson = JSON_MAPPER.writeValueAsString(completion); - Span.current().setAttribute("braintrust.output_json", completionJson); - } catch (JsonProcessingException e) { - log.error("Error mapping completion json", e); + if (completion.choices().isEmpty()) { + log.debug("no choices in OAI response"); + } else if (completion.choices().size() > 1) { + log.debug("multiple choices in OAI response: {}", completion.choices().size()); + } else { + Span.current() + .setAttribute( + "braintrust.output_json", + JSON_MAPPER.writeValueAsString( + new ChatCompletionMessage[] { + completion.choices().get(0).message() + })); } for (ChatCompletion.Choice choice : completion.choices()) { ChatCompletionMessage choiceMsg = choice.message(); diff --git a/src/test/java/dev/braintrust/instrumentation/anthropic/BraintrustAnthropicTest.java b/src/test/java/dev/braintrust/instrumentation/anthropic/BraintrustAnthropicTest.java new file mode 100644 index 0000000..a145b74 --- /dev/null +++ b/src/test/java/dev/braintrust/instrumentation/anthropic/BraintrustAnthropicTest.java @@ -0,0 +1,175 @@ +package dev.braintrust.instrumentation.anthropic; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static dev.braintrust.trace.BraintrustTracingTest.getExportedBraintrustSpans; +import static org.junit.jupiter.api.Assertions.*; + +import com.anthropic.client.AnthropicClient; +import com.anthropic.client.okhttp.AnthropicOkHttpClient; +import com.anthropic.models.messages.MessageCreateParams; +import com.anthropic.models.messages.Model; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.junit5.WireMockExtension; +import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.trace.BraintrustTracing; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import java.util.concurrent.TimeUnit; +import lombok.SneakyThrows; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +public class BraintrustAnthropicTest { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + + @RegisterExtension + static WireMockExtension wireMock = + WireMockExtension.newInstance().options(wireMockConfig().dynamicPort()).build(); + + private final BraintrustConfig config = + BraintrustConfig.of( + "BRAINTRUST_API_KEY", "foobar", + "BRAINTRUST_DEFAULT_PROJECT_NAME", "unit-test-project", + "BRAINTRUST_JAVA_EXPORT_SPANS_IN_MEMORY_FOR_UNIT_TEST", "true"); + + @BeforeEach + void beforeEach() { + GlobalOpenTelemetry.resetForTest(); + getExportedBraintrustSpans().clear(); + wireMock.resetAll(); + } + + @Test + @SneakyThrows + void testWrapAnthropic() { + // Mock the Anthropic API response + wireMock.stubFor( + post(urlEqualTo("/v1/messages")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + """ + { + "id": "msg_test123", + "type": "message", + "role": "assistant", + "model": "claude-3-5-haiku-20241022", + "content": [ + { + "type": "text", + "text": "The capital of France is Paris." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 20, + "output_tokens": 8 + } + } + """))); + + var openTelemetry = (OpenTelemetrySdk) BraintrustTracing.of(config, true); + + // Create Anthropic client pointing to WireMock server + AnthropicClient anthropicClient = + AnthropicOkHttpClient.builder() + .baseUrl("http://localhost:" + wireMock.getPort()) + .apiKey("test-api-key") + .build(); + + // Wrap with Braintrust instrumentation + anthropicClient = BraintrustAnthropic.wrap(openTelemetry, anthropicClient); + + var request = + MessageCreateParams.builder() + .model(Model.CLAUDE_3_5_HAIKU_20241022) + .system("You are a helpful assistant") + .addUserMessage("What is the capital of France?") + .maxTokens(50) + .temperature(0.0) + .build(); + + var response = anthropicClient.messages().create(request); + + // Verify the response + assertNotNull(response); + wireMock.verify(1, postRequestedFor(urlEqualTo("/v1/messages"))); + assertEquals("msg_test123", response.id()); + var contentBlock = response.content().get(0); + assertTrue(contentBlock.isText()); + assertEquals("The capital of France is Paris.", contentBlock.asText().text()); + + // Verify spans were exported + assertTrue( + openTelemetry + .getSdkTracerProvider() + .forceFlush() + .join(10, TimeUnit.SECONDS) + .isSuccess()); + var spanData = + getExportedBraintrustSpans().get(config.getBraintrustParentValue().orElseThrow()); + assertNotNull(spanData); + assertEquals(1, spanData.size()); + var span = spanData.get(0); + + // Verify standard GenAI attributes + assertEquals( + "claude-3-5-haiku-20241022", + span.getAttributes().get(AttributeKey.stringKey("gen_ai.request.model"))); + assertEquals( + "claude-3-5-haiku-20241022", + span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.model"))); + assertEquals( + "[end_turn]", + span.getAttributes() + .get(AttributeKey.stringArrayKey("gen_ai.response.finish_reasons")) + .toString()); + assertEquals( + "chat", span.getAttributes().get(AttributeKey.stringKey("gen_ai.operation.name"))); + assertEquals( + "msg_test123", + span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); + assertEquals( + "project_name:unit-test-project", + span.getAttributes().get(AttributeKey.stringKey("braintrust.parent"))); + assertEquals( + 20L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.input_tokens"))); + assertEquals( + 8L, span.getAttributes().get(AttributeKey.longKey("gen_ai.usage.output_tokens"))); + assertEquals( + 0.0, + span.getAttributes().get(AttributeKey.doubleKey("gen_ai.request.temperature"))); + assertEquals( + 50L, span.getAttributes().get(AttributeKey.longKey("gen_ai.request.max_tokens"))); + + // Verify Braintrust-specific attributes + assertEquals( + "[{\"content\":\"You are a helpful" + + " assistant\",\"role\":\"system\",\"valid\":false},{\"content\":\"What is the" + + " capital of France?\",\"role\":\"user\",\"valid\":true}]", + span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json"))); + + // Verify output JSON + String outputJson = + span.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); + assertNotNull(outputJson); + + var outputMessages = JSON_MAPPER.readTree(outputJson); + assertEquals(1, outputMessages.size()); + var messageZero = outputMessages.get(0); + assertEquals("msg_test123", messageZero.get("id").asText()); + assertEquals("message", messageZero.get("type").asText()); + assertEquals("assistant", messageZero.get("role").asText()); + assertEquals( + "The capital of France is Paris.", + messageZero.get("content").get(0).get("text").asText()); + assertEquals(8, messageZero.get("usage").get("output_tokens").asInt()); + assertEquals(20, messageZero.get("usage").get("input_tokens").asInt()); + } +} diff --git a/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java b/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java index 6010106..af71876 100644 --- a/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java +++ b/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java @@ -17,6 +17,7 @@ import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.sdk.OpenTelemetrySdk; import java.util.concurrent.TimeUnit; +import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -43,6 +44,7 @@ void beforeEach() { } @Test + @SneakyThrows void testWrapOpenAi() { // Mock the OpenAI API response wireMock.stubFor( @@ -137,10 +139,9 @@ void testWrapOpenAi() { "chatcmpl-test123", span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); assertEquals( - "You are a helpful assistant", - span.getAttributes().get(AttributeKey.stringKey("instructions"))); - assertEquals( - "\"What is the capital of France?\"", + "[{\"content\":\"You are a helpful" + + " assistant\",\"role\":\"system\",\"valid\":true},{\"content\":\"What is the" + + " capital of France?\",\"role\":\"user\",\"valid\":true}]", span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json"))); assertEquals( "project_name:unit-test-project", @@ -152,20 +153,17 @@ void testWrapOpenAi() { assertEquals( 0.0, span.getAttributes().get(AttributeKey.doubleKey("gen_ai.request.temperature"))); + String outputJson = span.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); assertNotNull(outputJson); - try { - var jsonNode = JSON_MAPPER.readTree(outputJson); - assertEquals("chatcmpl-test123", jsonNode.get("id").asText()); - assertEquals( - "The capital of France is Paris.", - jsonNode.get("choices").get(0).get("message").get("content").asText()); - assertEquals(8, jsonNode.get("usage").get("completion_tokens").asInt()); - assertEquals(20, jsonNode.get("usage").get("prompt_tokens").asInt()); - assertEquals(28, jsonNode.get("usage").get("total_tokens").asInt()); - } catch (Exception e) { - fail("Failed to parse output JSON: " + e.getMessage()); - } + var outputMessages = JSON_MAPPER.readTree(outputJson); + assertEquals(1, outputMessages.size()); + var messageZero = outputMessages.get(0); + assertEquals("The capital of France is Paris.", messageZero.get("content").asText()); + + assertEquals( + "chatcmpl-test123", + span.getAttributes().get(AttributeKey.stringKey("gen_ai.response.id"))); } }