diff --git a/docs/streaming.md b/docs/streaming.md index b34deda..da50b7b 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -66,13 +66,18 @@ See [examples/README.md](../examples/README.md) for details on supported instrum ### OTLP/JSON Support -Native OpenTelemetry format — no conversion to Jaeger needed: +Native OpenTelemetry format. The CLI auto-detects Jaeger vs OTLP from +file contents, so `.json` and `.jsonl` exports from Tempo, Jaeger, or +the OTel collector all work without a `--format` flag: ```bash -# Load OTLP files directly -agentevals run trace.otlp.json --format otlp-json --eval-set eval.json +# Load any trace file directly; format is auto-detected +agentevals run trace.otlp.json --eval-set eval.json ``` +Pass `--format otlp-json` (or `jaeger-json`) only as an override when +auto-detection fails on a non-standard export. + ### Real-time Span Streaming The `AgentEvalsStreamingProcessor` is an OTel `SpanProcessor` that streams spans over WebSocket as they complete: @@ -311,6 +316,7 @@ This installs `opentelemetry-sdk>=1.20.0`. Agent code also needs `websockets` fo ## Compatibility All existing workflows continue to work: -- Jaeger JSON files still supported: `agentevals run trace.json --eval-set ...` -- OTLP/JSON files: `agentevals run trace.otlp.json --format otlp-json --eval-set ...` -- Web UI upload flow unchanged +- Trace files (Jaeger or OTLP, including Tempo exports) auto-detect by + content: `agentevals run trace.json --eval-set ...` +- Pass `--format` only to override detection on non-standard exports. +- Web UI upload flow unchanged. diff --git a/samples/tempo_export_with_batches.json b/samples/tempo_export_with_batches.json new file mode 100644 index 0000000..dbd79a7 --- /dev/null +++ b/samples/tempo_export_with_batches.json @@ -0,0 +1 @@ +{"batches":[{"resource":{"attributes":[{"key":"k8s.node.name","value":{"stringValue":"k3d-playground-server-0"}},{"key":"service.instance.id","value":{"stringValue":"kagent-controller-55bdf57754-j845v"}},{"key":"service.namespace","value":{"stringValue":"kagent"}},{"key":"service.version","value":{"stringValue":"0.9.0"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.43.0"}},{"key":"service.name","value":{"stringValue":"kagent-controller"}},{"key":"k8s.namespace.name","value":{"stringValue":"kagent"}},{"key":"k8s.pod.name","value":{"stringValue":"kagent-controller-55bdf57754-j845v"}}],"droppedAttributesCount":0},"instrumentationLibrarySpans":[{"spans":[{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"21beb323d66610f7","parentSpanId":"cbd1b46f7435c01a","traceState":"","name":"GET /api/tasks/60494115-84c3-4c3c-a5a4-6cdd1e033e8d","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053780274700,"endTimeUnixNano":1777555053781172000,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"GET"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58452}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/tasks/60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.response.body.size","value":{"intValue":107}},{"key":"http.response.status_code","value":{"intValue":404}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}}],"instrumentationLibrary":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.68.0"}}]},{"resource":{"attributes":[{"key":"k8s.node.name","value":{"stringValue":"k3d-playground-server-0"}},{"key":"service.instance.id","value":{"stringValue":"kagent-controller-55bdf57754-j845v"}},{"key":"service.namespace","value":{"stringValue":"kagent"}},{"key":"service.version","value":{"stringValue":"0.9.0"}},{"key":"telemetry.sdk.language","value":{"stringValue":"go"}},{"key":"telemetry.sdk.name","value":{"stringValue":"opentelemetry"}},{"key":"telemetry.sdk.version","value":{"stringValue":"1.43.0"}},{"key":"service.name","value":{"stringValue":"kagent-controller"}},{"key":"k8s.namespace.name","value":{"stringValue":"kagent"}},{"key":"k8s.pod.name","value":{"stringValue":"kagent-controller-55bdf57754-j845v"}}],"droppedAttributesCount":0},"instrumentationLibrarySpans":[{"spans":[{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"6ee4d24cdd290f96","parentSpanId":"d424ac1bd701b78b","traceState":"","name":"GET /api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053780360200,"endTimeUnixNano":1777555053782409500,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"GET"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.response.body.size","value":{"intValue":323}},{"key":"http.response.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"bc38ad52ea67d5cb","parentSpanId":"27b41a6c0b925dff","traceState":"","name":"POST /api/tasks","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053782472400,"endTimeUnixNano":1777555053783400700,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58452}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/tasks"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":563}},{"key":"http.response.body.size","value":{"intValue":501}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"2210ea302d9c4f72","parentSpanId":"0e5feac38c144e3f","traceState":"","name":"POST /api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053783389400,"endTimeUnixNano":1777555053784635400,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":1344}},{"key":"http.response.body.size","value":{"intValue":1572}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"02e8c608db2d8426","parentSpanId":"103e37ba3346cbfd","traceState":"","name":"POST /api/tasks","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053784038100,"endTimeUnixNano":1777555053784767700,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58452}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/tasks"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":929}},{"key":"http.response.body.size","value":{"intValue":841}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"8f1ee9d0088ad9dc","parentSpanId":"da34d563e1c8eaff","traceState":"","name":"POST /api/tasks","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053786945000,"endTimeUnixNano":1777555053787402500,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58452}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/tasks"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":1071}},{"key":"http.response.body.size","value":{"intValue":984}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"5d7c994b536ef2f7","parentSpanId":"0649360b6ea9e714","traceState":"","name":"GET /api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053786816500,"endTimeUnixNano":1777555053787946200,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"GET"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.response.body.size","value":{"intValue":1838}},{"key":"http.response.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"f921fb9155315cfd","parentSpanId":"1019b56f89af25b9","traceState":"","name":"POST /api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053788824300,"endTimeUnixNano":1777555053789479700,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":1331}},{"key":"http.response.body.size","value":{"intValue":1559}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"bd050c929ef80beb","parentSpanId":"fb115e7d3347090d","traceState":"","name":"POST /api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055630320400,"endTimeUnixNano":1777555055631026400,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":1785}},{"key":"http.response.body.size","value":{"intValue":2013}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"9c121fe676a934fd","parentSpanId":"9582bfd90ee62cd1","traceState":"","name":"POST /api/tasks","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055632428000,"endTimeUnixNano":1777555055633255000,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/tasks"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":1968}},{"key":"http.response.body.size","value":{"intValue":1823}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"c542df14d5edefb5","parentSpanId":"105f194ca3e87a3a","traceState":"","name":"POST /api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055708264400,"endTimeUnixNano":1777555055709010400,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":1899}},{"key":"http.response.body.size","value":{"intValue":2127}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"27f2e480bf698e3d","parentSpanId":"11a02cdb9b309a95","traceState":"","name":"POST /api/tasks","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055710899700,"endTimeUnixNano":1777555055711456500,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/tasks"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":2971}},{"key":"http.response.body.size","value":{"intValue":2753}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"97ea9631e05881bc","parentSpanId":"ae7c169dfd5a4de5","traceState":"","name":"POST /api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058420126700,"endTimeUnixNano":1777555058421828000,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":2091}},{"key":"http.response.body.size","value":{"intValue":2319}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"14540555da0867f5","parentSpanId":"9692d2073638903c","traceState":"","name":"POST /api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058423689700,"endTimeUnixNano":1777555058424613400,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":1029}},{"key":"http.response.body.size","value":{"intValue":1257}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"ee7193800032f7b4","parentSpanId":"ca82dc6753899960","traceState":"","name":"POST /api/tasks","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058423944400,"endTimeUnixNano":1777555058424772600,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58452}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/tasks"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":3972}},{"key":"http.response.body.size","value":{"intValue":3665}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"b435c6a22f6b565d","parentSpanId":"1402eb8f871d9534","traceState":"","name":"POST /api/tasks","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058426807300,"endTimeUnixNano":1777555058427439900,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/tasks"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":4556}},{"key":"http.response.body.size","value":{"intValue":4185}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"80ea058cd3db7415","parentSpanId":"bb0edd95f81e9032","traceState":"","name":"POST /api/tasks","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058428932600,"endTimeUnixNano":1777555058429606400,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.37"}},{"key":"network.peer.port","value":{"intValue":58468}},{"key":"user_agent.original","value":{"stringValue":"python-httpx/0.28.1"}},{"key":"client.address","value":{"stringValue":"10.42.0.37"}},{"key":"url.path","value":{"stringValue":"/api/tasks"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":4563}},{"key":"http.response.body.size","value":{"intValue":4177}},{"key":"http.response.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"529566afa763bd4d","parentSpanId":"","traceState":"","name":"POST /api/a2a/kagent/helm-agent/","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053769217800,"endTimeUnixNano":1777555058430533600,"attributes":[{"key":"server.address","value":{"stringValue":"kagent-controller.kagent.svc.cluster.local"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.port","value":{"intValue":8083}},{"key":"network.peer.address","value":{"stringValue":"10.42.0.15"}},{"key":"network.peer.port","value":{"intValue":44692}},{"key":"user_agent.original","value":{"stringValue":"kagent-ui"}},{"key":"client.address","value":{"stringValue":"10.42.0.15"}},{"key":"url.path","value":{"stringValue":"/api/a2a/kagent/helm-agent/"}},{"key":"network.protocol.version","value":{"stringValue":"1.1"}},{"key":"http.request.body.size","value":{"intValue":357}},{"key":"http.response.body.size","value":{"intValue":6975}},{"key":"http.response.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"79f1c6b28f13ea1c","parentSpanId":"529566afa763bd4d","traceState":"","name":"invoke_agent","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053769345800,"endTimeUnixNano":1777555058430498800,"attributes":[{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"gen_ai.provider.name","value":{"stringValue":"openai"}},{"key":"gen_ai.agent.name","value":{"stringValue":"helm-agent"}},{"key":"gen_ai.agent.id","value":{"stringValue":"kagent/helm-agent"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}}],"instrumentationLibrary":{"name":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp","version":"0.68.0"}}]},{"resource":{"attributes":[{"key":"service.namespace","value":{"stringValue":"kagent"}},{"key":"service.name","value":{"stringValue":"helm_agent"}}],"droppedAttributesCount":0},"instrumentationLibrarySpans":[{"spans":[{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"00f4683d7842d93d","parentSpanId":"0e5deee1c91f77f8","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555053812873700,"endTimeUnixNano":1777555055622452200,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"https://api.openai.com/v1/chat/completions"}},{"key":"http.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"ec833f1aa1885d59","parentSpanId":"eb7f99f3e3ec5041","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555055626496300,"endTimeUnixNano":1777555055627517400,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-tools.kagent:8084/mcp"}},{"key":"http.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"fb115e7d3347090d","parentSpanId":"6ed9a13ee02d7cab","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555055630066200,"endTimeUnixNano":1777555055631201500,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-controller.kagent:8083/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events?user_id=admin@kagent.dev"}},{"key":"http.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"9582bfd90ee62cd1","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555055631933400,"endTimeUnixNano":1777555055633530000,"attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-controller.kagent:8083/api/tasks"}},{"key":"http.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"fdf1a098bd82cb4d","parentSpanId":"eb7f99f3e3ec5041","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555055632509700,"endTimeUnixNano":1777555055706847700,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-tools.kagent:8084/mcp"}},{"key":"http.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"105f194ca3e87a3a","parentSpanId":"6ed9a13ee02d7cab","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555055707955000,"endTimeUnixNano":1777555055709376800,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-controller.kagent:8083/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events?user_id=admin@kagent.dev"}},{"key":"http.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"11a02cdb9b309a95","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555055710262800,"endTimeUnixNano":1777555055711626800,"attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-controller.kagent:8083/api/tasks"}},{"key":"http.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"82b0f573c1c5e246","parentSpanId":"eb7f99f3e3ec5041","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555055710713000,"endTimeUnixNano":1777555055712163800,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-tools.kagent:8084/mcp"}},{"key":"http.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"d8dee7d32c9b8241","parentSpanId":"2373d7ea8819e064","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555055716542200,"endTimeUnixNano":1777555058416513500,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"https://api.openai.com/v1/chat/completions"}},{"key":"http.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"ae7c169dfd5a4de5","parentSpanId":"0eada367953b97e2","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555058419624000,"endTimeUnixNano":1777555058422067000,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-controller.kagent:8083/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events?user_id=admin@kagent.dev"}},{"key":"http.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"9692d2073638903c","parentSpanId":"eb7f99f3e3ec5041","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555058422930000,"endTimeUnixNano":1777555058424905700,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-controller.kagent:8083/api/sessions/ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f/events?user_id=admin@kagent.dev"}},{"key":"http.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"ca82dc6753899960","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555058423258400,"endTimeUnixNano":1777555058425677000,"attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-controller.kagent:8083/api/tasks"}},{"key":"http.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"1d0fd983890b0cd5","parentSpanId":"eb7f99f3e3ec5041","traceState":"","name":"DELETE","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555058426627000,"endTimeUnixNano":1777555058427198500,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"http.method","value":{"stringValue":"DELETE"}},{"key":"http.url","value":{"stringValue":"http://kagent-tools.kagent:8084/mcp"}},{"key":"http.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"1402eb8f871d9534","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555058426377200,"endTimeUnixNano":1777555058427742200,"attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-controller.kagent:8083/api/tasks"}},{"key":"http.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"bb0edd95f81e9032","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555058428554800,"endTimeUnixNano":1777555058429758200,"attributes":[{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://kagent-controller.kagent:8083/api/tasks"}},{"key":"http.status_code","value":{"intValue":201}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"0e5deee1c91f77f8","parentSpanId":"6ed9a13ee02d7cab","traceState":"","name":"openai.chat","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555053809433900,"endTimeUnixNano":1777555055625694500,"attributes":[{"key":"llm.request.type","value":{"stringValue":"chat"}},{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"gen_ai.system","value":{"stringValue":"openai"}},{"key":"gen_ai.request.model","value":{"stringValue":"gpt-4.1-mini"}},{"key":"llm.headers","value":{"stringValue":"None"}},{"key":"llm.is_streaming","value":{"boolValue":false}},{"key":"gen_ai.openai.api_base","value":{"stringValue":"https://api.openai.com/v1/"}},{"key":"gen_ai.prompt.0.role","value":{"stringValue":"system"}},{"key":"gen_ai.prompt.0.content","value":{"stringValue":"# Helm AI Agent System Prompt\n\nYou are an advanced AI agent specialized in Helm package management for Kubernetes. You possess deep expertise in Helm charts, releases, repositories, and best practices for deploying applications on Kubernetes using Helm. Your purpose is to help users manage, troubleshoot, and optimize their Helm deployments while following Kubernetes and Helm best practices.\n\n## Core Capabilities\n\n- **Helm Expertise**: You understand Helm architecture, chart structure, templating, dependencies, and release management.\n- **Chart Knowledge**: You can assist with using public charts, private repositories, and creating custom charts.\n- **Deployment Strategy**: You understand upgrade strategies, rollbacks, hooks, and release management.\n- **Kubernetes Integration**: You comprehend how Helm interacts with Kubernetes resources and API.\n- **Troubleshooting Skills**: You can diagnose and resolve common Helm-related issues effectively.\n\n## Operational Guidelines\n\n## Kubernetes Operational Context\n\nYou are operating within a Kubernetes cluster. Follow this methodology:\n\n### Investigation Protocol\n1. **Start non-intrusively**: Begin with read-only operations (get, describe, logs,\n events) before any modifications. Gather information before acting.\n2. **Progressive escalation**: Start with broad resource checks, then narrow down to\n specific resources, pods, containers, or logs as you identify the problem area.\n3. **Verify before acting**: Consider the potential impact of any change. Check\n dependencies, related resources, and downstream effects before modifying anything.\n\n### Problem-Solving Framework\n1. **Initial assessment**: Check cluster health, node status, and recent events. Identify\n the scope of the issue (single pod, deployment, namespace, or cluster-wide).\n2. **Classify the problem**: Determine if it's an application issue (crashes, errors),\n infrastructure problem (node failures, resource exhaustion), networking issue\n (connectivity, DNS, policies), or configuration error (invalid specs, missing secrets).\n3. **Analyze resources**: Examine pod status, container logs, resource metrics, events,\n and network connectivity relevant to the issue.\n4. **Implement solutions**: Propose targeted fixes. Present multiple options when\n appropriate, explain trade-offs, and include verification steps.\n\n### Key Principles\n- Always be explicit about which namespace you are operating in.\n- Remember that your actions impact real workloads — prioritize stability.\n- Use labels and selectors to target resources precisely.\n- Check resource quotas and limits before creating or scaling resources.\n\n\n## Available Tools\n\nYou have access to the following tools to help manage and troubleshoot Helm:\n\n### Helm Tools\n- `ListReleases`: List all Helm releases in a namespace with optional filtering.\n- `GetRelease`: Retrieve detailed information about a specific release, including manifests, hooks, values, and notes.\n- `Upgrade`: Upgrade or install a release to a new version of a chart.\n- `RepoUpdate`: Update the local Helm repositories to sync with the latest available charts.\n- `RepoAdd`: Add a new chart repository to the local configuration.\n\n### Kubernetes Tools\n- `GetResources`: Retrieve information about Kubernetes resources created by Helm releases.\n- `GetAvailableAPIResources`: View supported API resources in the cluster to verify compatibility with Helm charts.\n- `ApplyManifest`: Apply a YAML resource file to the cluster (useful for customizations).\n\n### Documentation Tools\n- `query_documentation`: Search documentation related to Helm, charts, and Kubernetes integration.\n\n## Tool Usage Best Practices\n\n## Tool Usage Best Practices\n\nFollow these principles when using tools:\n\n1. **Read before write**: Always use informational tools (get, describe, list, status)\n before modification tools. Understand the current state before making changes.\n2. **Explain before acting**: Before calling any modification tool, explain to the user\n what you intend to do and why. Wait for confirmation on destructive operations.\n3. **Dry-run when possible**: Use dry-run flags or preview modes when available to\n validate changes before applying them.\n4. **Use the right tool**: Select the most specific tool for the task. Check tool\n descriptions carefully and ensure you understand the expected parameters.\n5. **Backup current state**: Before modifications, capture the current state (e.g.,\n export YAML, save configuration) so changes can be reverted if needed.\n6. **Verify after changes**: After any modification, use informational tools to confirm\n the change took effect and didn't cause unintended side effects.\n7. **Handle errors gracefully**: If a tool call fails, analyze the error, adjust your\n approach, and retry. Do not repeat the exact same failing call.\n8. **Minimize calls**: Plan your approach to minimize tool calls. Batch operations when\n possible and avoid redundant calls.\n\n\n## Safety Protocols\n\n## Safety Guidelines\n\nYou must follow these safety principles in all interactions:\n\n1. **No destructive operations without confirmation**: Never delete resources, modify\n production systems, or take irreversible actions without explicit user confirmation.\n Always explain what will happen and ask for approval first.\n2. **Least privilege**: Apply changes to the minimum scope necessary. Prefer targeted\n operations over broad ones. Avoid cluster-wide changes when namespace-scoped changes\n will suffice.\n3. **Rollback planning**: Before making changes, ensure you have a plan to revert them.\n Capture the current state, explain the rollback procedure, and verify the rollback\n path exists before proceeding.\n4. **Protect sensitive data**: Never expose secrets, credentials, API keys, tokens, or\n certificates in your responses. Redact sensitive information before presenting output\n to the user.\n5. **Stay within scope**: Only perform actions within your defined capabilities and the\n user's stated intent. Do not take autonomous actions beyond what was requested.\n6. **Report uncertainties**: If you are unsure about an action or its consequences,\n communicate your uncertainty to the user and ask for guidance before proceeding.\n\n\n## Common Helm Operations\n\n### Adding and Managing Repositories\n```\n# Add a repository\nRepoAdd(name, url, [username], [password])\n\n# Update repositories\nRepoUpdate()\n```\n\n### Working with Releases\n```\n# List releases\nListReleases([namespace], [filter])\n\n# Get release details\nGetRelease(release_name, [option]) # Options: all, hooks, manifest, notes, values\n```\n\n### Installing and Upgrading\n```\n# Upgrade or install a release\nUpgrade(release_name, chart, [values], [version], [namespace])\n```\n\n### After Operations\n```\n# Verify Kubernetes resources\nGetResources(\"pods\", namespace)\nGetResources(\"services\", namespace)\nGetResources(\"deployments\", namespace)\n```\n\n## Limitations\n\n1. You cannot directly execute shell commands or use the Helm CLI directly.\n2. You must use the provided tools rather than suggesting raw kubectl or Helm commands.\n3. You cannot access local files on the user's system to read or create chart files.\n4. You cannot access external systems outside the Kubernetes cluster unless through configured repositories.\n\nAlways prioritize stability and correctness in Helm operations, and provide clear guidance on how to verify the success of operations.\n\nYou are an agent. Your internal name is \"helm_agent\". The description about you is \"The Helm Expert AI Agent specializing in using Helm for Kubernetes cluster management and operations. This agent is equipped with a range of tools to manage Helm releases and troubleshoot Helm-related issues.\"."}},{"key":"gen_ai.prompt.1.role","value":{"stringValue":"user"}},{"key":"gen_ai.prompt.1.content","value":{"stringValue":"list all helm releases\n"}},{"key":"llm.request.functions.0.name","value":{"stringValue":"helm_get_release"}},{"key":"llm.request.functions.0.description","value":{"stringValue":"Get extended information about a Helm release"}},{"key":"llm.request.functions.0.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"name\": {\"description\": \"The name of the release\", \"type\": \"string\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"string\"}, \"resource\": {\"description\": \"The resource to get (all, hooks, manifest, notes, values)\", \"type\": \"string\"}}, \"required\": [\"name\", \"namespace\"]}"}},{"key":"llm.request.functions.1.name","value":{"stringValue":"helm_list_releases"}},{"key":"llm.request.functions.1.description","value":{"stringValue":"List Helm releases in a namespace"}},{"key":"llm.request.functions.1.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"all\": {\"description\": \"Show all releases without any filter applied\", \"type\": \"string\"}, \"all_namespaces\": {\"description\": \"List releases from all namespaces\", \"type\": \"string\"}, \"deployed\": {\"description\": \"List deployed releases\", \"type\": \"string\"}, \"failed\": {\"description\": \"List failed releases\", \"type\": \"string\"}, \"filter\": {\"description\": \"A regular expression to filter releases by\", \"type\": \"string\"}, \"namespace\": {\"description\": \"The namespace to list releases from\", \"type\": \"string\"}, \"output\": {\"description\": \"The output format (e.g., 'json', 'yaml', 'table')\", \"type\": \"string\"}, \"pending\": {\"description\": \"List pending releases\", \"type\": \"string\"}, \"uninstalled\": {\"description\": \"List uninstalled releases\", \"type\": \"string\"}, \"uninstalling\": {\"description\": \"List uninstalling releases\", \"type\": \"string\"}}, \"required\": []}"}},{"key":"llm.request.functions.2.name","value":{"stringValue":"helm_repo_add"}},{"key":"llm.request.functions.2.description","value":{"stringValue":"Add a Helm repository"}},{"key":"llm.request.functions.2.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"name\": {\"description\": \"The name of the repository\", \"type\": \"string\"}, \"url\": {\"description\": \"The URL of the repository\", \"type\": \"string\"}}, \"required\": [\"name\", \"url\"]}"}},{"key":"llm.request.functions.3.name","value":{"stringValue":"helm_repo_update"}},{"key":"llm.request.functions.3.description","value":{"stringValue":"Update information of available charts locally from chart repositories"}},{"key":"llm.request.functions.3.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {}, \"required\": []}"}},{"key":"llm.request.functions.4.name","value":{"stringValue":"helm_uninstall"}},{"key":"llm.request.functions.4.description","value":{"stringValue":"Uninstall a Helm release"}},{"key":"llm.request.functions.4.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"dry_run\": {\"description\": \"Simulate an uninstall\", \"type\": \"string\"}, \"name\": {\"description\": \"The name of the release to uninstall\", \"type\": \"string\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"string\"}, \"wait\": {\"description\": \"Wait for the uninstall to complete\", \"type\": \"string\"}}, \"required\": [\"name\", \"namespace\"]}"}},{"key":"llm.request.functions.5.name","value":{"stringValue":"helm_upgrade"}},{"key":"llm.request.functions.5.description","value":{"stringValue":"Upgrade or install a Helm release"}},{"key":"llm.request.functions.5.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"chart\": {\"description\": \"The chart to install or upgrade to\", \"type\": \"string\"}, \"dry_run\": {\"description\": \"Simulate an upgrade\", \"type\": \"string\"}, \"install\": {\"description\": \"Run an install if the release is not present\", \"type\": \"string\"}, \"name\": {\"description\": \"The name of the release\", \"type\": \"string\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"string\"}, \"set\": {\"description\": \"Set values on the command line (e.g., 'key1=val1,key2=val2')\", \"type\": \"string\"}, \"values\": {\"description\": \"Path to a values file\", \"type\": \"string\"}, \"version\": {\"description\": \"The version of the chart to upgrade to\", \"type\": \"string\"}, \"wait\": {\"description\": \"Wait for the upgrade to complete\", \"type\": \"string\"}}, \"required\": [\"name\", \"chart\"]}"}},{"key":"llm.request.functions.6.name","value":{"stringValue":"k8s_apply_manifest"}},{"key":"llm.request.functions.6.description","value":{"stringValue":"Apply a YAML manifest to the Kubernetes cluster"}},{"key":"llm.request.functions.6.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"manifest\": {\"description\": \"YAML manifest content\", \"type\": \"string\"}}, \"required\": [\"manifest\"]}"}},{"key":"llm.request.functions.7.name","value":{"stringValue":"k8s_get_available_api_resources"}},{"key":"llm.request.functions.7.description","value":{"stringValue":"Get available Kubernetes API resources"}},{"key":"llm.request.functions.7.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {}, \"required\": []}"}},{"key":"llm.request.functions.8.name","value":{"stringValue":"k8s_get_resources"}},{"key":"llm.request.functions.8.description","value":{"stringValue":"Get Kubernetes resources using kubectl"}},{"key":"llm.request.functions.8.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"all_namespaces\": {\"description\": \"Query all namespaces (true/false)\", \"type\": \"string\"}, \"namespace\": {\"description\": \"Namespace to query (optional)\", \"type\": \"string\"}, \"output\": {\"default\": \"wide\", \"description\": \"Output format (json, yaml, wide)\", \"type\": \"string\"}, \"resource_name\": {\"description\": \"Name of specific resource (optional)\", \"type\": \"string\"}, \"resource_type\": {\"description\": \"Type of resource (pod, service, deployment, etc.)\", \"type\": \"string\"}}, \"required\": [\"resource_type\"]}"}},{"key":"llm.request.functions.9.name","value":{"stringValue":"ask_user"}},{"key":"llm.request.functions.9.description","value":{"stringValue":"Ask the user one or more questions and wait for their answers before continuing. Use this when you need clarifying information, preferences, or explicit confirmation from the user."}},{"key":"llm.request.functions.9.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"questions\": {\"description\": \"List of questions to ask the user.\", \"items\": {\"properties\": {\"question\": {\"description\": \"The question text to display to the user.\", \"type\": \"string\"}, \"choices\": {\"description\": \"Predefined answer choices shown as selectable chips. Leave empty for a free-text-only question.\", \"items\": {\"type\": \"string\"}, \"type\": \"array\"}, \"multiple\": {\"description\": \"If true, the user can select multiple choices. Defaults to false (single-select).\", \"type\": \"boolean\"}}, \"required\": [\"question\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"questions\"]}"}},{"key":"llm.request.reasoning_effort","value":{"stringValue":"[]"}},{"key":"gen_ai.response.model","value":{"stringValue":"gpt-4.1-mini-2025-04-14"}},{"key":"gen_ai.response.id","value":{"stringValue":"chatcmpl-DaLYlqJDD4djXUGg9TjyClK9BdeVV"}},{"key":"gen_ai.openai.system_fingerprint","value":{"stringValue":"fp_a1923f6a47"}},{"key":"openai.response.service_tier","value":{"stringValue":"default"}},{"key":"llm.usage.total_tokens","value":{"intValue":2269}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":13}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":2256}},{"key":"gen_ai.usage.cache_read_input_tokens","value":{"intValue":0}},{"key":"llm.usage.reasoning_tokens","value":{"intValue":0}},{"key":"gen_ai.completion.0.finish_reason","value":{"stringValue":"tool_calls"}},{"key":"gen_ai.completion.0.role","value":{"stringValue":"assistant"}},{"key":"gen_ai.completion.0.tool_calls.0.id","value":{"stringValue":"call_w0eKlvnaE7S9GQJeSSs0gn05"}},{"key":"gen_ai.completion.0.tool_calls.0.name","value":{"stringValue":"helm_list_releases"}},{"key":"gen_ai.completion.0.tool_calls.0.arguments","value":{"stringValue":"{}"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"2373d7ea8819e064","parentSpanId":"0eada367953b97e2","traceState":"","name":"openai.chat","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":1777555055715299000,"endTimeUnixNano":1777555058418614500,"attributes":[{"key":"llm.request.type","value":{"stringValue":"chat"}},{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"gen_ai.system","value":{"stringValue":"openai"}},{"key":"gen_ai.request.model","value":{"stringValue":"gpt-4.1-mini"}},{"key":"llm.headers","value":{"stringValue":"None"}},{"key":"llm.is_streaming","value":{"boolValue":false}},{"key":"gen_ai.openai.api_base","value":{"stringValue":"https://api.openai.com/v1/"}},{"key":"gen_ai.prompt.0.role","value":{"stringValue":"system"}},{"key":"gen_ai.prompt.0.content","value":{"stringValue":"# Helm AI Agent System Prompt\n\nYou are an advanced AI agent specialized in Helm package management for Kubernetes. You possess deep expertise in Helm charts, releases, repositories, and best practices for deploying applications on Kubernetes using Helm. Your purpose is to help users manage, troubleshoot, and optimize their Helm deployments while following Kubernetes and Helm best practices.\n\n## Core Capabilities\n\n- **Helm Expertise**: You understand Helm architecture, chart structure, templating, dependencies, and release management.\n- **Chart Knowledge**: You can assist with using public charts, private repositories, and creating custom charts.\n- **Deployment Strategy**: You understand upgrade strategies, rollbacks, hooks, and release management.\n- **Kubernetes Integration**: You comprehend how Helm interacts with Kubernetes resources and API.\n- **Troubleshooting Skills**: You can diagnose and resolve common Helm-related issues effectively.\n\n## Operational Guidelines\n\n## Kubernetes Operational Context\n\nYou are operating within a Kubernetes cluster. Follow this methodology:\n\n### Investigation Protocol\n1. **Start non-intrusively**: Begin with read-only operations (get, describe, logs,\n events) before any modifications. Gather information before acting.\n2. **Progressive escalation**: Start with broad resource checks, then narrow down to\n specific resources, pods, containers, or logs as you identify the problem area.\n3. **Verify before acting**: Consider the potential impact of any change. Check\n dependencies, related resources, and downstream effects before modifying anything.\n\n### Problem-Solving Framework\n1. **Initial assessment**: Check cluster health, node status, and recent events. Identify\n the scope of the issue (single pod, deployment, namespace, or cluster-wide).\n2. **Classify the problem**: Determine if it's an application issue (crashes, errors),\n infrastructure problem (node failures, resource exhaustion), networking issue\n (connectivity, DNS, policies), or configuration error (invalid specs, missing secrets).\n3. **Analyze resources**: Examine pod status, container logs, resource metrics, events,\n and network connectivity relevant to the issue.\n4. **Implement solutions**: Propose targeted fixes. Present multiple options when\n appropriate, explain trade-offs, and include verification steps.\n\n### Key Principles\n- Always be explicit about which namespace you are operating in.\n- Remember that your actions impact real workloads — prioritize stability.\n- Use labels and selectors to target resources precisely.\n- Check resource quotas and limits before creating or scaling resources.\n\n\n## Available Tools\n\nYou have access to the following tools to help manage and troubleshoot Helm:\n\n### Helm Tools\n- `ListReleases`: List all Helm releases in a namespace with optional filtering.\n- `GetRelease`: Retrieve detailed information about a specific release, including manifests, hooks, values, and notes.\n- `Upgrade`: Upgrade or install a release to a new version of a chart.\n- `RepoUpdate`: Update the local Helm repositories to sync with the latest available charts.\n- `RepoAdd`: Add a new chart repository to the local configuration.\n\n### Kubernetes Tools\n- `GetResources`: Retrieve information about Kubernetes resources created by Helm releases.\n- `GetAvailableAPIResources`: View supported API resources in the cluster to verify compatibility with Helm charts.\n- `ApplyManifest`: Apply a YAML resource file to the cluster (useful for customizations).\n\n### Documentation Tools\n- `query_documentation`: Search documentation related to Helm, charts, and Kubernetes integration.\n\n## Tool Usage Best Practices\n\n## Tool Usage Best Practices\n\nFollow these principles when using tools:\n\n1. **Read before write**: Always use informational tools (get, describe, list, status)\n before modification tools. Understand the current state before making changes.\n2. **Explain before acting**: Before calling any modification tool, explain to the user\n what you intend to do and why. Wait for confirmation on destructive operations.\n3. **Dry-run when possible**: Use dry-run flags or preview modes when available to\n validate changes before applying them.\n4. **Use the right tool**: Select the most specific tool for the task. Check tool\n descriptions carefully and ensure you understand the expected parameters.\n5. **Backup current state**: Before modifications, capture the current state (e.g.,\n export YAML, save configuration) so changes can be reverted if needed.\n6. **Verify after changes**: After any modification, use informational tools to confirm\n the change took effect and didn't cause unintended side effects.\n7. **Handle errors gracefully**: If a tool call fails, analyze the error, adjust your\n approach, and retry. Do not repeat the exact same failing call.\n8. **Minimize calls**: Plan your approach to minimize tool calls. Batch operations when\n possible and avoid redundant calls.\n\n\n## Safety Protocols\n\n## Safety Guidelines\n\nYou must follow these safety principles in all interactions:\n\n1. **No destructive operations without confirmation**: Never delete resources, modify\n production systems, or take irreversible actions without explicit user confirmation.\n Always explain what will happen and ask for approval first.\n2. **Least privilege**: Apply changes to the minimum scope necessary. Prefer targeted\n operations over broad ones. Avoid cluster-wide changes when namespace-scoped changes\n will suffice.\n3. **Rollback planning**: Before making changes, ensure you have a plan to revert them.\n Capture the current state, explain the rollback procedure, and verify the rollback\n path exists before proceeding.\n4. **Protect sensitive data**: Never expose secrets, credentials, API keys, tokens, or\n certificates in your responses. Redact sensitive information before presenting output\n to the user.\n5. **Stay within scope**: Only perform actions within your defined capabilities and the\n user's stated intent. Do not take autonomous actions beyond what was requested.\n6. **Report uncertainties**: If you are unsure about an action or its consequences,\n communicate your uncertainty to the user and ask for guidance before proceeding.\n\n\n## Common Helm Operations\n\n### Adding and Managing Repositories\n```\n# Add a repository\nRepoAdd(name, url, [username], [password])\n\n# Update repositories\nRepoUpdate()\n```\n\n### Working with Releases\n```\n# List releases\nListReleases([namespace], [filter])\n\n# Get release details\nGetRelease(release_name, [option]) # Options: all, hooks, manifest, notes, values\n```\n\n### Installing and Upgrading\n```\n# Upgrade or install a release\nUpgrade(release_name, chart, [values], [version], [namespace])\n```\n\n### After Operations\n```\n# Verify Kubernetes resources\nGetResources(\"pods\", namespace)\nGetResources(\"services\", namespace)\nGetResources(\"deployments\", namespace)\n```\n\n## Limitations\n\n1. You cannot directly execute shell commands or use the Helm CLI directly.\n2. You must use the provided tools rather than suggesting raw kubectl or Helm commands.\n3. You cannot access local files on the user's system to read or create chart files.\n4. You cannot access external systems outside the Kubernetes cluster unless through configured repositories.\n\nAlways prioritize stability and correctness in Helm operations, and provide clear guidance on how to verify the success of operations.\n\nYou are an agent. Your internal name is \"helm_agent\". The description about you is \"The Helm Expert AI Agent specializing in using Helm for Kubernetes cluster management and operations. This agent is equipped with a range of tools to manage Helm releases and troubleshoot Helm-related issues.\"."}},{"key":"gen_ai.prompt.1.role","value":{"stringValue":"user"}},{"key":"gen_ai.prompt.1.content","value":{"stringValue":"list all helm releases\n"}},{"key":"gen_ai.prompt.2.role","value":{"stringValue":"assistant"}},{"key":"gen_ai.prompt.2.tool_calls.0.id","value":{"stringValue":"call_w0eKlvnaE7S9GQJeSSs0gn05"}},{"key":"gen_ai.prompt.2.tool_calls.0.name","value":{"stringValue":"helm_list_releases"}},{"key":"gen_ai.prompt.2.tool_calls.0.arguments","value":{"stringValue":"{}"}},{"key":"gen_ai.prompt.3.role","value":{"stringValue":"tool"}},{"key":"gen_ai.prompt.3.content","value":{"stringValue":"NAME \tNAMESPACE\tREVISION\tUPDATED \tSTATUS \tCHART \tAPP VERSION\nkagent \tkagent \t5 \t2026-04-30 15:14:13.141794486 +0200 +0200\tdeployed\tkagent-0.9.0 \t \nkagent-crds\tkagent \t1 \t2026-04-30 14:51:26.764317537 +0200 +0200\tdeployed\tkagent-crds-0.9.0\t \n"}},{"key":"gen_ai.prompt.3.tool_call_id","value":{"stringValue":"call_w0eKlvnaE7S9GQJeSSs0gn05"}},{"key":"llm.request.functions.0.name","value":{"stringValue":"helm_get_release"}},{"key":"llm.request.functions.0.description","value":{"stringValue":"Get extended information about a Helm release"}},{"key":"llm.request.functions.0.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"name\": {\"description\": \"The name of the release\", \"type\": \"string\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"string\"}, \"resource\": {\"description\": \"The resource to get (all, hooks, manifest, notes, values)\", \"type\": \"string\"}}, \"required\": [\"name\", \"namespace\"]}"}},{"key":"llm.request.functions.1.name","value":{"stringValue":"helm_list_releases"}},{"key":"llm.request.functions.1.description","value":{"stringValue":"List Helm releases in a namespace"}},{"key":"llm.request.functions.1.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"all\": {\"description\": \"Show all releases without any filter applied\", \"type\": \"string\"}, \"all_namespaces\": {\"description\": \"List releases from all namespaces\", \"type\": \"string\"}, \"deployed\": {\"description\": \"List deployed releases\", \"type\": \"string\"}, \"failed\": {\"description\": \"List failed releases\", \"type\": \"string\"}, \"filter\": {\"description\": \"A regular expression to filter releases by\", \"type\": \"string\"}, \"namespace\": {\"description\": \"The namespace to list releases from\", \"type\": \"string\"}, \"output\": {\"description\": \"The output format (e.g., 'json', 'yaml', 'table')\", \"type\": \"string\"}, \"pending\": {\"description\": \"List pending releases\", \"type\": \"string\"}, \"uninstalled\": {\"description\": \"List uninstalled releases\", \"type\": \"string\"}, \"uninstalling\": {\"description\": \"List uninstalling releases\", \"type\": \"string\"}}, \"required\": []}"}},{"key":"llm.request.functions.2.name","value":{"stringValue":"helm_repo_add"}},{"key":"llm.request.functions.2.description","value":{"stringValue":"Add a Helm repository"}},{"key":"llm.request.functions.2.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"name\": {\"description\": \"The name of the repository\", \"type\": \"string\"}, \"url\": {\"description\": \"The URL of the repository\", \"type\": \"string\"}}, \"required\": [\"name\", \"url\"]}"}},{"key":"llm.request.functions.3.name","value":{"stringValue":"helm_repo_update"}},{"key":"llm.request.functions.3.description","value":{"stringValue":"Update information of available charts locally from chart repositories"}},{"key":"llm.request.functions.3.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {}, \"required\": []}"}},{"key":"llm.request.functions.4.name","value":{"stringValue":"helm_uninstall"}},{"key":"llm.request.functions.4.description","value":{"stringValue":"Uninstall a Helm release"}},{"key":"llm.request.functions.4.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"dry_run\": {\"description\": \"Simulate an uninstall\", \"type\": \"string\"}, \"name\": {\"description\": \"The name of the release to uninstall\", \"type\": \"string\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"string\"}, \"wait\": {\"description\": \"Wait for the uninstall to complete\", \"type\": \"string\"}}, \"required\": [\"name\", \"namespace\"]}"}},{"key":"llm.request.functions.5.name","value":{"stringValue":"helm_upgrade"}},{"key":"llm.request.functions.5.description","value":{"stringValue":"Upgrade or install a Helm release"}},{"key":"llm.request.functions.5.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"chart\": {\"description\": \"The chart to install or upgrade to\", \"type\": \"string\"}, \"dry_run\": {\"description\": \"Simulate an upgrade\", \"type\": \"string\"}, \"install\": {\"description\": \"Run an install if the release is not present\", \"type\": \"string\"}, \"name\": {\"description\": \"The name of the release\", \"type\": \"string\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"string\"}, \"set\": {\"description\": \"Set values on the command line (e.g., 'key1=val1,key2=val2')\", \"type\": \"string\"}, \"values\": {\"description\": \"Path to a values file\", \"type\": \"string\"}, \"version\": {\"description\": \"The version of the chart to upgrade to\", \"type\": \"string\"}, \"wait\": {\"description\": \"Wait for the upgrade to complete\", \"type\": \"string\"}}, \"required\": [\"name\", \"chart\"]}"}},{"key":"llm.request.functions.6.name","value":{"stringValue":"k8s_apply_manifest"}},{"key":"llm.request.functions.6.description","value":{"stringValue":"Apply a YAML manifest to the Kubernetes cluster"}},{"key":"llm.request.functions.6.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"manifest\": {\"description\": \"YAML manifest content\", \"type\": \"string\"}}, \"required\": [\"manifest\"]}"}},{"key":"llm.request.functions.7.name","value":{"stringValue":"k8s_get_available_api_resources"}},{"key":"llm.request.functions.7.description","value":{"stringValue":"Get available Kubernetes API resources"}},{"key":"llm.request.functions.7.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {}, \"required\": []}"}},{"key":"llm.request.functions.8.name","value":{"stringValue":"k8s_get_resources"}},{"key":"llm.request.functions.8.description","value":{"stringValue":"Get Kubernetes resources using kubectl"}},{"key":"llm.request.functions.8.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"all_namespaces\": {\"description\": \"Query all namespaces (true/false)\", \"type\": \"string\"}, \"namespace\": {\"description\": \"Namespace to query (optional)\", \"type\": \"string\"}, \"output\": {\"default\": \"wide\", \"description\": \"Output format (json, yaml, wide)\", \"type\": \"string\"}, \"resource_name\": {\"description\": \"Name of specific resource (optional)\", \"type\": \"string\"}, \"resource_type\": {\"description\": \"Type of resource (pod, service, deployment, etc.)\", \"type\": \"string\"}}, \"required\": [\"resource_type\"]}"}},{"key":"llm.request.functions.9.name","value":{"stringValue":"ask_user"}},{"key":"llm.request.functions.9.description","value":{"stringValue":"Ask the user one or more questions and wait for their answers before continuing. Use this when you need clarifying information, preferences, or explicit confirmation from the user."}},{"key":"llm.request.functions.9.parameters","value":{"stringValue":"{\"type\": \"object\", \"properties\": {\"questions\": {\"description\": \"List of questions to ask the user.\", \"items\": {\"properties\": {\"question\": {\"description\": \"The question text to display to the user.\", \"type\": \"string\"}, \"choices\": {\"description\": \"Predefined answer choices shown as selectable chips. Leave empty for a free-text-only question.\", \"items\": {\"type\": \"string\"}, \"type\": \"array\"}, \"multiple\": {\"description\": \"If true, the user can select multiple choices. Defaults to false (single-select).\", \"type\": \"boolean\"}}, \"required\": [\"question\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"questions\"]}"}},{"key":"llm.request.reasoning_effort","value":{"stringValue":"[]"}},{"key":"gen_ai.response.model","value":{"stringValue":"gpt-4.1-mini-2025-04-14"}},{"key":"gen_ai.response.id","value":{"stringValue":"chatcmpl-DaLYmIS19l0ClnUwPDWAYAmn2jz66"}},{"key":"gen_ai.openai.system_fingerprint","value":{"stringValue":"fp_49f260aaf7"}},{"key":"openai.response.service_tier","value":{"stringValue":"default"}},{"key":"llm.usage.total_tokens","value":{"intValue":2508}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":116}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":2392}},{"key":"gen_ai.usage.cache_read_input_tokens","value":{"intValue":0}},{"key":"llm.usage.reasoning_tokens","value":{"intValue":0}},{"key":"gen_ai.completion.0.finish_reason","value":{"stringValue":"stop"}},{"key":"gen_ai.completion.0.role","value":{"stringValue":"assistant"}},{"key":"gen_ai.completion.0.content","value":{"stringValue":"There are two Helm releases currently deployed:\n\n1. Release Name: kagent\n - Namespace: kagent\n - Revision: 5\n - Status: deployed\n - Chart: kagent version 0.9.0\n\n2. Release Name: kagent-crds\n - Namespace: kagent\n - Revision: 1\n - Status: deployed\n - Chart: kagent-crds version 0.9.0\n\nIf you need detailed information about any of these releases or further assistance, please let me know!"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"6120c51e098a9930","parentSpanId":"6ed9a13ee02d7cab","traceState":"","name":"a2a.server.events.event_queue.EventQueue.enqueue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055631622700,"endTimeUnixNano":1777555055631653400,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"f590c14cb43b5cf9","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.dequeue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055295051500,"endTimeUnixNano":1777555055631695600,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"3289ed232aac7849","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.request_handlers.default_request_handler.DefaultRequestHandler._send_push_notification_if_needed","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055633932000,"endTimeUnixNano":1777555055633941000,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"a8bc1c5117c846e2","parentSpanId":"6ed9a13ee02d7cab","traceState":"","name":"a2a.server.events.event_queue.EventQueue.enqueue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055709691400,"endTimeUnixNano":1777555055709708000,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"6ce87965d13532fa","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.dequeue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055634148600,"endTimeUnixNano":1777555055710057500,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"44dab2823a79392f","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.request_handlers.default_request_handler.DefaultRequestHandler._send_push_notification_if_needed","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055711903500,"endTimeUnixNano":1777555055711909000,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"555bcfa255cded6d","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.dequeue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055712050400,"endTimeUnixNano":1777555056212825600,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""},"events":[{"timeUnixNano":1777555056212815600,"attributes":[{"key":"exception.type","value":{"stringValue":"asyncio.exceptions.CancelledError"}},{"key":"exception.message","value":{"stringValue":""}},{"key":"exception.stacktrace","value":{"stringValue":"Traceback (most recent call last):\n File \"/.kagent/.venv/lib/python3.13/site-packages/a2a/utils/telemetry.py\", line 228, in async_wrapper\n result = await func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/.kagent/.venv/lib/python3.13/site-packages/a2a/server/events/event_queue.py\", line 111, in dequeue_event\n event = await self.queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^\n File \"/python/cpython-3.13.13-linux-aarch64-gnu/lib/python3.13/asyncio/queues.py\", line 186, in get\n await getter\nasyncio.exceptions.CancelledError\n"}},{"key":"exception.escaped","value":{"stringValue":"False"}}],"droppedAttributesCount":0,"name":"exception"}]},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"3e65429f0e5cda1d","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.dequeue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555056212903200,"endTimeUnixNano":1777555056714237700,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""},"events":[{"timeUnixNano":1777555056714214400,"attributes":[{"key":"exception.type","value":{"stringValue":"asyncio.exceptions.CancelledError"}},{"key":"exception.message","value":{"stringValue":""}},{"key":"exception.stacktrace","value":{"stringValue":"Traceback (most recent call last):\n File \"/.kagent/.venv/lib/python3.13/site-packages/a2a/utils/telemetry.py\", line 228, in async_wrapper\n result = await func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/.kagent/.venv/lib/python3.13/site-packages/a2a/server/events/event_queue.py\", line 111, in dequeue_event\n event = await self.queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^\n File \"/python/cpython-3.13.13-linux-aarch64-gnu/lib/python3.13/asyncio/queues.py\", line 186, in get\n await getter\nasyncio.exceptions.CancelledError\n"}},{"key":"exception.escaped","value":{"stringValue":"False"}}],"droppedAttributesCount":0,"name":"exception"}]},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"b834dd2e2f4cfd89","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.dequeue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555056714398700,"endTimeUnixNano":1777555057215979300,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""},"events":[{"timeUnixNano":1777555057215963100,"attributes":[{"key":"exception.type","value":{"stringValue":"asyncio.exceptions.CancelledError"}},{"key":"exception.message","value":{"stringValue":""}},{"key":"exception.stacktrace","value":{"stringValue":"Traceback (most recent call last):\n File \"/.kagent/.venv/lib/python3.13/site-packages/a2a/utils/telemetry.py\", line 228, in async_wrapper\n result = await func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/.kagent/.venv/lib/python3.13/site-packages/a2a/server/events/event_queue.py\", line 111, in dequeue_event\n event = await self.queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^\n File \"/python/cpython-3.13.13-linux-aarch64-gnu/lib/python3.13/asyncio/queues.py\", line 186, in get\n await getter\nasyncio.exceptions.CancelledError\n"}},{"key":"exception.escaped","value":{"stringValue":"False"}}],"droppedAttributesCount":0,"name":"exception"}]},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"099a6af72758513c","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.dequeue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555057216098000,"endTimeUnixNano":1777555057717266400,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""},"events":[{"timeUnixNano":1777555057717251600,"attributes":[{"key":"exception.type","value":{"stringValue":"asyncio.exceptions.CancelledError"}},{"key":"exception.message","value":{"stringValue":""}},{"key":"exception.stacktrace","value":{"stringValue":"Traceback (most recent call last):\n File \"/.kagent/.venv/lib/python3.13/site-packages/a2a/utils/telemetry.py\", line 228, in async_wrapper\n result = await func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/.kagent/.venv/lib/python3.13/site-packages/a2a/server/events/event_queue.py\", line 111, in dequeue_event\n event = await self.queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^\n File \"/python/cpython-3.13.13-linux-aarch64-gnu/lib/python3.13/asyncio/queues.py\", line 186, in get\n await getter\nasyncio.exceptions.CancelledError\n"}},{"key":"exception.escaped","value":{"stringValue":"False"}}],"droppedAttributesCount":0,"name":"exception"}]},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"acd2bfc954ec4d03","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.dequeue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555057717385700,"endTimeUnixNano":1777555058219534000,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""},"events":[{"timeUnixNano":1777555058219524000,"attributes":[{"key":"exception.type","value":{"stringValue":"asyncio.exceptions.CancelledError"}},{"key":"exception.message","value":{"stringValue":""}},{"key":"exception.stacktrace","value":{"stringValue":"Traceback (most recent call last):\n File \"/.kagent/.venv/lib/python3.13/site-packages/a2a/utils/telemetry.py\", line 228, in async_wrapper\n result = await func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/.kagent/.venv/lib/python3.13/site-packages/a2a/server/events/event_queue.py\", line 111, in dequeue_event\n event = await self.queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^\n File \"/python/cpython-3.13.13-linux-aarch64-gnu/lib/python3.13/asyncio/queues.py\", line 186, in get\n await getter\nasyncio.exceptions.CancelledError\n"}},{"key":"exception.escaped","value":{"stringValue":"False"}}],"droppedAttributesCount":0,"name":"exception"}]},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"c920635836d51fa8","parentSpanId":"0eada367953b97e2","traceState":"","name":"a2a.server.events.event_queue.EventQueue.enqueue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058422540300,"endTimeUnixNano":1777555058422563600,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"13dda15df808b10a","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.dequeue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058219613200,"endTimeUnixNano":1777555058423021000,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"b82eb8ddd06948e6","parentSpanId":"c05a7cb43a326b4f","traceState":"","name":"a2a.server.events.event_queue.EventQueue.enqueue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058425385200,"endTimeUnixNano":1777555058425402400,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"f99433b985713932","parentSpanId":"c05a7cb43a326b4f","traceState":"","name":"a2a.server.events.event_queue.EventQueue.enqueue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058425439200,"endTimeUnixNano":1777555058425448400,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"5420dbdd83fdefd2","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.request_handlers.default_request_handler.DefaultRequestHandler._send_push_notification_if_needed","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058425892900,"endTimeUnixNano":1777555058425899300,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"0dc02d1e22ee46fa","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.dequeue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058426071600,"endTimeUnixNano":1777555058426081500,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"f8898b20de40ae03","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.request_handlers.default_request_handler.DefaultRequestHandler._send_push_notification_if_needed","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058427983000,"endTimeUnixNano":1777555058427991800,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"76e65538a0a150d4","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.dequeue_event","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058428235500,"endTimeUnixNano":1777555058428246800,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"7a2cd31698d336dc","parentSpanId":"c1e29d3f22c8a474","traceState":"","name":"a2a.server.events.event_queue.EventQueue.clear_events","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058428312000,"endTimeUnixNano":1777555058428320800,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"c1e29d3f22c8a474","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.close","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058428293600,"endTimeUnixNano":1777555058428324400,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"ae384d96703fbf0f","parentSpanId":"c05a7cb43a326b4f","traceState":"","name":"a2a.server.events.event_queue.EventQueue.close","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058428985300,"endTimeUnixNano":1777555058428993300,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"c05a7cb43a326b4f","parentSpanId":"633f5402693da75e","traceState":"","name":"a2a.server.request_handlers.default_request_handler.DefaultRequestHandler._run_event_stream","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053777104000,"endTimeUnixNano":1777555058428999700,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"26069280b2cb2c51","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.request_handlers.default_request_handler.DefaultRequestHandler._send_push_notification_if_needed","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058430033400,"endTimeUnixNano":1777555058430041900,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"aec8f1671713cded","parentSpanId":"0746addd26dcf26f","traceState":"","name":"a2a.server.events.event_queue.EventQueue.close","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058430309600,"endTimeUnixNano":1777555058430314800,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"0746addd26dcf26f","parentSpanId":"875b650ecb1653ad","traceState":"","name":"a2a.server.events.in_memory_queue_manager.InMemoryQueueManager.close","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058430295000,"endTimeUnixNano":1777555058430320400,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"875b650ecb1653ad","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.request_handlers.default_request_handler.DefaultRequestHandler._cleanup_producer","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058430277600,"endTimeUnixNano":1777555058430325500,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"9ef5d889e6242016","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.task_done","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055631717600,"endTimeUnixNano":1777555055631723300,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"986b181d98c29eca","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.task_done","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555055710082600,"endTimeUnixNano":1777555055710087700,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"5d60752bd9b96960","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.task_done","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058423053800,"endTimeUnixNano":1777555058423060000,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"9e8137a16a5c483d","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.task_done","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058426109400,"endTimeUnixNano":1777555058426114600,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"5e9eafc9dfac6bc2","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.utils.helpers.append_artifact_to_task","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555058426135300,"endTimeUnixNano":1777555058426156000,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"85b7ef57360d299b","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_queue.EventQueue.task_done","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058428273200,"endTimeUnixNano":1777555058428278000,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"41bca363c5c0fbd3","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.events.event_consumer.EventConsumer.agent_task_callback","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058429020000,"endTimeUnixNano":1777555058429025000,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"2705c8df1e58d3f1","parentSpanId":"e42666557c259fbf","traceState":"","name":"a2a.server.request_handlers.default_request_handler.DefaultRequestHandler._track_background_task","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555058430219500,"endTimeUnixNano":1777555058430227000,"attributes":[],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":1,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"d0122def955bb799","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST / http send","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555055634050600,"endTimeUnixNano":1777555055634057200,"attributes":[{"key":"asgi.event.type","value":{"stringValue":"http.response.body"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"4e04aa9e2449978f","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST / http send","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555055711984600,"endTimeUnixNano":1777555055711989500,"attributes":[{"key":"asgi.event.type","value":{"stringValue":"http.response.body"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"a68773cbe6a3a06a","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST / http send","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555058425986600,"endTimeUnixNano":1777555058425992700,"attributes":[{"key":"asgi.event.type","value":{"stringValue":"http.response.body"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"ffd63338b6ec4c09","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST / http send","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555058428160000,"endTimeUnixNano":1777555058428167200,"attributes":[{"key":"asgi.event.type","value":{"stringValue":"http.response.body"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"536ce00ef86dc4e0","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST / http send","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555058430109700,"endTimeUnixNano":1777555058430115800,"attributes":[{"key":"asgi.event.type","value":{"stringValue":"http.response.body"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"10517cc3adc99a21","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST / http send","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555058430344200,"endTimeUnixNano":1777555058430348500,"attributes":[{"key":"asgi.event.type","value":{"stringValue":"http.response.body"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"e42666557c259fbf","parentSpanId":"79f1c6b28f13ea1c","traceState":"","name":"POST /","kind":"SPAN_KIND_SERVER","startTimeUnixNano":1777555053771001900,"endTimeUnixNano":1777555058430403600,"attributes":[{"key":"http.scheme","value":{"stringValue":"http"}},{"key":"http.host","value":{"stringValue":"10.42.0.37:8080"}},{"key":"net.host.port","value":{"intValue":8080}},{"key":"http.flavor","value":{"stringValue":"1.1"}},{"key":"http.target","value":{"stringValue":"/"}},{"key":"http.server_name","value":{"stringValue":"helm-agent.kagent:8080"}},{"key":"http.user_agent","value":{"stringValue":"trpc-a2a-go-client/0.1"}},{"key":"net.peer.ip","value":{"stringValue":"10.42.0.31"}},{"key":"net.peer.port","value":{"intValue":39776}},{"key":"http.route","value":{"stringValue":"/"}},{"key":"http.method","value":{"stringValue":"POST"}},{"key":"http.url","value":{"stringValue":"http://helm-agent.kagent:8080/"}},{"key":"http.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"dfd6b6cf79ba9e51","parentSpanId":"e42666557c259fbf","traceState":"","name":"POST / http receive","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555053777080300,"endTimeUnixNano":1777555058430435800,"attributes":[{"key":"asgi.event.type","value":{"stringValue":"http.disconnect"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"def5801af66aaae7","parentSpanId":"6ed9a13ee02d7cab","traceState":"","name":"execute_tool helm_list_releases","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555055632028200,"endTimeUnixNano":1777555055707612400,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"gen_ai.operation.name","value":{"stringValue":"execute_tool"}},{"key":"gen_ai.tool.description","value":{"stringValue":"List Helm releases in a namespace"}},{"key":"gen_ai.tool.name","value":{"stringValue":"helm_list_releases"}},{"key":"gen_ai.tool.type","value":{"stringValue":"ConnectionSafeMcpTool"}},{"key":"gcp.vertex.agent.llm_request","value":{"stringValue":"{}"}},{"key":"gcp.vertex.agent.llm_response","value":{"stringValue":"{}"}},{"key":"gcp.vertex.agent.tool_call_args","value":{"stringValue":"{}"}},{"key":"gen_ai.tool.call.id","value":{"stringValue":"call_w0eKlvnaE7S9GQJeSSs0gn05"}},{"key":"gcp.vertex.agent.event_id","value":{"stringValue":"3ac0b872-69a5-4835-b53e-a7adad01e75f"}},{"key":"gcp.vertex.agent.tool_response","value":{"stringValue":"{\"content\": [{\"type\": \"text\", \"text\": \"NAME \\tNAMESPACE\\tREVISION\\tUPDATED \\tSTATUS \\tCHART \\tAPP VERSION\\nkagent \\tkagent \\t5 \\t2026-04-30 15:14:13.141794486 +0200 +0200\\tdeployed\\tkagent-0.9.0 \\t \\nkagent-crds\\tkagent \\t1 \\t2026-04-30 14:51:26.764317537 +0200 +0200\\tdeployed\\tkagent-crds-0.9.0\\t \\n\"}], \"isError\": false}"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"6ed9a13ee02d7cab","parentSpanId":"ef7e626b81d68000","traceState":"","name":"generate_content gpt-4.1-mini","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555053806271700,"endTimeUnixNano":1777555055709734400,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.system","value":{"stringValue":"gemini"}},{"key":"gen_ai.operation.name","value":{"stringValue":"generate_content"}},{"key":"gen_ai.request.model","value":{"stringValue":"gpt-4.1-mini"}},{"key":"gen_ai.agent.name","value":{"stringValue":"helm_agent"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"user.id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gcp.vertex.agent.event_id","value":{"stringValue":"54a39347-2b5f-47cc-af44-2c15df27934c"}},{"key":"gcp.vertex.agent.invocation_id","value":{"stringValue":"e-fd4599ca-7ab2-47c2-9efe-0c61a5d579f3"}},{"key":"gen_ai.response.finish_reasons","value":{"stringValue":"[\"stop\"]"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":2256}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":13}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"ef7e626b81d68000","parentSpanId":"eb7f99f3e3ec5041","traceState":"","name":"call_llm","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555053806128400,"endTimeUnixNano":1777555055709739800,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"gen_ai.system","value":{"stringValue":"gcp.vertex.agent"}},{"key":"gen_ai.request.model","value":{"stringValue":"gpt-4.1-mini"}},{"key":"gcp.vertex.agent.invocation_id","value":{"stringValue":"e-fd4599ca-7ab2-47c2-9efe-0c61a5d579f3"}},{"key":"gcp.vertex.agent.session_id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"gcp.vertex.agent.event_id","value":{"stringValue":"54a39347-2b5f-47cc-af44-2c15df27934c"}},{"key":"gcp.vertex.agent.llm_request","value":{"stringValue":"{\"model\": \"gpt-4.1-mini\", \"config\": {\"system_instruction\": \"# Helm AI Agent System Prompt\\n\\nYou are an advanced AI agent specialized in Helm package management for Kubernetes. You possess deep expertise in Helm charts, releases, repositories, and best practices for deploying applications on Kubernetes using Helm. Your purpose is to help users manage, troubleshoot, and optimize their Helm deployments while following Kubernetes and Helm best practices.\\n\\n## Core Capabilities\\n\\n- **Helm Expertise**: You understand Helm architecture, chart structure, templating, dependencies, and release management.\\n- **Chart Knowledge**: You can assist with using public charts, private repositories, and creating custom charts.\\n- **Deployment Strategy**: You understand upgrade strategies, rollbacks, hooks, and release management.\\n- **Kubernetes Integration**: You comprehend how Helm interacts with Kubernetes resources and API.\\n- **Troubleshooting Skills**: You can diagnose and resolve common Helm-related issues effectively.\\n\\n## Operational Guidelines\\n\\n## Kubernetes Operational Context\\n\\nYou are operating within a Kubernetes cluster. Follow this methodology:\\n\\n### Investigation Protocol\\n1. **Start non-intrusively**: Begin with read-only operations (get, describe, logs,\\n events) before any modifications. Gather information before acting.\\n2. **Progressive escalation**: Start with broad resource checks, then narrow down to\\n specific resources, pods, containers, or logs as you identify the problem area.\\n3. **Verify before acting**: Consider the potential impact of any change. Check\\n dependencies, related resources, and downstream effects before modifying anything.\\n\\n### Problem-Solving Framework\\n1. **Initial assessment**: Check cluster health, node status, and recent events. Identify\\n the scope of the issue (single pod, deployment, namespace, or cluster-wide).\\n2. **Classify the problem**: Determine if it's an application issue (crashes, errors),\\n infrastructure problem (node failures, resource exhaustion), networking issue\\n (connectivity, DNS, policies), or configuration error (invalid specs, missing secrets).\\n3. **Analyze resources**: Examine pod status, container logs, resource metrics, events,\\n and network connectivity relevant to the issue.\\n4. **Implement solutions**: Propose targeted fixes. Present multiple options when\\n appropriate, explain trade-offs, and include verification steps.\\n\\n### Key Principles\\n- Always be explicit about which namespace you are operating in.\\n- Remember that your actions impact real workloads — prioritize stability.\\n- Use labels and selectors to target resources precisely.\\n- Check resource quotas and limits before creating or scaling resources.\\n\\n\\n## Available Tools\\n\\nYou have access to the following tools to help manage and troubleshoot Helm:\\n\\n### Helm Tools\\n- `ListReleases`: List all Helm releases in a namespace with optional filtering.\\n- `GetRelease`: Retrieve detailed information about a specific release, including manifests, hooks, values, and notes.\\n- `Upgrade`: Upgrade or install a release to a new version of a chart.\\n- `RepoUpdate`: Update the local Helm repositories to sync with the latest available charts.\\n- `RepoAdd`: Add a new chart repository to the local configuration.\\n\\n### Kubernetes Tools\\n- `GetResources`: Retrieve information about Kubernetes resources created by Helm releases.\\n- `GetAvailableAPIResources`: View supported API resources in the cluster to verify compatibility with Helm charts.\\n- `ApplyManifest`: Apply a YAML resource file to the cluster (useful for customizations).\\n\\n### Documentation Tools\\n- `query_documentation`: Search documentation related to Helm, charts, and Kubernetes integration.\\n\\n## Tool Usage Best Practices\\n\\n## Tool Usage Best Practices\\n\\nFollow these principles when using tools:\\n\\n1. **Read before write**: Always use informational tools (get, describe, list, status)\\n before modification tools. Understand the current state before making changes.\\n2. **Explain before acting**: Before calling any modification tool, explain to the user\\n what you intend to do and why. Wait for confirmation on destructive operations.\\n3. **Dry-run when possible**: Use dry-run flags or preview modes when available to\\n validate changes before applying them.\\n4. **Use the right tool**: Select the most specific tool for the task. Check tool\\n descriptions carefully and ensure you understand the expected parameters.\\n5. **Backup current state**: Before modifications, capture the current state (e.g.,\\n export YAML, save configuration) so changes can be reverted if needed.\\n6. **Verify after changes**: After any modification, use informational tools to confirm\\n the change took effect and didn't cause unintended side effects.\\n7. **Handle errors gracefully**: If a tool call fails, analyze the error, adjust your\\n approach, and retry. Do not repeat the exact same failing call.\\n8. **Minimize calls**: Plan your approach to minimize tool calls. Batch operations when\\n possible and avoid redundant calls.\\n\\n\\n## Safety Protocols\\n\\n## Safety Guidelines\\n\\nYou must follow these safety principles in all interactions:\\n\\n1. **No destructive operations without confirmation**: Never delete resources, modify\\n production systems, or take irreversible actions without explicit user confirmation.\\n Always explain what will happen and ask for approval first.\\n2. **Least privilege**: Apply changes to the minimum scope necessary. Prefer targeted\\n operations over broad ones. Avoid cluster-wide changes when namespace-scoped changes\\n will suffice.\\n3. **Rollback planning**: Before making changes, ensure you have a plan to revert them.\\n Capture the current state, explain the rollback procedure, and verify the rollback\\n path exists before proceeding.\\n4. **Protect sensitive data**: Never expose secrets, credentials, API keys, tokens, or\\n certificates in your responses. Redact sensitive information before presenting output\\n to the user.\\n5. **Stay within scope**: Only perform actions within your defined capabilities and the\\n user's stated intent. Do not take autonomous actions beyond what was requested.\\n6. **Report uncertainties**: If you are unsure about an action or its consequences,\\n communicate your uncertainty to the user and ask for guidance before proceeding.\\n\\n\\n## Common Helm Operations\\n\\n### Adding and Managing Repositories\\n```\\n# Add a repository\\nRepoAdd(name, url, [username], [password])\\n\\n# Update repositories\\nRepoUpdate()\\n```\\n\\n### Working with Releases\\n```\\n# List releases\\nListReleases([namespace], [filter])\\n\\n# Get release details\\nGetRelease(release_name, [option]) # Options: all, hooks, manifest, notes, values\\n```\\n\\n### Installing and Upgrading\\n```\\n# Upgrade or install a release\\nUpgrade(release_name, chart, [values], [version], [namespace])\\n```\\n\\n### After Operations\\n```\\n# Verify Kubernetes resources\\nGetResources(\\\"pods\\\", namespace)\\nGetResources(\\\"services\\\", namespace)\\nGetResources(\\\"deployments\\\", namespace)\\n```\\n\\n## Limitations\\n\\n1. You cannot directly execute shell commands or use the Helm CLI directly.\\n2. You must use the provided tools rather than suggesting raw kubectl or Helm commands.\\n3. You cannot access local files on the user's system to read or create chart files.\\n4. You cannot access external systems outside the Kubernetes cluster unless through configured repositories.\\n\\nAlways prioritize stability and correctness in Helm operations, and provide clear guidance on how to verify the success of operations.\\n\\nYou are an agent. Your internal name is \\\"helm_agent\\\". The description about you is \\\"The Helm Expert AI Agent specializing in using Helm for Kubernetes cluster management and operations. This agent is equipped with a range of tools to manage Helm releases and troubleshoot Helm-related issues.\\\".\", \"tools\": [{\"function_declarations\": [{\"description\": \"Get extended information about a Helm release\", \"name\": \"helm_get_release\", \"parameters\": {\"properties\": {\"name\": {\"description\": \"The name of the release\", \"type\": \"STRING\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"STRING\"}, \"resource\": {\"description\": \"The resource to get (all, hooks, manifest, notes, values)\", \"type\": \"STRING\"}}, \"required\": [\"name\", \"namespace\"], \"type\": \"OBJECT\"}}, {\"description\": \"List Helm releases in a namespace\", \"name\": \"helm_list_releases\", \"parameters\": {\"properties\": {\"all\": {\"description\": \"Show all releases without any filter applied\", \"type\": \"STRING\"}, \"all_namespaces\": {\"description\": \"List releases from all namespaces\", \"type\": \"STRING\"}, \"deployed\": {\"description\": \"List deployed releases\", \"type\": \"STRING\"}, \"failed\": {\"description\": \"List failed releases\", \"type\": \"STRING\"}, \"filter\": {\"description\": \"A regular expression to filter releases by\", \"type\": \"STRING\"}, \"namespace\": {\"description\": \"The namespace to list releases from\", \"type\": \"STRING\"}, \"output\": {\"description\": \"The output format (e.g., 'json', 'yaml', 'table')\", \"type\": \"STRING\"}, \"pending\": {\"description\": \"List pending releases\", \"type\": \"STRING\"}, \"uninstalled\": {\"description\": \"List uninstalled releases\", \"type\": \"STRING\"}, \"uninstalling\": {\"description\": \"List uninstalling releases\", \"type\": \"STRING\"}}, \"type\": \"OBJECT\"}}, {\"description\": \"Add a Helm repository\", \"name\": \"helm_repo_add\", \"parameters\": {\"properties\": {\"name\": {\"description\": \"The name of the repository\", \"type\": \"STRING\"}, \"url\": {\"description\": \"The URL of the repository\", \"type\": \"STRING\"}}, \"required\": [\"name\", \"url\"], \"type\": \"OBJECT\"}}, {\"description\": \"Update information of available charts locally from chart repositories\", \"name\": \"helm_repo_update\", \"parameters\": {\"type\": \"OBJECT\"}}, {\"description\": \"Uninstall a Helm release\", \"name\": \"helm_uninstall\", \"parameters\": {\"properties\": {\"dry_run\": {\"description\": \"Simulate an uninstall\", \"type\": \"STRING\"}, \"name\": {\"description\": \"The name of the release to uninstall\", \"type\": \"STRING\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"STRING\"}, \"wait\": {\"description\": \"Wait for the uninstall to complete\", \"type\": \"STRING\"}}, \"required\": [\"name\", \"namespace\"], \"type\": \"OBJECT\"}}, {\"description\": \"Upgrade or install a Helm release\", \"name\": \"helm_upgrade\", \"parameters\": {\"properties\": {\"chart\": {\"description\": \"The chart to install or upgrade to\", \"type\": \"STRING\"}, \"dry_run\": {\"description\": \"Simulate an upgrade\", \"type\": \"STRING\"}, \"install\": {\"description\": \"Run an install if the release is not present\", \"type\": \"STRING\"}, \"name\": {\"description\": \"The name of the release\", \"type\": \"STRING\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"STRING\"}, \"set\": {\"description\": \"Set values on the command line (e.g., 'key1=val1,key2=val2')\", \"type\": \"STRING\"}, \"values\": {\"description\": \"Path to a values file\", \"type\": \"STRING\"}, \"version\": {\"description\": \"The version of the chart to upgrade to\", \"type\": \"STRING\"}, \"wait\": {\"description\": \"Wait for the upgrade to complete\", \"type\": \"STRING\"}}, \"required\": [\"name\", \"chart\"], \"type\": \"OBJECT\"}}, {\"description\": \"Apply a YAML manifest to the Kubernetes cluster\", \"name\": \"k8s_apply_manifest\", \"parameters\": {\"properties\": {\"manifest\": {\"description\": \"YAML manifest content\", \"type\": \"STRING\"}}, \"required\": [\"manifest\"], \"type\": \"OBJECT\"}}, {\"description\": \"Get available Kubernetes API resources\", \"name\": \"k8s_get_available_api_resources\", \"parameters\": {\"type\": \"OBJECT\"}}, {\"description\": \"Get Kubernetes resources using kubectl\", \"name\": \"k8s_get_resources\", \"parameters\": {\"properties\": {\"all_namespaces\": {\"description\": \"Query all namespaces (true/false)\", \"type\": \"STRING\"}, \"namespace\": {\"description\": \"Namespace to query (optional)\", \"type\": \"STRING\"}, \"output\": {\"default\": \"wide\", \"description\": \"Output format (json, yaml, wide)\", \"type\": \"STRING\"}, \"resource_name\": {\"description\": \"Name of specific resource (optional)\", \"type\": \"STRING\"}, \"resource_type\": {\"description\": \"Type of resource (pod, service, deployment, etc.)\", \"type\": \"STRING\"}}, \"required\": [\"resource_type\"], \"type\": \"OBJECT\"}}, {\"description\": \"Ask the user one or more questions and wait for their answers before continuing. Use this when you need clarifying information, preferences, or explicit confirmation from the user.\", \"name\": \"ask_user\", \"parameters\": {\"properties\": {\"questions\": {\"description\": \"List of questions to ask the user.\", \"items\": {\"properties\": {\"question\": {\"description\": \"The question text to display to the user.\", \"type\": \"STRING\"}, \"choices\": {\"description\": \"Predefined answer choices shown as selectable chips. Leave empty for a free-text-only question.\", \"items\": {\"type\": \"STRING\"}, \"type\": \"ARRAY\"}, \"multiple\": {\"description\": \"If true, the user can select multiple choices. Defaults to false (single-select).\", \"type\": \"BOOLEAN\"}}, \"required\": [\"question\"], \"type\": \"OBJECT\"}, \"type\": \"ARRAY\"}}, \"required\": [\"questions\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"helm_agent\"}}, \"contents\": [{\"parts\": [{\"text\": \"list all helm releases\\n\"}], \"role\": \"user\"}]}"}},{"key":"gcp.vertex.agent.llm_response","value":{"stringValue":"{\"content\":{\"parts\":[{\"function_call\":{\"id\":\"call_w0eKlvnaE7S9GQJeSSs0gn05\",\"args\":{},\"name\":\"helm_list_releases\"}}],\"role\":\"model\"},\"finish_reason\":\"STOP\",\"usage_metadata\":{\"candidates_token_count\":13,\"prompt_token_count\":2256,\"total_token_count\":2269}}"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":2256}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":13}},{"key":"gen_ai.response.finish_reasons","value":{"stringValue":"[\"stop\"]"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"0eada367953b97e2","parentSpanId":"c8186a2f55581ff1","traceState":"","name":"generate_content gpt-4.1-mini","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555055715197400,"endTimeUnixNano":1777555058422600700,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.system","value":{"stringValue":"gemini"}},{"key":"gen_ai.operation.name","value":{"stringValue":"generate_content"}},{"key":"gen_ai.request.model","value":{"stringValue":"gpt-4.1-mini"}},{"key":"gen_ai.agent.name","value":{"stringValue":"helm_agent"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"user.id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gcp.vertex.agent.event_id","value":{"stringValue":"a1c478a2-7013-44ae-ba4f-01fd7b9f3e0c"}},{"key":"gcp.vertex.agent.invocation_id","value":{"stringValue":"e-fd4599ca-7ab2-47c2-9efe-0c61a5d579f3"}},{"key":"gen_ai.response.finish_reasons","value":{"stringValue":"[\"stop\"]"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":2392}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":116}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"c8186a2f55581ff1","parentSpanId":"eb7f99f3e3ec5041","traceState":"","name":"call_llm","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555055715163400,"endTimeUnixNano":1777555058422608400,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"gen_ai.system","value":{"stringValue":"gcp.vertex.agent"}},{"key":"gen_ai.request.model","value":{"stringValue":"gpt-4.1-mini"}},{"key":"gcp.vertex.agent.invocation_id","value":{"stringValue":"e-fd4599ca-7ab2-47c2-9efe-0c61a5d579f3"}},{"key":"gcp.vertex.agent.session_id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}},{"key":"gcp.vertex.agent.event_id","value":{"stringValue":"a1c478a2-7013-44ae-ba4f-01fd7b9f3e0c"}},{"key":"gcp.vertex.agent.llm_request","value":{"stringValue":"{\"model\": \"gpt-4.1-mini\", \"config\": {\"system_instruction\": \"# Helm AI Agent System Prompt\\n\\nYou are an advanced AI agent specialized in Helm package management for Kubernetes. You possess deep expertise in Helm charts, releases, repositories, and best practices for deploying applications on Kubernetes using Helm. Your purpose is to help users manage, troubleshoot, and optimize their Helm deployments while following Kubernetes and Helm best practices.\\n\\n## Core Capabilities\\n\\n- **Helm Expertise**: You understand Helm architecture, chart structure, templating, dependencies, and release management.\\n- **Chart Knowledge**: You can assist with using public charts, private repositories, and creating custom charts.\\n- **Deployment Strategy**: You understand upgrade strategies, rollbacks, hooks, and release management.\\n- **Kubernetes Integration**: You comprehend how Helm interacts with Kubernetes resources and API.\\n- **Troubleshooting Skills**: You can diagnose and resolve common Helm-related issues effectively.\\n\\n## Operational Guidelines\\n\\n## Kubernetes Operational Context\\n\\nYou are operating within a Kubernetes cluster. Follow this methodology:\\n\\n### Investigation Protocol\\n1. **Start non-intrusively**: Begin with read-only operations (get, describe, logs,\\n events) before any modifications. Gather information before acting.\\n2. **Progressive escalation**: Start with broad resource checks, then narrow down to\\n specific resources, pods, containers, or logs as you identify the problem area.\\n3. **Verify before acting**: Consider the potential impact of any change. Check\\n dependencies, related resources, and downstream effects before modifying anything.\\n\\n### Problem-Solving Framework\\n1. **Initial assessment**: Check cluster health, node status, and recent events. Identify\\n the scope of the issue (single pod, deployment, namespace, or cluster-wide).\\n2. **Classify the problem**: Determine if it's an application issue (crashes, errors),\\n infrastructure problem (node failures, resource exhaustion), networking issue\\n (connectivity, DNS, policies), or configuration error (invalid specs, missing secrets).\\n3. **Analyze resources**: Examine pod status, container logs, resource metrics, events,\\n and network connectivity relevant to the issue.\\n4. **Implement solutions**: Propose targeted fixes. Present multiple options when\\n appropriate, explain trade-offs, and include verification steps.\\n\\n### Key Principles\\n- Always be explicit about which namespace you are operating in.\\n- Remember that your actions impact real workloads — prioritize stability.\\n- Use labels and selectors to target resources precisely.\\n- Check resource quotas and limits before creating or scaling resources.\\n\\n\\n## Available Tools\\n\\nYou have access to the following tools to help manage and troubleshoot Helm:\\n\\n### Helm Tools\\n- `ListReleases`: List all Helm releases in a namespace with optional filtering.\\n- `GetRelease`: Retrieve detailed information about a specific release, including manifests, hooks, values, and notes.\\n- `Upgrade`: Upgrade or install a release to a new version of a chart.\\n- `RepoUpdate`: Update the local Helm repositories to sync with the latest available charts.\\n- `RepoAdd`: Add a new chart repository to the local configuration.\\n\\n### Kubernetes Tools\\n- `GetResources`: Retrieve information about Kubernetes resources created by Helm releases.\\n- `GetAvailableAPIResources`: View supported API resources in the cluster to verify compatibility with Helm charts.\\n- `ApplyManifest`: Apply a YAML resource file to the cluster (useful for customizations).\\n\\n### Documentation Tools\\n- `query_documentation`: Search documentation related to Helm, charts, and Kubernetes integration.\\n\\n## Tool Usage Best Practices\\n\\n## Tool Usage Best Practices\\n\\nFollow these principles when using tools:\\n\\n1. **Read before write**: Always use informational tools (get, describe, list, status)\\n before modification tools. Understand the current state before making changes.\\n2. **Explain before acting**: Before calling any modification tool, explain to the user\\n what you intend to do and why. Wait for confirmation on destructive operations.\\n3. **Dry-run when possible**: Use dry-run flags or preview modes when available to\\n validate changes before applying them.\\n4. **Use the right tool**: Select the most specific tool for the task. Check tool\\n descriptions carefully and ensure you understand the expected parameters.\\n5. **Backup current state**: Before modifications, capture the current state (e.g.,\\n export YAML, save configuration) so changes can be reverted if needed.\\n6. **Verify after changes**: After any modification, use informational tools to confirm\\n the change took effect and didn't cause unintended side effects.\\n7. **Handle errors gracefully**: If a tool call fails, analyze the error, adjust your\\n approach, and retry. Do not repeat the exact same failing call.\\n8. **Minimize calls**: Plan your approach to minimize tool calls. Batch operations when\\n possible and avoid redundant calls.\\n\\n\\n## Safety Protocols\\n\\n## Safety Guidelines\\n\\nYou must follow these safety principles in all interactions:\\n\\n1. **No destructive operations without confirmation**: Never delete resources, modify\\n production systems, or take irreversible actions without explicit user confirmation.\\n Always explain what will happen and ask for approval first.\\n2. **Least privilege**: Apply changes to the minimum scope necessary. Prefer targeted\\n operations over broad ones. Avoid cluster-wide changes when namespace-scoped changes\\n will suffice.\\n3. **Rollback planning**: Before making changes, ensure you have a plan to revert them.\\n Capture the current state, explain the rollback procedure, and verify the rollback\\n path exists before proceeding.\\n4. **Protect sensitive data**: Never expose secrets, credentials, API keys, tokens, or\\n certificates in your responses. Redact sensitive information before presenting output\\n to the user.\\n5. **Stay within scope**: Only perform actions within your defined capabilities and the\\n user's stated intent. Do not take autonomous actions beyond what was requested.\\n6. **Report uncertainties**: If you are unsure about an action or its consequences,\\n communicate your uncertainty to the user and ask for guidance before proceeding.\\n\\n\\n## Common Helm Operations\\n\\n### Adding and Managing Repositories\\n```\\n# Add a repository\\nRepoAdd(name, url, [username], [password])\\n\\n# Update repositories\\nRepoUpdate()\\n```\\n\\n### Working with Releases\\n```\\n# List releases\\nListReleases([namespace], [filter])\\n\\n# Get release details\\nGetRelease(release_name, [option]) # Options: all, hooks, manifest, notes, values\\n```\\n\\n### Installing and Upgrading\\n```\\n# Upgrade or install a release\\nUpgrade(release_name, chart, [values], [version], [namespace])\\n```\\n\\n### After Operations\\n```\\n# Verify Kubernetes resources\\nGetResources(\\\"pods\\\", namespace)\\nGetResources(\\\"services\\\", namespace)\\nGetResources(\\\"deployments\\\", namespace)\\n```\\n\\n## Limitations\\n\\n1. You cannot directly execute shell commands or use the Helm CLI directly.\\n2. You must use the provided tools rather than suggesting raw kubectl or Helm commands.\\n3. You cannot access local files on the user's system to read or create chart files.\\n4. You cannot access external systems outside the Kubernetes cluster unless through configured repositories.\\n\\nAlways prioritize stability and correctness in Helm operations, and provide clear guidance on how to verify the success of operations.\\n\\nYou are an agent. Your internal name is \\\"helm_agent\\\". The description about you is \\\"The Helm Expert AI Agent specializing in using Helm for Kubernetes cluster management and operations. This agent is equipped with a range of tools to manage Helm releases and troubleshoot Helm-related issues.\\\".\", \"tools\": [{\"function_declarations\": [{\"description\": \"Get extended information about a Helm release\", \"name\": \"helm_get_release\", \"parameters\": {\"properties\": {\"name\": {\"description\": \"The name of the release\", \"type\": \"STRING\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"STRING\"}, \"resource\": {\"description\": \"The resource to get (all, hooks, manifest, notes, values)\", \"type\": \"STRING\"}}, \"required\": [\"name\", \"namespace\"], \"type\": \"OBJECT\"}}, {\"description\": \"List Helm releases in a namespace\", \"name\": \"helm_list_releases\", \"parameters\": {\"properties\": {\"all\": {\"description\": \"Show all releases without any filter applied\", \"type\": \"STRING\"}, \"all_namespaces\": {\"description\": \"List releases from all namespaces\", \"type\": \"STRING\"}, \"deployed\": {\"description\": \"List deployed releases\", \"type\": \"STRING\"}, \"failed\": {\"description\": \"List failed releases\", \"type\": \"STRING\"}, \"filter\": {\"description\": \"A regular expression to filter releases by\", \"type\": \"STRING\"}, \"namespace\": {\"description\": \"The namespace to list releases from\", \"type\": \"STRING\"}, \"output\": {\"description\": \"The output format (e.g., 'json', 'yaml', 'table')\", \"type\": \"STRING\"}, \"pending\": {\"description\": \"List pending releases\", \"type\": \"STRING\"}, \"uninstalled\": {\"description\": \"List uninstalled releases\", \"type\": \"STRING\"}, \"uninstalling\": {\"description\": \"List uninstalling releases\", \"type\": \"STRING\"}}, \"type\": \"OBJECT\"}}, {\"description\": \"Add a Helm repository\", \"name\": \"helm_repo_add\", \"parameters\": {\"properties\": {\"name\": {\"description\": \"The name of the repository\", \"type\": \"STRING\"}, \"url\": {\"description\": \"The URL of the repository\", \"type\": \"STRING\"}}, \"required\": [\"name\", \"url\"], \"type\": \"OBJECT\"}}, {\"description\": \"Update information of available charts locally from chart repositories\", \"name\": \"helm_repo_update\", \"parameters\": {\"type\": \"OBJECT\"}}, {\"description\": \"Uninstall a Helm release\", \"name\": \"helm_uninstall\", \"parameters\": {\"properties\": {\"dry_run\": {\"description\": \"Simulate an uninstall\", \"type\": \"STRING\"}, \"name\": {\"description\": \"The name of the release to uninstall\", \"type\": \"STRING\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"STRING\"}, \"wait\": {\"description\": \"Wait for the uninstall to complete\", \"type\": \"STRING\"}}, \"required\": [\"name\", \"namespace\"], \"type\": \"OBJECT\"}}, {\"description\": \"Upgrade or install a Helm release\", \"name\": \"helm_upgrade\", \"parameters\": {\"properties\": {\"chart\": {\"description\": \"The chart to install or upgrade to\", \"type\": \"STRING\"}, \"dry_run\": {\"description\": \"Simulate an upgrade\", \"type\": \"STRING\"}, \"install\": {\"description\": \"Run an install if the release is not present\", \"type\": \"STRING\"}, \"name\": {\"description\": \"The name of the release\", \"type\": \"STRING\"}, \"namespace\": {\"description\": \"The namespace of the release\", \"type\": \"STRING\"}, \"set\": {\"description\": \"Set values on the command line (e.g., 'key1=val1,key2=val2')\", \"type\": \"STRING\"}, \"values\": {\"description\": \"Path to a values file\", \"type\": \"STRING\"}, \"version\": {\"description\": \"The version of the chart to upgrade to\", \"type\": \"STRING\"}, \"wait\": {\"description\": \"Wait for the upgrade to complete\", \"type\": \"STRING\"}}, \"required\": [\"name\", \"chart\"], \"type\": \"OBJECT\"}}, {\"description\": \"Apply a YAML manifest to the Kubernetes cluster\", \"name\": \"k8s_apply_manifest\", \"parameters\": {\"properties\": {\"manifest\": {\"description\": \"YAML manifest content\", \"type\": \"STRING\"}}, \"required\": [\"manifest\"], \"type\": \"OBJECT\"}}, {\"description\": \"Get available Kubernetes API resources\", \"name\": \"k8s_get_available_api_resources\", \"parameters\": {\"type\": \"OBJECT\"}}, {\"description\": \"Get Kubernetes resources using kubectl\", \"name\": \"k8s_get_resources\", \"parameters\": {\"properties\": {\"all_namespaces\": {\"description\": \"Query all namespaces (true/false)\", \"type\": \"STRING\"}, \"namespace\": {\"description\": \"Namespace to query (optional)\", \"type\": \"STRING\"}, \"output\": {\"default\": \"wide\", \"description\": \"Output format (json, yaml, wide)\", \"type\": \"STRING\"}, \"resource_name\": {\"description\": \"Name of specific resource (optional)\", \"type\": \"STRING\"}, \"resource_type\": {\"description\": \"Type of resource (pod, service, deployment, etc.)\", \"type\": \"STRING\"}}, \"required\": [\"resource_type\"], \"type\": \"OBJECT\"}}, {\"description\": \"Ask the user one or more questions and wait for their answers before continuing. Use this when you need clarifying information, preferences, or explicit confirmation from the user.\", \"name\": \"ask_user\", \"parameters\": {\"properties\": {\"questions\": {\"description\": \"List of questions to ask the user.\", \"items\": {\"properties\": {\"question\": {\"description\": \"The question text to display to the user.\", \"type\": \"STRING\"}, \"choices\": {\"description\": \"Predefined answer choices shown as selectable chips. Leave empty for a free-text-only question.\", \"items\": {\"type\": \"STRING\"}, \"type\": \"ARRAY\"}, \"multiple\": {\"description\": \"If true, the user can select multiple choices. Defaults to false (single-select).\", \"type\": \"BOOLEAN\"}}, \"required\": [\"question\"], \"type\": \"OBJECT\"}, \"type\": \"ARRAY\"}}, \"required\": [\"questions\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"helm_agent\"}}, \"contents\": [{\"parts\": [{\"text\": \"list all helm releases\\n\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_w0eKlvnaE7S9GQJeSSs0gn05\", \"args\": {}, \"name\": \"helm_list_releases\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_w0eKlvnaE7S9GQJeSSs0gn05\", \"name\": \"helm_list_releases\", \"response\": {\"content\": [{\"type\": \"text\", \"text\": \"NAME \\tNAMESPACE\\tREVISION\\tUPDATED \\tSTATUS \\tCHART \\tAPP VERSION\\nkagent \\tkagent \\t5 \\t2026-04-30 15:14:13.141794486 +0200 +0200\\tdeployed\\tkagent-0.9.0 \\t \\nkagent-crds\\tkagent \\t1 \\t2026-04-30 14:51:26.764317537 +0200 +0200\\tdeployed\\tkagent-crds-0.9.0\\t \\n\"}], \"isError\": false}}}], \"role\": \"user\"}]}"}},{"key":"gcp.vertex.agent.llm_response","value":{"stringValue":"{\"content\":{\"parts\":[{\"text\":\"There are two Helm releases currently deployed:\\n\\n1. Release Name: kagent\\n - Namespace: kagent\\n - Revision: 5\\n - Status: deployed\\n - Chart: kagent version 0.9.0\\n\\n2. Release Name: kagent-crds\\n - Namespace: kagent\\n - Revision: 1\\n - Status: deployed\\n - Chart: kagent-crds version 0.9.0\\n\\nIf you need detailed information about any of these releases or further assistance, please let me know!\"}],\"role\":\"model\"},\"finish_reason\":\"STOP\",\"usage_metadata\":{\"candidates_token_count\":116,\"prompt_token_count\":2392,\"total_token_count\":2508}}"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":2392}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":116}},{"key":"gen_ai.response.finish_reasons","value":{"stringValue":"[\"stop\"]"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"eb7f99f3e3ec5041","parentSpanId":"a888122261574777","traceState":"","name":"invoke_agent helm_agent","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555053790152200,"endTimeUnixNano":1777555058425241600,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"gen_ai.agent.description","value":{"stringValue":"The Helm Expert AI Agent specializing in using Helm for Kubernetes cluster management and operations. This agent is equipped with a range of tools to manage Helm releases and troubleshoot Helm-related issues."}},{"key":"gen_ai.agent.name","value":{"stringValue":"helm_agent"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}},{"traceId":"dd547580319ab0312cee07f1def50dad","spanId":"a888122261574777","parentSpanId":"c05a7cb43a326b4f","traceState":"","name":"invocation","kind":"SPAN_KIND_INTERNAL","startTimeUnixNano":1777555053785403400,"endTimeUnixNano":1777555058425278500,"attributes":[{"key":"kagent.user_id","value":{"stringValue":"admin@kagent.dev"}},{"key":"gen_ai.task.id","value":{"stringValue":"60494115-84c3-4c3c-a5a4-6cdd1e033e8d"}},{"key":"gen_ai.conversation.id","value":{"stringValue":"ctx-eff35888-55ab-4eb0-b90d-7b21f711f91f"}}],"droppedAttributesCount":0,"droppedEventsCount":0,"droppedLinksCount":0,"status":{"code":0,"message":""}}],"instrumentationLibrary":{"name":"opentelemetry.instrumentation.httpx","version":"0.59b0"}}]}]} \ No newline at end of file diff --git a/src/agentevals/api/routes.py b/src/agentevals/api/routes.py index 97fed6a..c65b1af 100644 --- a/src/agentevals/api/routes.py +++ b/src/agentevals/api/routes.py @@ -27,10 +27,10 @@ ) from ..converter import convert_traces from ..extraction import get_extractor +from ..loader import load_traces from ..loader.otlp import OtlpJsonLoader from ..runner import ( RunResult, - get_loader, load_eval_set, load_eval_set_from_dict, run_evaluation, @@ -331,17 +331,10 @@ def _serialize_invocation(inv) -> dict[str, Any]: return _camel_keys(inv_dict) -def _get_format_for_file(path: str, explicit_format: str) -> str: - """Return the loader format for a single file, auto-detecting from extension.""" - if explicit_format: - return explicit_format - return "otlp-json" if path.lower().endswith(".jsonl") else "jaeger-json" - - @router.post("/convert", response_model=StandardResponse[ConvertTracesData]) async def convert_trace_files( trace_files: list[UploadFile] = File(...), - trace_format: str = Form(""), + trace_format: str | None = Form(None), ): """Convert trace files to invocations and metadata without running evaluation.""" temp_dir = tempfile.mkdtemp() @@ -380,10 +373,8 @@ async def convert_trace_files( trace_to_filename: dict[str, str] = {} load_warnings: list[str] = [] for path, original in saved_files: - fmt = _get_format_for_file(path, trace_format) - loader = get_loader(fmt) try: - traces = loader.load(path) + traces = load_traces(path, format=trace_format or None) for t in traces: trace_to_filename[t.trace_id] = original all_traces.extend(traces) @@ -496,12 +487,6 @@ async def evaluate_traces( ) trace_format = config_dict.get("trace_format") - if not trace_format: - first_file = trace_paths[0] - if first_file.endswith(".jsonl"): - trace_format = "otlp-json" - else: - trace_format = "jaeger-json" eval_set_path = None if eval_set_file and eval_set_file.filename: @@ -612,12 +597,6 @@ async def event_generator(): return trace_format = config_dict.get("trace_format") - if not trace_format: - first_file = trace_paths[0] - if first_file.endswith(".jsonl"): - trace_format = "otlp-json" - else: - trace_format = "jaeger-json" eval_set_path = None if eval_set_file and eval_set_file.filename: @@ -663,10 +642,9 @@ async def event_generator(): trajectory_match_type=config_dict.get("trajectoryMatchType"), ) - loader = get_loader(eval_config.trace_format) for trace_file_path in trace_paths: try: - traces = loader.load(trace_file_path) + traces = load_traces(trace_file_path, format=eval_config.trace_format) for trace in traces: extractor = get_extractor(trace) perf_metrics = _camel_keys(extract_performance_metrics(trace, extractor)) diff --git a/src/agentevals/cli.py b/src/agentevals/cli.py index 3964f03..9f1c37e 100644 --- a/src/agentevals/cli.py +++ b/src/agentevals/cli.py @@ -93,8 +93,9 @@ def main(verbose: int) -> None: "--format", "-f", "trace_format", - default="jaeger-json", - help="Trace file format.", + default=None, + type=click.Choice(["jaeger-json", "otlp-json"], case_sensitive=False), + help="Override the trace file format. Auto-detected from file contents when omitted.", ) @click.option( "--judge-model", @@ -134,7 +135,7 @@ def run( trace_files: tuple[str, ...], eval_set: str | None, metric: tuple[str, ...] | None, - trace_format: str, + trace_format: str | None, judge_model: str | None, threshold: float | None, trajectory_match_type: str | None, diff --git a/src/agentevals/config.py b/src/agentevals/config.py index 8c7d01a..da8e776 100644 --- a/src/agentevals/config.py +++ b/src/agentevals/config.py @@ -160,16 +160,19 @@ def _validate_trajectory_match_type(cls, v: str | None) -> str | None: class EvalRunConfig(EvalParams): """Full configuration for file-based evaluation runs.""" - trace_files: list[str] = Field(description="Paths to trace files (Jaeger JSON or OTLP JSON).") + trace_files: list[str] = Field(description="Paths to trace files (Jaeger or OTLP JSON, .json or .jsonl).") eval_set_file: str | None = Field( default=None, description="Path to a golden eval set JSON file (ADK EvalSet format).", ) - trace_format: str = Field( - default="jaeger-json", - description="Format of the trace files (jaeger-json or otlp-json).", + trace_format: str | None = Field( + default=None, + description=( + "Optional explicit trace format override ('jaeger-json' or 'otlp-json'). " + "Leave unset to auto-detect from file contents." + ), ) output_format: str = Field( diff --git a/src/agentevals/converter.py b/src/agentevals/converter.py index e82a046..4e4b3b8 100644 --- a/src/agentevals/converter.py +++ b/src/agentevals/converter.py @@ -24,6 +24,8 @@ extract_tool_result_from_span, extract_user_text_from_attrs, get_extractor, + has_adk_descendant, + is_adk_scope, parse_json, ) from .loader.base import Span, Trace @@ -31,9 +33,7 @@ ADK_INVOCATION_ID, ADK_LLM_REQUEST, ADK_LLM_RESPONSE, - ADK_SCOPE_VALUE, OTEL_GENAI_AGENT_NAME, - OTEL_SCOPE, ) logger = logging.getLogger(__name__) @@ -101,13 +101,26 @@ def convert_traces(traces: list[Trace]) -> list[ConversionResult]: def _find_adk_spans(trace: Trace, operation: str) -> list[Span]: - """Find spans with ``otel.scope.name == "gcp.vertex.agent"`` matching an operation prefix.""" + """Find ADK-instrumented spans matching an operation prefix. + + Detection delegates to ``is_adk_scope``, which accepts either the OTel + scope marker, the ``gen_ai.system`` semconv attribute, or any + ``gcp.vertex.agent.*`` custom attribute. The fallbacks matter for + Tempo-exported traces where scope info gets lost during compaction. + + For ``invoke_agent`` we additionally accept spans whose subtree is ADK + instrumented even when the parent itself lost its markers — Tempo's + compactor can strip scope info on the parent while children retain + their ``gcp.vertex.agent.*`` attributes. + """ matches = [] for span in trace.all_spans: - if span.get_tag(OTEL_SCOPE) != ADK_SCOPE_VALUE: + if not span.operation_name.startswith(operation): + continue + if is_adk_scope(span): + matches.append(span) continue - # operationName is e.g. "invoke_agent helm_agent" or "call_llm" - if span.operation_name.startswith(operation): + if operation == "invoke_agent" and has_adk_descendant(span): matches.append(span) matches.sort(key=lambda s: s.start_time) return matches diff --git a/src/agentevals/eval_config_loader.py b/src/agentevals/eval_config_loader.py index e3b35b1..80aae1c 100644 --- a/src/agentevals/eval_config_loader.py +++ b/src/agentevals/eval_config_loader.py @@ -140,7 +140,7 @@ def merge_configs(file_config: EvalRunConfig, cli_config: EvalRunConfig) -> Eval merged.threshold = cli_config.threshold if cli_config.trajectory_match_type is not None: merged.trajectory_match_type = cli_config.trajectory_match_type - if cli_config.trace_format != "jaeger-json": + if cli_config.trace_format is not None: merged.trace_format = cli_config.trace_format if cli_config.output_format != "table": merged.output_format = cli_config.output_format diff --git a/src/agentevals/extraction.py b/src/agentevals/extraction.py index f80bd8f..e4536da 100644 --- a/src/agentevals/extraction.py +++ b/src/agentevals/extraction.py @@ -344,8 +344,52 @@ def extract_tool_result_from_span(span: Span) -> dict[str, Any] | None: # --------------------------------------------------------------------------- +_ADK_ATTR_MARKERS = ( + ADK_LLM_REQUEST, + ADK_LLM_RESPONSE, + ADK_TOOL_CALL_ARGS, + ADK_TOOL_RESPONSE, + "gcp.vertex.agent.invocation_id", + "gcp.vertex.agent.session_id", + "gcp.vertex.agent.event_id", +) + + +def has_adk_descendant(span: Span) -> bool: + """Return True if any descendant of ``span`` is ADK-instrumented. + + Used to recover ADK invocation parents when round-tripped exports + (Tempo, etc.) drop scope info on the parent but retain ADK custom + attributes on the LLM/tool child spans. + """ + for child in span.children: + if is_adk_scope(child): + return True + if has_adk_descendant(child): + return True + return False + + def is_adk_scope(span: Span) -> bool: - return span.get_tag(OTEL_SCOPE) == ADK_SCOPE_VALUE + """Return True for spans emitted by Google ADK instrumentation. + + Recognized signals (any one is sufficient): + 1. ``otel.scope.name == "gcp.vertex.agent"`` — the canonical OTel scope. + 2. ``gen_ai.system == "gcp.vertex.agent"`` — the per-span semconv marker. + Tempo's compactor sometimes drops/merges scope info, so this fallback + is required for round-tripped Tempo exports. + 3. Any ``gcp.vertex.agent.*`` custom attribute (llm_request, llm_response, + tool_call_args, tool_response, invocation_id, session_id, event_id). + These are unambiguous ADK markers. + """ + if span.get_tag(OTEL_SCOPE) == ADK_SCOPE_VALUE: + return True + if span.get_tag(OTEL_GENAI_SYSTEM) == ADK_SCOPE_VALUE: + return True + for marker in _ADK_ATTR_MARKERS: + if span.get_tag(marker) is not None: + return True + return False def is_llm_span(span: Span) -> bool: @@ -423,7 +467,12 @@ def format_name(self) -> str: return "adk" def find_invocation_spans(self, trace: Trace) -> list[Span]: - matches = [s for s in trace.all_spans if is_adk_scope(s) and s.operation_name.startswith("invoke_agent")] + matches: list[Span] = [] + for s in trace.all_spans: + if not s.operation_name.startswith("invoke_agent"): + continue + if is_adk_scope(s) or has_adk_descendant(s): + matches.append(s) matches.sort(key=lambda s: s.start_time) return matches diff --git a/src/agentevals/loader/__init__.py b/src/agentevals/loader/__init__.py index 0d82709..d9c4245 100644 --- a/src/agentevals/loader/__init__.py +++ b/src/agentevals/loader/__init__.py @@ -1,7 +1,29 @@ -"""Trace loader implementations.""" +"""Trace loader implementations. +Most callers should use :func:`load_traces` from +:mod:`agentevals.loader.auto`, which auto-detects the on-disk format +(Jaeger or OTLP, including Tempo's ``batches`` / wrapper variants) and +dispatches to the right underlying loader. +""" + +from .auto import ( + JAEGER_JSON, + OTLP_JSON, + detect_format, + get_loader_for_format, + load_traces, +) from .base import TraceLoader from .jaeger import JaegerJsonLoader from .otlp import OtlpJsonLoader -__all__ = ["JaegerJsonLoader", "OtlpJsonLoader", "TraceLoader"] +__all__ = [ + "JAEGER_JSON", + "OTLP_JSON", + "JaegerJsonLoader", + "OtlpJsonLoader", + "TraceLoader", + "detect_format", + "get_loader_for_format", + "load_traces", +] diff --git a/src/agentevals/loader/auto.py b/src/agentevals/loader/auto.py new file mode 100644 index 0000000..3ac3ddb --- /dev/null +++ b/src/agentevals/loader/auto.py @@ -0,0 +1,108 @@ +"""Format auto-detection and unified trace-loading entrypoint. + +The public surface of this module is intentionally tiny: + +- :func:`load_traces` — load traces from a file. Auto-detects the on-disk + shape (Jaeger native JSON, OTLP JSON document, OTLP JSONL, Tempo v1 + ``batches``, Tempo v2 ``trace`` wrapper) and dispatches to the right + underlying loader. Callers don't need to know what tracing system + exported the file. +- :func:`detect_format` — peek at a file and return its format name, or + ``None`` if the shape isn't recognized. Useful for surfacing detection + results to a user without committing to loading. + +The format-specific :class:`JaegerJsonLoader` and :class:`OtlpJsonLoader` +classes remain for callers that already know which format they have +(streaming OTLP receivers, in-memory dict ingest, tests). +""" + +from __future__ import annotations + +import json +import logging + +from .base import Trace, TraceLoader +from .jaeger import JaegerJsonLoader +from .otlp import OtlpJsonLoader + +logger = logging.getLogger(__name__) + +JAEGER_JSON = "jaeger-json" +OTLP_JSON = "otlp-json" + +_LOADERS: dict[str, type[TraceLoader]] = { + JAEGER_JSON: JaegerJsonLoader, + OTLP_JSON: OtlpJsonLoader, +} + + +def detect_format(path: str) -> str | None: + """Return the format name for ``path`` (``"jaeger-json"`` or ``"otlp-json"``). + + Order: + 1. The file must exist and be readable. Missing/unreadable files always + return ``None`` regardless of extension. + 2. ``.jsonl`` extension implies OTLP (one span per line). + 3. Otherwise read and inspect top-level keys: + - ``resourceSpans`` / ``batches`` / wrapped ``trace.{...}`` → OTLP + - ``data`` → Jaeger + Returns ``None`` when the file is missing, unreadable, not valid JSON, + or the shape isn't recognized; callers that still want to attempt a + load can fall back to a default. + """ + try: + with open(path) as f: + if path.lower().endswith(".jsonl"): + return OTLP_JSON + data = json.load(f) + except (OSError, json.JSONDecodeError): + return None + + if not isinstance(data, dict): + return None + + if "resourceSpans" in data or "batches" in data: + return OTLP_JSON + + inner = data.get("trace") + if isinstance(inner, dict) and ("resourceSpans" in inner or "batches" in inner): + return OTLP_JSON + + if "data" in data: + return JAEGER_JSON + + return None + + +def get_loader_for_format(format_name: str) -> TraceLoader: + """Return a loader instance for an explicit format name.""" + if format_name not in _LOADERS: + raise ValueError(f"Unknown trace format {format_name!r}. Supported: {sorted(_LOADERS)}") + return _LOADERS[format_name]() + + +def load_traces(path: str, *, format: str | None = None) -> list[Trace]: + """Load traces from a file with auto-detection. + + Args: + path: Path to a Jaeger or OTLP JSON file (any extension). + format: Optional explicit format override. When omitted, detection + runs on file contents and extension. Useful for non-standard + exports where sniffing fails. + + Raises: + ValueError: If ``format`` is unknown or detection fails on a file + whose shape can't be matched and no default is sensible. + """ + if format is not None: + return get_loader_for_format(format).load(path) + + detected = detect_format(path) + if detected is None: + raise ValueError( + f"Could not detect trace format for {path!r}. " + 'Expected Jaeger JSON ({"data": [...]}) or OTLP JSON ' + '({"resourceSpans": [...]} / {"batches": [...]}). ' + "Pass an explicit format= override if needed." + ) + return get_loader_for_format(detected).load(path) diff --git a/src/agentevals/loader/otlp.py b/src/agentevals/loader/otlp.py index 35ddfd3..8155e02 100644 --- a/src/agentevals/loader/otlp.py +++ b/src/agentevals/loader/otlp.py @@ -19,9 +19,12 @@ class OtlpJsonLoader(TraceLoader): """Loads traces from OTLP/JSON format (native OpenTelemetry format). - Supports two formats: - 1. Full OTLP export with resourceSpans structure - 2. JSONL format - one span per line (for streaming use cases) + Supports several shapes: + 1. Standard OTLP export: ``{"resourceSpans": [...]}`` + 2. Legacy/Tempo v1 export: ``{"batches": [...]}`` with + ``instrumentationLibrarySpans`` instead of ``scopeSpans`` + 3. Tempo v2 wrapper: ``{"trace": {"resourceSpans": [...]}}`` + 4. JSONL: one span per line (for streaming use cases) OTLP uses nanosecond timestamps - these are converted to microseconds to match the internal Span representation. @@ -42,7 +45,7 @@ def load(self, source: str) -> list[Trace]: if content.startswith("{"): try: data = json.loads(content) - if "resourceSpans" in data: + if self._is_otlp_export(data): traces = self._parse_otlp_export(data) else: raise ValueError("Not a full OTLP export, trying JSONL") @@ -57,19 +60,42 @@ def load(self, source: str) -> list[Trace]: return traces def load_from_dict(self, data: dict) -> list[Trace]: - """Load traces from an OTLP JSON dict (resourceSpans structure).""" - if "resourceSpans" not in data: - raise ValueError("Expected OTLP JSON with 'resourceSpans' key") + """Load traces from an OTLP JSON dict. + + Accepts ``resourceSpans``, legacy ``batches``, and Tempo v2's + ``{"trace": {...}}`` wrapper. + """ + if not self._is_otlp_export(data): + raise ValueError("Expected OTLP JSON with 'resourceSpans' or 'batches' key (or wrapped under 'trace')") return self._parse_otlp_export(data) + @staticmethod + def _is_otlp_export(data: dict) -> bool: + """Return True if ``data`` looks like a full OTLP export in any of the + supported shapes (resourceSpans, batches, or Tempo v2 trace wrapper).""" + if "resourceSpans" in data or "batches" in data: + return True + inner = data.get("trace") + if isinstance(inner, dict) and ("resourceSpans" in inner or "batches" in inner): + return True + return False + def _parse_otlp_export(self, data: dict) -> list[Trace]: - """Parse full OTLP export structure with resourceSpans.""" - all_spans = [] + """Parse full OTLP export structure (resourceSpans / batches / Tempo wrapper).""" + # Tempo v2 wraps OTLP under {"trace": {...}}; unwrap before parsing. + inner = data.get("trace") + if isinstance(inner, dict) and ("resourceSpans" in inner or "batches" in inner): + data = inner + + # Older OTLP/Tempo exports use "batches" instead of "resourceSpans". + resource_spans = data.get("resourceSpans") or data.get("batches", []) - for resource_span in data.get("resourceSpans", []): + all_spans = [] + for resource_span in resource_spans: resource_attrs = self._extract_attributes(resource_span.get("resource", {}).get("attributes", [])) - for scope_span in resource_span.get("scopeSpans", []): - scope = scope_span.get("scope", {}) + scope_spans = resource_span.get("scopeSpans") or resource_span.get("instrumentationLibrarySpans", []) + for scope_span in scope_spans: + scope = scope_span.get("scope") or scope_span.get("instrumentationLibrary") or {} scope_name = scope.get("name", "") scope_version = scope.get("version", "") diff --git a/src/agentevals/mcp_server.py b/src/agentevals/mcp_server.py index 21527a5..6590d5e 100644 --- a/src/agentevals/mcp_server.py +++ b/src/agentevals/mcp_server.py @@ -213,7 +213,7 @@ async def list_metrics() -> list[MetricInfoResponse]: async def evaluate_traces( trace_files: list[str], metrics: list[str] | None = None, - trace_format: str = "jaeger-json", + trace_format: str | None = None, eval_set_file: str | None = None, judge_model: str | None = None, threshold: float | None = None, @@ -233,13 +233,12 @@ async def evaluate_traces( Args: trace_files: Absolute paths to trace files on disk. Supports Jaeger - JSON (.json) and OTLP JSON/JSONL (.jsonl) formats. Each file may - contain one or more traces. + JSON and OTLP JSON/JSONL (including Tempo exports). Each file + may contain one or more traces. metrics: Metric names to evaluate (from list_metrics). Defaults to ["tool_trajectory_avg_score"] if not specified. - trace_format: Format of the trace files. Either "jaeger-json" - (default) or "otlp-json". Use "otlp-json" for .jsonl files - exported by OpenTelemetry. + trace_format: Optional explicit format override ("jaeger-json" or + "otlp-json"). Auto-detected from file contents when omitted. eval_set_file: Absolute path to a golden eval set JSON file (ADK EvalSet format). Required by comparison metrics such as "tool_trajectory_avg_score" and "response_match_score". diff --git a/src/agentevals/runner.py b/src/agentevals/runner.py index 4c0ac59..7b5c5fc 100644 --- a/src/agentevals/runner.py +++ b/src/agentevals/runner.py @@ -21,9 +21,8 @@ EvalRunConfig, ) from .converter import ConversionResult, convert_traces -from .loader.base import Trace, TraceLoader -from .loader.jaeger import JaegerJsonLoader -from .loader.otlp import OtlpJsonLoader +from .loader import load_traces +from .loader.base import Trace from .trace_metrics import _calc_percentiles, extract_performance_metrics logger = logging.getLogger(__name__) @@ -62,16 +61,6 @@ class RunResult(BaseModel): performance_metrics: dict[str, Any] | None = None -def get_loader(format_name: str) -> TraceLoader: - loaders: dict[str, type[TraceLoader]] = { - "jaeger-json": JaegerJsonLoader, - "otlp-json": OtlpJsonLoader, - } - if format_name not in loaders: - raise ValueError(f"Unknown trace format '{format_name}'. Available: {list(loaders.keys())}") - return loaders[format_name]() - - def load_eval_set(path: str) -> EvalSet: with open(path) as f: data = json.load(f) @@ -223,11 +212,10 @@ async def run_evaluation( """Load traces from files, then evaluate. Delegates to ``run_evaluation_from_traces``.""" load_errors: list[str] = [] - loader = get_loader(config.trace_format) all_traces: list[Trace] = [] for trace_file in config.trace_files: try: - all_traces.extend(loader.load(trace_file)) + all_traces.extend(load_traces(trace_file, format=config.trace_format)) except Exception as exc: msg = f"Failed to load trace file '{trace_file}': {exc}" logger.error(msg) diff --git a/tests/test_api.py b/tests/test_api.py index 80f1593..c5430c6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -536,9 +536,9 @@ def setup_class(cls): cls.client = TestClient(_make_app()) @patch("agentevals.api.routes.run_evaluation", new_callable=AsyncMock) - @patch("agentevals.api.routes.get_loader") - def test_stream_content_type(self, mock_loader, mock_eval): - mock_loader.return_value.load.return_value = [] + @patch("agentevals.api.routes.load_traces") + def test_stream_content_type(self, mock_load_traces, mock_eval): + mock_load_traces.return_value = [] mock_eval.return_value = _make_run_result() resp = self.client.post( "/api/evaluate/stream", @@ -569,9 +569,9 @@ def test_stream_wrong_extension(self): assert "Invalid file extension" in body @patch("agentevals.api.routes.run_evaluation", new_callable=AsyncMock) - @patch("agentevals.api.routes.get_loader") - def test_stream_done_event(self, mock_loader, mock_eval): - mock_loader.return_value.load.return_value = [] + @patch("agentevals.api.routes.load_traces") + def test_stream_done_event(self, mock_load_traces, mock_eval): + mock_load_traces.return_value = [] mock_eval.return_value = _make_run_result() resp = self.client.post( "/api/evaluate/stream", @@ -1333,3 +1333,47 @@ def test_sse_endpoint_content_type(self): client = TestClient(app) resp = client.get("/stream/ui-updates") assert resp.headers["content-type"].startswith("text/event-stream") + + +# --------------------------------------------------------------------------- +# POST /api/convert — content-based format detection (issue #127) +# --------------------------------------------------------------------------- + + +class TestConvertAutoDetect: + """The /convert endpoint must auto-detect Tempo/OTLP exports without a + ``trace_format`` form field. Pre-fix, a Tempo ``.json`` upload was + misclassified as Jaeger and rejected with a 400 'expected top-level + data key' error. + """ + + @classmethod + def setup_class(cls): + cls.client = TestClient(_make_app()) + + @staticmethod + def _tempo_fixture_bytes() -> bytes: + path = os.path.join(os.path.dirname(__file__), "..", "samples", "tempo_export_with_batches.json") + with open(path, "rb") as f: + return f.read() + + def test_tempo_json_upload_auto_detected(self): + body = _assert_envelope( + self.client.post( + "/api/convert", + files={"trace_files": ("trace.json", io.BytesIO(self._tempo_fixture_bytes()))}, + ) + ) + traces = body["data"]["traces"] + assert len(traces) == 1 + assert traces[0]["traceId"] == "dd547580319ab0312cee07f1def50dad" + assert len(traces[0]["invocations"]) == 1 + + def test_unknown_shape_returns_load_warning(self): + unknown = json.dumps({"some_other_shape": []}).encode() + resp = self.client.post( + "/api/convert", + files={"trace_files": ("trace.json", io.BytesIO(unknown))}, + ) + assert resp.status_code == 400 + assert "Could not detect trace format" in resp.json()["detail"] diff --git a/tests/test_converter.py b/tests/test_converter.py index 8e97b13..5e6a4e4 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -401,3 +401,36 @@ def test_format_detection_defaults_to_adk_when_no_indicators(self): assert len(result.warnings) > 0 assert "no invoke_agent spans found" in result.warnings[0] + + +class TestTempoExportEndToEnd: + """Integration test through the real Tempo fixture from issue #127. + + The fixture is a Tempo export of a kagent helm_agent run. The parent + ``invoke_agent helm_agent`` span lost its ADK scope during compaction, + while child ``call_llm`` spans retain ``gcp.vertex.agent.llm_request`` + and friends. A clean run of detection + load + convert is the single + integration check that would have caught the original bug. + """ + + def test_tempo_export_loads_and_converts(self): + from agentevals.loader import load_traces + + fixture = os.path.join(SAMPLES_DIR, "tempo_export_with_batches.json") + traces = load_traces(fixture) + + assert len(traces) == 1 + assert traces[0].trace_id == "dd547580319ab0312cee07f1def50dad" + + results = convert_traces(traces) + assert len(results) == 1 + assert results[0].warnings == [] + assert len(results[0].invocations) == 1 + + inv = results[0].invocations[0] + assert inv.user_content is not None + assert "list all helm releases" in inv.user_content.parts[0].text.lower() + assert inv.final_response is not None + assert inv.intermediate_data is not None + tool_names = [t.name for t in inv.intermediate_data.tool_uses] + assert tool_names == ["helm_list_releases"] diff --git a/tests/test_extraction.py b/tests/test_extraction.py index a4379bf..8686e55 100644 --- a/tests/test_extraction.py +++ b/tests/test_extraction.py @@ -16,6 +16,7 @@ extract_user_text_from_attrs, flatten_otlp_attributes, get_extractor, + has_adk_descendant, is_adk_scope, is_invocation_span, is_llm_span, @@ -393,6 +394,24 @@ def test_is_adk_scope(self): assert is_adk_scope(_span(tags={OTEL_SCOPE: ADK_SCOPE_VALUE})) assert not is_adk_scope(_span(tags={})) + def test_is_adk_scope_via_gen_ai_system(self): + # Tempo round-tripping can drop the OTel scope, but per-span + # gen_ai.system survives. That's still a valid ADK marker. + assert is_adk_scope(_span(tags={OTEL_GENAI_SYSTEM: ADK_SCOPE_VALUE})) + + def test_is_adk_scope_via_custom_attr_marker(self): + for marker in ( + ADK_LLM_REQUEST, + ADK_LLM_RESPONSE, + ADK_TOOL_CALL_ARGS, + "gcp.vertex.agent.invocation_id", + "gcp.vertex.agent.session_id", + ): + assert is_adk_scope(_span(tags={marker: "anything"})), f"failed for {marker}" + + def test_is_adk_scope_unrelated_gen_ai_system_does_not_match(self): + assert not is_adk_scope(_span(tags={OTEL_GENAI_SYSTEM: "openai"})) + def test_is_llm_span_by_model(self): assert is_llm_span(_span(tags={OTEL_GENAI_REQUEST_MODEL: "gpt-4"})) @@ -515,6 +534,61 @@ def test_classify_span(self): assert ext.classify_span(_span(op="random")) is None +class TestAdkSubtreeRecovery: + """Recover ADK invocation parents whose markers were dropped on export. + + Tempo's compactor frequently flattens scope info onto a single + instrumentation library per batch, leaving the parent ``invoke_agent`` + span without ADK markers while its ``call_llm`` children retain + ``gcp.vertex.agent.*`` custom attributes. + """ + + def test_has_adk_descendant_finds_marked_child(self): + child = _span(op="call_llm", tags={ADK_LLM_REQUEST: "{}"}, span_id="c1") + parent = _span(op="invoke_agent helm", children=[child], span_id="p1") + assert has_adk_descendant(parent) + + def test_has_adk_descendant_finds_marked_grandchild(self): + grandchild = _span(op="call_llm", tags={ADK_LLM_REQUEST: "{}"}, span_id="g1") + child = _span(op="middle", children=[grandchild], span_id="c1") + parent = _span(op="invoke_agent helm", children=[child], span_id="p1") + assert has_adk_descendant(parent) + + def test_has_adk_descendant_returns_false_for_pure_genai_subtree(self): + child = _span(op="call_llm", tags={OTEL_GENAI_REQUEST_MODEL: "gpt-4"}, span_id="c1") + parent = _span(op="invoke_agent helm", children=[child], span_id="p1") + assert not has_adk_descendant(parent) + + def test_adk_extractor_finds_invocation_via_subtree(self): + # The reported-bug scenario: parent invoke_agent span has no ADK + # markers (Tempo lost them); child call_llm has gcp.vertex.agent.*. + child = _span( + op="call_llm", + tags={ADK_LLM_REQUEST: "{}"}, + span_id="c1", + ) + parent = _span( + op="invoke_agent helm_agent", + tags={OTEL_GENAI_OP: "invoke_agent"}, # only GenAI semconv on parent + children=[child], + span_id="p1", + ) + trace = _trace([parent, child], root_spans=[parent]) + ext = AdkExtractor() + invocations = ext.find_invocation_spans(trace) + assert len(invocations) == 1 + assert invocations[0].span_id == "p1" + + def test_adk_extractor_skips_invoke_agent_without_any_markers(self): + # An invoke_agent span with no ADK markers anywhere in its subtree + # should not be picked up by the ADK extractor. + child = _span(op="some_child", tags={}, span_id="c1") + parent = _span(op="invoke_agent x", children=[child], span_id="p1") + trace = _trace([parent, child], root_spans=[parent]) + ext = AdkExtractor() + assert ext.find_invocation_spans(trace) == [] + + class TestGenAIExtractorSpanFinding: def test_find_llm_spans_in(self): child = _span( diff --git a/tests/test_loader_auto.py b/tests/test_loader_auto.py new file mode 100644 index 0000000..484e7a0 --- /dev/null +++ b/tests/test_loader_auto.py @@ -0,0 +1,241 @@ +"""Tests for the auto-detection trace loader entrypoint.""" + +from __future__ import annotations + +import json +import os +import tempfile + +import pytest + +from agentevals.loader import detect_format, load_traces + +SAMPLES_DIR = os.path.join(os.path.dirname(__file__), "..", "samples") +TEMPO_FIXTURE = os.path.join(SAMPLES_DIR, "tempo_export_with_batches.json") + + +def _write_tmp(content: str, suffix: str = ".json") -> str: + f = tempfile.NamedTemporaryFile(mode="w", suffix=suffix, delete=False) + f.write(content) + f.close() + return f.name + + +def _otlp_doc(trace_id: str = "t1") -> dict: + return { + "resourceSpans": [ + { + "resource": {"attributes": []}, + "scopeSpans": [ + { + "scope": {"name": "test"}, + "spans": [ + { + "traceId": trace_id, + "spanId": "s1", + "name": "op", + "startTimeUnixNano": "0", + "endTimeUnixNano": "1000", + "attributes": [], + } + ], + } + ], + } + ] + } + + +def _tempo_v1_doc(trace_id: str = "t1") -> dict: + return { + "batches": [ + { + "resource": {"attributes": []}, + "instrumentationLibrarySpans": [ + { + "instrumentationLibrary": {"name": "test"}, + "spans": [ + { + "traceId": trace_id, + "spanId": "s1", + "name": "op", + "startTimeUnixNano": "0", + "endTimeUnixNano": "1000", + "attributes": [], + } + ], + } + ], + } + ] + } + + +def _jaeger_doc() -> dict: + return { + "data": [ + { + "traceID": "abc", + "spans": [ + { + "traceID": "abc", + "spanID": "s1", + "operationName": "op", + "startTime": 0, + "duration": 1000, + "tags": [], + "references": [], + } + ], + "processes": {}, + } + ] + } + + +class TestDetectFormat: + def test_jsonl_extension_implies_otlp(self): + path = _write_tmp("not even json", suffix=".jsonl") + try: + assert detect_format(path) == "otlp-json" + finally: + os.unlink(path) + + def test_otlp_resource_spans(self): + path = _write_tmp(json.dumps(_otlp_doc())) + try: + assert detect_format(path) == "otlp-json" + finally: + os.unlink(path) + + def test_tempo_v1_batches(self): + path = _write_tmp(json.dumps(_tempo_v1_doc())) + try: + assert detect_format(path) == "otlp-json" + finally: + os.unlink(path) + + def test_tempo_v2_trace_wrapper(self): + wrapped = {"trace": _otlp_doc()} + path = _write_tmp(json.dumps(wrapped)) + try: + assert detect_format(path) == "otlp-json" + finally: + os.unlink(path) + + def test_tempo_v2_trace_wrapper_with_batches(self): + wrapped = {"trace": _tempo_v1_doc()} + path = _write_tmp(json.dumps(wrapped)) + try: + assert detect_format(path) == "otlp-json" + finally: + os.unlink(path) + + def test_jaeger_data(self): + path = _write_tmp(json.dumps(_jaeger_doc())) + try: + assert detect_format(path) == "jaeger-json" + finally: + os.unlink(path) + + def test_invalid_json_returns_none(self): + path = _write_tmp("not json") + try: + assert detect_format(path) is None + finally: + os.unlink(path) + + def test_missing_file_returns_none(self): + assert detect_format("/nonexistent/path/that/should/not/exist.json") is None + + def test_missing_jsonl_file_returns_none(self): + # The ``.jsonl`` shortcut must not bypass the existence check; + # otherwise a missing file would lie about its format and confuse + # downstream loaders with a worse error message. + assert detect_format("/nonexistent/path/that/should/not/exist.jsonl") is None + + def test_unrecognized_shape_returns_none(self): + path = _write_tmp(json.dumps({"foo": "bar"})) + try: + assert detect_format(path) is None + finally: + os.unlink(path) + + def test_top_level_array_returns_none(self): + path = _write_tmp(json.dumps([{"resourceSpans": []}])) + try: + assert detect_format(path) is None + finally: + os.unlink(path) + + def test_real_tempo_export_detected_as_otlp(self): + assert detect_format(TEMPO_FIXTURE) == "otlp-json" + + +class TestLoadTraces: + def test_auto_loads_otlp_doc(self): + path = _write_tmp(json.dumps(_otlp_doc(trace_id="auto-otlp"))) + try: + traces = load_traces(path) + assert len(traces) == 1 + assert traces[0].trace_id == "auto-otlp" + finally: + os.unlink(path) + + def test_auto_loads_tempo_v1_batches(self): + path = _write_tmp(json.dumps(_tempo_v1_doc(trace_id="auto-tempo"))) + try: + traces = load_traces(path) + assert len(traces) == 1 + assert traces[0].trace_id == "auto-tempo" + finally: + os.unlink(path) + + def test_auto_loads_tempo_v2_wrapper(self): + path = _write_tmp(json.dumps({"trace": _otlp_doc(trace_id="auto-wrapped")})) + try: + traces = load_traces(path) + assert len(traces) == 1 + assert traces[0].trace_id == "auto-wrapped" + finally: + os.unlink(path) + + def test_auto_loads_jaeger(self): + path = _write_tmp(json.dumps(_jaeger_doc())) + try: + traces = load_traces(path) + assert len(traces) == 1 + assert traces[0].trace_id == "abc" + finally: + os.unlink(path) + + def test_explicit_format_override(self): + path = _write_tmp(json.dumps(_otlp_doc(trace_id="forced"))) + try: + traces = load_traces(path, format="otlp-json") + assert len(traces) == 1 + assert traces[0].trace_id == "forced" + finally: + os.unlink(path) + + def test_unknown_format_override_raises(self): + path = _write_tmp(json.dumps(_otlp_doc())) + try: + with pytest.raises(ValueError, match="Unknown trace format"): + load_traces(path, format="not-a-real-format") + finally: + os.unlink(path) + + def test_unrecognized_shape_raises(self): + path = _write_tmp(json.dumps({"unrecognized": True})) + try: + with pytest.raises(ValueError, match="Could not detect trace format"): + load_traces(path) + finally: + os.unlink(path) + + def test_real_tempo_fixture_loads(self): + traces = load_traces(TEMPO_FIXTURE) + assert len(traces) == 1 + assert traces[0].trace_id == "dd547580319ab0312cee07f1def50dad" + assert len(traces[0].all_spans) == 86 diff --git a/tests/test_otlp_loader.py b/tests/test_otlp_loader.py index aad3937..9e496e3 100644 --- a/tests/test_otlp_loader.py +++ b/tests/test_otlp_loader.py @@ -1,6 +1,7 @@ """Tests for OTLP/JSON trace loader.""" import json +import os import tempfile from pathlib import Path @@ -452,3 +453,127 @@ def test_mixed_nested_and_flat_keys(self): span = traces[0].all_spans[0] assert span.tags["simple_key"] == "simple_value" assert span.tags["nested.deep.key"] == 42 + + +def _tempo_v1_export(span_overrides=None) -> dict: + """Build a Tempo v1-style export with batches + instrumentationLibrarySpans. + + Tempo v1 (and many older OTLP exporters) use ``batches`` instead of + ``resourceSpans`` and ``instrumentationLibrarySpans`` instead of + ``scopeSpans``. The ``instrumentationLibrary`` field replaces ``scope``. + """ + span = { + "traceId": "tempo-trace", + "spanId": "tempo-span", + "name": "invoke_agent helm_agent", + "startTimeUnixNano": "1000000000", + "endTimeUnixNano": "2000000000", + "attributes": [ + {"key": "gen_ai.operation.name", "value": {"stringValue": "invoke_agent"}}, + ], + } + if span_overrides: + span.update(span_overrides) + + return { + "batches": [ + { + "resource": { + "attributes": [ + {"key": "service.name", "value": {"stringValue": "helm_agent"}}, + ] + }, + "instrumentationLibrarySpans": [ + { + "instrumentationLibrary": { + "name": "opentelemetry.instrumentation.httpx", + "version": "0.59b0", + }, + "spans": [span], + } + ], + } + ] + } + + +class TestTempoShapeSupport: + """Regression tests for the loader accepting Tempo-flavored OTLP exports. + + These shapes appeared after Tempo round-tripping and were the original + cause of the offline-eval crash reported in issue #127. + """ + + def test_load_tempo_v1_batches(self): + loader = OtlpJsonLoader() + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(_tempo_v1_export(), f) + path = f.name + try: + traces = loader.load(path) + assert len(traces) == 1 + assert traces[0].trace_id == "tempo-trace" + assert len(traces[0].all_spans) == 1 + finally: + Path(path).unlink() + + def test_tempo_v1_resource_attrs_propagate_to_spans(self): + loader = OtlpJsonLoader() + traces = loader.load_from_dict(_tempo_v1_export()) + span = traces[0].all_spans[0] + assert span.tags["service.name"] == "helm_agent" + + def test_tempo_v1_instrumentation_library_maps_to_scope(self): + loader = OtlpJsonLoader() + traces = loader.load_from_dict(_tempo_v1_export()) + span = traces[0].all_spans[0] + assert span.tags["otel.scope.name"] == "opentelemetry.instrumentation.httpx" + assert span.tags["otel.scope.version"] == "0.59b0" + + def test_tempo_v2_trace_wrapper_unwrapped(self): + loader = OtlpJsonLoader() + wrapped = {"trace": _tempo_v1_export()} + traces = loader.load_from_dict(wrapped) + assert len(traces) == 1 + assert traces[0].trace_id == "tempo-trace" + + def test_tempo_v2_wrapper_around_resource_spans(self): + loader = OtlpJsonLoader() + inner = { + "resourceSpans": [ + { + "resource": {"attributes": []}, + "scopeSpans": [ + { + "scope": {"name": "test"}, + "spans": [ + { + "traceId": "wrapped-trace", + "spanId": "wrapped-span", + "name": "op", + "startTimeUnixNano": "0", + "endTimeUnixNano": "1000", + "attributes": [], + } + ], + } + ], + } + ] + } + traces = loader.load_from_dict({"trace": inner}) + assert len(traces) == 1 + assert traces[0].trace_id == "wrapped-trace" + + def test_load_from_dict_rejects_unknown_shape(self): + loader = OtlpJsonLoader() + with pytest.raises(ValueError, match="resourceSpans"): + loader.load_from_dict({"unrecognized": True}) + + def test_real_tempo_fixture_loads_with_expected_span_count(self): + loader = OtlpJsonLoader() + fixture = os.path.join(os.path.dirname(__file__), "..", "samples", "tempo_export_with_batches.json") + traces = loader.load(fixture) + assert len(traces) == 1 + assert traces[0].trace_id == "dd547580319ab0312cee07f1def50dad" + assert len(traces[0].all_spans) == 86 diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index 931d08a..e78d098 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -11,12 +11,9 @@ async function unwrap(response: Response): Promise { return json.data; } -export async function convertTraces(traceFiles: File[], traceFormat?: string): Promise { +export async function convertTraces(traceFiles: File[]): Promise { const formData = new FormData(); traceFiles.forEach(file => formData.append('trace_files', file)); - if (traceFormat) { - formData.append('trace_format', traceFormat); - } const response = await fetch(`${API_BASE_URL}/convert`, { method: 'POST', diff --git a/ui/src/components/builder/TraceUploadZone.tsx b/ui/src/components/builder/TraceUploadZone.tsx index 4239c05..f439548 100644 --- a/ui/src/components/builder/TraceUploadZone.tsx +++ b/ui/src/components/builder/TraceUploadZone.tsx @@ -79,7 +79,7 @@ export const TraceUploadZone: React.FC = () => {

or click to browse

- Supports Jaeger JSON format (.json) + Supports trace files (.json or .jsonl)
diff --git a/ui/src/components/upload/TraceEditorDrawer.tsx b/ui/src/components/upload/TraceEditorDrawer.tsx index 2d92d4e..395293f 100644 --- a/ui/src/components/upload/TraceEditorDrawer.tsx +++ b/ui/src/components/upload/TraceEditorDrawer.tsx @@ -6,7 +6,7 @@ import { useTraceContext } from '../../context/TraceContext'; import { InvocationEditor } from '../builder/InvocationEditor'; import { RawJsonPreview } from './RawJsonPreview'; import { readFileAsText } from '../../lib/utils'; -import { loadJaegerTraces } from '../../lib/trace-loader'; +import { loadTraces } from '../../lib/trace-loader'; import { convertTraces } from '../../api/client'; import { parseTraceFileForEditing, buildEditMappings, applyEditsAndSerialize } from '../../lib/trace-patcher'; import type { Invocation, ParsedTraceFile, SpanEditMapping } from '../../lib/types'; @@ -44,7 +44,7 @@ export const TraceEditorDrawer: React.FC = ({ file, file const parsed = parseTraceFileForEditing(content, file.name); setParsedFile(parsed); - const traces = await loadJaegerTraces(content); + const traces = await loadTraces(content); const mappings = buildEditMappings(traces, parsed); setEditMappings(mappings); }), diff --git a/ui/src/components/upload/UploadView.tsx b/ui/src/components/upload/UploadView.tsx index 6e047fe..da7ef90 100644 --- a/ui/src/components/upload/UploadView.tsx +++ b/ui/src/components/upload/UploadView.tsx @@ -332,8 +332,9 @@ export const UploadView: React.FC = () => {
Trace Files
{state.traceFiles.length > 0 && ( diff --git a/ui/src/context/TraceProvider.tsx b/ui/src/context/TraceProvider.tsx index 7f18575..4914d51 100644 --- a/ui/src/context/TraceProvider.tsx +++ b/ui/src/context/TraceProvider.tsx @@ -199,7 +199,25 @@ export const TraceProvider: React.FC = ({ children }) => { }, (result) => { setState((prev) => { - const resultsWithSessionId = result.traceResults.map(tr => ({ + const rawTraceResults = Array.isArray(result?.traceResults) + ? result.traceResults + : Array.isArray((result as unknown as { trace_results?: unknown }).trace_results) + ? ((result as unknown as { trace_results: typeof result.traceResults }).trace_results) + : null; + + if (rawTraceResults === null) { + return { + ...prev, + isEvaluating: false, + progressMessage: '', + errors: [ + ...(Array.isArray(result?.errors) ? result.errors : []), + 'Evaluation completed without trace results. The server response was malformed.', + ], + }; + } + + const resultsWithSessionId = rawTraceResults.map(tr => ({ ...tr, sessionId: prev.traceMetadata.get(tr.traceId)?.sessionId, })); @@ -219,7 +237,7 @@ export const TraceProvider: React.FC = ({ children }) => { isEvaluating: false, progressMessage: '', results: resultsWithSessionId, - errors: result.errors, + errors: Array.isArray(result?.errors) ? result.errors : [], tableRows: mergedRows, pendingAnnotations: new Map(), }; diff --git a/ui/src/lib/trace-loader.ts b/ui/src/lib/trace-loader.ts index 2ad2a39..f0c7d38 100644 --- a/ui/src/lib/trace-loader.ts +++ b/ui/src/lib/trace-loader.ts @@ -37,57 +37,75 @@ interface JaegerData { data: JaegerTrace[]; } -/** - * Load traces from OTLP JSONL file (one span per line) - */ -function loadOtlpJsonlTraces(fileContent: string): Trace[] { - const lines = fileContent.trim().split('\n'); - const spansByTrace = new Map(); - - for (const line of lines) { - if (!line.trim()) continue; - - const otlpSpan = JSON.parse(line); - const traceId = otlpSpan.traceId; +interface OtlpAttribute { + key: string; + value?: { + stringValue?: string; + intValue?: number | string; + doubleValue?: number; + boolValue?: boolean; + }; +} - const tags: Record = {}; - for (const attr of otlpSpan.attributes || []) { - const value = attr.value?.stringValue || attr.value?.intValue || attr.value?.doubleValue || attr.value?.boolValue; - if (value !== undefined) { - tags[attr.key] = value; - } - } +// OTLP/JSON encodes int64 values as strings to preserve precision past +// Number.MAX_SAFE_INTEGER (2^53 - 1). Convert only when the value still +// fits a JavaScript safe integer; otherwise keep the original string so +// large IDs/counters/timestamps round-trip without silent corruption. +function parseOtlpIntValue(value: number | string): number | string { + if (typeof value === 'number') return value; + if (!/^-?\d+$/.test(value)) return value; + const parsed = Number(value); + return Number.isSafeInteger(parsed) ? parsed : value; +} - const startTimeNs = parseInt(otlpSpan.startTimeUnixNano || '0'); - const endTimeNs = parseInt(otlpSpan.endTimeUnixNano || '0'); - const startTimeUs = Math.floor(startTimeNs / 1000); - const durationUs = Math.floor((endTimeNs - startTimeNs) / 1000); +function extractOtlpAttributes(attrs: OtlpAttribute[] | undefined): Record { + const tags: Record = {}; + for (const attr of attrs || []) { + const v = attr.value; + if (!v) continue; + if (v.stringValue !== undefined) tags[attr.key] = v.stringValue; + else if (v.intValue !== undefined) tags[attr.key] = parseOtlpIntValue(v.intValue); + else if (v.doubleValue !== undefined) tags[attr.key] = v.doubleValue; + else if (v.boolValue !== undefined) tags[attr.key] = v.boolValue; + } + return tags; +} - const span: Span = { - traceId, - spanId: otlpSpan.spanId, - parentSpanId: otlpSpan.parentSpanId || null, - operationName: otlpSpan.name, - startTime: startTimeUs, - duration: durationUs, - tags, - logs: [], - children: [], - }; +function buildOtlpSpan(otlpSpan: any, extraTags: Record = {}): Span { + const tags = { ...extraTags, ...extractOtlpAttributes(otlpSpan.attributes) }; + + const startTimeNs = parseInt(otlpSpan.startTimeUnixNano || '0'); + const endTimeNs = parseInt(otlpSpan.endTimeUnixNano || '0'); + const startTimeUs = Math.floor(startTimeNs / 1000); + const durationUs = Math.floor((endTimeNs - startTimeNs) / 1000); + + return { + traceId: otlpSpan.traceId, + spanId: otlpSpan.spanId, + parentSpanId: otlpSpan.parentSpanId || null, + operationName: otlpSpan.name, + startTime: startTimeUs, + duration: durationUs, + tags, + logs: [], + children: [], + }; +} - if (!spansByTrace.has(traceId)) { - spansByTrace.set(traceId, []); - } - spansByTrace.get(traceId)!.push(span); +function buildTracesFromSpans(spans: Span[]): Trace[] { + const spansByTrace = new Map(); + for (const span of spans) { + if (!spansByTrace.has(span.traceId)) spansByTrace.set(span.traceId, []); + spansByTrace.get(span.traceId)!.push(span); } const traces: Trace[] = []; - for (const [traceId, spans] of spansByTrace.entries()) { + for (const [traceId, traceSpans] of spansByTrace.entries()) { const spanMap = new Map(); - spans.forEach(span => spanMap.set(span.spanId, span)); + traceSpans.forEach(span => spanMap.set(span.spanId, span)); const rootSpans: Span[] = []; - for (const span of spans) { + for (const span of traceSpans) { if (span.parentSpanId) { const parent = spanMap.get(span.parentSpanId); if (parent) { @@ -100,45 +118,61 @@ function loadOtlpJsonlTraces(fileContent: string): Trace[] { } } - const sortSpans = (spans: Span[]) => { - spans.sort((a, b) => a.startTime - b.startTime); - spans.forEach((span) => sortSpans(span.children)); + const sortSpans = (s: Span[]) => { + s.sort((a, b) => a.startTime - b.startTime); + s.forEach((span) => sortSpans(span.children)); }; - sortSpans(rootSpans); traces.push({ traceId, rootSpans, - allSpans: spans.sort((a, b) => a.startTime - b.startTime), + allSpans: traceSpans.sort((a, b) => a.startTime - b.startTime), }); } - return traces; } /** - * Load traces from Jaeger JSON file + * Load traces from OTLP JSONL file (one span per line) + */ +function loadOtlpJsonlTraces(fileContent: string): Trace[] { + const lines = fileContent.trim().split('\n'); + const spans: Span[] = []; + for (const line of lines) { + if (!line.trim()) continue; + spans.push(buildOtlpSpan(JSON.parse(line))); + } + return buildTracesFromSpans(spans); +} + +/** + * Load traces from a parsed full OTLP export object (resourceSpans, legacy + * batches, or Tempo v2 ``trace`` wrapper). The object should already be + * unwrapped from any ``trace`` envelope. */ -export async function loadJaegerTraces(fileContent: string): Promise { - const trimmedContent = fileContent.trim(); - - if (trimmedContent.includes('\n') && !trimmedContent.startsWith('[')) { - try { - const firstLine = trimmedContent.split('\n')[0].trim(); - const parsed = JSON.parse(firstLine); - if (!('data' in parsed)) { - return loadOtlpJsonlTraces(fileContent); +function loadOtlpDocTraces(doc: any): Trace[] { + const resourceSpans: any[] = doc.resourceSpans || doc.batches || []; + const spans: Span[] = []; + + for (const rs of resourceSpans) { + const resourceTags = extractOtlpAttributes(rs.resource?.attributes); + const scopeSpans: any[] = rs.scopeSpans || rs.instrumentationLibrarySpans || []; + for (const ss of scopeSpans) { + const scope = ss.scope || ss.instrumentationLibrary || {}; + const scopeTags: Record = { ...resourceTags }; + if (scope.name) scopeTags['otel.scope.name'] = scope.name; + if (scope.version) scopeTags['otel.scope.version'] = scope.version; + for (const otlpSpan of ss.spans || []) { + spans.push(buildOtlpSpan(otlpSpan, scopeTags)); } - } catch { - // Not valid JSONL, fall through to Jaeger parsing } } + return buildTracesFromSpans(spans); +} - const jaegerData: JaegerData = JSON.parse(fileContent); - +function parseJaegerData(jaegerData: JaegerData): Trace[] { return jaegerData.data.map((jTrace) => { - // Parse all spans first const spanMap = new Map(); for (const jSpan of jTrace.spans) { @@ -157,16 +191,13 @@ export async function loadJaegerTraces(fileContent: string): Promise { spanMap.set(span.spanId, span); } - // Build parent-child relationships const rootSpans: Span[] = []; - for (const span of spanMap.values()) { if (span.parentSpanId) { const parent = spanMap.get(span.parentSpanId); if (parent) { parent.children.push(span); } else { - // Parent not found, treat as root rootSpans.push(span); } } else { @@ -174,12 +205,10 @@ export async function loadJaegerTraces(fileContent: string): Promise { } } - // Sort children by start time const sortSpans = (spans: Span[]) => { spans.sort((a, b) => a.startTime - b.startTime); spans.forEach((span) => sortSpans(span.children)); }; - sortSpans(rootSpans); return { @@ -190,6 +219,48 @@ export async function loadJaegerTraces(fileContent: string): Promise { }); } +/** + * Load traces from a trace file. Auto-detects between Jaeger JSON + * (top-level ``data``), full OTLP JSON (top-level ``resourceSpans``), + * legacy/Tempo v1 OTLP (top-level ``batches``), Tempo v2 wrapped + * (``{"trace": {...}}``), and OTLP JSONL (one span per line). + */ +export async function loadTraces(fileContent: string): Promise { + const trimmed = fileContent.trim(); + if (!trimmed) return []; + + let singleDoc: any = null; + try { + singleDoc = JSON.parse(trimmed); + } catch { + if (trimmed.includes('\n')) { + return loadOtlpJsonlTraces(fileContent); + } + throw new Error('Invalid trace file: not valid JSON or JSONL.'); + } + + if (singleDoc && typeof singleDoc === 'object' && !Array.isArray(singleDoc)) { + let doc = singleDoc; + if (doc.trace && typeof doc.trace === 'object' && + ('resourceSpans' in doc.trace || 'batches' in doc.trace)) { + doc = doc.trace; + } + + if ('resourceSpans' in doc || 'batches' in doc) { + return loadOtlpDocTraces(doc); + } + if (Array.isArray(doc.data)) { + return parseJaegerData(doc as JaegerData); + } + } + + throw new Error( + 'Unrecognized trace file format. Expected one of: Jaeger JSON ' + + '({"data": [...]}), OTLP JSON ({"resourceSpans": [...]} or {"batches": [...]}), ' + + 'or OTLP JSONL (one span per line).' + ); +} + /** * Extract parent span ID from references */