Skip to content

feat: add OTel GenAI semantic conventions to agent span #28511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pelikhan merged 4 commits into from
Apr 25, 2026
+161 −58
Merged

feat: add OTel GenAI semantic conventions to agent span #28511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ac2f1b0
Initial plan
Copilot Apr 25, 2026
8e4193e
feat: add OTel GenAI semantic convention attributes to agent span
Copilot Apr 25, 2026
c312aef
fix: use correct OTel GenAI spec attribute names on agent span
Copilot Apr 25, 2026
ce76a46
fix: clarify gen_ai.provider.name comment — may be omitted when engin…
Copilot Apr 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 31 additions & 20 deletions actions/setup/js/send_otlp_span.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -742,31 +742,12 @@ async function sendJobConclusionSpan(spanName, options = {}) {

if (jobName) attributes.push(buildAttr("gh-aw.job.name", jobName));
if (engineId) attributes.push(buildAttr("gh-aw.engine.id", engineId));
if (model) attributes.push(buildAttr("gh-aw.model", model));
if (eventName) attributes.push(buildAttr("gh-aw.event_name", eventName));
attributes.push(buildAttr("gh-aw.staged", staged));
if (!isNaN(effectiveTokens) && effectiveTokens > 0) {
attributes.push(buildAttr("gh-aw.effective_tokens", effectiveTokens));
}

// Enrich span with per-type token breakdown from agent_usage.json (written by
// parse_token_usage.cjs). These four attributes enable cache-hit-rate panels,
// per-type cost attribution, and fine-grained threshold alerts in Grafana /
// Honeycomb / Datadog without requiring the step summary HTML.
const agentUsage = readJSONIfExists("/tmp/gh-aw/agent_usage.json") || {};
if (typeof agentUsage.input_tokens === "number" && agentUsage.input_tokens > 0) {
attributes.push(buildAttr("gh-aw.tokens.input", agentUsage.input_tokens));
}
if (typeof agentUsage.output_tokens === "number" && agentUsage.output_tokens > 0) {
attributes.push(buildAttr("gh-aw.tokens.output", agentUsage.output_tokens));
}
if (typeof agentUsage.cache_read_tokens === "number" && agentUsage.cache_read_tokens > 0) {
attributes.push(buildAttr("gh-aw.tokens.cache_read", agentUsage.cache_read_tokens));
}
if (typeof agentUsage.cache_write_tokens === "number" && agentUsage.cache_write_tokens > 0) {
attributes.push(buildAttr("gh-aw.tokens.cache_write", agentUsage.cache_write_tokens));
}

if (agentConclusion) {
attributes.push(buildAttr("gh-aw.agent.conclusion", agentConclusion));
}
Expand Down Expand Up @@ -877,6 +858,35 @@ async function sendJobConclusionSpan(spanName, options = {}) {
const conclusionSpanId = generateSpanId();
if (jobName === "agent" && typeof agentStartMs === "number" && agentStartMs > 0 && typeof agentEndMs === "number" && agentEndMs > agentStartMs) {
const agentSpanEvents = buildSpanEvents(agentEndMs);

// Build OTel GenAI semantic convention attributes for the dedicated agent span.
// These follow the OpenTelemetry GenAI specification and enable out-of-the-box
// LLM dashboards in Grafana, Datadog, and Honeycomb without custom mappings.
const agentUsage = readJSONIfExists("/tmp/gh-aw/agent_usage.json") || {};
const agentAttributes = [...attributes];
// gen_ai.operation.name is Required by the OTel GenAI spec for inference spans.
// All gh-aw agent executions are chat-style LLM completions.
agentAttributes.push(buildAttr("gen_ai.operation.name", "chat"));
if (model) agentAttributes.push(buildAttr("gen_ai.request.model", model));
// Emit gen_ai.provider.name when engineId is available; it may be omitted when
// engine metadata is unavailable, so this span does not guarantee full GenAI spec compliance.
if (engineId) agentAttributes.push(buildAttr("gen_ai.provider.name", engineId));
// gen_ai.workflow.name identifies the agentic workflow, matching the OTel spec example
// use-cases (e.g. "multi_agent_rag", "customer_support_pipeline").
if (workflowName) agentAttributes.push(buildAttr("gen_ai.workflow.name", workflowName));
if (typeof agentUsage.input_tokens === "number" && agentUsage.input_tokens > 0) {
agentAttributes.push(buildAttr("gen_ai.usage.input_tokens", agentUsage.input_tokens));
}
if (typeof agentUsage.output_tokens === "number" && agentUsage.output_tokens > 0) {
agentAttributes.push(buildAttr("gen_ai.usage.output_tokens", agentUsage.output_tokens));
}
if (typeof agentUsage.cache_read_tokens === "number" && agentUsage.cache_read_tokens > 0) {
agentAttributes.push(buildAttr("gen_ai.usage.cache_read.input_tokens", agentUsage.cache_read_tokens));
}
if (typeof agentUsage.cache_write_tokens === "number" && agentUsage.cache_write_tokens > 0) {
agentAttributes.push(buildAttr("gen_ai.usage.cache_creation.input_tokens", agentUsage.cache_write_tokens));
}

const agentPayload = buildOTLPPayload({
traceId,
spanId: generateSpanId(),
Expand All @@ -886,11 +896,12 @@ async function sendJobConclusionSpan(spanName, options = {}) {
endMs: agentEndMs,
serviceName,
scopeVersion: version,
attributes,
attributes: agentAttributes,
resourceAttributes,
statusCode,
statusMessage,
events: agentSpanEvents,
kind: SPAN_KIND_CLIENT,
});
appendToOTLPJSONL(agentPayload);
if (endpoint) {
Expand Down
168 changes: 130 additions & 38 deletions actions/setup/js/send_otlp_span.test.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1787,6 +1787,93 @@ describe("sendJobConclusionSpan", () => {
expect(span.name).toBe("gh-aw.safe-outputs.conclusion");
});

it("emits the agent span with SPAN_KIND_CLIENT (3)", async () => {
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
vi.stubGlobal("fetch", mockFetch);

process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "https://traces.example.com";
process.env.INPUT_JOB_NAME = "agent";

const startMs = 1_700_000_000_000;
const endMs = 1_700_000_005_000;
const statSpy = vi.spyOn(fs, "statSync").mockReturnValue(/** @type {Partial<fs.Stats>} */ { mtimeMs: endMs });
const readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation(() => {
throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
});

await sendJobConclusionSpan("gh-aw.agent.conclusion", { startMs });

statSpy.mockRestore();
readFileSpy.mockRestore();

const agentBody = JSON.parse(mockFetch.mock.calls[0][1].body);
const agentSpan = agentBody.resourceSpans[0].scopeSpans[0].spans[0];
expect(agentSpan.name).toBe("gh-aw.agent.agent");
expect(agentSpan.kind).toBe(3); // SPAN_KIND_CLIENT
});

it("includes gen_ai.request.model, gen_ai.provider.name, gen_ai.operation.name and gen_ai.workflow.name on the agent span from aw_info.json", async () => {
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
vi.stubGlobal("fetch", mockFetch);

process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "https://traces.example.com";
process.env.INPUT_JOB_NAME = "agent";

const startMs = 1_700_000_000_000;
const endMs = 1_700_000_005_000;
const statSpy = vi.spyOn(fs, "statSync").mockReturnValue(/** @type {Partial<fs.Stats>} */ { mtimeMs: endMs });
const readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation(filePath => {
if (filePath === "/tmp/gh-aw/aw_info.json") {
return JSON.stringify({ model: "claude-3-5-sonnet-20241022", engine_id: "claude", workflow_name: "otel-advisor" });
}
throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
});

await sendJobConclusionSpan("gh-aw.agent.conclusion", { startMs });

statSpy.mockRestore();
readFileSpy.mockRestore();

const agentBody = JSON.parse(mockFetch.mock.calls[0][1].body);
const agentSpan = agentBody.resourceSpans[0].scopeSpans[0].spans[0];
expect(agentSpan.name).toBe("gh-aw.agent.agent");
const attrs = Object.fromEntries(agentSpan.attributes.map(a => [a.key, a.value.stringValue ?? a.value.intValue]));
expect(attrs["gen_ai.operation.name"]).toBe("chat");
expect(attrs["gen_ai.request.model"]).toBe("claude-3-5-sonnet-20241022");
expect(attrs["gen_ai.provider.name"]).toBe("claude");
expect(attrs["gen_ai.workflow.name"]).toBe("otel-advisor");
});

it("omits gen_ai.request.model, gen_ai.provider.name and gen_ai.workflow.name from the agent span when model, engine_id and workflow_name are absent", async () => {
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
vi.stubGlobal("fetch", mockFetch);

process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "https://traces.example.com";
process.env.INPUT_JOB_NAME = "agent";

const startMs = 1_700_000_000_000;
const endMs = 1_700_000_005_000;
const statSpy = vi.spyOn(fs, "statSync").mockReturnValue(/** @type {Partial<fs.Stats>} */ { mtimeMs: endMs });
const readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation(() => {
throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
});

await sendJobConclusionSpan("gh-aw.agent.conclusion", { startMs });

statSpy.mockRestore();
readFileSpy.mockRestore();

const agentBody = JSON.parse(mockFetch.mock.calls[0][1].body);
const agentSpan = agentBody.resourceSpans[0].scopeSpans[0].spans[0];
const attrs = Object.fromEntries(agentSpan.attributes.map(a => [a.key, a.value.stringValue ?? a.value.intValue]));
// gen_ai.operation.name is always present
expect(attrs["gen_ai.operation.name"]).toBe("chat");
const keys = agentSpan.attributes.map(a => a.key);
expect(keys).not.toContain("gen_ai.request.model");
expect(keys).not.toContain("gen_ai.provider.name");
expect(keys).not.toContain("gen_ai.workflow.name");
});

it("includes gh-aw.run.attempt attribute from GITHUB_RUN_ATTEMPT env var", async () => {
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
vi.stubGlobal("fetch", mockFetch);
Expand Down Expand Up @@ -2836,20 +2923,25 @@ describe("sendJobConclusionSpan", () => {
});
});

describe("token breakdown enrichment in conclusion span", () => {
describe("token breakdown enrichment in agent span", () => {
let readFileSpy;
let statSpy;

beforeEach(() => {
process.env.INPUT_JOB_NAME = "agent";
const agentEndMs = 1_700_000_005_000;
statSpy = vi.spyOn(fs, "statSync").mockReturnValue(/** @type {Partial<fs.Stats>} */ { mtimeMs: agentEndMs });
readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation(() => {
throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
});
});

afterEach(() => {
readFileSpy.mockRestore();
statSpy.mockRestore();
});

it("includes all four token breakdown attributes when agent_usage.json is present", async () => {
it("includes all four gen_ai token breakdown attributes on the agent span when agent_usage.json is present", async () => {
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
vi.stubGlobal("fetch", mockFetch);

Expand All @@ -2863,37 +2955,37 @@ describe("sendJobConclusionSpan", () => {
throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
});

await sendJobConclusionSpan("gh-aw.job.conclusion");
await sendJobConclusionSpan("gh-aw.agent.conclusion", { startMs: 1_700_000_000_000 });

const body = JSON.parse(mockFetch.mock.calls[0][1].body);
const span = body.resourceSpans[0].scopeSpans[0].spans[0];
const attrs = Object.fromEntries(span.attributes.map(a => [a.key, a.value.intValue ?? a.value.stringValue]));
expect(attrs["gh-aw.tokens.input"]).toBe(48200);
expect(attrs["gh-aw.tokens.output"]).toBe(1350);
expect(attrs["gh-aw.tokens.cache_read"]).toBe(41000);
expect(attrs["gh-aw.tokens.cache_write"]).toBe(3100);
const agentBody = JSON.parse(mockFetch.mock.calls[0][1].body);
const agentSpan = agentBody.resourceSpans[0].scopeSpans[0].spans[0];
const attrs = Object.fromEntries(agentSpan.attributes.map(a => [a.key, a.value.intValue ?? a.value.stringValue]));
expect(attrs["gen_ai.usage.input_tokens"]).toBe(48200);
expect(attrs["gen_ai.usage.output_tokens"]).toBe(1350);
expect(attrs["gen_ai.usage.cache_read.input_tokens"]).toBe(41000);
expect(attrs["gen_ai.usage.cache_creation.input_tokens"]).toBe(3100);
});

it("omits all token breakdown attributes when agent_usage.json is absent", async () => {
it("omits all gen_ai token breakdown attributes when agent_usage.json is absent", async () => {
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
vi.stubGlobal("fetch", mockFetch);

process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "https://traces.example.com";

// readFileSpy already throws ENOENT for all paths

await sendJobConclusionSpan("gh-aw.job.conclusion");
await sendJobConclusionSpan("gh-aw.agent.conclusion", { startMs: 1_700_000_000_000 });

const body = JSON.parse(mockFetch.mock.calls[0][1].body);
const span = body.resourceSpans[0].scopeSpans[0].spans[0];
const keys = span.attributes.map(a => a.key);
expect(keys).not.toContain("gh-aw.tokens.input");
expect(keys).not.toContain("gh-aw.tokens.output");
expect(keys).not.toContain("gh-aw.tokens.cache_read");
expect(keys).not.toContain("gh-aw.tokens.cache_write");
const agentBody = JSON.parse(mockFetch.mock.calls[0][1].body);
const agentSpan = agentBody.resourceSpans[0].scopeSpans[0].spans[0];
const keys = agentSpan.attributes.map(a => a.key);
expect(keys).not.toContain("gen_ai.usage.input_tokens");
expect(keys).not.toContain("gen_ai.usage.output_tokens");
expect(keys).not.toContain("gen_ai.usage.cache_read.input_tokens");
expect(keys).not.toContain("gen_ai.usage.cache_creation.input_tokens");
});

it("omits a token attribute when its value is zero", async () => {
it("omits a gen_ai token attribute when its value is zero", async () => {
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
vi.stubGlobal("fetch", mockFetch);

Expand All @@ -2907,19 +2999,19 @@ describe("sendJobConclusionSpan", () => {
throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
});

await sendJobConclusionSpan("gh-aw.job.conclusion");
await sendJobConclusionSpan("gh-aw.agent.conclusion", { startMs: 1_700_000_000_000 });

const body = JSON.parse(mockFetch.mock.calls[0][1].body);
const span = body.resourceSpans[0].scopeSpans[0].spans[0];
const attrs = Object.fromEntries(span.attributes.map(a => [a.key, a.value.intValue ?? a.value.stringValue]));
expect(attrs["gh-aw.tokens.input"]).toBe(1000);
expect(attrs["gh-aw.tokens.cache_read"]).toBe(500);
const keys = span.attributes.map(a => a.key);
expect(keys).not.toContain("gh-aw.tokens.output");
expect(keys).not.toContain("gh-aw.tokens.cache_write");
const agentBody = JSON.parse(mockFetch.mock.calls[0][1].body);
const agentSpan = agentBody.resourceSpans[0].scopeSpans[0].spans[0];
const attrs = Object.fromEntries(agentSpan.attributes.map(a => [a.key, a.value.intValue ?? a.value.stringValue]));
expect(attrs["gen_ai.usage.input_tokens"]).toBe(1000);
expect(attrs["gen_ai.usage.cache_read.input_tokens"]).toBe(500);
const keys = agentSpan.attributes.map(a => a.key);
expect(keys).not.toContain("gen_ai.usage.output_tokens");
expect(keys).not.toContain("gen_ai.usage.cache_creation.input_tokens");
});

it("omits token breakdown attributes when agent_usage.json contains invalid JSON", async () => {
it("omits gen_ai token breakdown attributes when agent_usage.json contains invalid JSON", async () => {
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
vi.stubGlobal("fetch", mockFetch);

Expand All @@ -2932,15 +3024,15 @@ describe("sendJobConclusionSpan", () => {
throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
});

await sendJobConclusionSpan("gh-aw.job.conclusion");
await sendJobConclusionSpan("gh-aw.agent.conclusion", { startMs: 1_700_000_000_000 });

const body = JSON.parse(mockFetch.mock.calls[0][1].body);
const span = body.resourceSpans[0].scopeSpans[0].spans[0];
const keys = span.attributes.map(a => a.key);
expect(keys).not.toContain("gh-aw.tokens.input");
expect(keys).not.toContain("gh-aw.tokens.output");
expect(keys).not.toContain("gh-aw.tokens.cache_read");
expect(keys).not.toContain("gh-aw.tokens.cache_write");
const agentBody = JSON.parse(mockFetch.mock.calls[0][1].body);
const agentSpan = agentBody.resourceSpans[0].scopeSpans[0].spans[0];
const keys = agentSpan.attributes.map(a => a.key);
expect(keys).not.toContain("gen_ai.usage.input_tokens");
expect(keys).not.toContain("gen_ai.usage.output_tokens");
expect(keys).not.toContain("gen_ai.usage.cache_read.input_tokens");
expect(keys).not.toContain("gen_ai.usage.cache_creation.input_tokens");
});
});

Expand Down