From 480fe5d61e51dddc66dab1efdfaaa740ecf2368c Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Wed, 6 May 2026 15:10:03 +0000 Subject: [PATCH 1/5] feat(usage): classify finish_reason values and flag error reasons in has_error Introduce a shared zod enum + helper (isErrorFinishReason) for the set of finish_reason / stop_reason / status values we observe across OpenAI chat completions, OpenRouter, Anthropic Messages API, OpenAI Responses API, and Vercel AI SDK style responses. Reasons that indicate truncation, refusal, content filtering, upstream failure, or an interrupted in_progress stream now flip has_error to true in all three usage parsers (chat completions, messages, responses string path). Normal completion reasons (stop, end_turn, tool_use, completed, stop_sequence, tool_calls/tool-calls) and unclassified catch-alls (unknown, other, null) keep has_error driven only by status code and abort signals as before. --- .../src/lib/ai-gateway/finishReason.test.ts | 47 +++++++++++++++ apps/web/src/lib/ai-gateway/finishReason.ts | 60 +++++++++++++++++++ .../lib/ai-gateway/processUsage.messages.ts | 8 ++- .../lib/ai-gateway/processUsage.responses.ts | 3 +- apps/web/src/lib/ai-gateway/processUsage.ts | 8 ++- 5 files changed, 119 insertions(+), 7 deletions(-) create mode 100644 apps/web/src/lib/ai-gateway/finishReason.test.ts create mode 100644 apps/web/src/lib/ai-gateway/finishReason.ts diff --git a/apps/web/src/lib/ai-gateway/finishReason.test.ts b/apps/web/src/lib/ai-gateway/finishReason.test.ts new file mode 100644 index 000000000..5d7783ed2 --- /dev/null +++ b/apps/web/src/lib/ai-gateway/finishReason.test.ts @@ -0,0 +1,47 @@ +import { + ERROR_FINISH_REASONS, + FINISH_REASONS, + FinishReasonSchema, + NON_ERROR_FINISH_REASONS, + isErrorFinishReason, +} from '@/lib/ai-gateway/finishReason'; + +describe('finishReason', () => { + it('classifies known error finish_reasons as errors', () => { + for (const reason of ERROR_FINISH_REASONS) { + expect(isErrorFinishReason(reason)).toBe(true); + } + }); + + it('classifies known non-error finish_reasons as non-errors', () => { + for (const reason of NON_ERROR_FINISH_REASONS) { + expect(isErrorFinishReason(reason)).toBe(false); + } + }); + + it('treats null/undefined as non-error', () => { + expect(isErrorFinishReason(null)).toBe(false); + expect(isErrorFinishReason(undefined)).toBe(false); + }); + + it('treats unrecognised string values as non-error', () => { + // Unknown values should not flip hasError; other signals (statusCode, + // wasAborted) handle those cases. This also keeps us from creating + // spurious error rows when a new provider adds a new stop reason. + expect(isErrorFinishReason('something_new_from_provider')).toBe(false); + }); + + it('exposes a zod enum covering both error and non-error reasons', () => { + for (const reason of FINISH_REASONS) { + expect(FinishReasonSchema.safeParse(reason).success).toBe(true); + } + expect(FinishReasonSchema.safeParse('not_a_real_reason').success).toBe(false); + }); + + it('does not double-count any reason in both lists', () => { + const intersection = NON_ERROR_FINISH_REASONS.filter(r => + (ERROR_FINISH_REASONS as readonly string[]).includes(r) + ); + expect(intersection).toEqual([]); + }); +}); diff --git a/apps/web/src/lib/ai-gateway/finishReason.ts b/apps/web/src/lib/ai-gateway/finishReason.ts new file mode 100644 index 000000000..55db6fa28 --- /dev/null +++ b/apps/web/src/lib/ai-gateway/finishReason.ts @@ -0,0 +1,60 @@ +import { z } from 'zod'; + +/** + * Finish reasons that indicate the model completed its turn in an expected way. + * Includes: + * - OpenAI/OpenRouter chat completion: `stop`, `tool_calls`, `stop_sequence` + * - Vercel AI SDK style: `tool-calls` + * - Anthropic Messages API: `end_turn`, `tool_use`, `stop_sequence` + * - OpenAI Responses API: `completed` + * - Catch-alls we cannot classify: `unknown`, `other` + */ +export const NON_ERROR_FINISH_REASONS = [ + 'stop', + 'tool_calls', + 'tool-calls', + 'end_turn', + 'completed', + 'tool_use', + 'stop_sequence', + 'unknown', + 'other', +] as const; + +/** + * Finish reasons that mean the response was truncated, refused, or upstream + * failed in some way. Records carrying these values should be flagged as + * errors so they show up in dashboards / alerts. + */ +export const ERROR_FINISH_REASONS = [ + 'length', + 'max_tokens', + 'content_filter', + 'content-filter', + 'error', + 'network_error', + 'failed', + 'model_context_window_exceeded', + 'engine_overloaded', + 'refusal', + 'incomplete', + 'in_progress', +] as const; + +export const FINISH_REASONS = [...NON_ERROR_FINISH_REASONS, ...ERROR_FINISH_REASONS] as const; + +export const FinishReasonSchema = z.enum(FINISH_REASONS); +export type FinishReason = z.infer; + +const errorFinishReasonSet: ReadonlySet = new Set(ERROR_FINISH_REASONS); + +/** + * Returns true if the given finish_reason indicates an upstream error, + * truncation, refusal, or other failure. `null` is treated as non-error + * (it means we never observed a finish_reason, which is handled separately + * by the `wasAborted` / `reportedError` signals). + */ +export function isErrorFinishReason(finish_reason: string | null | undefined): boolean { + if (finish_reason == null) return false; + return errorFinishReasonSet.has(finish_reason); +} diff --git a/apps/web/src/lib/ai-gateway/processUsage.messages.ts b/apps/web/src/lib/ai-gateway/processUsage.messages.ts index 67af35004..dc343e464 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.messages.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.messages.ts @@ -17,6 +17,7 @@ import { drainSseStream, extractVercelIsByok, } from '@/lib/ai-gateway/processUsage.shared'; +import { isErrorFinishReason } from '@/lib/ai-gateway/finishReason'; import type Anthropic from '@anthropic-ai/sdk'; type MaybeHasVercelProviderMetadata = { @@ -170,7 +171,7 @@ export async function parseMessagesMicrodollarUsageFromStream( const coreProps = { messageId, - hasError: reportedError || wasAborted, + hasError: reportedError || wasAborted || isErrorFinishReason(finish_reason), model, responseContent, inference_provider, @@ -205,14 +206,15 @@ export function parseMessagesMicrodollarUsageFromString( .map(c => c.text) .join(''); + const finish_reason = responseJson?.stop_reason ?? null; const coreProps = { messageId: responseJson?.id ?? null, - hasError: !responseJson?.model || statusCode >= 400, + hasError: !responseJson?.model || statusCode >= 400 || isErrorFinishReason(finish_reason), model: responseJson?.model ?? null, responseContent, inference_provider, upstream_id: null, - finish_reason: responseJson?.stop_reason ?? null, + finish_reason, latency: null, moderation_latency: null, generation_time: null, diff --git a/apps/web/src/lib/ai-gateway/processUsage.responses.ts b/apps/web/src/lib/ai-gateway/processUsage.responses.ts index eb8e2eae3..b5fefef3e 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.responses.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.responses.ts @@ -18,6 +18,7 @@ import { drainSseStream, extractVercelIsByok, } from '@/lib/ai-gateway/processUsage.shared'; +import { isErrorFinishReason } from '@/lib/ai-gateway/finishReason'; // OpenRouter adds cost fields to the standard Responses API usage object. // ref: https://openrouter.ai/docs/use-cases/usage-accounting#response-format @@ -204,7 +205,7 @@ export async function parseResponsesMicrodollarUsageFromStream( const coreProps = { messageId, - hasError: reportedError || wasAborted, + hasError: reportedError || wasAborted || isErrorFinishReason(finish_reason), model, responseContent, inference_provider, diff --git a/apps/web/src/lib/ai-gateway/processUsage.ts b/apps/web/src/lib/ai-gateway/processUsage.ts index f45ff1cd0..50039eacf 100644 --- a/apps/web/src/lib/ai-gateway/processUsage.ts +++ b/apps/web/src/lib/ai-gateway/processUsage.ts @@ -54,6 +54,7 @@ import { parseMessagesMicrodollarUsageFromString, } from '@/lib/ai-gateway/processUsage.messages'; import { OPENROUTER_BYOK_COST_MULTIPLIER } from '@/lib/ai-gateway/processUsage.constants'; +import { isErrorFinishReason } from '@/lib/ai-gateway/finishReason'; import { computeOpenRouterCostFields, drainSseStream, @@ -785,7 +786,7 @@ export async function parseMicrodollarUsageFromStream( const coreProps = { kiloUserId, messageId, - hasError: reportedError || wasAborted, + hasError: reportedError || wasAborted || isErrorFinishReason(finish_reason), model, responseContent, inference_provider, @@ -822,10 +823,11 @@ export function parseMicrodollarUsageFromString( }); } const choice = responseJson?.choices?.[0]; + const finish_reason = choice?.finish_reason ?? null; const coreProps = { kiloUserId, messageId: responseJson?.id ?? null, - hasError: !responseJson?.model || statusCode >= 400, + hasError: !responseJson?.model || statusCode >= 400 || isErrorFinishReason(finish_reason), model: responseJson?.model ?? null, responseContent: choice?.message.content ?? '', inference_provider: @@ -833,7 +835,7 @@ export function parseMicrodollarUsageFromString( choice?.message?.provider_metadata?.gateway?.routing?.finalProvider ?? null, upstream_id: null, - finish_reason: choice?.finish_reason ?? null, + finish_reason, latency: null, moderation_latency: null, generation_time: null, From 53d15bb88367c602e6736f2e622ccf289d9194b6 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 09:37:44 +0000 Subject: [PATCH 2/5] refactor(usage): drop unused zod enum from finishReason module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The zod schema had no runtime consumer — isErrorFinishReason uses a plain Set and the pipeline intentionally keeps finish_reason: string | null end-to-end so unknown upstream values flow through unchanged. Keeping the const arrays gives us the same type-level safety without adding a zod dependency nobody calls. --- apps/web/src/lib/ai-gateway/finishReason.test.ts | 9 --------- apps/web/src/lib/ai-gateway/finishReason.ts | 7 ------- 2 files changed, 16 deletions(-) diff --git a/apps/web/src/lib/ai-gateway/finishReason.test.ts b/apps/web/src/lib/ai-gateway/finishReason.test.ts index 5d7783ed2..8ec6f0cd7 100644 --- a/apps/web/src/lib/ai-gateway/finishReason.test.ts +++ b/apps/web/src/lib/ai-gateway/finishReason.test.ts @@ -1,7 +1,5 @@ import { ERROR_FINISH_REASONS, - FINISH_REASONS, - FinishReasonSchema, NON_ERROR_FINISH_REASONS, isErrorFinishReason, } from '@/lib/ai-gateway/finishReason'; @@ -31,13 +29,6 @@ describe('finishReason', () => { expect(isErrorFinishReason('something_new_from_provider')).toBe(false); }); - it('exposes a zod enum covering both error and non-error reasons', () => { - for (const reason of FINISH_REASONS) { - expect(FinishReasonSchema.safeParse(reason).success).toBe(true); - } - expect(FinishReasonSchema.safeParse('not_a_real_reason').success).toBe(false); - }); - it('does not double-count any reason in both lists', () => { const intersection = NON_ERROR_FINISH_REASONS.filter(r => (ERROR_FINISH_REASONS as readonly string[]).includes(r) diff --git a/apps/web/src/lib/ai-gateway/finishReason.ts b/apps/web/src/lib/ai-gateway/finishReason.ts index 55db6fa28..a5530d50f 100644 --- a/apps/web/src/lib/ai-gateway/finishReason.ts +++ b/apps/web/src/lib/ai-gateway/finishReason.ts @@ -1,5 +1,3 @@ -import { z } from 'zod'; - /** * Finish reasons that indicate the model completed its turn in an expected way. * Includes: @@ -41,11 +39,6 @@ export const ERROR_FINISH_REASONS = [ 'in_progress', ] as const; -export const FINISH_REASONS = [...NON_ERROR_FINISH_REASONS, ...ERROR_FINISH_REASONS] as const; - -export const FinishReasonSchema = z.enum(FINISH_REASONS); -export type FinishReason = z.infer; - const errorFinishReasonSet: ReadonlySet = new Set(ERROR_FINISH_REASONS); /** From 475a7f4fb0967a309de458df0f857f7592720fdf Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 10:00:48 +0000 Subject: [PATCH 3/5] refactor(usage): trim obvious comments in finishReason module --- apps/web/src/lib/ai-gateway/finishReason.ts | 30 +++++++-------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/apps/web/src/lib/ai-gateway/finishReason.ts b/apps/web/src/lib/ai-gateway/finishReason.ts index a5530d50f..6926975f4 100644 --- a/apps/web/src/lib/ai-gateway/finishReason.ts +++ b/apps/web/src/lib/ai-gateway/finishReason.ts @@ -1,12 +1,11 @@ -/** - * Finish reasons that indicate the model completed its turn in an expected way. - * Includes: - * - OpenAI/OpenRouter chat completion: `stop`, `tool_calls`, `stop_sequence` - * - Vercel AI SDK style: `tool-calls` - * - Anthropic Messages API: `end_turn`, `tool_use`, `stop_sequence` - * - OpenAI Responses API: `completed` - * - Catch-alls we cannot classify: `unknown`, `other` - */ +// Finish reason / stop_reason / Responses API status values observed across +// OpenAI & OpenRouter chat completions (stop, tool_calls, stop_sequence, +// length, content_filter), Vercel AI SDK (tool-calls, content-filter), +// Anthropic Messages (end_turn, tool_use, refusal, model_context_window_exceeded), +// and the OpenAI Responses API (completed, failed, incomplete, in_progress). +// `unknown` / `other` are kept as non-error catch-alls so a novel upstream +// value does not immediately inflate the error rate. + export const NON_ERROR_FINISH_REASONS = [ 'stop', 'tool_calls', @@ -19,11 +18,6 @@ export const NON_ERROR_FINISH_REASONS = [ 'other', ] as const; -/** - * Finish reasons that mean the response was truncated, refused, or upstream - * failed in some way. Records carrying these values should be flagged as - * errors so they show up in dashboards / alerts. - */ export const ERROR_FINISH_REASONS = [ 'length', 'max_tokens', @@ -41,12 +35,8 @@ export const ERROR_FINISH_REASONS = [ const errorFinishReasonSet: ReadonlySet = new Set(ERROR_FINISH_REASONS); -/** - * Returns true if the given finish_reason indicates an upstream error, - * truncation, refusal, or other failure. `null` is treated as non-error - * (it means we never observed a finish_reason, which is handled separately - * by the `wasAborted` / `reportedError` signals). - */ +// `null` / `undefined` return false: an absent finish_reason is handled by +// the `wasAborted` / `reportedError` signals in the parsers, not here. export function isErrorFinishReason(finish_reason: string | null | undefined): boolean { if (finish_reason == null) return false; return errorFinishReasonSet.has(finish_reason); From 4442d533f57778c8dae1ab1d3344090809763280 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 11:51:44 +0000 Subject: [PATCH 4/5] docs(usage): note that finish_reason lists come from production logs --- apps/web/src/lib/ai-gateway/finishReason.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/web/src/lib/ai-gateway/finishReason.ts b/apps/web/src/lib/ai-gateway/finishReason.ts index 6926975f4..aacdb5612 100644 --- a/apps/web/src/lib/ai-gateway/finishReason.ts +++ b/apps/web/src/lib/ai-gateway/finishReason.ts @@ -1,8 +1,7 @@ -// Finish reason / stop_reason / Responses API status values observed across -// OpenAI & OpenRouter chat completions (stop, tool_calls, stop_sequence, -// length, content_filter), Vercel AI SDK (tool-calls, content-filter), -// Anthropic Messages (end_turn, tool_use, refusal, model_context_window_exceeded), -// and the OpenAI Responses API (completed, failed, incomplete, in_progress). +// The two lists below enumerate every distinct finish_reason / stop_reason / +// Responses API status value observed in production microdollar_usage logs, +// spanning OpenAI & OpenRouter chat completions, Vercel AI SDK, Anthropic +// Messages, and the OpenAI Responses API. // `unknown` / `other` are kept as non-error catch-alls so a novel upstream // value does not immediately inflate the error rate. From 0fa03f67a9a2cf8a9579db15fed75d1d58cb01b7 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 12:49:35 +0000 Subject: [PATCH 5/5] docs(usage): drop Responses API reference from finish_reason comment The comment was mentioning specific provider APIs, which is noise when the list is sourced from production log distinct values. Keep the comment focused on where the data comes from and the unknown/other policy. --- apps/web/src/lib/ai-gateway/finishReason.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/web/src/lib/ai-gateway/finishReason.ts b/apps/web/src/lib/ai-gateway/finishReason.ts index aacdb5612..27706f602 100644 --- a/apps/web/src/lib/ai-gateway/finishReason.ts +++ b/apps/web/src/lib/ai-gateway/finishReason.ts @@ -1,7 +1,5 @@ -// The two lists below enumerate every distinct finish_reason / stop_reason / -// Responses API status value observed in production microdollar_usage logs, -// spanning OpenAI & OpenRouter chat completions, Vercel AI SDK, Anthropic -// Messages, and the OpenAI Responses API. +// The two lists below enumerate every distinct value observed for the +// `finish_reason` column in production `microdollar_usage` logs. // `unknown` / `other` are kept as non-error catch-alls so a novel upstream // value does not immediately inflate the error rate.