From c4f606bfdb9a02a10587f2eacf7afb51e9aca823 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 6 Mar 2026 15:22:56 -0700 Subject: [PATCH] Fix JSONL parsing warnings and add missing model pricing Claude Code analyzer: - Add catch-all #[serde(other)] variant to ClaudeCodeEntry enum to silently skip unknown entry types like 'last-prompt' - Add catch-all #[serde(other)] variant to ContentBlock enum to handle new block types like 'tool_reference' and 'redacted_thinking' - Make ToolResult.content optional to handle tool results with no content field - Add deserialize_u64_or_null helper to handle null token counts in Usage struct (OpenRouter sends null instead of omitting fields) Model pricing: - Add model name normalization to get_model_info() that strips provider prefixes (minimax/, z-ai/, openrouter/, etc.) and handles :free and -free suffixes with $0 pricing - Add pricing for GPT-5.4, GLM-5, GLM-5-Code, GLM-4.5-Air, MiniMax M2.5, Step 3.5 Flash, Solar Pro 3, Aurora Alpha - Add date-suffixed aliases for new models Reduces startup warnings from 130+ to 2 (unrelated to Claude Code). --- src/analyzers/claude_code.rs | 27 ++++- src/analyzers/codex_cli.rs | 70 ++++++------- src/analyzers/copilot.rs | 11 +-- src/models.rs | 184 +++++++++++++++++++++++++++++++++-- src/tui.rs | 152 ++++++++++++++--------------- 5 files changed, 313 insertions(+), 131 deletions(-) diff --git a/src/analyzers/claude_code.rs b/src/analyzers/claude_code.rs index 8011c66..f154106 100644 --- a/src/analyzers/claude_code.rs +++ b/src/analyzers/claude_code.rs @@ -297,7 +297,8 @@ pub enum ContentBlock { }, ToolResult { tool_use_id: String, // e.g. "toolu_01K7hbuwktKtti8mQb1wH2q8" - content: Content, // e.g. "Found 4 files\nC:\\..." + #[serde(default)] + content: Option, // e.g. "Found 4 files\nC:\\..." — absent for empty results }, Text { text: serde_bytes::ByteBuf, @@ -309,6 +310,9 @@ pub enum ContentBlock { Image { source: ImageSource, }, + // Catch-all for unknown/new content block types (e.g. tool_reference, redacted_thinking) + #[serde(other)] + Other, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -324,15 +328,25 @@ pub enum ImageSource { Base64 { media_type: String, data: String }, } +/// Deserializes a JSON value as u64, treating null as 0. +/// Needed because some providers (e.g. OpenRouter) send `null` for token counts +/// instead of omitting the field, and `#[serde(default)]` only handles missing fields. +fn deserialize_u64_or_null<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + Ok(Option::::deserialize(deserializer)?.unwrap_or(0)) +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Usage { - #[serde(default)] + #[serde(default, deserialize_with = "deserialize_u64_or_null")] pub input_tokens: u64, - #[serde(default)] + #[serde(default, deserialize_with = "deserialize_u64_or_null")] pub output_tokens: u64, - #[serde(default)] + #[serde(default, deserialize_with = "deserialize_u64_or_null")] pub cache_creation_input_tokens: u64, - #[serde(default)] + #[serde(default, deserialize_with = "deserialize_u64_or_null")] pub cache_read_input_tokens: u64, } @@ -411,6 +425,9 @@ enum ClaudeCodeEntry { QueueOperation(ClaudeCodeQueueOperationEntry), #[serde(rename = "progress")] Progress(ClaudeCodeProgressEntry), + // Catch-all for unknown/new entry types (e.g. last-prompt) + #[serde(other)] + Other, } pub mod tool_schema { diff --git a/src/analyzers/codex_cli.rs b/src/analyzers/codex_cli.rs index 69de15e..ee6c704 100644 --- a/src/analyzers/codex_cli.rs +++ b/src/analyzers/codex_cli.rs @@ -392,45 +392,45 @@ pub(crate) fn parse_codex_cli_jsonl_file( session_name: effective_name, }); } - "assistant" => { + "assistant" // Token usage is now emitted immediately when processing token_count // events. We still track assistant messages without additional stats // to avoid double-counting when Codex emits separate reasoning/tool // outputs. - if !saw_token_usage { - let model_state = session_model.clone().unwrap_or_else(|| { - let fallback = SessionModel::inferred( - DEFAULT_FALLBACK_MODEL.to_string(), - ); - warn_once(format!( - "WARNING: session {file_path_str} missing model metadata; using fallback model {} for cost estimation.", - fallback.name - )); - session_model = Some(fallback.clone()); - fallback - }); - - entries.push(ConversationMessage { - application: Application::CodexCli, - model: Some(model_state.name.clone()), - global_hash: hash_text(&format!( - "{}_{}_assistant_{}", - file_path_str, - wrapper.timestamp.to_rfc3339(), - entries.len() - )), - local_hash: None, - conversation_hash: hash_text(&file_path_str), - date: wrapper.timestamp, - project_hash: "".to_string(), - stats: Stats::default(), - role: MessageRole::Assistant, - uuid: None, - session_name: session_name - .clone() - .or_else(|| fallback_session_name.clone()), - }); - } + if !saw_token_usage => + { + let model_state = session_model.clone().unwrap_or_else(|| { + let fallback = SessionModel::inferred( + DEFAULT_FALLBACK_MODEL.to_string(), + ); + warn_once(format!( + "WARNING: session {file_path_str} missing model metadata; using fallback model {} for cost estimation.", + fallback.name + )); + session_model = Some(fallback.clone()); + fallback + }); + + entries.push(ConversationMessage { + application: Application::CodexCli, + model: Some(model_state.name.clone()), + global_hash: hash_text(&format!( + "{}_{}_assistant_{}", + file_path_str, + wrapper.timestamp.to_rfc3339(), + entries.len() + )), + local_hash: None, + conversation_hash: hash_text(&file_path_str), + date: wrapper.timestamp, + project_hash: "".to_string(), + stats: Stats::default(), + role: MessageRole::Assistant, + uuid: None, + session_name: session_name + .clone() + .or_else(|| fallback_session_name.clone()), + }); } _ => {} } diff --git a/src/analyzers/copilot.rs b/src/analyzers/copilot.rs index 1ec28f2..7a8b88a 100644 --- a/src/analyzers/copilot.rs +++ b/src/analyzers/copilot.rs @@ -149,15 +149,14 @@ fn count_tokens(text: &str) -> u64 { // Recursively extract all text content from a nested JSON structure fn extract_text_from_value(value: &simd_json::OwnedValue, accumulated_text: &mut String) { match value { - simd_json::OwnedValue::String(s) => { + simd_json::OwnedValue::String(s) // Only accumulate if it's a "text" field value, not metadata like URIs if !s.starts_with("vscode-") && !s.starts_with("file://") - && !s.starts_with("ssh-remote") - { - accumulated_text.push_str(s); - accumulated_text.push(' '); - } + && !s.starts_with("ssh-remote") => + { + accumulated_text.push_str(s); + accumulated_text.push(' '); } simd_json::OwnedValue::Object(obj) => { // Look for "text" fields specifically diff --git a/src/models.rs b/src/models.rs index 18f2228..06de394 100644 --- a/src/models.rs +++ b/src/models.rs @@ -344,6 +344,35 @@ static MODEL_INDEX: phf::Map<&'static str, ModelInfo> = phf_map! { caching: CachingSupport::None, is_estimated: false, }, + "gpt-5.4" => ModelInfo { + pricing: PricingStructure::Tiered { + tiers: &[ + PricingTier { + max_tokens: Some(272_000), + input_per_1m: 2.50, + output_per_1m: 15.0, + }, + PricingTier { + max_tokens: None, + input_per_1m: 5.0, + output_per_1m: 22.5, + }, + ], + }, + caching: CachingSupport::Google { + tiers: &[ + CachingTier { + max_tokens: Some(272_000), + cached_input_per_1m: 0.25, + }, + CachingTier { + max_tokens: None, + cached_input_per_1m: 0.50, + }, + ], + }, + is_estimated: false, + }, // Anthropic Models "claude-opus-4-6" => ModelInfo { @@ -780,6 +809,78 @@ static MODEL_INDEX: phf::Map<&'static str, ModelInfo> = phf_map! { caching: CachingSupport::None, is_estimated: false, }, + + // Z.AI (Zhipu AI) - Additional Models + "glm-5" => ModelInfo { + pricing: PricingStructure::Flat { + input_per_1m: 1.0, + output_per_1m: 3.2, + }, + caching: CachingSupport::OpenAI { + cached_input_per_1m: 0.2, + }, + is_estimated: false, + }, + "glm-5-code" => ModelInfo { + pricing: PricingStructure::Flat { + input_per_1m: 1.2, + output_per_1m: 5.0, + }, + caching: CachingSupport::OpenAI { + cached_input_per_1m: 0.3, + }, + is_estimated: false, + }, + "glm-4.5-air" => ModelInfo { + pricing: PricingStructure::Flat { + input_per_1m: 0.2, + output_per_1m: 1.1, + }, + caching: CachingSupport::OpenAI { + cached_input_per_1m: 0.03, + }, + is_estimated: false, + }, + + // MiniMax Models + "minimax-m2.5" => ModelInfo { + pricing: PricingStructure::Flat { + input_per_1m: 0.30, + output_per_1m: 1.10, + }, + caching: CachingSupport::None, + is_estimated: false, + }, + + // StepFun Models + "step-3.5-flash" => ModelInfo { + pricing: PricingStructure::Flat { + input_per_1m: 0.10, + output_per_1m: 0.30, + }, + caching: CachingSupport::None, + is_estimated: false, + }, + + // Upstage Models + "solar-pro-3" => ModelInfo { + pricing: PricingStructure::Flat { + input_per_1m: 0.15, + output_per_1m: 0.60, + }, + caching: CachingSupport::None, + is_estimated: false, + }, + + // OpenRouter Models + "aurora-alpha" => ModelInfo { + pricing: PricingStructure::Flat { + input_per_1m: 0.0, + output_per_1m: 0.0, + }, + caching: CachingSupport::None, + is_estimated: false, + }, }; static MODEL_ALIASES: phf::Map<&'static str, &'static str> = phf_map! { @@ -912,19 +1013,88 @@ static MODEL_ALIASES: phf::Map<&'static str, &'static str> = phf_map! { // Zhipu AI aliases "zai-glm-4.6" => "glm-4.6", + "glm-5-20260211" => "glm-5", + "glm-5-code" => "glm-5-code", + "glm-5-code-20260211" => "glm-5-code", + "glm-4.5-air-20260211" => "glm-4.5-air", + + // OpenAI aliases (continued) + "gpt-5.4" => "gpt-5.4", + "gpt-5.4-2026-03-05" => "gpt-5.4", + + // MiniMax aliases + "minimax-m2.5" => "minimax-m2.5", + "minimax-m2.5-20260211" => "minimax-m2.5", + + // StepFun aliases + "step-3.5-flash" => "step-3.5-flash", + + // Upstage aliases + "solar-pro-3" => "solar-pro-3", + + // Aurora aliases + "aurora-alpha" => "aurora-alpha", }; -/// Get model info by any valid name (canonical or alias) -pub fn get_model_info(model_name: &str) -> Option<&ModelInfo> { - // First try direct lookup in model index - if let Some(model_info) = MODEL_INDEX.get(model_name) { +/// Free-tier model pricing for models accessed via OpenRouter's `:free` suffix +/// or other free-tier naming patterns. +static FREE_MODEL_INFO: ModelInfo = ModelInfo { + pricing: PricingStructure::Flat { + input_per_1m: 0.0, + output_per_1m: 0.0, + }, + caching: CachingSupport::None, + is_estimated: false, +}; + +/// Look up a model name directly in the index and alias tables. +fn lookup_model(name: &str) -> Option<&'static ModelInfo> { + if let Some(model_info) = MODEL_INDEX.get(name) { return Some(model_info); } - - // Then try alias lookup - if let Some(&canonical_name) = MODEL_ALIASES.get(model_name) { + if let Some(&canonical_name) = MODEL_ALIASES.get(name) { return MODEL_INDEX.get(canonical_name); } + None +} + +/// Get model info by any valid name (canonical or alias). +/// +/// Handles provider-prefixed model names (e.g. `minimax/minimax-m2.5`, +/// `z-ai/glm-5`, `openrouter/aurora-alpha`) by stripping the prefix before +/// lookup. Models with a `:free` suffix (OpenRouter free tier) always +/// return $0 pricing. +pub fn get_model_info(model_name: &str) -> Option<&'static ModelInfo> { + // Fast path: direct lookup + if let Some(info) = lookup_model(model_name) { + return Some(info); + } + + // Normalize: strip provider prefix (everything before last `/`) + let after_slash = model_name + .rsplit_once('/') + .map(|(_, name)| name) + .unwrap_or(model_name); + + // Handle `:free` suffix → always $0 + if after_slash.strip_suffix(":free").is_some() { + return Some(&FREE_MODEL_INFO); + } + + // Handle other suffixes like `:extended` + let base_name = after_slash.strip_suffix(":extended").unwrap_or(after_slash); + + // Try the normalized name (only if different from original) + if base_name != model_name + && let Some(info) = lookup_model(base_name) + { + return Some(info); + } + + // Also handle patterns like "minimax-m2.5-free" (without colon) + if base_name.strip_suffix("-free").is_some() { + return Some(&FREE_MODEL_INFO); + } None } diff --git a/src/tui.rs b/src/tui.rs index e9b8ca6..59329f9 100644 --- a/src/tui.rs +++ b/src/tui.rs @@ -365,61 +365,59 @@ async fn run_app( } match key.code { - KeyCode::Left | KeyCode::Char('h') => { - if *selected_tab > 0 { - *selected_tab -= 1; + KeyCode::Left | KeyCode::Char('h') if *selected_tab > 0 => { + *selected_tab -= 1; - if let StatsViewMode::Session = *stats_view_mode - && let Some(table_state) = table_states.get_mut(*selected_tab) - && let Some(view) = filtered_stats.get(*selected_tab) + if let StatsViewMode::Session = *stats_view_mode + && let Some(table_state) = table_states.get_mut(*selected_tab) + && let Some(view) = filtered_stats.get(*selected_tab) + { + let view = view.read(); + let target_len = match session_day_filters + .get(*selected_tab) + .and_then(|f| f.as_ref()) { - let view = view.read(); - let target_len = match session_day_filters - .get(*selected_tab) - .and_then(|f| f.as_ref()) - { - Some(day) => view - .session_aggregates - .iter() - .filter(|s| &s.date == day) - .count(), - None => view.session_aggregates.len(), - }; - if target_len > 0 { - table_state.select(Some(target_len.saturating_sub(1))); - } + Some(day) => view + .session_aggregates + .iter() + .filter(|s| &s.date == day) + .count(), + None => view.session_aggregates.len(), + }; + if target_len > 0 { + table_state.select(Some(target_len.saturating_sub(1))); } - - needs_redraw = true; } + + needs_redraw = true; } - KeyCode::Right | KeyCode::Char('l') => { - if *selected_tab < filtered_stats.len().saturating_sub(1) { - *selected_tab += 1; + KeyCode::Right | KeyCode::Char('l') + if *selected_tab < filtered_stats.len().saturating_sub(1) => + { + *selected_tab += 1; - if let StatsViewMode::Session = *stats_view_mode - && let Some(table_state) = table_states.get_mut(*selected_tab) - && let Some(view) = filtered_stats.get(*selected_tab) + if let StatsViewMode::Session = *stats_view_mode + && let Some(table_state) = table_states.get_mut(*selected_tab) + && let Some(view) = filtered_stats.get(*selected_tab) + { + let view = view.read(); + let target_len = match session_day_filters + .get(*selected_tab) + .and_then(|f| f.as_ref()) { - let view = view.read(); - let target_len = match session_day_filters - .get(*selected_tab) - .and_then(|f| f.as_ref()) - { - Some(day) => view - .session_aggregates - .iter() - .filter(|s| &s.date == day) - .count(), - None => view.session_aggregates.len(), - }; - if target_len > 0 { - table_state.select(Some(target_len.saturating_sub(1))); - } + Some(day) => view + .session_aggregates + .iter() + .filter(|s| &s.date == day) + .count(), + None => view.session_aggregates.len(), + }; + if target_len > 0 { + table_state.select(Some(target_len.saturating_sub(1))); } - - needs_redraw = true; } + + needs_redraw = true; } KeyCode::Down | KeyCode::Char('j') => { if let Some(table_state) = table_states.get_mut(*selected_tab) @@ -629,43 +627,41 @@ async fn run_app( needs_redraw = true; } } - KeyCode::Char('t') => { - if key.modifiers.contains(KeyModifiers::CONTROL) { - *stats_view_mode = match *stats_view_mode { - StatsViewMode::Daily => { - session_day_filters[*selected_tab] = None; - StatsViewMode::Session - } - StatsViewMode::Session => StatsViewMode::Daily, - }; + KeyCode::Char('t') if key.modifiers.contains(KeyModifiers::CONTROL) => { + *stats_view_mode = match *stats_view_mode { + StatsViewMode::Daily => { + session_day_filters[*selected_tab] = None; + StatsViewMode::Session + } + StatsViewMode::Session => StatsViewMode::Daily, + }; - date_jump_active = false; - date_jump_buffer.clear(); + date_jump_active = false; + date_jump_buffer.clear(); - if let StatsViewMode::Session = *stats_view_mode - && let Some(table_state) = table_states.get_mut(*selected_tab) - && let Some(view) = filtered_stats.get(*selected_tab) - { - let v = view.read(); - if !v.session_aggregates.is_empty() { - let target_len = session_day_filters - .get(*selected_tab) - .and_then(|f| f.as_ref()) - .map(|day| { - v.session_aggregates - .iter() - .filter(|s| &s.date == day) - .count() - }) - .unwrap_or_else(|| v.session_aggregates.len()); - if target_len > 0 { - table_state.select(Some(target_len.saturating_sub(1))); - } + if let StatsViewMode::Session = *stats_view_mode + && let Some(table_state) = table_states.get_mut(*selected_tab) + && let Some(view) = filtered_stats.get(*selected_tab) + { + let v = view.read(); + if !v.session_aggregates.is_empty() { + let target_len = session_day_filters + .get(*selected_tab) + .and_then(|f| f.as_ref()) + .map(|day| { + v.session_aggregates + .iter() + .filter(|s| &s.date == day) + .count() + }) + .unwrap_or_else(|| v.session_aggregates.len()); + if target_len > 0 { + table_state.select(Some(target_len.saturating_sub(1))); } } - - needs_redraw = true; } + + needs_redraw = true; } KeyCode::Enter => { if let StatsViewMode::Daily = *stats_view_mode