agent stats: track tool calls by type with EWMA, add Stats pane

- RunStats now includes tool_calls_by_type HashMap
- AutoAgent tracks runs, last_stats, and EWMA for tool calls/failures
- Removed duplicate stats fields from individual agent structs
- Fixed provenance to use bare agent name (no "agent:" prefix)
- Subconscious screen now displays both agent types consistently
- Added Stats pane showing tool call breakdown sorted by count

Co-Authored-By: Proof of Concept <poc@bcachefs.org>
This commit is contained in:
Kent Overstreet 2026-04-11 22:12:46 -04:00
parent e9e7458013
commit 314ae9c4cb
4 changed files with 169 additions and 55 deletions

View file

@ -22,10 +22,11 @@ use super::Agent;
// Agent logging — shared by Mind and CLI paths // Agent logging — shared by Mind and CLI paths
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
#[derive(Clone, serde::Serialize)] #[derive(Clone, Default, serde::Serialize)]
pub struct RunStats { pub struct RunStats {
pub messages: usize, pub messages: usize,
pub tool_calls: usize, pub tool_calls: usize,
pub tool_failures: usize,
pub tool_calls_by_type: HashMap<String, usize>, pub tool_calls_by_type: HashMap<String, usize>,
} }
@ -57,6 +58,7 @@ fn compute_run_stats(conversation: &[super::context::AstNode]) -> RunStats {
let mut messages = 0usize; let mut messages = 0usize;
let mut tool_calls = 0usize; let mut tool_calls = 0usize;
let mut tool_failures = 0usize;
let mut by_type: HashMap<String, usize> = HashMap::new(); let mut by_type: HashMap<String, usize> = HashMap::new();
for node in conversation { for node in conversation {
@ -64,16 +66,27 @@ fn compute_run_stats(conversation: &[super::context::AstNode]) -> RunStats {
messages += 1; messages += 1;
for child in children { for child in children {
if let AstNode::Leaf(leaf) = child { if let AstNode::Leaf(leaf) = child {
if let NodeBody::ToolCall { name, .. } = leaf.body() { match leaf.body() {
tool_calls += 1; NodeBody::ToolCall { name, .. } => {
*by_type.entry(name.to_string()).or_default() += 1; tool_calls += 1;
*by_type.entry(name.to_string()).or_default() += 1;
}
NodeBody::ToolResult(text) => {
// Detect failures from error patterns in result
let t = text.trim_start();
if t.starts_with("Error") || t.starts_with("error:") ||
t.starts_with("Failed") || t.contains("not found") {
tool_failures += 1;
}
}
_ => {}
} }
} }
} }
} }
} }
RunStats { messages, tool_calls, tool_calls_by_type: by_type } RunStats { messages, tool_calls, tool_failures, tool_calls_by_type: by_type }
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -96,6 +109,11 @@ pub struct AutoAgent {
pub current_phase: String, pub current_phase: String,
pub turn: usize, pub turn: usize,
pub enabled: bool, pub enabled: bool,
// Stats tracking
pub runs: usize,
pub last_stats: Option<RunStats>,
pub tool_calls_ewma: f64,
pub tool_failures_ewma: f64,
} }
/// Per-run conversation backend — wraps a forked agent. /// Per-run conversation backend — wraps a forked agent.
@ -168,6 +186,10 @@ impl AutoAgent {
current_phase: String::new(), current_phase: String::new(),
turn: 0, turn: 0,
enabled: true, enabled: true,
runs: 0,
last_stats: None,
tool_calls_ewma: 0.0,
tool_failures_ewma: 0.0,
} }
} }
@ -242,6 +264,17 @@ impl AutoAgent {
result result
} }
/// Update stats after a run completes. Called with the stats from save_agent_log.
pub fn update_stats(&mut self, stats: RunStats) {
const ALPHA: f64 = 0.3;
self.runs += 1;
self.tool_calls_ewma = ALPHA * (stats.tool_calls as f64)
+ (1.0 - ALPHA) * self.tool_calls_ewma;
self.tool_failures_ewma = ALPHA * (stats.tool_failures as f64)
+ (1.0 - ALPHA) * self.tool_failures_ewma;
self.last_stats = Some(stats);
}
async fn run_with_backend( async fn run_with_backend(
&mut self, &mut self,
backend: &mut Backend, backend: &mut Backend,

View file

@ -294,6 +294,7 @@ pub struct SubconsciousSnapshot {
pub enabled: bool, pub enabled: bool,
pub current_phase: String, pub current_phase: String,
pub turn: usize, pub turn: usize,
pub runs: usize,
pub last_run_secs_ago: Option<f64>, pub last_run_secs_ago: Option<f64>,
/// Shared handle to the forked agent — UI locks to read entries. /// Shared handle to the forked agent — UI locks to read entries.
pub forked_agent: Option<Arc<crate::agent::Agent>>, pub forked_agent: Option<Arc<crate::agent::Agent>>,
@ -303,6 +304,9 @@ pub struct SubconsciousSnapshot {
pub state: std::collections::BTreeMap<String, String>, pub state: std::collections::BTreeMap<String, String>,
/// Recent store activity for this agent: (key, timestamp), newest first. /// Recent store activity for this agent: (key, timestamp), newest first.
pub history: Vec<(String, i64)>, pub history: Vec<(String, i64)>,
pub last_stats: Option<crate::agent::oneshot::RunStats>,
pub tool_calls_ewma: f64,
pub tool_failures_ewma: f64,
} }
struct SubconsciousAgent { struct SubconsciousAgent {
@ -367,11 +371,15 @@ impl SubconsciousAgent {
enabled: self.auto.enabled, enabled: self.auto.enabled,
current_phase: self.auto.current_phase.clone(), current_phase: self.auto.current_phase.clone(),
turn: self.auto.turn, turn: self.auto.turn,
runs: self.auto.runs,
last_run_secs_ago: self.last_run.map(|t| t.elapsed().as_secs_f64()), last_run_secs_ago: self.last_run.map(|t| t.elapsed().as_secs_f64()),
forked_agent: self.forked_agent.clone(), forked_agent: self.forked_agent.clone(),
fork_point: self.fork_point, fork_point: self.fork_point,
state: state.clone(), state: state.clone(),
history, history,
last_stats: self.auto.last_stats.clone(),
tool_calls_ewma: self.auto.tool_calls_ewma,
tool_failures_ewma: self.auto.tool_failures_ewma,
} }
} }
} }
@ -614,7 +622,8 @@ impl Subconscious {
self.agents[idx].handle = Some(tokio::spawn(async move { self.agents[idx].handle = Some(tokio::spawn(async move {
let result = auto.run_forked_shared(&forked, &keys, &st, &recent).await; let result = auto.run_forked_shared(&forked, &keys, &st, &recent).await;
crate::agent::oneshot::save_agent_log(&auto.name, &forked).await; let stats = crate::agent::oneshot::save_agent_log(&auto.name, &forked).await;
auto.update_stats(stats);
(auto, result) (auto, result)
})); }));
} }

View file

@ -34,12 +34,10 @@ struct UnconsciousAgent {
name: String, name: String,
enabled: bool, enabled: bool,
auto: AutoAgent, auto: AutoAgent,
handle: Option<tokio::task::JoinHandle<(AutoAgent, Result<String, String>, RunStats)>>, handle: Option<tokio::task::JoinHandle<(AutoAgent, Result<String, String>)>>,
/// Shared agent handle — UI locks to read context live. /// Shared agent handle — UI locks to read context live.
pub agent: Option<std::sync::Arc<crate::agent::Agent>>, pub agent: Option<std::sync::Arc<crate::agent::Agent>>,
last_run: Option<Instant>, last_run: Option<Instant>,
runs: usize,
last_stats: Option<RunStats>,
} }
impl UnconsciousAgent { impl UnconsciousAgent {
@ -64,6 +62,8 @@ pub struct UnconsciousSnapshot {
pub last_stats: Option<RunStats>, pub last_stats: Option<RunStats>,
/// Recent store activity for this agent: (key, timestamp), newest first. /// Recent store activity for this agent: (key, timestamp), newest first.
pub history: Vec<(String, i64)>, pub history: Vec<(String, i64)>,
pub tool_calls_ewma: f64,
pub tool_failures_ewma: f64,
} }
pub struct Unconscious { pub struct Unconscious {
@ -107,8 +107,6 @@ impl Unconscious {
handle: None, handle: None,
agent: None, agent: None,
last_run: None, last_run: None,
runs: 0,
last_stats: None,
}); });
} }
agents.sort_by(|a, b| a.name.cmp(&b.name)); agents.sort_by(|a, b| a.name.cmp(&b.name));
@ -148,11 +146,13 @@ impl Unconscious {
name: a.name.clone(), name: a.name.clone(),
running: a.is_running(), running: a.is_running(),
enabled: a.enabled, enabled: a.enabled,
runs: a.runs, runs: a.auto.runs,
last_run_secs_ago: a.last_run.map(|t| t.elapsed().as_secs_f64()), last_run_secs_ago: a.last_run.map(|t| t.elapsed().as_secs_f64()),
agent: a.agent.clone(), agent: a.agent.clone(),
last_stats: a.last_stats.clone(), last_stats: a.auto.last_stats.clone(),
history, history,
tool_calls_ewma: a.auto.tool_calls_ewma,
tool_failures_ewma: a.auto.tool_failures_ewma,
} }
}).collect() }).collect()
} }
@ -180,15 +180,13 @@ impl Unconscious {
if agent.handle.as_ref().is_some_and(|h| h.is_finished()) { if agent.handle.as_ref().is_some_and(|h| h.is_finished()) {
let handle = agent.handle.take().unwrap(); let handle = agent.handle.take().unwrap();
agent.last_run = Some(Instant::now()); agent.last_run = Some(Instant::now());
agent.runs += 1; // Get the AutoAgent back from the finished task (stats already updated)
// Get the AutoAgent back from the finished task
match handle.now_or_never() { match handle.now_or_never() {
Some(Ok((auto_back, result, stats))) => { Some(Ok((auto_back, result))) => {
agent.auto = auto_back; agent.auto = auto_back;
agent.last_stats = Some(stats);
match result { match result {
Ok(_) => dbglog!("[unconscious] {} completed (run {})", Ok(_) => dbglog!("[unconscious] {} completed (run {})",
agent.name, agent.runs), agent.name, agent.auto.runs),
Err(e) => dbglog!("[unconscious] {} failed: {}", agent.name, e), Err(e) => dbglog!("[unconscious] {} failed: {}", agent.name, e),
} }
} }
@ -301,8 +299,9 @@ impl Unconscious {
self.agents[idx].handle = Some(tokio::spawn(async move { self.agents[idx].handle = Some(tokio::spawn(async move {
let result = auto.run_shared(&agent).await; let result = auto.run_shared(&agent).await;
let stats = crate::agent::oneshot::save_agent_log(&auto.name, &agent).await; let stats = crate::agent::oneshot::save_agent_log(&auto.name, &agent).await;
auto.update_stats(stats);
auto.steps = orig_steps; auto.steps = orig_steps;
(auto, result, stats) (auto, result)
})); }));
} }
} }

View file

@ -18,10 +18,10 @@ use super::{App, ScreenView, screen_legend};
use super::widgets::{SectionTree, SectionView, section_to_view, pane_block_focused, tree_legend, format_age, format_ts_age}; use super::widgets::{SectionTree, SectionView, section_to_view, pane_block_focused, tree_legend, format_age, format_ts_age};
#[derive(Clone, Copy, PartialEq)] #[derive(Clone, Copy, PartialEq)]
enum Pane { Agents, Outputs, History, Context } enum Pane { Agents, Outputs, Stats, History, Context }
// Clockwise: top-left → right → bottom-left → middle-left // Clockwise: top-left → right → bottom-left → middle-left
const PANE_ORDER: &[Pane] = &[Pane::Agents, Pane::Context, Pane::History, Pane::Outputs]; const PANE_ORDER: &[Pane] = &[Pane::Agents, Pane::Context, Pane::History, Pane::Stats, Pane::Outputs];
pub(crate) struct SubconsciousScreen { pub(crate) struct SubconsciousScreen {
focus: Pane, focus: Pane,
@ -29,6 +29,7 @@ pub(crate) struct SubconsciousScreen {
output_tree: SectionTree, output_tree: SectionTree,
context_tree: SectionTree, context_tree: SectionTree,
history_scroll: super::scroll_pane::ScrollPaneState, history_scroll: super::scroll_pane::ScrollPaneState,
stats_scroll: super::scroll_pane::ScrollPaneState,
} }
impl SubconsciousScreen { impl SubconsciousScreen {
@ -41,6 +42,7 @@ impl SubconsciousScreen {
output_tree: SectionTree::new(), output_tree: SectionTree::new(),
context_tree: SectionTree::new(), context_tree: SectionTree::new(),
history_scroll: super::scroll_pane::ScrollPaneState::new(), history_scroll: super::scroll_pane::ScrollPaneState::new(),
stats_scroll: super::scroll_pane::ScrollPaneState::new(),
} }
} }
@ -87,6 +89,13 @@ impl ScreenView for SubconsciousScreen {
_ => {} _ => {}
} }
Pane::Outputs => self.output_tree.handle_nav(code, &output_sections, area.height), Pane::Outputs => self.output_tree.handle_nav(code, &output_sections, area.height),
Pane::Stats => match code {
KeyCode::Up => self.stats_scroll.scroll_up(3),
KeyCode::Down => self.stats_scroll.scroll_down(3),
KeyCode::PageUp => self.stats_scroll.scroll_up(20),
KeyCode::PageDown => self.stats_scroll.scroll_down(20),
_ => {}
}
Pane::History => match code { Pane::History => match code {
KeyCode::Up => self.history_scroll.scroll_up(3), KeyCode::Up => self.history_scroll.scroll_up(3),
KeyCode::Down => self.history_scroll.scroll_down(3), KeyCode::Down => self.history_scroll.scroll_down(3),
@ -106,7 +115,7 @@ impl ScreenView for SubconsciousScreen {
Constraint::Percentage(62), Constraint::Percentage(62),
]).areas(area); ]).areas(area);
// Left column: agent list (top) | outputs (middle) | history (bottom, main) // Left column: agent list | outputs | stats | history
let unc_count = if app.unconscious_state.is_empty() { 0 } let unc_count = if app.unconscious_state.is_empty() { 0 }
else { app.unconscious_state.len() + 1 }; // +1 for separator else { app.unconscious_state.len() + 1 }; // +1 for separator
let agent_count = (app.agent_state.len() + unc_count).max(1) as u16; let agent_count = (app.agent_state.len() + unc_count).max(1) as u16;
@ -114,15 +123,21 @@ impl ScreenView for SubconsciousScreen {
let output_lines = app.agent_state.get(self.selected()) let output_lines = app.agent_state.get(self.selected())
.map(|s| s.state.values().map(|v| v.lines().count() + 1).sum::<usize>()) .map(|s| s.state.values().map(|v| v.lines().count() + 1).sum::<usize>())
.unwrap_or(0); .unwrap_or(0);
let output_height = (output_lines as u16 + 2).min(left.height / 4).max(3); let output_height = (output_lines as u16 + 2).min(left.height / 5).max(3);
let [list_area, output_area, history_area] = Layout::vertical([ let stats_lines = self.selected_stats(app)
.map(|s| s.tool_calls_by_type.len())
.unwrap_or(0);
let stats_height = (stats_lines as u16 + 2).min(left.height / 5).max(3);
let [list_area, output_area, stats_area, history_area] = Layout::vertical([
Constraint::Length(list_height), Constraint::Length(list_height),
Constraint::Length(output_height), Constraint::Length(output_height),
Constraint::Length(stats_height),
Constraint::Min(5), Constraint::Min(5),
]).areas(left); ]).areas(left);
self.draw_list(frame, list_area, app); self.draw_list(frame, list_area, app);
self.draw_outputs(frame, output_area, app); self.draw_outputs(frame, output_area, app);
self.draw_stats(frame, stats_area, app);
self.draw_history(frame, history_area, app); self.draw_history(frame, history_area, app);
self.draw_context(frame, right, &context_sections, app); self.draw_context(frame, right, &context_sections, app);
} }
@ -147,6 +162,7 @@ impl SubconsciousScreen {
self.output_tree = SectionTree::new(); self.output_tree = SectionTree::new();
self.context_tree = SectionTree::new(); self.context_tree = SectionTree::new();
self.history_scroll = super::scroll_pane::ScrollPaneState::new(); self.history_scroll = super::scroll_pane::ScrollPaneState::new();
self.stats_scroll = super::scroll_pane::ScrollPaneState::new();
} }
/// Get the agent Arc for the selected item, whether subconscious or unconscious. /// Get the agent Arc for the selected item, whether subconscious or unconscious.
@ -175,6 +191,19 @@ impl SubconsciousScreen {
.unwrap_or(&[]) .unwrap_or(&[])
} }
/// Get last run stats for the selected agent.
fn selected_stats<'a>(&self, app: &'a App) -> Option<&'a crate::agent::oneshot::RunStats> {
let idx = self.selected();
let sub_count = app.agent_state.len();
if idx < sub_count {
return app.agent_state.get(idx)
.and_then(|s| s.last_stats.as_ref());
}
idx.checked_sub(sub_count + 1)
.and_then(|i| app.unconscious_state.get(i))
.and_then(|s| s.last_stats.as_ref())
}
fn output_sections(&self, app: &App) -> Vec<SectionView> { fn output_sections(&self, app: &App) -> Vec<SectionView> {
let snap = match app.agent_state.get(self.selected()) { let snap = match app.agent_state.get(self.selected()) {
Some(s) => s, Some(s) => s,
@ -211,37 +240,39 @@ impl SubconsciousScreen {
fn draw_list(&mut self, frame: &mut Frame, area: Rect, app: &App) { fn draw_list(&mut self, frame: &mut Frame, area: Rect, app: &App) {
let mut items: Vec<ListItem> = app.agent_state.iter().map(|snap| { let mut items: Vec<ListItem> = app.agent_state.iter().map(|snap| {
if !snap.enabled { let (name_color, indicator) = if !snap.enabled {
ListItem::from(Line::from(vec![ (Color::DarkGray, "")
Span::styled(&snap.name, Style::default().fg(Color::DarkGray)),
Span::styled(" ○ off", Style::default().fg(Color::DarkGray)),
]))
} else if snap.running { } else if snap.running {
ListItem::from(Line::from(vec![ (Color::Green, "")
Span::styled(&snap.name, Style::default().fg(Color::Green)),
Span::styled("", Style::default().fg(Color::Green)),
Span::styled(
format!("p:{} t:{}", snap.current_phase, snap.turn),
Style::default().fg(Color::DarkGray),
),
]))
} else { } else {
let ago = snap.last_run_secs_ago (Color::Gray, "")
.map(|s| format_age(s)) };
.unwrap_or_else(|| "".to_string()); let ago = snap.last_run_secs_ago
let entries = snap.forked_agent.as_ref() .map(|s| format_age(s))
.and_then(|a| a.context.try_lock().ok()) .unwrap_or_else(|| "".to_string());
.map(|ctx| ctx.conversation().len().saturating_sub(snap.fork_point)) let detail = if snap.running {
.unwrap_or(0); format!("p:{} t:{}", snap.current_phase, snap.turn)
ListItem::from(Line::from(vec![ } else if !snap.enabled {
Span::styled(&snap.name, Style::default().fg(Color::Gray)), "off".to_string()
Span::styled("", Style::default().fg(Color::DarkGray)), } else if let Some(ref stats) = snap.last_stats {
Span::styled( let fail_str = if stats.tool_failures > 0 {
format!("{} {}e", ago, entries), format!(" {}fail", stats.tool_failures)
Style::default().fg(Color::DarkGray), } else {
), String::new()
])) };
} format!("×{} {} {}tc{} avg:{:.1}",
snap.runs, ago,
stats.tool_calls, fail_str,
snap.tool_calls_ewma)
} else {
format!("×{} {}", snap.runs, ago)
};
ListItem::from(Line::from(vec![
Span::styled(&snap.name, Style::default().fg(name_color)),
Span::styled(format!(" {} ", indicator),
Style::default().fg(if snap.running { Color::Green } else { Color::DarkGray })),
Span::styled(detail, Style::default().fg(Color::DarkGray)),
]))
}).collect(); }).collect();
// Unconscious agents (graph maintenance) // Unconscious agents (graph maintenance)
@ -266,9 +297,15 @@ impl SubconsciousScreen {
} else if !snap.enabled { } else if !snap.enabled {
"off".to_string() "off".to_string()
} else if let Some(ref stats) = snap.last_stats { } else if let Some(ref stats) = snap.last_stats {
format!("×{} {} {}msg {}tc", let fail_str = if stats.tool_failures > 0 {
format!(" {}fail", stats.tool_failures)
} else {
String::new()
};
format!("×{} {} {}tc{} avg:{:.1}",
snap.runs, ago, snap.runs, ago,
stats.messages, stats.tool_calls) stats.tool_calls, fail_str,
snap.tool_calls_ewma)
} else { } else {
format!("×{} {}", snap.runs, ago) format!("×{} {}", snap.runs, ago)
}; };
@ -316,6 +353,42 @@ impl SubconsciousScreen {
frame.render_stateful_widget(widget, area, &mut self.output_tree.scroll); frame.render_stateful_widget(widget, area, &mut self.output_tree.scroll);
} }
fn draw_stats(&mut self, frame: &mut Frame, area: Rect, app: &App) {
let dim = Style::default().fg(Color::DarkGray);
let name_style = Style::default().fg(Color::Cyan);
let count_style = Style::default().fg(Color::Yellow);
let mut lines: Vec<Line> = Vec::new();
if let Some(stats) = self.selected_stats(app) {
// Sort by count descending
let mut tools: Vec<_> = stats.tool_calls_by_type.iter().collect();
tools.sort_by(|a, b| b.1.cmp(a.1));
for (name, count) in tools {
lines.push(Line::from(vec![
Span::styled(format!(" {:>3} ", count), count_style),
Span::styled(name.as_str(), name_style),
]));
}
}
if lines.is_empty() {
lines.push(Line::styled(" (no tool calls)", dim));
}
let mut block = pane_block_focused("tool calls", self.focus == Pane::Stats);
if self.focus == Pane::Stats {
block = block.title_bottom(Line::styled(
" ↑↓:scroll PgUp/Dn ",
Style::default().fg(Color::DarkGray),
));
}
let widget = super::scroll_pane::ScrollPane::new(&lines)
.block(block);
frame.render_stateful_widget(widget, area, &mut self.stats_scroll);
}
fn draw_history(&mut self, frame: &mut Frame, area: Rect, app: &App) { fn draw_history(&mut self, frame: &mut Frame, area: Rect, app: &App) {
let dim = Style::default().fg(Color::DarkGray); let dim = Style::default().fg(Color::DarkGray);
let key_style = Style::default().fg(Color::Yellow); let key_style = Style::default().fg(Color::Yellow);