forked from kent/consciousness
agent stats: track tool calls by type with EWMA, add Stats pane
- RunStats now includes tool_calls_by_type HashMap - AutoAgent tracks runs, last_stats, and EWMA for tool calls/failures - Removed duplicate stats fields from individual agent structs - Fixed provenance to use bare agent name (no "agent:" prefix) - Subconscious screen now displays both agent types consistently - Added Stats pane showing tool call breakdown sorted by count Co-Authored-By: Proof of Concept <poc@bcachefs.org>
This commit is contained in:
Kent Overstreet
parent
e9e7458013
commit
314ae9c4cb
4 changed files with 169 additions and 55 deletions
|
|
@ -22,10 +22,11 @@ use super::Agent;
|
|||
// Agent logging — shared by Mind and CLI paths
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Clone, serde::Serialize)]
|
||||
#[derive(Clone, Default, serde::Serialize)]
|
||||
pub struct RunStats {
|
||||
pub messages: usize,
|
||||
pub tool_calls: usize,
|
||||
pub tool_failures: usize,
|
||||
pub tool_calls_by_type: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
|
|
@ -57,6 +58,7 @@ fn compute_run_stats(conversation: &[super::context::AstNode]) -> RunStats {
|
|||
|
||||
let mut messages = 0usize;
|
||||
let mut tool_calls = 0usize;
|
||||
let mut tool_failures = 0usize;
|
||||
let mut by_type: HashMap<String, usize> = HashMap::new();
|
||||
|
||||
for node in conversation {
|
||||
|
|
@ -64,16 +66,27 @@ fn compute_run_stats(conversation: &[super::context::AstNode]) -> RunStats {
|
|||
messages += 1;
|
||||
for child in children {
|
||||
if let AstNode::Leaf(leaf) = child {
|
||||
if let NodeBody::ToolCall { name, .. } = leaf.body() {
|
||||
match leaf.body() {
|
||||
NodeBody::ToolCall { name, .. } => {
|
||||
tool_calls += 1;
|
||||
*by_type.entry(name.to_string()).or_default() += 1;
|
||||
}
|
||||
NodeBody::ToolResult(text) => {
|
||||
// Detect failures from error patterns in result
|
||||
let t = text.trim_start();
|
||||
if t.starts_with("Error") || t.starts_with("error:") ||
|
||||
t.starts_with("Failed") || t.contains("not found") {
|
||||
tool_failures += 1;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
RunStats { messages, tool_calls, tool_calls_by_type: by_type }
|
||||
RunStats { messages, tool_calls, tool_failures, tool_calls_by_type: by_type }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -96,6 +109,11 @@ pub struct AutoAgent {
|
|||
pub current_phase: String,
|
||||
pub turn: usize,
|
||||
pub enabled: bool,
|
||||
// Stats tracking
|
||||
pub runs: usize,
|
||||
pub last_stats: Option<RunStats>,
|
||||
pub tool_calls_ewma: f64,
|
||||
pub tool_failures_ewma: f64,
|
||||
}
|
||||
|
||||
/// Per-run conversation backend — wraps a forked agent.
|
||||
|
|
@ -168,6 +186,10 @@ impl AutoAgent {
|
|||
current_phase: String::new(),
|
||||
turn: 0,
|
||||
enabled: true,
|
||||
runs: 0,
|
||||
last_stats: None,
|
||||
tool_calls_ewma: 0.0,
|
||||
tool_failures_ewma: 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -242,6 +264,17 @@ impl AutoAgent {
|
|||
result
|
||||
}
|
||||
|
||||
/// Update stats after a run completes. Called with the stats from save_agent_log.
|
||||
pub fn update_stats(&mut self, stats: RunStats) {
|
||||
const ALPHA: f64 = 0.3;
|
||||
self.runs += 1;
|
||||
self.tool_calls_ewma = ALPHA * (stats.tool_calls as f64)
|
||||
+ (1.0 - ALPHA) * self.tool_calls_ewma;
|
||||
self.tool_failures_ewma = ALPHA * (stats.tool_failures as f64)
|
||||
+ (1.0 - ALPHA) * self.tool_failures_ewma;
|
||||
self.last_stats = Some(stats);
|
||||
}
|
||||
|
||||
async fn run_with_backend(
|
||||
&mut self,
|
||||
backend: &mut Backend,
|
||||
|
|
|
|||
|
|
@ -294,6 +294,7 @@ pub struct SubconsciousSnapshot {
|
|||
pub enabled: bool,
|
||||
pub current_phase: String,
|
||||
pub turn: usize,
|
||||
pub runs: usize,
|
||||
pub last_run_secs_ago: Option<f64>,
|
||||
/// Shared handle to the forked agent — UI locks to read entries.
|
||||
pub forked_agent: Option<Arc<crate::agent::Agent>>,
|
||||
|
|
@ -303,6 +304,9 @@ pub struct SubconsciousSnapshot {
|
|||
pub state: std::collections::BTreeMap<String, String>,
|
||||
/// Recent store activity for this agent: (key, timestamp), newest first.
|
||||
pub history: Vec<(String, i64)>,
|
||||
pub last_stats: Option<crate::agent::oneshot::RunStats>,
|
||||
pub tool_calls_ewma: f64,
|
||||
pub tool_failures_ewma: f64,
|
||||
}
|
||||
|
||||
struct SubconsciousAgent {
|
||||
|
|
@ -367,11 +371,15 @@ impl SubconsciousAgent {
|
|||
enabled: self.auto.enabled,
|
||||
current_phase: self.auto.current_phase.clone(),
|
||||
turn: self.auto.turn,
|
||||
runs: self.auto.runs,
|
||||
last_run_secs_ago: self.last_run.map(|t| t.elapsed().as_secs_f64()),
|
||||
forked_agent: self.forked_agent.clone(),
|
||||
fork_point: self.fork_point,
|
||||
state: state.clone(),
|
||||
history,
|
||||
last_stats: self.auto.last_stats.clone(),
|
||||
tool_calls_ewma: self.auto.tool_calls_ewma,
|
||||
tool_failures_ewma: self.auto.tool_failures_ewma,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -614,7 +622,8 @@ impl Subconscious {
|
|||
|
||||
self.agents[idx].handle = Some(tokio::spawn(async move {
|
||||
let result = auto.run_forked_shared(&forked, &keys, &st, &recent).await;
|
||||
crate::agent::oneshot::save_agent_log(&auto.name, &forked).await;
|
||||
let stats = crate::agent::oneshot::save_agent_log(&auto.name, &forked).await;
|
||||
auto.update_stats(stats);
|
||||
(auto, result)
|
||||
}));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,12 +34,10 @@ struct UnconsciousAgent {
|
|||
name: String,
|
||||
enabled: bool,
|
||||
auto: AutoAgent,
|
||||
handle: Option<tokio::task::JoinHandle<(AutoAgent, Result<String, String>, RunStats)>>,
|
||||
handle: Option<tokio::task::JoinHandle<(AutoAgent, Result<String, String>)>>,
|
||||
/// Shared agent handle — UI locks to read context live.
|
||||
pub agent: Option<std::sync::Arc<crate::agent::Agent>>,
|
||||
last_run: Option<Instant>,
|
||||
runs: usize,
|
||||
last_stats: Option<RunStats>,
|
||||
}
|
||||
|
||||
impl UnconsciousAgent {
|
||||
|
|
@ -64,6 +62,8 @@ pub struct UnconsciousSnapshot {
|
|||
pub last_stats: Option<RunStats>,
|
||||
/// Recent store activity for this agent: (key, timestamp), newest first.
|
||||
pub history: Vec<(String, i64)>,
|
||||
pub tool_calls_ewma: f64,
|
||||
pub tool_failures_ewma: f64,
|
||||
}
|
||||
|
||||
pub struct Unconscious {
|
||||
|
|
@ -107,8 +107,6 @@ impl Unconscious {
|
|||
handle: None,
|
||||
agent: None,
|
||||
last_run: None,
|
||||
runs: 0,
|
||||
last_stats: None,
|
||||
});
|
||||
}
|
||||
agents.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
|
|
@ -148,11 +146,13 @@ impl Unconscious {
|
|||
name: a.name.clone(),
|
||||
running: a.is_running(),
|
||||
enabled: a.enabled,
|
||||
runs: a.runs,
|
||||
runs: a.auto.runs,
|
||||
last_run_secs_ago: a.last_run.map(|t| t.elapsed().as_secs_f64()),
|
||||
agent: a.agent.clone(),
|
||||
last_stats: a.last_stats.clone(),
|
||||
last_stats: a.auto.last_stats.clone(),
|
||||
history,
|
||||
tool_calls_ewma: a.auto.tool_calls_ewma,
|
||||
tool_failures_ewma: a.auto.tool_failures_ewma,
|
||||
}
|
||||
}).collect()
|
||||
}
|
||||
|
|
@ -180,15 +180,13 @@ impl Unconscious {
|
|||
if agent.handle.as_ref().is_some_and(|h| h.is_finished()) {
|
||||
let handle = agent.handle.take().unwrap();
|
||||
agent.last_run = Some(Instant::now());
|
||||
agent.runs += 1;
|
||||
// Get the AutoAgent back from the finished task
|
||||
// Get the AutoAgent back from the finished task (stats already updated)
|
||||
match handle.now_or_never() {
|
||||
Some(Ok((auto_back, result, stats))) => {
|
||||
Some(Ok((auto_back, result))) => {
|
||||
agent.auto = auto_back;
|
||||
agent.last_stats = Some(stats);
|
||||
match result {
|
||||
Ok(_) => dbglog!("[unconscious] {} completed (run {})",
|
||||
agent.name, agent.runs),
|
||||
agent.name, agent.auto.runs),
|
||||
Err(e) => dbglog!("[unconscious] {} failed: {}", agent.name, e),
|
||||
}
|
||||
}
|
||||
|
|
@ -301,8 +299,9 @@ impl Unconscious {
|
|||
self.agents[idx].handle = Some(tokio::spawn(async move {
|
||||
let result = auto.run_shared(&agent).await;
|
||||
let stats = crate::agent::oneshot::save_agent_log(&auto.name, &agent).await;
|
||||
auto.update_stats(stats);
|
||||
auto.steps = orig_steps;
|
||||
(auto, result, stats)
|
||||
(auto, result)
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,10 +18,10 @@ use super::{App, ScreenView, screen_legend};
|
|||
use super::widgets::{SectionTree, SectionView, section_to_view, pane_block_focused, tree_legend, format_age, format_ts_age};
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Pane { Agents, Outputs, History, Context }
|
||||
enum Pane { Agents, Outputs, Stats, History, Context }
|
||||
|
||||
// Clockwise: top-left → right → bottom-left → middle-left
|
||||
const PANE_ORDER: &[Pane] = &[Pane::Agents, Pane::Context, Pane::History, Pane::Outputs];
|
||||
const PANE_ORDER: &[Pane] = &[Pane::Agents, Pane::Context, Pane::History, Pane::Stats, Pane::Outputs];
|
||||
|
||||
pub(crate) struct SubconsciousScreen {
|
||||
focus: Pane,
|
||||
|
|
@ -29,6 +29,7 @@ pub(crate) struct SubconsciousScreen {
|
|||
output_tree: SectionTree,
|
||||
context_tree: SectionTree,
|
||||
history_scroll: super::scroll_pane::ScrollPaneState,
|
||||
stats_scroll: super::scroll_pane::ScrollPaneState,
|
||||
}
|
||||
|
||||
impl SubconsciousScreen {
|
||||
|
|
@ -41,6 +42,7 @@ impl SubconsciousScreen {
|
|||
output_tree: SectionTree::new(),
|
||||
context_tree: SectionTree::new(),
|
||||
history_scroll: super::scroll_pane::ScrollPaneState::new(),
|
||||
stats_scroll: super::scroll_pane::ScrollPaneState::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -87,6 +89,13 @@ impl ScreenView for SubconsciousScreen {
|
|||
_ => {}
|
||||
}
|
||||
Pane::Outputs => self.output_tree.handle_nav(code, &output_sections, area.height),
|
||||
Pane::Stats => match code {
|
||||
KeyCode::Up => self.stats_scroll.scroll_up(3),
|
||||
KeyCode::Down => self.stats_scroll.scroll_down(3),
|
||||
KeyCode::PageUp => self.stats_scroll.scroll_up(20),
|
||||
KeyCode::PageDown => self.stats_scroll.scroll_down(20),
|
||||
_ => {}
|
||||
}
|
||||
Pane::History => match code {
|
||||
KeyCode::Up => self.history_scroll.scroll_up(3),
|
||||
KeyCode::Down => self.history_scroll.scroll_down(3),
|
||||
|
|
@ -106,7 +115,7 @@ impl ScreenView for SubconsciousScreen {
|
|||
Constraint::Percentage(62),
|
||||
]).areas(area);
|
||||
|
||||
// Left column: agent list (top) | outputs (middle) | history (bottom, main)
|
||||
// Left column: agent list | outputs | stats | history
|
||||
let unc_count = if app.unconscious_state.is_empty() { 0 }
|
||||
else { app.unconscious_state.len() + 1 }; // +1 for separator
|
||||
let agent_count = (app.agent_state.len() + unc_count).max(1) as u16;
|
||||
|
|
@ -114,15 +123,21 @@ impl ScreenView for SubconsciousScreen {
|
|||
let output_lines = app.agent_state.get(self.selected())
|
||||
.map(|s| s.state.values().map(|v| v.lines().count() + 1).sum::<usize>())
|
||||
.unwrap_or(0);
|
||||
let output_height = (output_lines as u16 + 2).min(left.height / 4).max(3);
|
||||
let [list_area, output_area, history_area] = Layout::vertical([
|
||||
let output_height = (output_lines as u16 + 2).min(left.height / 5).max(3);
|
||||
let stats_lines = self.selected_stats(app)
|
||||
.map(|s| s.tool_calls_by_type.len())
|
||||
.unwrap_or(0);
|
||||
let stats_height = (stats_lines as u16 + 2).min(left.height / 5).max(3);
|
||||
let [list_area, output_area, stats_area, history_area] = Layout::vertical([
|
||||
Constraint::Length(list_height),
|
||||
Constraint::Length(output_height),
|
||||
Constraint::Length(stats_height),
|
||||
Constraint::Min(5),
|
||||
]).areas(left);
|
||||
|
||||
self.draw_list(frame, list_area, app);
|
||||
self.draw_outputs(frame, output_area, app);
|
||||
self.draw_stats(frame, stats_area, app);
|
||||
self.draw_history(frame, history_area, app);
|
||||
self.draw_context(frame, right, &context_sections, app);
|
||||
}
|
||||
|
|
@ -147,6 +162,7 @@ impl SubconsciousScreen {
|
|||
self.output_tree = SectionTree::new();
|
||||
self.context_tree = SectionTree::new();
|
||||
self.history_scroll = super::scroll_pane::ScrollPaneState::new();
|
||||
self.stats_scroll = super::scroll_pane::ScrollPaneState::new();
|
||||
}
|
||||
|
||||
/// Get the agent Arc for the selected item, whether subconscious or unconscious.
|
||||
|
|
@ -175,6 +191,19 @@ impl SubconsciousScreen {
|
|||
.unwrap_or(&[])
|
||||
}
|
||||
|
||||
/// Get last run stats for the selected agent.
|
||||
fn selected_stats<'a>(&self, app: &'a App) -> Option<&'a crate::agent::oneshot::RunStats> {
|
||||
let idx = self.selected();
|
||||
let sub_count = app.agent_state.len();
|
||||
if idx < sub_count {
|
||||
return app.agent_state.get(idx)
|
||||
.and_then(|s| s.last_stats.as_ref());
|
||||
}
|
||||
idx.checked_sub(sub_count + 1)
|
||||
.and_then(|i| app.unconscious_state.get(i))
|
||||
.and_then(|s| s.last_stats.as_ref())
|
||||
}
|
||||
|
||||
fn output_sections(&self, app: &App) -> Vec<SectionView> {
|
||||
let snap = match app.agent_state.get(self.selected()) {
|
||||
Some(s) => s,
|
||||
|
|
@ -211,37 +240,39 @@ impl SubconsciousScreen {
|
|||
|
||||
fn draw_list(&mut self, frame: &mut Frame, area: Rect, app: &App) {
|
||||
let mut items: Vec<ListItem> = app.agent_state.iter().map(|snap| {
|
||||
if !snap.enabled {
|
||||
ListItem::from(Line::from(vec![
|
||||
Span::styled(&snap.name, Style::default().fg(Color::DarkGray)),
|
||||
Span::styled(" ○ off", Style::default().fg(Color::DarkGray)),
|
||||
]))
|
||||
let (name_color, indicator) = if !snap.enabled {
|
||||
(Color::DarkGray, "○")
|
||||
} else if snap.running {
|
||||
ListItem::from(Line::from(vec![
|
||||
Span::styled(&snap.name, Style::default().fg(Color::Green)),
|
||||
Span::styled(" ● ", Style::default().fg(Color::Green)),
|
||||
Span::styled(
|
||||
format!("p:{} t:{}", snap.current_phase, snap.turn),
|
||||
Style::default().fg(Color::DarkGray),
|
||||
),
|
||||
]))
|
||||
(Color::Green, "●")
|
||||
} else {
|
||||
(Color::Gray, "○")
|
||||
};
|
||||
let ago = snap.last_run_secs_ago
|
||||
.map(|s| format_age(s))
|
||||
.unwrap_or_else(|| "—".to_string());
|
||||
let entries = snap.forked_agent.as_ref()
|
||||
.and_then(|a| a.context.try_lock().ok())
|
||||
.map(|ctx| ctx.conversation().len().saturating_sub(snap.fork_point))
|
||||
.unwrap_or(0);
|
||||
let detail = if snap.running {
|
||||
format!("p:{} t:{}", snap.current_phase, snap.turn)
|
||||
} else if !snap.enabled {
|
||||
"off".to_string()
|
||||
} else if let Some(ref stats) = snap.last_stats {
|
||||
let fail_str = if stats.tool_failures > 0 {
|
||||
format!(" {}fail", stats.tool_failures)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
format!("×{} {} {}tc{} avg:{:.1}",
|
||||
snap.runs, ago,
|
||||
stats.tool_calls, fail_str,
|
||||
snap.tool_calls_ewma)
|
||||
} else {
|
||||
format!("×{} {}", snap.runs, ago)
|
||||
};
|
||||
ListItem::from(Line::from(vec![
|
||||
Span::styled(&snap.name, Style::default().fg(Color::Gray)),
|
||||
Span::styled(" ○ ", Style::default().fg(Color::DarkGray)),
|
||||
Span::styled(
|
||||
format!("{} {}e", ago, entries),
|
||||
Style::default().fg(Color::DarkGray),
|
||||
),
|
||||
Span::styled(&snap.name, Style::default().fg(name_color)),
|
||||
Span::styled(format!(" {} ", indicator),
|
||||
Style::default().fg(if snap.running { Color::Green } else { Color::DarkGray })),
|
||||
Span::styled(detail, Style::default().fg(Color::DarkGray)),
|
||||
]))
|
||||
}
|
||||
}).collect();
|
||||
|
||||
// Unconscious agents (graph maintenance)
|
||||
|
|
@ -266,9 +297,15 @@ impl SubconsciousScreen {
|
|||
} else if !snap.enabled {
|
||||
"off".to_string()
|
||||
} else if let Some(ref stats) = snap.last_stats {
|
||||
format!("×{} {} {}msg {}tc",
|
||||
let fail_str = if stats.tool_failures > 0 {
|
||||
format!(" {}fail", stats.tool_failures)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
format!("×{} {} {}tc{} avg:{:.1}",
|
||||
snap.runs, ago,
|
||||
stats.messages, stats.tool_calls)
|
||||
stats.tool_calls, fail_str,
|
||||
snap.tool_calls_ewma)
|
||||
} else {
|
||||
format!("×{} {}", snap.runs, ago)
|
||||
};
|
||||
|
|
@ -316,6 +353,42 @@ impl SubconsciousScreen {
|
|||
frame.render_stateful_widget(widget, area, &mut self.output_tree.scroll);
|
||||
}
|
||||
|
||||
fn draw_stats(&mut self, frame: &mut Frame, area: Rect, app: &App) {
|
||||
let dim = Style::default().fg(Color::DarkGray);
|
||||
let name_style = Style::default().fg(Color::Cyan);
|
||||
let count_style = Style::default().fg(Color::Yellow);
|
||||
|
||||
let mut lines: Vec<Line> = Vec::new();
|
||||
|
||||
if let Some(stats) = self.selected_stats(app) {
|
||||
// Sort by count descending
|
||||
let mut tools: Vec<_> = stats.tool_calls_by_type.iter().collect();
|
||||
tools.sort_by(|a, b| b.1.cmp(a.1));
|
||||
|
||||
for (name, count) in tools {
|
||||
lines.push(Line::from(vec![
|
||||
Span::styled(format!(" {:>3} ", count), count_style),
|
||||
Span::styled(name.as_str(), name_style),
|
||||
]));
|
||||
}
|
||||
}
|
||||
|
||||
if lines.is_empty() {
|
||||
lines.push(Line::styled(" (no tool calls)", dim));
|
||||
}
|
||||
|
||||
let mut block = pane_block_focused("tool calls", self.focus == Pane::Stats);
|
||||
if self.focus == Pane::Stats {
|
||||
block = block.title_bottom(Line::styled(
|
||||
" ↑↓:scroll PgUp/Dn ",
|
||||
Style::default().fg(Color::DarkGray),
|
||||
));
|
||||
}
|
||||
let widget = super::scroll_pane::ScrollPane::new(&lines)
|
||||
.block(block);
|
||||
frame.render_stateful_widget(widget, area, &mut self.stats_scroll);
|
||||
}
|
||||
|
||||
fn draw_history(&mut self, frame: &mut Frame, area: Rect, app: &App) {
|
||||
let dim = Style::default().fg(Color::DarkGray);
|
||||
let key_style = Style::default().fg(Color::Yellow);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue