forked from kent/consciousness
agent stats: track tool calls by type with EWMA, add Stats pane
- RunStats now includes tool_calls_by_type HashMap - AutoAgent tracks runs, last_stats, and EWMA for tool calls/failures - Removed duplicate stats fields from individual agent structs - Fixed provenance to use bare agent name (no "agent:" prefix) - Subconscious screen now displays both agent types consistently - Added Stats pane showing tool call breakdown sorted by count Co-Authored-By: Proof of Concept <poc@bcachefs.org>
This commit is contained in:
parent
e9e7458013
commit
314ae9c4cb
4 changed files with 169 additions and 55 deletions
|
|
@ -22,10 +22,11 @@ use super::Agent;
|
|||
// Agent logging — shared by Mind and CLI paths
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Clone, serde::Serialize)]
|
||||
#[derive(Clone, Default, serde::Serialize)]
|
||||
pub struct RunStats {
|
||||
pub messages: usize,
|
||||
pub tool_calls: usize,
|
||||
pub tool_failures: usize,
|
||||
pub tool_calls_by_type: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
|
|
@ -57,6 +58,7 @@ fn compute_run_stats(conversation: &[super::context::AstNode]) -> RunStats {
|
|||
|
||||
let mut messages = 0usize;
|
||||
let mut tool_calls = 0usize;
|
||||
let mut tool_failures = 0usize;
|
||||
let mut by_type: HashMap<String, usize> = HashMap::new();
|
||||
|
||||
for node in conversation {
|
||||
|
|
@ -64,16 +66,27 @@ fn compute_run_stats(conversation: &[super::context::AstNode]) -> RunStats {
|
|||
messages += 1;
|
||||
for child in children {
|
||||
if let AstNode::Leaf(leaf) = child {
|
||||
if let NodeBody::ToolCall { name, .. } = leaf.body() {
|
||||
tool_calls += 1;
|
||||
*by_type.entry(name.to_string()).or_default() += 1;
|
||||
match leaf.body() {
|
||||
NodeBody::ToolCall { name, .. } => {
|
||||
tool_calls += 1;
|
||||
*by_type.entry(name.to_string()).or_default() += 1;
|
||||
}
|
||||
NodeBody::ToolResult(text) => {
|
||||
// Detect failures from error patterns in result
|
||||
let t = text.trim_start();
|
||||
if t.starts_with("Error") || t.starts_with("error:") ||
|
||||
t.starts_with("Failed") || t.contains("not found") {
|
||||
tool_failures += 1;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
RunStats { messages, tool_calls, tool_calls_by_type: by_type }
|
||||
RunStats { messages, tool_calls, tool_failures, tool_calls_by_type: by_type }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -96,6 +109,11 @@ pub struct AutoAgent {
|
|||
pub current_phase: String,
|
||||
pub turn: usize,
|
||||
pub enabled: bool,
|
||||
// Stats tracking
|
||||
pub runs: usize,
|
||||
pub last_stats: Option<RunStats>,
|
||||
pub tool_calls_ewma: f64,
|
||||
pub tool_failures_ewma: f64,
|
||||
}
|
||||
|
||||
/// Per-run conversation backend — wraps a forked agent.
|
||||
|
|
@ -168,6 +186,10 @@ impl AutoAgent {
|
|||
current_phase: String::new(),
|
||||
turn: 0,
|
||||
enabled: true,
|
||||
runs: 0,
|
||||
last_stats: None,
|
||||
tool_calls_ewma: 0.0,
|
||||
tool_failures_ewma: 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -242,6 +264,17 @@ impl AutoAgent {
|
|||
result
|
||||
}
|
||||
|
||||
/// Update stats after a run completes. Called with the stats from save_agent_log.
|
||||
pub fn update_stats(&mut self, stats: RunStats) {
|
||||
const ALPHA: f64 = 0.3;
|
||||
self.runs += 1;
|
||||
self.tool_calls_ewma = ALPHA * (stats.tool_calls as f64)
|
||||
+ (1.0 - ALPHA) * self.tool_calls_ewma;
|
||||
self.tool_failures_ewma = ALPHA * (stats.tool_failures as f64)
|
||||
+ (1.0 - ALPHA) * self.tool_failures_ewma;
|
||||
self.last_stats = Some(stats);
|
||||
}
|
||||
|
||||
async fn run_with_backend(
|
||||
&mut self,
|
||||
backend: &mut Backend,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue