forked from kent/consciousness
Persist agent stats across restarts, add per-tool metrics grid
Stats now survive daemon restarts via ~/.consciousness/agent-stats.json, loaded into a global Mutex<HashMap> on first access. Each tool type tracks last count, EWMA (alpha=0.3), and total calls. UI shows a grid view: tool | last | avg | total, sorted by total desc. Failures row appears at bottom if any occurred. Also fixes temperature/priority not being applied to spawned agents. Co-Authored-By: Proof of Concept <poc@bcachefs.org>
This commit is contained in:
parent
314ae9c4cb
commit
f408bb5d86
4 changed files with 148 additions and 56 deletions
|
|
@ -22,7 +22,8 @@ use super::Agent;
|
||||||
// Agent logging — shared by Mind and CLI paths
|
// Agent logging — shared by Mind and CLI paths
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
#[derive(Clone, Default, serde::Serialize)]
|
/// Stats from a single run.
|
||||||
|
#[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||||
pub struct RunStats {
|
pub struct RunStats {
|
||||||
pub messages: usize,
|
pub messages: usize,
|
||||||
pub tool_calls: usize,
|
pub tool_calls: usize,
|
||||||
|
|
@ -30,6 +31,57 @@ pub struct RunStats {
|
||||||
pub tool_calls_by_type: HashMap<String, usize>,
|
pub tool_calls_by_type: HashMap<String, usize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Per-tool accumulated stats.
|
||||||
|
#[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct ToolStats {
|
||||||
|
pub last: usize,
|
||||||
|
pub ewma: f64,
|
||||||
|
pub total: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persisted stats for an agent (survives restarts).
|
||||||
|
#[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct PersistedStats {
|
||||||
|
pub runs: usize,
|
||||||
|
pub last_stats: Option<RunStats>,
|
||||||
|
/// Per-tool-type stats: last, ewma, total.
|
||||||
|
pub by_tool: HashMap<String, ToolStats>,
|
||||||
|
/// Failed calls stats.
|
||||||
|
pub failures: ToolStats,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stats_path() -> std::path::PathBuf {
|
||||||
|
dirs::home_dir().unwrap_or_default()
|
||||||
|
.join(".consciousness/agent-stats.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
static AGENT_STATS: std::sync::OnceLock<std::sync::Mutex<HashMap<String, PersistedStats>>> =
|
||||||
|
std::sync::OnceLock::new();
|
||||||
|
|
||||||
|
fn stats_map() -> &'static std::sync::Mutex<HashMap<String, PersistedStats>> {
|
||||||
|
AGENT_STATS.get_or_init(|| {
|
||||||
|
let map: HashMap<String, PersistedStats> = std::fs::read_to_string(stats_path()).ok()
|
||||||
|
.and_then(|s| serde_json::from_str(&s).ok())
|
||||||
|
.unwrap_or_default();
|
||||||
|
std::sync::Mutex::new(map)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_stats(name: &str) -> PersistedStats {
|
||||||
|
stats_map().lock().ok()
|
||||||
|
.and_then(|m| m.get(name).cloned())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_stats(name: &str, stats: PersistedStats) {
|
||||||
|
if let Ok(mut map) = stats_map().lock() {
|
||||||
|
map.insert(name.to_string(), stats);
|
||||||
|
if let Ok(json) = serde_json::to_string_pretty(&*map) {
|
||||||
|
let _ = std::fs::write(stats_path(), json);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Save agent conversation to JSON log file.
|
/// Save agent conversation to JSON log file.
|
||||||
/// Used by both mind-run agents and CLI-run agents.
|
/// Used by both mind-run agents and CLI-run agents.
|
||||||
pub async fn save_agent_log(name: &str, agent: &std::sync::Arc<Agent>) -> RunStats {
|
pub async fn save_agent_log(name: &str, agent: &std::sync::Arc<Agent>) -> RunStats {
|
||||||
|
|
@ -109,13 +161,11 @@ pub struct AutoAgent {
|
||||||
pub current_phase: String,
|
pub current_phase: String,
|
||||||
pub turn: usize,
|
pub turn: usize,
|
||||||
pub enabled: bool,
|
pub enabled: bool,
|
||||||
// Stats tracking
|
pub temperature: f32,
|
||||||
pub runs: usize,
|
pub priority: i32,
|
||||||
pub last_stats: Option<RunStats>,
|
|
||||||
pub tool_calls_ewma: f64,
|
|
||||||
pub tool_failures_ewma: f64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// Per-run conversation backend — wraps a forked agent.
|
/// Per-run conversation backend — wraps a forked agent.
|
||||||
struct Backend(std::sync::Arc<Agent>);
|
struct Backend(std::sync::Arc<Agent>);
|
||||||
|
|
||||||
|
|
@ -178,18 +228,16 @@ impl AutoAgent {
|
||||||
name: String,
|
name: String,
|
||||||
tools: Vec<agent_tools::Tool>,
|
tools: Vec<agent_tools::Tool>,
|
||||||
steps: Vec<AutoStep>,
|
steps: Vec<AutoStep>,
|
||||||
_temperature: f32,
|
temperature: f32,
|
||||||
_priority: i32,
|
priority: i32,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
name, tools, steps,
|
name, tools, steps,
|
||||||
current_phase: String::new(),
|
current_phase: String::new(),
|
||||||
turn: 0,
|
turn: 0,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
runs: 0,
|
temperature,
|
||||||
last_stats: None,
|
priority,
|
||||||
tool_calls_ewma: 0.0,
|
|
||||||
tool_failures_ewma: 0.0,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -225,7 +273,8 @@ impl AutoAgent {
|
||||||
let mut st = agent.state.lock().await;
|
let mut st = agent.state.lock().await;
|
||||||
st.provenance = format!("standalone:{}", self.name);
|
st.provenance = format!("standalone:{}", self.name);
|
||||||
st.tools = self.tools.clone();
|
st.tools = self.tools.clone();
|
||||||
st.priority = Some(10);
|
st.temperature = self.temperature;
|
||||||
|
st.priority = Some(self.priority);
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut backend = Backend(agent.clone());
|
let mut backend = Backend(agent.clone());
|
||||||
|
|
@ -265,14 +314,33 @@ impl AutoAgent {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Update stats after a run completes. Called with the stats from save_agent_log.
|
/// Update stats after a run completes. Called with the stats from save_agent_log.
|
||||||
pub fn update_stats(&mut self, stats: RunStats) {
|
pub fn update_stats(&self, run_stats: RunStats) {
|
||||||
const ALPHA: f64 = 0.3;
|
const ALPHA: f64 = 0.3;
|
||||||
self.runs += 1;
|
let old = get_stats(&self.name);
|
||||||
self.tool_calls_ewma = ALPHA * (stats.tool_calls as f64)
|
|
||||||
+ (1.0 - ALPHA) * self.tool_calls_ewma;
|
// Update per-tool stats
|
||||||
self.tool_failures_ewma = ALPHA * (stats.tool_failures as f64)
|
let mut by_tool = old.by_tool;
|
||||||
+ (1.0 - ALPHA) * self.tool_failures_ewma;
|
for (tool, count) in &run_stats.tool_calls_by_type {
|
||||||
self.last_stats = Some(stats);
|
let entry = by_tool.entry(tool.clone()).or_default();
|
||||||
|
entry.last = *count;
|
||||||
|
entry.ewma = ALPHA * (*count as f64) + (1.0 - ALPHA) * entry.ewma;
|
||||||
|
entry.total += count;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update failure stats
|
||||||
|
let failures = ToolStats {
|
||||||
|
last: run_stats.tool_failures,
|
||||||
|
ewma: ALPHA * (run_stats.tool_failures as f64) + (1.0 - ALPHA) * old.failures.ewma,
|
||||||
|
total: old.failures.total + run_stats.tool_failures,
|
||||||
|
};
|
||||||
|
|
||||||
|
let new = PersistedStats {
|
||||||
|
runs: old.runs + 1,
|
||||||
|
last_stats: Some(run_stats),
|
||||||
|
by_tool,
|
||||||
|
failures,
|
||||||
|
};
|
||||||
|
set_stats(&self.name, new);
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_with_backend(
|
async fn run_with_backend(
|
||||||
|
|
|
||||||
|
|
@ -365,21 +365,23 @@ impl SubconsciousAgent {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn snapshot(&self, state: &std::collections::BTreeMap<String, String>, history: Vec<(String, i64)>) -> SubconsciousSnapshot {
|
fn snapshot(&self, state: &std::collections::BTreeMap<String, String>, history: Vec<(String, i64)>) -> SubconsciousSnapshot {
|
||||||
|
let stats = crate::agent::oneshot::get_stats(&self.name);
|
||||||
|
let tool_calls_ewma: f64 = stats.by_tool.values().map(|t| t.ewma).sum();
|
||||||
SubconsciousSnapshot {
|
SubconsciousSnapshot {
|
||||||
name: self.name.clone(),
|
name: self.name.clone(),
|
||||||
running: self.is_running(),
|
running: self.is_running(),
|
||||||
enabled: self.auto.enabled,
|
enabled: self.auto.enabled,
|
||||||
current_phase: self.auto.current_phase.clone(),
|
current_phase: self.auto.current_phase.clone(),
|
||||||
turn: self.auto.turn,
|
turn: self.auto.turn,
|
||||||
runs: self.auto.runs,
|
runs: stats.runs,
|
||||||
last_run_secs_ago: self.last_run.map(|t| t.elapsed().as_secs_f64()),
|
last_run_secs_ago: self.last_run.map(|t| t.elapsed().as_secs_f64()),
|
||||||
forked_agent: self.forked_agent.clone(),
|
forked_agent: self.forked_agent.clone(),
|
||||||
fork_point: self.fork_point,
|
fork_point: self.fork_point,
|
||||||
state: state.clone(),
|
state: state.clone(),
|
||||||
history,
|
history,
|
||||||
last_stats: self.auto.last_stats.clone(),
|
last_stats: stats.last_stats.clone(),
|
||||||
tool_calls_ewma: self.auto.tool_calls_ewma,
|
tool_calls_ewma,
|
||||||
tool_failures_ewma: self.auto.tool_failures_ewma,
|
tool_failures_ewma: stats.failures.ewma,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -485,7 +487,7 @@ impl Subconscious {
|
||||||
any_finished = true;
|
any_finished = true;
|
||||||
|
|
||||||
let (auto_back, result) = handle.await.unwrap_or_else(
|
let (auto_back, result) = handle.await.unwrap_or_else(
|
||||||
|e| (AutoAgent::new(String::new(), vec![], vec![], 0.0, 0),
|
|e| (AutoAgent::new(String::new(), vec![], vec![], 0.6, 0),
|
||||||
Err(format!("task panicked: {}", e))));
|
Err(format!("task panicked: {}", e))));
|
||||||
self.agents[i].auto = auto_back;
|
self.agents[i].auto = auto_back;
|
||||||
|
|
||||||
|
|
@ -584,7 +586,7 @@ impl Subconscious {
|
||||||
self.agents[i].last_trigger_bytes = conversation_bytes;
|
self.agents[i].last_trigger_bytes = conversation_bytes;
|
||||||
|
|
||||||
let auto = std::mem::replace(&mut self.agents[i].auto,
|
let auto = std::mem::replace(&mut self.agents[i].auto,
|
||||||
AutoAgent::new(String::new(), vec![], vec![], 0.0, 0));
|
AutoAgent::new(String::new(), vec![], vec![], 0.6, 0));
|
||||||
to_run.push((i, auto));
|
to_run.push((i, auto));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -604,9 +606,10 @@ impl Subconscious {
|
||||||
{
|
{
|
||||||
let mut st = forked.state.lock().await;
|
let mut st = forked.state.lock().await;
|
||||||
st.provenance = auto.name.clone();
|
st.provenance = auto.name.clone();
|
||||||
|
st.temperature = auto.temperature;
|
||||||
// Surface agent gets near-interactive priority;
|
// Surface agent gets near-interactive priority;
|
||||||
// other subconscious agents get lower priority.
|
// other subconscious agents get lower priority.
|
||||||
st.priority = Some(if auto.name == "surface" { 1 } else { 2 });
|
st.priority = Some(if auto.name == "surface" { 1 } else { auto.priority });
|
||||||
}
|
}
|
||||||
let fork_point = forked.context.lock().await.conversation().len();
|
let fork_point = forked.context.lock().await.conversation().len();
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -142,17 +142,19 @@ impl Unconscious {
|
||||||
self.agents.iter().map(|a| {
|
self.agents.iter().map(|a| {
|
||||||
let history = store.map(|st| st.recent_by_provenance(&a.name, 30))
|
let history = store.map(|st| st.recent_by_provenance(&a.name, 30))
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
let stats = crate::agent::oneshot::get_stats(&a.name);
|
||||||
|
let tool_calls_ewma: f64 = stats.by_tool.values().map(|t| t.ewma).sum();
|
||||||
UnconsciousSnapshot {
|
UnconsciousSnapshot {
|
||||||
name: a.name.clone(),
|
name: a.name.clone(),
|
||||||
running: a.is_running(),
|
running: a.is_running(),
|
||||||
enabled: a.enabled,
|
enabled: a.enabled,
|
||||||
runs: a.auto.runs,
|
runs: stats.runs,
|
||||||
last_run_secs_ago: a.last_run.map(|t| t.elapsed().as_secs_f64()),
|
last_run_secs_ago: a.last_run.map(|t| t.elapsed().as_secs_f64()),
|
||||||
agent: a.agent.clone(),
|
agent: a.agent.clone(),
|
||||||
last_stats: a.auto.last_stats.clone(),
|
last_stats: stats.last_stats.clone(),
|
||||||
history,
|
history,
|
||||||
tool_calls_ewma: a.auto.tool_calls_ewma,
|
tool_calls_ewma,
|
||||||
tool_failures_ewma: a.auto.tool_failures_ewma,
|
tool_failures_ewma: stats.failures.ewma,
|
||||||
}
|
}
|
||||||
}).collect()
|
}).collect()
|
||||||
}
|
}
|
||||||
|
|
@ -186,7 +188,7 @@ impl Unconscious {
|
||||||
agent.auto = auto_back;
|
agent.auto = auto_back;
|
||||||
match result {
|
match result {
|
||||||
Ok(_) => dbglog!("[unconscious] {} completed (run {})",
|
Ok(_) => dbglog!("[unconscious] {} completed (run {})",
|
||||||
agent.name, agent.auto.runs),
|
agent.name, crate::agent::oneshot::get_stats(&agent.name).runs),
|
||||||
Err(e) => dbglog!("[unconscious] {} failed: {}", agent.name, e),
|
Err(e) => dbglog!("[unconscious] {} failed: {}", agent.name, e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -242,7 +244,7 @@ impl Unconscious {
|
||||||
|
|
||||||
// Swap auto out, replace steps with resolved prompts
|
// Swap auto out, replace steps with resolved prompts
|
||||||
let mut auto = std::mem::replace(&mut self.agents[idx].auto,
|
let mut auto = std::mem::replace(&mut self.agents[idx].auto,
|
||||||
AutoAgent::new(String::new(), vec![], vec![], 0.0, 0));
|
AutoAgent::new(String::new(), vec![], vec![], 0.6, 0));
|
||||||
let orig_steps = std::mem::replace(&mut auto.steps,
|
let orig_steps = std::mem::replace(&mut auto.steps,
|
||||||
batch.steps.iter().map(|s| AutoStep {
|
batch.steps.iter().map(|s| AutoStep {
|
||||||
prompt: s.prompt.clone(),
|
prompt: s.prompt.clone(),
|
||||||
|
|
@ -291,7 +293,8 @@ impl Unconscious {
|
||||||
{
|
{
|
||||||
let mut st = agent.state.lock().await;
|
let mut st = agent.state.lock().await;
|
||||||
st.provenance = auto.name.clone();
|
st.provenance = auto.name.clone();
|
||||||
st.priority = Some(10);
|
st.priority = Some(auto.priority);
|
||||||
|
st.temperature = auto.temperature;
|
||||||
}
|
}
|
||||||
|
|
||||||
self.agents[idx].agent = Some(agent.clone());
|
self.agents[idx].agent = Some(agent.clone());
|
||||||
|
|
|
||||||
|
|
@ -124,8 +124,8 @@ impl ScreenView for SubconsciousScreen {
|
||||||
.map(|s| s.state.values().map(|v| v.lines().count() + 1).sum::<usize>())
|
.map(|s| s.state.values().map(|v| v.lines().count() + 1).sum::<usize>())
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
let output_height = (output_lines as u16 + 2).min(left.height / 5).max(3);
|
let output_height = (output_lines as u16 + 2).min(left.height / 5).max(3);
|
||||||
let stats_lines = self.selected_stats(app)
|
let stats_lines = self.selected_persisted_stats(app)
|
||||||
.map(|s| s.tool_calls_by_type.len())
|
.map(|s| s.by_tool.len())
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
let stats_height = (stats_lines as u16 + 2).min(left.height / 5).max(3);
|
let stats_height = (stats_lines as u16 + 2).min(left.height / 5).max(3);
|
||||||
let [list_area, output_area, stats_area, history_area] = Layout::vertical([
|
let [list_area, output_area, stats_area, history_area] = Layout::vertical([
|
||||||
|
|
@ -191,17 +191,10 @@ impl SubconsciousScreen {
|
||||||
.unwrap_or(&[])
|
.unwrap_or(&[])
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get last run stats for the selected agent.
|
/// Get persisted stats for the selected agent.
|
||||||
fn selected_stats<'a>(&self, app: &'a App) -> Option<&'a crate::agent::oneshot::RunStats> {
|
fn selected_persisted_stats(&self, app: &App) -> Option<crate::agent::oneshot::PersistedStats> {
|
||||||
let idx = self.selected();
|
let name = self.selected_agent_name(app)?;
|
||||||
let sub_count = app.agent_state.len();
|
Some(crate::agent::oneshot::get_stats(&name))
|
||||||
if idx < sub_count {
|
|
||||||
return app.agent_state.get(idx)
|
|
||||||
.and_then(|s| s.last_stats.as_ref());
|
|
||||||
}
|
|
||||||
idx.checked_sub(sub_count + 1)
|
|
||||||
.and_then(|i| app.unconscious_state.get(i))
|
|
||||||
.and_then(|s| s.last_stats.as_ref())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn output_sections(&self, app: &App) -> Vec<SectionView> {
|
fn output_sections(&self, app: &App) -> Vec<SectionView> {
|
||||||
|
|
@ -355,21 +348,46 @@ impl SubconsciousScreen {
|
||||||
|
|
||||||
fn draw_stats(&mut self, frame: &mut Frame, area: Rect, app: &App) {
|
fn draw_stats(&mut self, frame: &mut Frame, area: Rect, app: &App) {
|
||||||
let dim = Style::default().fg(Color::DarkGray);
|
let dim = Style::default().fg(Color::DarkGray);
|
||||||
|
let header_style = Style::default().fg(Color::DarkGray);
|
||||||
let name_style = Style::default().fg(Color::Cyan);
|
let name_style = Style::default().fg(Color::Cyan);
|
||||||
let count_style = Style::default().fg(Color::Yellow);
|
let num_style = Style::default().fg(Color::Yellow);
|
||||||
|
|
||||||
let mut lines: Vec<Line> = Vec::new();
|
let mut lines: Vec<Line> = Vec::new();
|
||||||
|
|
||||||
if let Some(stats) = self.selected_stats(app) {
|
if let Some(stats) = self.selected_persisted_stats(app) {
|
||||||
// Sort by count descending
|
if !stats.by_tool.is_empty() {
|
||||||
let mut tools: Vec<_> = stats.tool_calls_by_type.iter().collect();
|
// Header
|
||||||
tools.sort_by(|a, b| b.1.cmp(a.1));
|
|
||||||
|
|
||||||
for (name, count) in tools {
|
|
||||||
lines.push(Line::from(vec![
|
lines.push(Line::from(vec![
|
||||||
Span::styled(format!(" {:>3} ", count), count_style),
|
Span::styled(" tool ", header_style),
|
||||||
Span::styled(name.as_str(), name_style),
|
Span::styled("last ", header_style),
|
||||||
|
Span::styled(" avg ", header_style),
|
||||||
|
Span::styled("total", header_style),
|
||||||
]));
|
]));
|
||||||
|
|
||||||
|
// Sort by total descending
|
||||||
|
let mut tools: Vec<_> = stats.by_tool.iter().collect();
|
||||||
|
tools.sort_by(|a, b| b.1.total.cmp(&a.1.total));
|
||||||
|
|
||||||
|
for (name, tool_stats) in tools {
|
||||||
|
let short_name = name.strip_prefix("memory_").unwrap_or(name);
|
||||||
|
lines.push(Line::from(vec![
|
||||||
|
Span::styled(format!(" {:<20} ", short_name), name_style),
|
||||||
|
Span::styled(format!("{:>4} ", tool_stats.last), num_style),
|
||||||
|
Span::styled(format!("{:>4.1} ", tool_stats.ewma), dim),
|
||||||
|
Span::styled(format!("{:>5}", tool_stats.total), num_style),
|
||||||
|
]));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Failures row if any
|
||||||
|
if stats.failures.total > 0 {
|
||||||
|
lines.push(Line::raw(""));
|
||||||
|
lines.push(Line::from(vec![
|
||||||
|
Span::styled(" failures ", Style::default().fg(Color::Red)),
|
||||||
|
Span::styled(format!("{:>4} ", stats.failures.last), num_style),
|
||||||
|
Span::styled(format!("{:>4.1} ", stats.failures.ewma), dim),
|
||||||
|
Span::styled(format!("{:>5}", stats.failures.total), num_style),
|
||||||
|
]));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue