llm: full per-agent usage logging with prompts and responses
Log every model call to ~/.claude/memory/llm-logs/YYYY-MM-DD.md with full prompt, response, agent type, model, duration, and status. One file per day, markdown formatted for easy reading. Agent types: fact-mine, experience-mine, consolidate, knowledge, digest, enrich, audit. This gives visibility into what each agent is doing and whether to adjust prompts or frequency.
This commit is contained in:
parent
e33fd4ffbc
commit
82b33c449c
7 changed files with 51 additions and 17 deletions
|
|
@ -211,7 +211,7 @@ pub fn link_audit(store: &mut Store, apply: bool) -> Result<AuditStats, String>
|
||||||
// Run batches in parallel via rayon
|
// Run batches in parallel via rayon
|
||||||
let batch_results: Vec<_> = batch_data.par_iter()
|
let batch_results: Vec<_> = batch_data.par_iter()
|
||||||
.map(|(batch_idx, batch_infos, prompt)| {
|
.map(|(batch_idx, batch_infos, prompt)| {
|
||||||
let response = call_sonnet(prompt, 300);
|
let response = call_sonnet("audit", prompt);
|
||||||
let completed = done.fetch_add(1, Ordering::Relaxed) + 1;
|
let completed = done.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
eprint!("\r Batches: {}/{} done", completed, total_batches);
|
eprint!("\r Batches: {}/{} done", completed, total_batches);
|
||||||
(*batch_idx, batch_infos, response)
|
(*batch_idx, batch_infos, response)
|
||||||
|
|
|
||||||
|
|
@ -134,7 +134,7 @@ pub fn consolidate_full_with_progress(
|
||||||
log_line(&mut log_buf, &format!(" Prompt: {} chars (~{} tokens)",
|
log_line(&mut log_buf, &format!(" Prompt: {} chars (~{} tokens)",
|
||||||
prompt.len(), prompt.len() / 4));
|
prompt.len(), prompt.len() / 4));
|
||||||
|
|
||||||
let response = match call_sonnet(&prompt, 300) {
|
let response = match call_sonnet("consolidate", &prompt) {
|
||||||
Ok(r) => r,
|
Ok(r) => r,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!(" ERROR from Sonnet: {}", e);
|
let msg = format!(" ERROR from Sonnet: {}", e);
|
||||||
|
|
@ -314,7 +314,7 @@ pub fn apply_consolidation(store: &mut Store, do_apply: bool, report_key: Option
|
||||||
let prompt = build_consolidation_prompt(store, &reports)?;
|
let prompt = build_consolidation_prompt(store, &reports)?;
|
||||||
println!(" Prompt: {} chars", prompt.len());
|
println!(" Prompt: {} chars", prompt.len());
|
||||||
|
|
||||||
let response = call_sonnet(&prompt, 300)?;
|
let response = call_sonnet("consolidate", &prompt)?;
|
||||||
|
|
||||||
let actions_value = parse_json_response(&response)?;
|
let actions_value = parse_json_response(&response)?;
|
||||||
let actions = actions_value.as_array()
|
let actions = actions_value.as_array()
|
||||||
|
|
|
||||||
|
|
@ -227,7 +227,7 @@ fn generate_digest(
|
||||||
println!(" Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);
|
println!(" Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);
|
||||||
|
|
||||||
println!(" Calling Sonnet...");
|
println!(" Calling Sonnet...");
|
||||||
let digest = call_sonnet(&prompt, level.timeout)?;
|
let digest = call_sonnet("digest", &prompt)?;
|
||||||
|
|
||||||
let key = digest_node_key(level.name, label);
|
let key = digest_node_key(level.name, label);
|
||||||
store.upsert_provenance(&key, &digest, store::Provenance::AgentDigest)?;
|
store.upsert_provenance(&key, &digest, store::Provenance::AgentDigest)?;
|
||||||
|
|
|
||||||
|
|
@ -175,7 +175,7 @@ pub fn journal_enrich(
|
||||||
println!(" Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);
|
println!(" Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);
|
||||||
|
|
||||||
println!(" Calling Sonnet...");
|
println!(" Calling Sonnet...");
|
||||||
let response = call_sonnet(&prompt, 300)?;
|
let response = call_sonnet("enrich", &prompt)?;
|
||||||
|
|
||||||
let result = parse_json_response(&response)?;
|
let result = parse_json_response(&response)?;
|
||||||
|
|
||||||
|
|
@ -304,7 +304,7 @@ pub fn experience_mine(
|
||||||
println!(" Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);
|
println!(" Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);
|
||||||
|
|
||||||
println!(" Calling Sonnet...");
|
println!(" Calling Sonnet...");
|
||||||
let response = call_sonnet(&prompt, 2000)?;
|
let response = call_sonnet("experience-mine", &prompt)?;
|
||||||
|
|
||||||
let entries = parse_json_response(&response)?;
|
let entries = parse_json_response(&response)?;
|
||||||
let entries = match entries.as_array() {
|
let entries = match entries.as_array() {
|
||||||
|
|
|
||||||
|
|
@ -248,7 +248,7 @@ pub fn mine_transcript(path: &Path, dry_run: bool) -> Result<Vec<Fact>, String>
|
||||||
eprint!(" Chunk {}/{} ({} chars)...", i + 1, chunks.len(), chunk.len());
|
eprint!(" Chunk {}/{} ({} chars)...", i + 1, chunks.len(), chunk.len());
|
||||||
|
|
||||||
let prompt = format!("{}{}", prompt_prefix, chunk);
|
let prompt = format!("{}{}", prompt_prefix, chunk);
|
||||||
let response = match llm::call_haiku(&prompt) {
|
let response = match llm::call_haiku("fact-mine", &prompt) {
|
||||||
Ok(r) => r,
|
Ok(r) => r,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!(" error: {}", e);
|
eprintln!(" error: {}", e);
|
||||||
|
|
|
||||||
|
|
@ -487,7 +487,7 @@ pub fn run_observation_extractor(store: &Store, graph: &Graph, batch_size: usize
|
||||||
.replace("{{TOPOLOGY}}", &topology)
|
.replace("{{TOPOLOGY}}", &topology)
|
||||||
.replace("{{CONVERSATIONS}}", &format!("### Session {}\n\n{}", session_id, text));
|
.replace("{{CONVERSATIONS}}", &format!("### Session {}\n\n{}", session_id, text));
|
||||||
|
|
||||||
let response = llm::call_sonnet(&prompt, 600)?;
|
let response = llm::call_sonnet("knowledge", &prompt)?;
|
||||||
results.push(format!("## Session: {}\n\n{}", session_id, response));
|
results.push(format!("## Session: {}\n\n{}", session_id, response));
|
||||||
}
|
}
|
||||||
Ok(results.join("\n\n---\n\n"))
|
Ok(results.join("\n\n---\n\n"))
|
||||||
|
|
@ -569,7 +569,7 @@ pub fn run_extractor(store: &Store, graph: &Graph, batch_size: usize) -> Result<
|
||||||
.replace("{{TOPOLOGY}}", &topology)
|
.replace("{{TOPOLOGY}}", &topology)
|
||||||
.replace("{{NODES}}", &node_texts.join("\n\n"));
|
.replace("{{NODES}}", &node_texts.join("\n\n"));
|
||||||
|
|
||||||
let response = llm::call_sonnet(&prompt, 600)?;
|
let response = llm::call_sonnet("knowledge", &prompt)?;
|
||||||
results.push(format!("## Cluster {}: {}...\n\n{}", i + 1,
|
results.push(format!("## Cluster {}: {}...\n\n{}", i + 1,
|
||||||
cluster.iter().take(3).cloned().collect::<Vec<_>>().join(", "), response));
|
cluster.iter().take(3).cloned().collect::<Vec<_>>().join(", "), response));
|
||||||
}
|
}
|
||||||
|
|
@ -643,7 +643,7 @@ pub fn run_connector(store: &Store, graph: &Graph, batch_size: usize) -> Result<
|
||||||
.replace("{{NODES_A}}", &nodes_a.join("\n\n"))
|
.replace("{{NODES_A}}", &nodes_a.join("\n\n"))
|
||||||
.replace("{{NODES_B}}", &nodes_b.join("\n\n"));
|
.replace("{{NODES_B}}", &nodes_b.join("\n\n"));
|
||||||
|
|
||||||
let response = llm::call_sonnet(&prompt, 600)?;
|
let response = llm::call_sonnet("knowledge", &prompt)?;
|
||||||
results.push(format!("## Pair {}: {} ↔ {}\n\n{}",
|
results.push(format!("## Pair {}: {} ↔ {}\n\n{}",
|
||||||
i + 1, group_a.join(", "), group_b.join(", "), response));
|
i + 1, group_a.join(", "), group_b.join(", "), response));
|
||||||
}
|
}
|
||||||
|
|
@ -677,7 +677,7 @@ pub fn run_challenger(store: &Store, graph: &Graph, batch_size: usize) -> Result
|
||||||
.replace("{{NODE_KEY}}", key)
|
.replace("{{NODE_KEY}}", key)
|
||||||
.replace("{{NODE_CONTENT}}", content);
|
.replace("{{NODE_CONTENT}}", content);
|
||||||
|
|
||||||
let response = llm::call_sonnet(&prompt, 600)?;
|
let response = llm::call_sonnet("knowledge", &prompt)?;
|
||||||
results.push(format!("## Challenge: {}\n\n{}", key, response));
|
results.push(format!("## Challenge: {}\n\n{}", key, response));
|
||||||
}
|
}
|
||||||
Ok(results.join("\n\n---\n\n"))
|
Ok(results.join("\n\n---\n\n"))
|
||||||
|
|
|
||||||
46
src/llm.rs
46
src/llm.rs
|
|
@ -10,11 +10,39 @@ use std::fs;
|
||||||
use std::os::unix::process::CommandExt;
|
use std::os::unix::process::CommandExt;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
|
|
||||||
|
fn log_usage(agent: &str, model: &str, prompt: &str, response: &str,
|
||||||
|
duration_ms: u128, ok: bool) {
|
||||||
|
let dir = crate::config::get().data_dir.join("llm-logs");
|
||||||
|
let _ = fs::create_dir_all(&dir);
|
||||||
|
|
||||||
|
let date = chrono::Local::now().format("%Y-%m-%d");
|
||||||
|
let path = dir.join(format!("{}.md", date));
|
||||||
|
|
||||||
|
let ts = chrono::Local::now().format("%H:%M:%S");
|
||||||
|
let status = if ok { "ok" } else { "ERROR" };
|
||||||
|
|
||||||
|
let entry = format!(
|
||||||
|
"\n## {} — {} ({}, {:.1}s, {})\n\n\
|
||||||
|
### Prompt ({} chars)\n\n\
|
||||||
|
```\n{}\n```\n\n\
|
||||||
|
### Response ({} chars)\n\n\
|
||||||
|
```\n{}\n```\n\n---\n",
|
||||||
|
ts, agent, model, duration_ms as f64 / 1000.0, status,
|
||||||
|
prompt.len(), prompt,
|
||||||
|
response.len(), response,
|
||||||
|
);
|
||||||
|
|
||||||
|
use std::io::Write;
|
||||||
|
if let Ok(mut f) = fs::OpenOptions::new().create(true).append(true).open(&path) {
|
||||||
|
let _ = f.write_all(entry.as_bytes());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Call a model via claude CLI. Returns the response text.
|
/// Call a model via claude CLI. Returns the response text.
|
||||||
///
|
///
|
||||||
/// Sets PR_SET_PDEATHSIG on the child so it gets SIGTERM if the
|
/// Sets PR_SET_PDEATHSIG on the child so it gets SIGTERM if the
|
||||||
/// parent daemon exits — no more orphaned claude processes.
|
/// parent daemon exits — no more orphaned claude processes.
|
||||||
fn call_model(model: &str, prompt: &str) -> Result<String, String> {
|
fn call_model(agent: &str, model: &str, prompt: &str) -> Result<String, String> {
|
||||||
// Write prompt to temp file (claude CLI needs file input for large prompts)
|
// Write prompt to temp file (claude CLI needs file input for large prompts)
|
||||||
let tmp = std::env::temp_dir().join(format!("poc-llm-{}-{:?}.txt",
|
let tmp = std::env::temp_dir().join(format!("poc-llm-{}-{:?}.txt",
|
||||||
std::process::id(), std::thread::current().id()));
|
std::process::id(), std::thread::current().id()));
|
||||||
|
|
@ -39,15 +67,21 @@ fn call_model(model: &str, prompt: &str) -> Result<String, String> {
|
||||||
.output()
|
.output()
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
fs::remove_file(&tmp).ok();
|
fs::remove_file(&tmp).ok();
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(output) => {
|
Ok(output) => {
|
||||||
|
let elapsed = start.elapsed().as_millis();
|
||||||
if output.status.success() {
|
if output.status.success() {
|
||||||
Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
|
let response = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||||
|
log_usage(agent, model, prompt, &response, elapsed, true);
|
||||||
|
Ok(response)
|
||||||
} else {
|
} else {
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
let preview: String = stderr.chars().take(500).collect();
|
let preview: String = stderr.chars().take(500).collect();
|
||||||
|
log_usage(agent, model, prompt, &preview, elapsed, false);
|
||||||
Err(format!("claude exited {}: {}", output.status, preview.trim()))
|
Err(format!("claude exited {}: {}", output.status, preview.trim()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -56,13 +90,13 @@ fn call_model(model: &str, prompt: &str) -> Result<String, String> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Call Sonnet via claude CLI.
|
/// Call Sonnet via claude CLI.
|
||||||
pub(crate) fn call_sonnet(prompt: &str, _timeout_secs: u64) -> Result<String, String> {
|
pub(crate) fn call_sonnet(agent: &str, prompt: &str) -> Result<String, String> {
|
||||||
call_model("sonnet", prompt)
|
call_model(agent, "sonnet", prompt)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Call Haiku via claude CLI (cheaper, faster — good for high-volume extraction).
|
/// Call Haiku via claude CLI (cheaper, faster — good for high-volume extraction).
|
||||||
pub(crate) fn call_haiku(prompt: &str) -> Result<String, String> {
|
pub(crate) fn call_haiku(agent: &str, prompt: &str) -> Result<String, String> {
|
||||||
call_model("haiku", prompt)
|
call_model(agent, "haiku", prompt)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse a JSON response, handling markdown fences.
|
/// Parse a JSON response, handling markdown fences.
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue