llm: full per-agent usage logging with prompts and responses

Log every model call to ~/.claude/memory/llm-logs/YYYY-MM-DD.md with
full prompt, response, agent type, model, duration, and status. One
file per day, markdown formatted for easy reading.

Agent types: fact-mine, experience-mine, consolidate, knowledge,
digest, enrich, audit. This gives visibility into what each agent
is doing and whether to adjust prompts or frequency.
This commit is contained in:
ProofOfConcept 2026-03-05 22:52:08 -05:00
parent e33fd4ffbc
commit 82b33c449c
7 changed files with 51 additions and 17 deletions

View file

@ -211,7 +211,7 @@ pub fn link_audit(store: &mut Store, apply: bool) -> Result<AuditStats, String>
// Run batches in parallel via rayon
let batch_results: Vec<_> = batch_data.par_iter()
.map(|(batch_idx, batch_infos, prompt)| {
let response = call_sonnet(prompt, 300);
let response = call_sonnet("audit", prompt);
let completed = done.fetch_add(1, Ordering::Relaxed) + 1;
eprint!("\r Batches: {}/{} done", completed, total_batches);
(*batch_idx, batch_infos, response)

View file

@ -134,7 +134,7 @@ pub fn consolidate_full_with_progress(
log_line(&mut log_buf, &format!(" Prompt: {} chars (~{} tokens)",
prompt.len(), prompt.len() / 4));
let response = match call_sonnet(&prompt, 300) {
let response = match call_sonnet("consolidate", &prompt) {
Ok(r) => r,
Err(e) => {
let msg = format!(" ERROR from Sonnet: {}", e);
@ -314,7 +314,7 @@ pub fn apply_consolidation(store: &mut Store, do_apply: bool, report_key: Option
let prompt = build_consolidation_prompt(store, &reports)?;
println!(" Prompt: {} chars", prompt.len());
let response = call_sonnet(&prompt, 300)?;
let response = call_sonnet("consolidate", &prompt)?;
let actions_value = parse_json_response(&response)?;
let actions = actions_value.as_array()

View file

@ -227,7 +227,7 @@ fn generate_digest(
println!(" Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);
println!(" Calling Sonnet...");
let digest = call_sonnet(&prompt, level.timeout)?;
let digest = call_sonnet("digest", &prompt)?;
let key = digest_node_key(level.name, label);
store.upsert_provenance(&key, &digest, store::Provenance::AgentDigest)?;

View file

@ -175,7 +175,7 @@ pub fn journal_enrich(
println!(" Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);
println!(" Calling Sonnet...");
let response = call_sonnet(&prompt, 300)?;
let response = call_sonnet("enrich", &prompt)?;
let result = parse_json_response(&response)?;
@ -304,7 +304,7 @@ pub fn experience_mine(
println!(" Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);
println!(" Calling Sonnet...");
let response = call_sonnet(&prompt, 2000)?;
let response = call_sonnet("experience-mine", &prompt)?;
let entries = parse_json_response(&response)?;
let entries = match entries.as_array() {

View file

@ -248,7 +248,7 @@ pub fn mine_transcript(path: &Path, dry_run: bool) -> Result<Vec<Fact>, String>
eprint!(" Chunk {}/{} ({} chars)...", i + 1, chunks.len(), chunk.len());
let prompt = format!("{}{}", prompt_prefix, chunk);
let response = match llm::call_haiku(&prompt) {
let response = match llm::call_haiku("fact-mine", &prompt) {
Ok(r) => r,
Err(e) => {
eprintln!(" error: {}", e);

View file

@ -487,7 +487,7 @@ pub fn run_observation_extractor(store: &Store, graph: &Graph, batch_size: usize
.replace("{{TOPOLOGY}}", &topology)
.replace("{{CONVERSATIONS}}", &format!("### Session {}\n\n{}", session_id, text));
let response = llm::call_sonnet(&prompt, 600)?;
let response = llm::call_sonnet("knowledge", &prompt)?;
results.push(format!("## Session: {}\n\n{}", session_id, response));
}
Ok(results.join("\n\n---\n\n"))
@ -569,7 +569,7 @@ pub fn run_extractor(store: &Store, graph: &Graph, batch_size: usize) -> Result<
.replace("{{TOPOLOGY}}", &topology)
.replace("{{NODES}}", &node_texts.join("\n\n"));
let response = llm::call_sonnet(&prompt, 600)?;
let response = llm::call_sonnet("knowledge", &prompt)?;
results.push(format!("## Cluster {}: {}...\n\n{}", i + 1,
cluster.iter().take(3).cloned().collect::<Vec<_>>().join(", "), response));
}
@ -643,7 +643,7 @@ pub fn run_connector(store: &Store, graph: &Graph, batch_size: usize) -> Result<
.replace("{{NODES_A}}", &nodes_a.join("\n\n"))
.replace("{{NODES_B}}", &nodes_b.join("\n\n"));
let response = llm::call_sonnet(&prompt, 600)?;
let response = llm::call_sonnet("knowledge", &prompt)?;
results.push(format!("## Pair {}: {}{}\n\n{}",
i + 1, group_a.join(", "), group_b.join(", "), response));
}
@ -677,7 +677,7 @@ pub fn run_challenger(store: &Store, graph: &Graph, batch_size: usize) -> Result
.replace("{{NODE_KEY}}", key)
.replace("{{NODE_CONTENT}}", content);
let response = llm::call_sonnet(&prompt, 600)?;
let response = llm::call_sonnet("knowledge", &prompt)?;
results.push(format!("## Challenge: {}\n\n{}", key, response));
}
Ok(results.join("\n\n---\n\n"))

View file

@ -10,11 +10,39 @@ use std::fs;
use std::os::unix::process::CommandExt;
use std::process::Command;
fn log_usage(agent: &str, model: &str, prompt: &str, response: &str,
duration_ms: u128, ok: bool) {
let dir = crate::config::get().data_dir.join("llm-logs");
let _ = fs::create_dir_all(&dir);
let date = chrono::Local::now().format("%Y-%m-%d");
let path = dir.join(format!("{}.md", date));
let ts = chrono::Local::now().format("%H:%M:%S");
let status = if ok { "ok" } else { "ERROR" };
let entry = format!(
"\n## {} — {} ({}, {:.1}s, {})\n\n\
### Prompt ({} chars)\n\n\
```\n{}\n```\n\n\
### Response ({} chars)\n\n\
```\n{}\n```\n\n---\n",
ts, agent, model, duration_ms as f64 / 1000.0, status,
prompt.len(), prompt,
response.len(), response,
);
use std::io::Write;
if let Ok(mut f) = fs::OpenOptions::new().create(true).append(true).open(&path) {
let _ = f.write_all(entry.as_bytes());
}
}
/// Call a model via claude CLI. Returns the response text.
///
/// Sets PR_SET_PDEATHSIG on the child so it gets SIGTERM if the
/// parent daemon exits — no more orphaned claude processes.
fn call_model(model: &str, prompt: &str) -> Result<String, String> {
fn call_model(agent: &str, model: &str, prompt: &str) -> Result<String, String> {
// Write prompt to temp file (claude CLI needs file input for large prompts)
let tmp = std::env::temp_dir().join(format!("poc-llm-{}-{:?}.txt",
std::process::id(), std::thread::current().id()));
@ -39,15 +67,21 @@ fn call_model(model: &str, prompt: &str) -> Result<String, String> {
.output()
};
let start = std::time::Instant::now();
fs::remove_file(&tmp).ok();
match result {
Ok(output) => {
let elapsed = start.elapsed().as_millis();
if output.status.success() {
Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
let response = String::from_utf8_lossy(&output.stdout).trim().to_string();
log_usage(agent, model, prompt, &response, elapsed, true);
Ok(response)
} else {
let stderr = String::from_utf8_lossy(&output.stderr);
let preview: String = stderr.chars().take(500).collect();
log_usage(agent, model, prompt, &preview, elapsed, false);
Err(format!("claude exited {}: {}", output.status, preview.trim()))
}
}
@ -56,13 +90,13 @@ fn call_model(model: &str, prompt: &str) -> Result<String, String> {
}
/// Call Sonnet via claude CLI.
pub(crate) fn call_sonnet(prompt: &str, _timeout_secs: u64) -> Result<String, String> {
call_model("sonnet", prompt)
pub(crate) fn call_sonnet(agent: &str, prompt: &str) -> Result<String, String> {
call_model(agent, "sonnet", prompt)
}
/// Call Haiku via claude CLI (cheaper, faster — good for high-volume extraction).
pub(crate) fn call_haiku(prompt: &str) -> Result<String, String> {
call_model("haiku", prompt)
pub(crate) fn call_haiku(agent: &str, prompt: &str) -> Result<String, String> {
call_model(agent, "haiku", prompt)
}
/// Parse a JSON response, handling markdown fences.