llm: full per-agent usage logging with prompts and responses
Log every model call to ~/.claude/memory/llm-logs/YYYY-MM-DD.md with full prompt, response, agent type, model, duration, and status. One file per day, markdown formatted for easy reading. Agent types: fact-mine, experience-mine, consolidate, knowledge, digest, enrich, audit. This gives visibility into what each agent is doing and whether to adjust prompts or frequency.
This commit is contained in:
parent
e33fd4ffbc
commit
82b33c449c
7 changed files with 51 additions and 17 deletions
46
src/llm.rs
46
src/llm.rs
|
|
@ -10,11 +10,39 @@ use std::fs;
|
|||
use std::os::unix::process::CommandExt;
|
||||
use std::process::Command;
|
||||
|
||||
fn log_usage(agent: &str, model: &str, prompt: &str, response: &str,
|
||||
duration_ms: u128, ok: bool) {
|
||||
let dir = crate::config::get().data_dir.join("llm-logs");
|
||||
let _ = fs::create_dir_all(&dir);
|
||||
|
||||
let date = chrono::Local::now().format("%Y-%m-%d");
|
||||
let path = dir.join(format!("{}.md", date));
|
||||
|
||||
let ts = chrono::Local::now().format("%H:%M:%S");
|
||||
let status = if ok { "ok" } else { "ERROR" };
|
||||
|
||||
let entry = format!(
|
||||
"\n## {} — {} ({}, {:.1}s, {})\n\n\
|
||||
### Prompt ({} chars)\n\n\
|
||||
```\n{}\n```\n\n\
|
||||
### Response ({} chars)\n\n\
|
||||
```\n{}\n```\n\n---\n",
|
||||
ts, agent, model, duration_ms as f64 / 1000.0, status,
|
||||
prompt.len(), prompt,
|
||||
response.len(), response,
|
||||
);
|
||||
|
||||
use std::io::Write;
|
||||
if let Ok(mut f) = fs::OpenOptions::new().create(true).append(true).open(&path) {
|
||||
let _ = f.write_all(entry.as_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
/// Call a model via claude CLI. Returns the response text.
|
||||
///
|
||||
/// Sets PR_SET_PDEATHSIG on the child so it gets SIGTERM if the
|
||||
/// parent daemon exits — no more orphaned claude processes.
|
||||
fn call_model(model: &str, prompt: &str) -> Result<String, String> {
|
||||
fn call_model(agent: &str, model: &str, prompt: &str) -> Result<String, String> {
|
||||
// Write prompt to temp file (claude CLI needs file input for large prompts)
|
||||
let tmp = std::env::temp_dir().join(format!("poc-llm-{}-{:?}.txt",
|
||||
std::process::id(), std::thread::current().id()));
|
||||
|
|
@ -39,15 +67,21 @@ fn call_model(model: &str, prompt: &str) -> Result<String, String> {
|
|||
.output()
|
||||
};
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
fs::remove_file(&tmp).ok();
|
||||
|
||||
match result {
|
||||
Ok(output) => {
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
if output.status.success() {
|
||||
Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
|
||||
let response = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||
log_usage(agent, model, prompt, &response, elapsed, true);
|
||||
Ok(response)
|
||||
} else {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let preview: String = stderr.chars().take(500).collect();
|
||||
log_usage(agent, model, prompt, &preview, elapsed, false);
|
||||
Err(format!("claude exited {}: {}", output.status, preview.trim()))
|
||||
}
|
||||
}
|
||||
|
|
@ -56,13 +90,13 @@ fn call_model(model: &str, prompt: &str) -> Result<String, String> {
|
|||
}
|
||||
|
||||
/// Call Sonnet via claude CLI.
|
||||
pub(crate) fn call_sonnet(prompt: &str, _timeout_secs: u64) -> Result<String, String> {
|
||||
call_model("sonnet", prompt)
|
||||
pub(crate) fn call_sonnet(agent: &str, prompt: &str) -> Result<String, String> {
|
||||
call_model(agent, "sonnet", prompt)
|
||||
}
|
||||
|
||||
/// Call Haiku via claude CLI (cheaper, faster — good for high-volume extraction).
|
||||
pub(crate) fn call_haiku(prompt: &str) -> Result<String, String> {
|
||||
call_model("haiku", prompt)
|
||||
pub(crate) fn call_haiku(agent: &str, prompt: &str) -> Result<String, String> {
|
||||
call_model(agent, "haiku", prompt)
|
||||
}
|
||||
|
||||
/// Parse a JSON response, handling markdown fences.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue