flatten: move poc-memory contents to workspace root
No more subcrate nesting — src/, agents/, schema/, defaults/, build.rs all live at the workspace root. poc-daemon remains as the only workspace member. Crate name (poc-memory) and all imports unchanged. Co-Authored-By: Proof of Concept <poc@bcachefs.org>
This commit is contained in:
parent
891cca57f8
commit
998b71e52c
113 changed files with 79 additions and 78 deletions
45
src/bin/diag-key.rs
Normal file
45
src/bin/diag-key.rs
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
// Diagnostic: dump all entries matching a key pattern from a capnp log
|
||||
use std::io::BufReader;
|
||||
use std::fs;
|
||||
use capnp::{message, serialize};
|
||||
use poc_memory::memory_capnp;
|
||||
use poc_memory::store::Node;
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if args.len() != 3 {
|
||||
eprintln!("usage: diag-key <nodes.capnp> <key-substring>");
|
||||
std::process::exit(1);
|
||||
}
|
||||
let path = &args[1];
|
||||
let pattern = &args[2];
|
||||
|
||||
let file = fs::File::open(path).unwrap();
|
||||
let mut reader = BufReader::new(file);
|
||||
let mut entry_num = 0u64;
|
||||
let mut matches = 0u64;
|
||||
|
||||
while let Ok(msg) = serialize::read_message(&mut reader, message::ReaderOptions::new()) {
|
||||
let log = msg.get_root::<memory_capnp::node_log::Reader>().unwrap();
|
||||
for node_reader in log.get_nodes().unwrap() {
|
||||
entry_num += 1;
|
||||
let node = Node::from_capnp_migrate(node_reader).unwrap();
|
||||
|
||||
// Exact substring match, but exclude keys with trailing chars
|
||||
// (e.g. "kernel-patterns-foo") unless pattern itself has the dash
|
||||
if node.key == *pattern || (node.key.contains(pattern) && !node.key.contains(&format!("{}-", pattern))) {
|
||||
matches += 1;
|
||||
println!("Entry #{}: key={:?} (len={})", entry_num, node.key, node.key.len());
|
||||
println!(" key bytes: {:02x?}", node.key.as_bytes());
|
||||
println!(" uuid: {:02x?}", node.uuid);
|
||||
println!(" version: {}", node.version);
|
||||
println!(" deleted: {}", node.deleted);
|
||||
println!(" timestamp: {}", node.timestamp);
|
||||
println!(" content len: {}", node.content.len());
|
||||
println!(" provenance: {}", node.provenance);
|
||||
println!();
|
||||
}
|
||||
}
|
||||
}
|
||||
eprintln!("Scanned {} entries, {} matches for {:?}", entry_num, matches, pattern);
|
||||
}
|
||||
56
src/bin/find-deleted.rs
Normal file
56
src/bin/find-deleted.rs
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
// Find all deleted nodes that have no subsequent non-deleted version
|
||||
// (i.e., nodes that are currently dead).
|
||||
//
|
||||
// Also checks: is there a live node under the same key with a different UUID?
|
||||
// If not, the deletion was terminal — the node is gone.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::BufReader;
|
||||
use std::fs;
|
||||
use capnp::{message, serialize};
|
||||
use poc_memory::memory_capnp;
|
||||
use poc_memory::store::Node;
|
||||
|
||||
fn main() {
|
||||
let path = std::env::args().nth(1)
|
||||
.unwrap_or_else(|| {
|
||||
let dir = poc_memory::store::nodes_path();
|
||||
dir.to_string_lossy().to_string()
|
||||
});
|
||||
|
||||
let file = fs::File::open(&path).unwrap();
|
||||
let mut reader = BufReader::new(file);
|
||||
|
||||
// Collect ALL entries, tracking latest version per key
|
||||
let mut latest_by_key: HashMap<String, Node> = HashMap::new();
|
||||
let mut all_entries = 0u64;
|
||||
|
||||
while let Ok(msg) = serialize::read_message(&mut reader, message::ReaderOptions::new()) {
|
||||
let log = msg.get_root::<memory_capnp::node_log::Reader>().unwrap();
|
||||
for node_reader in log.get_nodes().unwrap() {
|
||||
all_entries += 1;
|
||||
let node = Node::from_capnp_migrate(node_reader).unwrap();
|
||||
let dominated = latest_by_key.get(&node.key)
|
||||
.map(|n| node.version >= n.version)
|
||||
.unwrap_or(true);
|
||||
if dominated {
|
||||
latest_by_key.insert(node.key.clone(), node);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Find keys where the latest version is deleted
|
||||
let mut dead: Vec<&Node> = latest_by_key.values()
|
||||
.filter(|n| n.deleted)
|
||||
.collect();
|
||||
dead.sort_by(|a, b| a.key.cmp(&b.key));
|
||||
|
||||
eprintln!("Scanned {} entries, {} unique keys", all_entries, latest_by_key.len());
|
||||
eprintln!("{} live nodes, {} deleted (terminal tombstones)\n",
|
||||
latest_by_key.len() - dead.len(), dead.len());
|
||||
|
||||
for node in &dead {
|
||||
println!("{:<60} v{:<4} {}b prov={}",
|
||||
node.key, node.version, node.content.len(), node.provenance);
|
||||
}
|
||||
}
|
||||
208
src/bin/memory-search.rs
Normal file
208
src/bin/memory-search.rs
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
// memory-search CLI — thin wrapper around poc_memory::memory_search
|
||||
//
|
||||
// --hook: run hook logic (for debugging; poc-hook calls the library directly)
|
||||
// surface/reflect: run agent, parse output, render memories to stdout
|
||||
// no args: show seen set for current session
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use std::fs;
|
||||
use std::io::{self, Read};
|
||||
use std::process::Command;
|
||||
|
||||
const STASH_PATH: &str = "/tmp/claude-memory-search/last-input.json";
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "memory-search")]
|
||||
struct Args {
|
||||
/// Run hook logic (reads JSON from stdin or stash file)
|
||||
#[arg(long)]
|
||||
hook: bool,
|
||||
|
||||
#[command(subcommand)]
|
||||
command: Option<Cmd>,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Cmd {
|
||||
/// Run surface agent, parse output, render memories
|
||||
Surface,
|
||||
/// Run reflect agent, dump output
|
||||
Reflect,
|
||||
}
|
||||
|
||||
fn show_seen() {
|
||||
let input = match fs::read_to_string(STASH_PATH) {
|
||||
Ok(s) => s,
|
||||
Err(_) => { eprintln!("No session state available"); return; }
|
||||
};
|
||||
let Some(session) = poc_memory::memory_search::Session::from_json(&input) else {
|
||||
eprintln!("No session state available");
|
||||
return;
|
||||
};
|
||||
|
||||
println!("Session: {}", session.session_id);
|
||||
|
||||
if let Ok(cookie) = fs::read_to_string(&session.path("cookie")) {
|
||||
println!("Cookie: {}", cookie.trim());
|
||||
}
|
||||
|
||||
match fs::read_to_string(&session.path("compaction")) {
|
||||
Ok(s) => {
|
||||
let offset: u64 = s.trim().parse().unwrap_or(0);
|
||||
let ts = poc_memory::transcript::compaction_timestamp(&session.transcript_path, offset);
|
||||
match ts {
|
||||
Some(t) => println!("Last compaction: offset {} ({})", offset, t),
|
||||
None => println!("Last compaction: offset {}", offset),
|
||||
}
|
||||
}
|
||||
Err(_) => println!("Last compaction: none detected"),
|
||||
}
|
||||
|
||||
let pending = fs::read_dir(&session.path("chunks")).ok()
|
||||
.map(|d| d.flatten().count()).unwrap_or(0);
|
||||
if pending > 0 {
|
||||
println!("Pending chunks: {}", pending);
|
||||
}
|
||||
|
||||
for (label, suffix) in [("Current seen set", ""), ("Previous seen set (pre-compaction)", "-prev")] {
|
||||
let path = session.state_dir.join(format!("seen{}-{}", suffix, session.session_id));
|
||||
let content = fs::read_to_string(&path).unwrap_or_default();
|
||||
let lines: Vec<&str> = content.lines().filter(|s| !s.is_empty()).collect();
|
||||
if lines.is_empty() { continue; }
|
||||
|
||||
println!("\n{} ({}):", label, lines.len());
|
||||
for line in &lines { println!(" {}", line); }
|
||||
}
|
||||
}
|
||||
|
||||
fn run_agent_and_parse(agent: &str) {
|
||||
let session_id = std::env::var("CLAUDE_SESSION_ID")
|
||||
.or_else(|_| {
|
||||
fs::read_to_string(STASH_PATH).ok()
|
||||
.and_then(|s| poc_memory::memory_search::Session::from_json(&s))
|
||||
.map(|s| s.session_id)
|
||||
.ok_or(std::env::VarError::NotPresent)
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
if session_id.is_empty() {
|
||||
eprintln!("No session ID available (set CLAUDE_SESSION_ID or run --hook first)");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
eprintln!("Running {} agent (session {})...", agent, &session_id[..8.min(session_id.len())]);
|
||||
|
||||
let output = Command::new("poc-memory")
|
||||
.args(["agent", "run", agent, "--count", "1", "--local"])
|
||||
.env("POC_SESSION_ID", &session_id)
|
||||
.output();
|
||||
|
||||
let output = match output {
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
eprintln!("Failed to run agent: {}", e);
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
let result = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
if !stderr.is_empty() {
|
||||
eprintln!("{}", stderr);
|
||||
}
|
||||
|
||||
// Extract the final response — after the last "=== RESPONSE ===" marker
|
||||
let response = result.rsplit_once("=== RESPONSE ===")
|
||||
.map(|(_, rest)| rest.trim())
|
||||
.unwrap_or(result.trim());
|
||||
|
||||
if agent == "reflect" {
|
||||
// Reflect: find REFLECTION marker and dump what follows
|
||||
if let Some(pos) = response.find("REFLECTION") {
|
||||
let after = &response[pos + "REFLECTION".len()..];
|
||||
let text = after.trim();
|
||||
if !text.is_empty() {
|
||||
println!("{}", text);
|
||||
}
|
||||
} else if response.contains("NO OUTPUT") {
|
||||
println!("(no reflection)");
|
||||
} else {
|
||||
eprintln!("Unexpected output format");
|
||||
println!("{}", response);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Surface: parse NEW RELEVANT MEMORIES, render them
|
||||
let tail_lines: Vec<&str> = response.lines().rev()
|
||||
.filter(|l| !l.trim().is_empty()).take(8).collect();
|
||||
let has_new = tail_lines.iter().any(|l| l.starts_with("NEW RELEVANT MEMORIES:"));
|
||||
let has_none = tail_lines.iter().any(|l| l.starts_with("NO NEW RELEVANT MEMORIES"));
|
||||
|
||||
if has_new {
|
||||
let after_marker = response.rsplit_once("NEW RELEVANT MEMORIES:")
|
||||
.map(|(_, rest)| rest).unwrap_or("");
|
||||
let keys: Vec<String> = after_marker.lines()
|
||||
.map(|l| l.trim().trim_start_matches("- ").trim().to_string())
|
||||
.filter(|l| !l.is_empty() && !l.starts_with("```")).collect();
|
||||
|
||||
if keys.is_empty() {
|
||||
println!("(no memories found)");
|
||||
return;
|
||||
}
|
||||
|
||||
let Ok(store) = poc_memory::store::Store::load() else {
|
||||
eprintln!("Failed to load store");
|
||||
return;
|
||||
};
|
||||
|
||||
for key in &keys {
|
||||
if let Some(content) = poc_memory::cli::node::render_node(&store, key) {
|
||||
if !content.trim().is_empty() {
|
||||
println!("--- {} (surfaced) ---", key);
|
||||
print!("{}", content);
|
||||
println!();
|
||||
}
|
||||
} else {
|
||||
eprintln!(" key not found: {}", key);
|
||||
}
|
||||
}
|
||||
} else if has_none {
|
||||
println!("(no new relevant memories)");
|
||||
} else {
|
||||
eprintln!("Unexpected output format");
|
||||
print!("{}", response);
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args = Args::parse();
|
||||
|
||||
if let Some(cmd) = args.command {
|
||||
match cmd {
|
||||
Cmd::Surface => run_agent_and_parse("surface"),
|
||||
Cmd::Reflect => run_agent_and_parse("reflect"),
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if args.hook {
|
||||
// Read from stdin if piped, otherwise from stash
|
||||
let input = {
|
||||
let mut buf = String::new();
|
||||
io::stdin().read_to_string(&mut buf).ok();
|
||||
if buf.trim().is_empty() {
|
||||
fs::read_to_string(STASH_PATH).unwrap_or_default()
|
||||
} else {
|
||||
let _ = fs::create_dir_all("/tmp/claude-memory-search");
|
||||
let _ = fs::write(STASH_PATH, &buf);
|
||||
buf
|
||||
}
|
||||
};
|
||||
|
||||
let output = poc_memory::memory_search::run_hook(&input);
|
||||
print!("{}", output);
|
||||
} else {
|
||||
show_seen()
|
||||
}
|
||||
}
|
||||
205
src/bin/merge-logs.rs
Normal file
205
src/bin/merge-logs.rs
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
// merge-logs: Recover historical entries from a checkpoint log and merge
|
||||
// with the current log into a NEW output file.
|
||||
//
|
||||
// This tool was written to recover history destroyed by rewrite_store()
|
||||
// (see persist.rs comment). It reads two capnp node logs, finds entries
|
||||
// in the old log that don't exist in the current log (by uuid+version),
|
||||
// and writes a merged log containing both.
|
||||
//
|
||||
// SAFETY: This tool never modifies either input file. The merged output
|
||||
// goes to a new directory specified by the user.
|
||||
//
|
||||
// Usage:
|
||||
// merge-logs <old_log> <current_log> <output_dir>
|
||||
//
|
||||
// Example:
|
||||
// merge-logs ~/.claude/memory/checkpoints/nodes.capnp \
|
||||
// ~/.claude/memory/nodes.capnp \
|
||||
// /tmp/merged-store
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs;
|
||||
use std::io::{BufReader, BufWriter};
|
||||
use std::path::Path;
|
||||
|
||||
use capnp::message;
|
||||
use capnp::serialize;
|
||||
|
||||
use poc_memory::memory_capnp;
|
||||
use poc_memory::store::Node;
|
||||
|
||||
/// Read all node entries from a capnp log file, preserving order.
|
||||
fn read_all_entries(path: &Path) -> Result<Vec<Node>, String> {
|
||||
let file = fs::File::open(path)
|
||||
.map_err(|e| format!("open {}: {}", path.display(), e))?;
|
||||
let mut reader = BufReader::new(file);
|
||||
let mut entries = Vec::new();
|
||||
|
||||
while let Ok(msg) = serialize::read_message(&mut reader, message::ReaderOptions::new()) {
|
||||
let log = msg.get_root::<memory_capnp::node_log::Reader>()
|
||||
.map_err(|e| format!("read log from {}: {}", path.display(), e))?;
|
||||
for node_reader in log.get_nodes()
|
||||
.map_err(|e| format!("get nodes from {}: {}", path.display(), e))? {
|
||||
let node = Node::from_capnp_migrate(node_reader)?;
|
||||
entries.push(node);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// Write node entries to a new capnp log file in chunks.
|
||||
fn write_entries(path: &Path, entries: &[Node]) -> Result<(), String> {
|
||||
let file = fs::File::create(path)
|
||||
.map_err(|e| format!("create {}: {}", path.display(), e))?;
|
||||
let mut writer = BufWriter::new(file);
|
||||
|
||||
for chunk in entries.chunks(100) {
|
||||
let mut msg = message::Builder::new_default();
|
||||
{
|
||||
let log = msg.init_root::<memory_capnp::node_log::Builder>();
|
||||
let mut list = log.init_nodes(chunk.len() as u32);
|
||||
for (i, node) in chunk.iter().enumerate() {
|
||||
node.to_capnp(list.reborrow().get(i as u32));
|
||||
}
|
||||
}
|
||||
serialize::write_message(&mut writer, &msg)
|
||||
.map_err(|e| format!("write: {}", e))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() -> Result<(), String> {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if args.len() != 4 {
|
||||
eprintln!("Usage: merge-logs <old_log> <current_log> <output_dir>");
|
||||
eprintln!();
|
||||
eprintln!("Merges historical entries from old_log with current_log,");
|
||||
eprintln!("writing the result to output_dir/nodes.capnp.");
|
||||
eprintln!("Neither input file is modified.");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let old_path = Path::new(&args[1]);
|
||||
let current_path = Path::new(&args[2]);
|
||||
let output_dir = Path::new(&args[3]);
|
||||
|
||||
// Validate inputs exist
|
||||
if !old_path.exists() {
|
||||
return Err(format!("old log not found: {}", old_path.display()));
|
||||
}
|
||||
if !current_path.exists() {
|
||||
return Err(format!("current log not found: {}", current_path.display()));
|
||||
}
|
||||
|
||||
// Create output directory (must not already contain nodes.capnp)
|
||||
fs::create_dir_all(output_dir)
|
||||
.map_err(|e| format!("create output dir: {}", e))?;
|
||||
let output_path = output_dir.join("nodes.capnp");
|
||||
if output_path.exists() {
|
||||
return Err(format!("output already exists: {} — refusing to overwrite",
|
||||
output_path.display()));
|
||||
}
|
||||
|
||||
eprintln!("Reading old log: {} ...", old_path.display());
|
||||
let old_entries = read_all_entries(old_path)?;
|
||||
eprintln!(" {} entries", old_entries.len());
|
||||
|
||||
eprintln!("Reading current log: {} ...", current_path.display());
|
||||
let current_entries = read_all_entries(current_path)?;
|
||||
eprintln!(" {} entries", current_entries.len());
|
||||
|
||||
// Build set of (uuid, version) pairs from current log
|
||||
let current_set: HashSet<([u8; 16], u32)> = current_entries.iter()
|
||||
.map(|n| (n.uuid, n.version))
|
||||
.collect();
|
||||
|
||||
// Find entries in old log not present in current log
|
||||
let recovered: Vec<&Node> = old_entries.iter()
|
||||
.filter(|n| !current_set.contains(&(n.uuid, n.version)))
|
||||
.collect();
|
||||
|
||||
eprintln!();
|
||||
eprintln!("Current log has {} unique (uuid, version) pairs", current_set.len());
|
||||
eprintln!("Old log entries already in current: {}", old_entries.len() - recovered.len());
|
||||
eprintln!("Old log entries to recover: {}", recovered.len());
|
||||
|
||||
// Count unique keys being recovered
|
||||
let recovered_keys: HashSet<&str> = recovered.iter()
|
||||
.map(|n| n.key.as_str())
|
||||
.collect();
|
||||
eprintln!("Unique keys with recovered history: {}", recovered_keys.len());
|
||||
|
||||
// Show some stats about what we're recovering
|
||||
let mut version_counts: HashMap<&str, Vec<u32>> = HashMap::new();
|
||||
for node in &recovered {
|
||||
version_counts.entry(&node.key)
|
||||
.or_default()
|
||||
.push(node.version);
|
||||
}
|
||||
let mut keys_by_versions: Vec<_> = version_counts.iter()
|
||||
.map(|(k, v)| (*k, v.len()))
|
||||
.collect();
|
||||
keys_by_versions.sort_by(|a, b| b.1.cmp(&a.1));
|
||||
eprintln!();
|
||||
eprintln!("Top 20 keys by recovered versions:");
|
||||
for (key, count) in keys_by_versions.iter().take(20) {
|
||||
eprintln!(" {:4} versions {}", count, key);
|
||||
}
|
||||
|
||||
// Build merged log: recovered entries (preserving order), then current entries
|
||||
let mut merged: Vec<Node> = Vec::with_capacity(recovered.len() + current_entries.len());
|
||||
for node in recovered {
|
||||
merged.push(node.clone());
|
||||
}
|
||||
for node in current_entries {
|
||||
merged.push(node);
|
||||
}
|
||||
|
||||
eprintln!();
|
||||
eprintln!("Writing merged log: {} ({} entries) ...",
|
||||
output_path.display(), merged.len());
|
||||
write_entries(&output_path, &merged)?;
|
||||
|
||||
let output_size = fs::metadata(&output_path).map(|m| m.len()).unwrap_or(0);
|
||||
eprintln!("Done. Output: {} ({:.1} MB)", output_path.display(),
|
||||
output_size as f64 / 1_048_576.0);
|
||||
|
||||
// Verify: replay the merged log and check node count
|
||||
eprintln!();
|
||||
eprintln!("Verifying merged log...");
|
||||
let verify_entries = read_all_entries(&output_path)?;
|
||||
eprintln!(" Read back {} entries (expected {})",
|
||||
verify_entries.len(), merged.len());
|
||||
|
||||
// Replay to get final state
|
||||
let mut final_nodes: HashMap<String, Node> = HashMap::new();
|
||||
for node in &verify_entries {
|
||||
let dominated = final_nodes.get(&node.key)
|
||||
.map(|n| node.version >= n.version)
|
||||
.unwrap_or(true);
|
||||
if dominated {
|
||||
if node.deleted {
|
||||
final_nodes.remove(&node.key);
|
||||
} else {
|
||||
final_nodes.insert(node.key.clone(), node.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
eprintln!(" Replay produces {} live nodes", final_nodes.len());
|
||||
|
||||
if verify_entries.len() != merged.len() {
|
||||
return Err(format!("Verification failed: wrote {} but read back {}",
|
||||
merged.len(), verify_entries.len()));
|
||||
}
|
||||
|
||||
eprintln!();
|
||||
eprintln!("Merge complete. To use the merged log:");
|
||||
eprintln!(" 1. Back up ~/.claude/memory/nodes.capnp");
|
||||
eprintln!(" 2. cp {} ~/.claude/memory/nodes.capnp", output_path.display());
|
||||
eprintln!(" 3. rm ~/.claude/memory/state.bin ~/.claude/memory/snapshot.rkyv");
|
||||
eprintln!(" 4. poc-memory admin fsck");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
328
src/bin/parse-claude-conversation.rs
Normal file
328
src/bin/parse-claude-conversation.rs
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
// parse-claude-conversation: debug tool for inspecting what's in the context window
|
||||
//
|
||||
// Two-layer design:
|
||||
// 1. extract_context_items() — walks JSONL from last compaction, yields
|
||||
// structured records representing what's in the context window
|
||||
// 2. format_as_context() — renders those records as they appear to Claude
|
||||
//
|
||||
// The transcript is mmap'd and scanned backwards from EOF using brace-depth
|
||||
// tracking to find complete JSON objects, avoiding a full forward scan of
|
||||
// what can be a 500MB+ file.
|
||||
//
|
||||
// Usage:
|
||||
// parse-claude-conversation [TRANSCRIPT_PATH]
|
||||
// parse-claude-conversation --last # use the last stashed session
|
||||
|
||||
use clap::Parser;
|
||||
use memmap2::Mmap;
|
||||
use poc_memory::transcript::{JsonlBackwardIter, find_last_compaction};
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "parse-claude-conversation")]
|
||||
struct Args {
|
||||
/// Transcript JSONL path (or --last to use stashed session)
|
||||
path: Option<String>,
|
||||
|
||||
/// Use the last stashed session from memory-search
|
||||
#[arg(long)]
|
||||
last: bool,
|
||||
|
||||
/// Dump raw JSONL objects. Optional integer: number of extra objects
|
||||
/// to include before the compaction boundary.
|
||||
#[arg(long, num_args = 0..=1, default_missing_value = "0")]
|
||||
raw: Option<usize>,
|
||||
}
|
||||
|
||||
// --- Context extraction ---
|
||||
|
||||
/// A single item in the context window, as Claude sees it.
|
||||
enum ContextItem {
|
||||
UserText(String),
|
||||
SystemReminder(String),
|
||||
AssistantText(String),
|
||||
AssistantThinking,
|
||||
ToolUse { name: String, input: String },
|
||||
ToolResult(String),
|
||||
}
|
||||
|
||||
/// Extract context items from the transcript, starting from the last compaction.
|
||||
fn extract_context_items(data: &[u8]) -> Vec<ContextItem> {
|
||||
let start = find_last_compaction(data).unwrap_or(0);
|
||||
let region = &data[start..];
|
||||
|
||||
let mut items = Vec::new();
|
||||
|
||||
// Forward scan through JSONL lines from compaction onward
|
||||
for line in region.split(|&b| b == b'\n') {
|
||||
if line.is_empty() { continue; }
|
||||
|
||||
let obj: Value = match serde_json::from_slice(line) {
|
||||
Ok(v) => v,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
let msg_type = obj.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
|
||||
match msg_type {
|
||||
"user" => {
|
||||
if let Some(content) = obj.get("message").and_then(|m| m.get("content")) {
|
||||
extract_user_content(content, &mut items);
|
||||
}
|
||||
}
|
||||
"assistant" => {
|
||||
if let Some(content) = obj.get("message").and_then(|m| m.get("content")) {
|
||||
extract_assistant_content(content, &mut items);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
items
|
||||
}
|
||||
|
||||
/// Parse user message content into context items.
|
||||
fn extract_user_content(content: &Value, items: &mut Vec<ContextItem>) {
|
||||
match content {
|
||||
Value::String(s) => {
|
||||
split_system_reminders(s, items, false);
|
||||
}
|
||||
Value::Array(arr) => {
|
||||
for block in arr {
|
||||
let btype = block.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
match btype {
|
||||
"text" => {
|
||||
if let Some(t) = block.get("text").and_then(|v| v.as_str()) {
|
||||
split_system_reminders(t, items, false);
|
||||
}
|
||||
}
|
||||
"tool_result" => {
|
||||
let result_text = extract_tool_result_text(block);
|
||||
if !result_text.is_empty() {
|
||||
split_system_reminders(&result_text, items, true);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract text from a tool_result block (content can be string or array).
|
||||
fn extract_tool_result_text(block: &Value) -> String {
|
||||
match block.get("content") {
|
||||
Some(Value::String(s)) => s.clone(),
|
||||
Some(Value::Array(arr)) => {
|
||||
arr.iter()
|
||||
.filter_map(|b| b.get("text").and_then(|v| v.as_str()))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split text on <system-reminder> tags. Non-reminder text emits UserText
|
||||
/// or ToolResult depending on `is_tool_result`.
|
||||
fn split_system_reminders(text: &str, items: &mut Vec<ContextItem>, is_tool_result: bool) {
|
||||
let mut remaining = text;
|
||||
|
||||
loop {
|
||||
if let Some(start) = remaining.find("<system-reminder>") {
|
||||
let before = remaining[..start].trim();
|
||||
if !before.is_empty() {
|
||||
if is_tool_result {
|
||||
items.push(ContextItem::ToolResult(before.to_string()));
|
||||
} else {
|
||||
items.push(ContextItem::UserText(before.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
let after_open = &remaining[start + "<system-reminder>".len()..];
|
||||
if let Some(end) = after_open.find("</system-reminder>") {
|
||||
let reminder = after_open[..end].trim();
|
||||
if !reminder.is_empty() {
|
||||
items.push(ContextItem::SystemReminder(reminder.to_string()));
|
||||
}
|
||||
remaining = &after_open[end + "</system-reminder>".len()..];
|
||||
} else {
|
||||
let reminder = after_open.trim();
|
||||
if !reminder.is_empty() {
|
||||
items.push(ContextItem::SystemReminder(reminder.to_string()));
|
||||
}
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
let trimmed = remaining.trim();
|
||||
if !trimmed.is_empty() {
|
||||
if is_tool_result {
|
||||
items.push(ContextItem::ToolResult(trimmed.to_string()));
|
||||
} else {
|
||||
items.push(ContextItem::UserText(trimmed.to_string()));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse assistant message content into context items.
|
||||
fn extract_assistant_content(content: &Value, items: &mut Vec<ContextItem>) {
|
||||
match content {
|
||||
Value::String(s) => {
|
||||
let trimmed = s.trim();
|
||||
if !trimmed.is_empty() {
|
||||
items.push(ContextItem::AssistantText(trimmed.to_string()));
|
||||
}
|
||||
}
|
||||
Value::Array(arr) => {
|
||||
for block in arr {
|
||||
let btype = block.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
match btype {
|
||||
"text" => {
|
||||
if let Some(t) = block.get("text").and_then(|v| v.as_str()) {
|
||||
let trimmed = t.trim();
|
||||
if !trimmed.is_empty() {
|
||||
items.push(ContextItem::AssistantText(trimmed.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
"tool_use" => {
|
||||
let name = block.get("name")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("?")
|
||||
.to_string();
|
||||
let input = block.get("input")
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_default();
|
||||
items.push(ContextItem::ToolUse { name, input });
|
||||
}
|
||||
"thinking" => {
|
||||
items.push(ContextItem::AssistantThinking);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Formatting layer ---
|
||||
|
||||
fn truncate(s: &str, max: usize) -> String {
|
||||
if s.len() <= max {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!("{}...({} total)", &s[..max], s.len())
|
||||
}
|
||||
}
|
||||
|
||||
fn format_as_context(items: &[ContextItem]) {
|
||||
for item in items {
|
||||
match item {
|
||||
ContextItem::UserText(text) => {
|
||||
println!("USER: {}", truncate(text, 300));
|
||||
println!();
|
||||
}
|
||||
ContextItem::SystemReminder(text) => {
|
||||
println!("<system-reminder>");
|
||||
println!("{}", truncate(text, 500));
|
||||
println!("</system-reminder>");
|
||||
println!();
|
||||
}
|
||||
ContextItem::AssistantText(text) => {
|
||||
println!("ASSISTANT: {}", truncate(text, 300));
|
||||
println!();
|
||||
}
|
||||
ContextItem::AssistantThinking => {
|
||||
println!("[thinking]");
|
||||
println!();
|
||||
}
|
||||
ContextItem::ToolUse { name, input } => {
|
||||
println!("TOOL_USE: {} {}", name, truncate(input, 200));
|
||||
println!();
|
||||
}
|
||||
ContextItem::ToolResult(text) => {
|
||||
println!("TOOL_RESULT: {}", truncate(text, 300));
|
||||
println!();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args = Args::parse();
|
||||
|
||||
let path = if args.last {
|
||||
let stash = fs::read_to_string("/tmp/claude-memory-search/last-input.json")
|
||||
.expect("No stashed input");
|
||||
let json: Value = serde_json::from_str(&stash).expect("Bad JSON");
|
||||
json["transcript_path"]
|
||||
.as_str()
|
||||
.expect("No transcript_path")
|
||||
.to_string()
|
||||
} else if let Some(p) = args.path {
|
||||
p
|
||||
} else {
|
||||
eprintln!("error: provide a transcript path or --last");
|
||||
std::process::exit(1);
|
||||
};
|
||||
|
||||
let file = fs::File::open(&path).expect("Can't open transcript");
|
||||
let mmap = unsafe { Mmap::map(&file).expect("Failed to mmap") };
|
||||
|
||||
eprintln!(
|
||||
"Transcript: {} ({:.1} MB)",
|
||||
&path,
|
||||
mmap.len() as f64 / 1_000_000.0
|
||||
);
|
||||
|
||||
let compaction_offset = find_last_compaction(&mmap).unwrap_or(0);
|
||||
eprintln!("Compaction at byte offset: {}", compaction_offset);
|
||||
|
||||
if let Some(extra) = args.raw {
|
||||
use std::io::Write;
|
||||
|
||||
// Collect `extra` JSON objects before the compaction boundary
|
||||
let mut before = Vec::new();
|
||||
if extra > 0 && compaction_offset > 0 {
|
||||
for obj_bytes in JsonlBackwardIter::new(&mmap[..compaction_offset]) {
|
||||
if let Ok(obj) = serde_json::from_slice::<Value>(obj_bytes) {
|
||||
let t = obj.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if t == "file-history-snapshot" { continue; }
|
||||
}
|
||||
before.push(obj_bytes.to_vec());
|
||||
if before.len() >= extra {
|
||||
break;
|
||||
}
|
||||
}
|
||||
before.reverse();
|
||||
}
|
||||
|
||||
for obj in &before {
|
||||
std::io::stdout().write_all(obj).ok();
|
||||
println!();
|
||||
}
|
||||
|
||||
// Then dump everything from compaction onward
|
||||
let region = &mmap[compaction_offset..];
|
||||
for line in region.split(|&b| b == b'\n') {
|
||||
if line.is_empty() { continue; }
|
||||
if let Ok(obj) = serde_json::from_slice::<Value>(line) {
|
||||
let t = obj.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if t == "file-history-snapshot" { continue; }
|
||||
std::io::stdout().write_all(line).ok();
|
||||
println!();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let items = extract_context_items(&mmap);
|
||||
eprintln!("Context items: {}", items.len());
|
||||
format_as_context(&items);
|
||||
}
|
||||
}
|
||||
1282
src/bin/poc-agent.rs
Normal file
1282
src/bin/poc-agent.rs
Normal file
File diff suppressed because it is too large
Load diff
214
src/bin/poc-hook.rs
Normal file
214
src/bin/poc-hook.rs
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
// Unified Claude Code hook.
|
||||
//
|
||||
// Single binary handling all hook events:
|
||||
// UserPromptSubmit — signal daemon, check notifications, check context
|
||||
// PostToolUse — check context (rate-limited)
|
||||
// Stop — signal daemon response
|
||||
//
|
||||
// Replaces: record-user-message-time.sh, check-notifications.sh,
|
||||
// check-context-usage.sh, notify-done.sh, context-check
|
||||
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
use std::io::{self, Read};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
const CONTEXT_THRESHOLD: u64 = 900_000;
|
||||
const RATE_LIMIT_SECS: u64 = 60;
|
||||
const SOCK_PATH: &str = ".claude/hooks/idle-timer.sock";
|
||||
/// How many bytes of new transcript before triggering an observation run.
|
||||
/// Override with POC_OBSERVATION_THRESHOLD env var.
|
||||
/// Default: 20KB ≈ 5K tokens. The observation agent's chunk_size (in .agent
|
||||
/// file) controls how much context it actually reads.
|
||||
fn observation_threshold() -> u64 {
|
||||
std::env::var("POC_OBSERVATION_THRESHOLD")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(20_000)
|
||||
}
|
||||
|
||||
fn now_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
fn home() -> PathBuf {
|
||||
PathBuf::from(std::env::var("HOME").unwrap_or_else(|_| "/root".into()))
|
||||
}
|
||||
|
||||
fn daemon_cmd(args: &[&str]) {
|
||||
Command::new("poc-daemon")
|
||||
.args(args)
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.ok();
|
||||
}
|
||||
|
||||
fn daemon_available() -> bool {
|
||||
home().join(SOCK_PATH).exists()
|
||||
}
|
||||
|
||||
fn signal_user() {
|
||||
let pane = std::env::var("TMUX_PANE").unwrap_or_default();
|
||||
if pane.is_empty() {
|
||||
daemon_cmd(&["user"]);
|
||||
} else {
|
||||
daemon_cmd(&["user", &pane]);
|
||||
}
|
||||
}
|
||||
|
||||
fn signal_response() {
|
||||
daemon_cmd(&["response"]);
|
||||
}
|
||||
|
||||
fn check_notifications() {
|
||||
if !daemon_available() {
|
||||
return;
|
||||
}
|
||||
let output = Command::new("poc-daemon")
|
||||
.arg("notifications")
|
||||
.output()
|
||||
.ok();
|
||||
if let Some(out) = output {
|
||||
let text = String::from_utf8_lossy(&out.stdout);
|
||||
if !text.trim().is_empty() {
|
||||
println!("You have pending notifications:");
|
||||
print!("{text}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if enough new conversation has accumulated to trigger an observation run.
|
||||
fn maybe_trigger_observation(transcript: &PathBuf) {
|
||||
let cursor_file = poc_memory::store::memory_dir().join("observation-cursor");
|
||||
|
||||
let last_pos: u64 = fs::read_to_string(&cursor_file)
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse().ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
let current_size = transcript.metadata()
|
||||
.map(|m| m.len())
|
||||
.unwrap_or(0);
|
||||
|
||||
if current_size > last_pos + observation_threshold() {
|
||||
// Queue observation via daemon RPC
|
||||
let _ = Command::new("poc-memory")
|
||||
.args(["agent", "daemon", "run", "observation", "1"])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.spawn();
|
||||
|
||||
eprintln!("[poc-hook] observation triggered ({} new bytes)", current_size - last_pos);
|
||||
|
||||
// Update cursor to current position
|
||||
let _ = fs::write(&cursor_file, current_size.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
fn check_context(transcript: &PathBuf, rate_limit: bool) {
|
||||
if rate_limit {
|
||||
let rate_file = PathBuf::from("/tmp/claude-context-check-last");
|
||||
if let Ok(s) = fs::read_to_string(&rate_file) {
|
||||
if let Ok(last) = s.trim().parse::<u64>() {
|
||||
if now_secs() - last < RATE_LIMIT_SECS {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
let _ = fs::write(&rate_file, now_secs().to_string());
|
||||
}
|
||||
|
||||
if !transcript.exists() {
|
||||
return;
|
||||
}
|
||||
|
||||
let content = match fs::read_to_string(transcript) {
|
||||
Ok(c) => c,
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
let mut usage: u64 = 0;
|
||||
for line in content.lines().rev().take(500) {
|
||||
if !line.contains("cache_read_input_tokens") {
|
||||
continue;
|
||||
}
|
||||
if let Ok(v) = serde_json::from_str::<Value>(line) {
|
||||
let u = &v["message"]["usage"];
|
||||
let input_tokens = u["input_tokens"].as_u64().unwrap_or(0);
|
||||
let cache_creation = u["cache_creation_input_tokens"].as_u64().unwrap_or(0);
|
||||
let cache_read = u["cache_read_input_tokens"].as_u64().unwrap_or(0);
|
||||
usage = input_tokens + cache_creation + cache_read;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if usage > CONTEXT_THRESHOLD {
|
||||
print!(
|
||||
"\
|
||||
CONTEXT WARNING: Compaction approaching ({usage} tokens). Write a journal entry NOW.
|
||||
|
||||
Use `poc-memory journal write \"entry text\"` to save a dated entry covering:
|
||||
- What you're working on and current state (done / in progress / blocked)
|
||||
- Key things learned this session (patterns, debugging insights)
|
||||
- Anything half-finished that needs pickup
|
||||
|
||||
Keep it narrative, not a task log."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let mut input = String::new();
|
||||
io::stdin().read_to_string(&mut input).ok();
|
||||
|
||||
let hook: Value = match serde_json::from_str(&input) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
let hook_type = hook["hook_event_name"].as_str().unwrap_or("unknown");
|
||||
let transcript = hook["transcript_path"]
|
||||
.as_str()
|
||||
.filter(|p| !p.is_empty())
|
||||
.map(PathBuf::from);
|
||||
|
||||
// Daemon agent calls set POC_AGENT=1 — skip all signaling.
|
||||
// Without this, the daemon's claude -p calls trigger hooks that
|
||||
// signal "user active", keeping the idle timer permanently reset.
|
||||
if std::env::var("POC_AGENT").is_ok() {
|
||||
return;
|
||||
}
|
||||
|
||||
match hook_type {
|
||||
"UserPromptSubmit" => {
|
||||
signal_user();
|
||||
check_notifications();
|
||||
print!("{}", poc_memory::memory_search::run_hook(&input));
|
||||
|
||||
if let Some(ref t) = transcript {
|
||||
check_context(t, false);
|
||||
maybe_trigger_observation(t);
|
||||
}
|
||||
}
|
||||
"PostToolUse" => {
|
||||
print!("{}", poc_memory::memory_search::run_hook(&input));
|
||||
|
||||
if let Some(ref t) = transcript {
|
||||
check_context(t, true);
|
||||
}
|
||||
}
|
||||
"Stop" => {
|
||||
let stop_hook_active = hook["stop_hook_active"].as_bool().unwrap_or(false);
|
||||
if !stop_hook_active {
|
||||
signal_response();
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
65
src/bin/test-conversation.rs
Normal file
65
src/bin/test-conversation.rs
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
// Test tool for the conversation resolver.
|
||||
// Usage: POC_SESSION_ID=<id> cargo run --bin test-conversation
|
||||
// or: cargo run --bin test-conversation -- <transcript-path>
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() {
|
||||
let path = std::env::args().nth(1).unwrap_or_else(|| {
|
||||
let session_id = std::env::var("POC_SESSION_ID")
|
||||
.expect("pass a transcript path or set POC_SESSION_ID");
|
||||
let projects = poc_memory::config::get().projects_dir.clone();
|
||||
eprintln!("session: {}", session_id);
|
||||
eprintln!("projects dir: {}", projects.display());
|
||||
|
||||
let mut found = None;
|
||||
if let Ok(dirs) = std::fs::read_dir(&projects) {
|
||||
for dir in dirs.filter_map(|e| e.ok()) {
|
||||
let path = dir.path().join(format!("{}.jsonl", session_id));
|
||||
eprintln!(" checking: {}", path.display());
|
||||
if path.exists() {
|
||||
found = Some(path);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let path = found.expect("transcript not found");
|
||||
path.to_string_lossy().to_string()
|
||||
});
|
||||
|
||||
let meta = std::fs::metadata(&path).expect("can't stat file");
|
||||
eprintln!("transcript: {} ({} bytes)", path, meta.len());
|
||||
|
||||
let t0 = Instant::now();
|
||||
let iter = poc_memory::transcript::TailMessages::open(&path)
|
||||
.expect("can't open transcript");
|
||||
|
||||
let mut count = 0;
|
||||
let mut total_bytes = 0;
|
||||
let mut last_report = Instant::now();
|
||||
|
||||
for (role, content, ts) in iter {
|
||||
count += 1;
|
||||
total_bytes += content.len();
|
||||
|
||||
if last_report.elapsed().as_secs() >= 2 {
|
||||
eprintln!(" ... {} messages, {}KB so far ({:.1}s)",
|
||||
count, total_bytes / 1024, t0.elapsed().as_secs_f64());
|
||||
last_report = Instant::now();
|
||||
}
|
||||
|
||||
if count <= 5 {
|
||||
let preview: String = content.chars().take(80).collect();
|
||||
eprintln!(" [{}] {} {}: {}...",
|
||||
count, &ts[..ts.len().min(19)], role, preview);
|
||||
}
|
||||
|
||||
if total_bytes >= 200_000 {
|
||||
eprintln!(" hit 200KB budget at {} messages", count);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed = t0.elapsed();
|
||||
eprintln!("done: {} messages, {}KB in {:.3}s", count, total_bytes / 1024, elapsed.as_secs_f64());
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue