restructure: hippocampus/ for memory, subconscious/ for agents
hippocampus/ — memory storage, retrieval, and consolidation: store, graph, query, similarity, spectral, neuro, counters, config, transcript, memory_search, lookups, cursor, migrate subconscious/ — autonomous agents that process without being asked: reflect, surface, consolidate, digest, audit, etc. All existing crate::X paths preserved via re-exports in lib.rs. Co-Authored-By: Proof of Concept <poc@bcachefs.org> Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
This commit is contained in:
ProofOfConcept
Kent Overstreet
parent
cfed85bd20
commit
d5c0e86700
39 changed files with 87 additions and 32 deletions
360
src/hippocampus/memory_search.rs
Normal file
360
src/hippocampus/memory_search.rs
Normal file
|
|
@ -0,0 +1,360 @@
|
|||
// memory-search: context loading + ambient memory retrieval
|
||||
//
|
||||
// Core hook logic lives here as a library module so poc-hook can call
|
||||
// it directly (no subprocess). The memory-search binary is a thin CLI
|
||||
// wrapper with --hook for debugging and show_seen for inspection.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::fs;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn now_secs() -> u64 {
|
||||
SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs()
|
||||
}
|
||||
|
||||
/// Max bytes per context chunk (hook output limit is ~10K chars)
|
||||
const CHUNK_SIZE: usize = 9000;
|
||||
|
||||
pub struct Session {
|
||||
pub session_id: String,
|
||||
pub transcript_path: String,
|
||||
pub hook_event: String,
|
||||
pub state_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub fn from_json(input: &str) -> Option<Self> {
|
||||
let state_dir = PathBuf::from("/tmp/claude-memory-search");
|
||||
fs::create_dir_all(&state_dir).ok();
|
||||
|
||||
let json: serde_json::Value = serde_json::from_str(input).ok()?;
|
||||
let session_id = json["session_id"].as_str().unwrap_or("").to_string();
|
||||
if session_id.is_empty() { return None; }
|
||||
let transcript_path = json["transcript_path"].as_str().unwrap_or("").to_string();
|
||||
let hook_event = json["hook_event_name"].as_str().unwrap_or("").to_string();
|
||||
|
||||
Some(Session { session_id, transcript_path, hook_event, state_dir })
|
||||
}
|
||||
|
||||
pub fn path(&self, prefix: &str) -> PathBuf {
|
||||
self.state_dir.join(format!("{}-{}", prefix, self.session_id))
|
||||
}
|
||||
|
||||
/// Load from POC_SESSION_ID environment variable
|
||||
pub fn from_env() -> Option<Self> {
|
||||
let session_id = std::env::var("POC_SESSION_ID").ok()?;
|
||||
if session_id.is_empty() { return None; }
|
||||
let state_dir = PathBuf::from("/tmp/claude-memory-search");
|
||||
Some(Session {
|
||||
session_id,
|
||||
transcript_path: String::new(),
|
||||
hook_event: String::new(),
|
||||
state_dir,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get the seen set for this session
|
||||
pub fn seen(&self) -> HashSet<String> {
|
||||
load_seen(&self.state_dir, &self.session_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the hook logic on parsed JSON input. Returns output to inject.
|
||||
pub fn run_hook(input: &str) -> String {
|
||||
// Daemon agent calls set POC_AGENT=1 — skip memory search.
|
||||
if std::env::var("POC_AGENT").is_ok() { return String::new(); }
|
||||
|
||||
let Some(session) = Session::from_json(input) else { return String::new() };
|
||||
hook(&session)
|
||||
}
|
||||
|
||||
/// Split context output into chunks of approximately `max_bytes`, breaking
|
||||
/// at section boundaries ("--- KEY (group) ---" lines).
|
||||
fn chunk_context(ctx: &str, max_bytes: usize) -> Vec<String> {
|
||||
let mut sections: Vec<String> = Vec::new();
|
||||
let mut current = String::new();
|
||||
|
||||
for line in ctx.lines() {
|
||||
if line.starts_with("--- ") && line.ends_with(" ---") && !current.is_empty() {
|
||||
sections.push(std::mem::take(&mut current));
|
||||
}
|
||||
if !current.is_empty() {
|
||||
current.push('\n');
|
||||
}
|
||||
current.push_str(line);
|
||||
}
|
||||
if !current.is_empty() {
|
||||
sections.push(current);
|
||||
}
|
||||
|
||||
let mut chunks: Vec<String> = Vec::new();
|
||||
let mut chunk = String::new();
|
||||
for section in sections {
|
||||
if !chunk.is_empty() && chunk.len() + section.len() + 1 > max_bytes {
|
||||
chunks.push(std::mem::take(&mut chunk));
|
||||
}
|
||||
if !chunk.is_empty() {
|
||||
chunk.push('\n');
|
||||
}
|
||||
chunk.push_str(§ion);
|
||||
}
|
||||
if !chunk.is_empty() {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
chunks
|
||||
}
|
||||
|
||||
fn save_pending_chunks(dir: &Path, session_id: &str, chunks: &[String]) {
|
||||
let chunks_dir = dir.join(format!("chunks-{}", session_id));
|
||||
let _ = fs::remove_dir_all(&chunks_dir);
|
||||
if chunks.is_empty() { return; }
|
||||
fs::create_dir_all(&chunks_dir).ok();
|
||||
for (i, chunk) in chunks.iter().enumerate() {
|
||||
let path = chunks_dir.join(format!("{:04}", i));
|
||||
fs::write(path, chunk).ok();
|
||||
}
|
||||
}
|
||||
|
||||
fn pop_pending_chunk(dir: &Path, session_id: &str) -> Option<String> {
|
||||
let chunks_dir = dir.join(format!("chunks-{}", session_id));
|
||||
if !chunks_dir.exists() { return None; }
|
||||
|
||||
let mut entries: Vec<_> = fs::read_dir(&chunks_dir).ok()?
|
||||
.flatten()
|
||||
.filter(|e| e.file_type().map(|t| t.is_file()).unwrap_or(false))
|
||||
.collect();
|
||||
entries.sort_by_key(|e| e.file_name());
|
||||
|
||||
let first = entries.first()?;
|
||||
let content = fs::read_to_string(first.path()).ok()?;
|
||||
fs::remove_file(first.path()).ok();
|
||||
|
||||
if fs::read_dir(&chunks_dir).ok().map(|mut d| d.next().is_none()).unwrap_or(true) {
|
||||
fs::remove_dir(&chunks_dir).ok();
|
||||
}
|
||||
|
||||
Some(content)
|
||||
}
|
||||
|
||||
fn generate_cookie() -> String {
|
||||
uuid::Uuid::new_v4().as_simple().to_string()[..12].to_string()
|
||||
}
|
||||
|
||||
fn parse_seen_line(line: &str) -> &str {
|
||||
line.split_once('\t').map(|(_, key)| key).unwrap_or(line)
|
||||
}
|
||||
|
||||
fn load_seen(dir: &Path, session_id: &str) -> HashSet<String> {
|
||||
let path = dir.join(format!("seen-{}", session_id));
|
||||
if path.exists() {
|
||||
fs::read_to_string(&path)
|
||||
.unwrap_or_default()
|
||||
.lines()
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| parse_seen_line(s).to_string())
|
||||
.collect()
|
||||
} else {
|
||||
HashSet::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn mark_seen(dir: &Path, session_id: &str, key: &str, seen: &mut HashSet<String>) {
|
||||
if !seen.insert(key.to_string()) { return; }
|
||||
let path = dir.join(format!("seen-{}", session_id));
|
||||
if let Ok(mut f) = fs::OpenOptions::new().create(true).append(true).open(path) {
|
||||
let ts = chrono::Local::now().format("%Y-%m-%dT%H:%M:%S");
|
||||
writeln!(f, "{}\t{}", ts, key).ok();
|
||||
}
|
||||
}
|
||||
|
||||
fn surface_agent_cycle(session: &Session, out: &mut String, log_f: &mut File) {
|
||||
let result_path = session.state_dir.join(format!("surface-result-{}", session.session_id));
|
||||
let pid_path = session.state_dir.join(format!("surface-pid-{}", session.session_id));
|
||||
|
||||
let surface_timeout = crate::config::get()
|
||||
.surface_timeout_secs
|
||||
.unwrap_or(120) as u64;
|
||||
|
||||
let agent_done = match fs::read_to_string(&pid_path) {
|
||||
Ok(content) => {
|
||||
let parts: Vec<&str> = content.split('\t').collect();
|
||||
let pid: u32 = parts.first().and_then(|s| s.trim().parse().ok()).unwrap_or(0);
|
||||
let start_ts: u64 = parts.get(1).and_then(|s| s.trim().parse().ok()).unwrap_or(0);
|
||||
if pid == 0 { true }
|
||||
else {
|
||||
let alive = unsafe { libc::kill(pid as i32, 0) == 0 };
|
||||
if !alive { true }
|
||||
else if now_secs().saturating_sub(start_ts) > surface_timeout {
|
||||
unsafe { libc::kill(pid as i32, libc::SIGTERM); }
|
||||
true
|
||||
} else { false }
|
||||
}
|
||||
}
|
||||
Err(_) => true,
|
||||
};
|
||||
|
||||
let _ = writeln!(log_f, "agent_done {agent_done}");
|
||||
|
||||
if !agent_done { return; }
|
||||
|
||||
if let Ok(result) = fs::read_to_string(&result_path) {
|
||||
if !result.trim().is_empty() {
|
||||
let tail_lines: Vec<&str> = result.lines().rev()
|
||||
.filter(|l| !l.trim().is_empty()).take(8).collect();
|
||||
let has_new = tail_lines.iter().any(|l| l.starts_with("NEW RELEVANT MEMORIES:"));
|
||||
let has_none = tail_lines.iter().any(|l| l.starts_with("NO NEW RELEVANT MEMORIES"));
|
||||
|
||||
let _ = writeln!(log_f, "has_new {has_new} has_none {has_none}");
|
||||
|
||||
if has_new {
|
||||
let after_marker = result.rsplit_once("NEW RELEVANT MEMORIES:")
|
||||
.map(|(_, rest)| rest).unwrap_or("");
|
||||
let keys: Vec<String> = after_marker.lines()
|
||||
.map(|l| l.trim().trim_start_matches("- ").trim().to_string())
|
||||
.filter(|l| !l.is_empty() && !l.starts_with("```")).collect();
|
||||
|
||||
let _ = writeln!(log_f, "keys {:?}", keys);
|
||||
|
||||
let Ok(store) = crate::store::Store::load() else { return; };
|
||||
let mut seen = session.seen();
|
||||
let seen_path = session.path("seen");
|
||||
for key in &keys {
|
||||
if !seen.insert(key.clone()) {
|
||||
let _ = writeln!(log_f, " skip (seen): {}", key);
|
||||
continue;
|
||||
}
|
||||
if let Some(content) = crate::cli::node::render_node(&store, key) {
|
||||
if !content.trim().is_empty() {
|
||||
use std::fmt::Write as _;
|
||||
writeln!(out, "--- {} (surfaced) ---", key).ok();
|
||||
write!(out, "{}", content).ok();
|
||||
let _ = writeln!(log_f, " rendered {}: {} bytes, out now {} bytes", key, content.len(), out.len());
|
||||
if let Ok(mut f) = fs::OpenOptions::new()
|
||||
.create(true).append(true).open(&seen_path) {
|
||||
let ts = chrono::Local::now().format("%Y-%m-%dT%H:%M:%S");
|
||||
writeln!(f, "{}\t{}", ts, key).ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if !has_none {
|
||||
let log_dir = crate::store::memory_dir().join("logs");
|
||||
fs::create_dir_all(&log_dir).ok();
|
||||
let log_path = log_dir.join("surface-errors.log");
|
||||
if let Ok(mut f) = fs::OpenOptions::new().create(true).append(true).open(&log_path) {
|
||||
let ts = chrono::Local::now().format("%Y-%m-%dT%H:%M:%S");
|
||||
let last = tail_lines.first().unwrap_or(&"");
|
||||
let _ = writeln!(f, "[{}] unexpected surface output: {}", ts, last);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fs::remove_file(&result_path).ok();
|
||||
fs::remove_file(&pid_path).ok();
|
||||
|
||||
if let Ok(output_file) = fs::File::create(&result_path) {
|
||||
if let Ok(child) = Command::new("poc-memory")
|
||||
.args(["agent", "run", "surface", "--count", "1", "--local"])
|
||||
.env("POC_SESSION_ID", &session.session_id)
|
||||
.stdout(output_file)
|
||||
.stderr(std::process::Stdio::null())
|
||||
.spawn()
|
||||
{
|
||||
let pid = child.id();
|
||||
let ts = now_secs();
|
||||
if let Ok(mut f) = fs::File::create(&pid_path) {
|
||||
write!(f, "{}\t{}", pid, ts).ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_stale_files(dir: &Path, max_age: Duration) {
|
||||
let entries = match fs::read_dir(dir) {
|
||||
Ok(e) => e,
|
||||
Err(_) => return,
|
||||
};
|
||||
let cutoff = SystemTime::now() - max_age;
|
||||
for entry in entries.flatten() {
|
||||
if let Ok(meta) = entry.metadata() {
|
||||
if let Ok(modified) = meta.modified() {
|
||||
if modified < cutoff {
|
||||
fs::remove_file(entry.path()).ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn hook(session: &Session) -> String {
|
||||
let mut out = String::new();
|
||||
let is_compaction = crate::transcript::detect_new_compaction(
|
||||
&session.state_dir, &session.session_id, &session.transcript_path,
|
||||
);
|
||||
let cookie_path = session.path("cookie");
|
||||
let is_first = !cookie_path.exists();
|
||||
|
||||
let log_path = session.state_dir.join(format!("hook-log-{}", session.session_id));
|
||||
let Ok(mut log_f) = fs::OpenOptions::new().create(true).append(true).open(log_path) else { return Default::default(); };
|
||||
let ts = chrono::Local::now().format("%Y-%m-%dT%H:%M:%S");
|
||||
let _ = writeln!(log_f, "\n=== {} ({}) {} bytes ===", ts, session.hook_event, out.len());
|
||||
|
||||
let _ = writeln!(log_f, "is_first {is_first} is_compaction {is_compaction}");
|
||||
|
||||
if is_first || is_compaction {
|
||||
if is_compaction {
|
||||
fs::rename(&session.path("seen"), &session.path("seen-prev")).ok();
|
||||
} else {
|
||||
fs::remove_file(&session.path("seen")).ok();
|
||||
fs::remove_file(&session.path("seen-prev")).ok();
|
||||
}
|
||||
fs::remove_file(&session.path("returned")).ok();
|
||||
|
||||
if is_first {
|
||||
fs::write(&cookie_path, generate_cookie()).ok();
|
||||
}
|
||||
|
||||
if let Ok(output) = Command::new("poc-memory").args(["admin", "load-context"]).output() {
|
||||
if output.status.success() {
|
||||
let ctx = String::from_utf8_lossy(&output.stdout).to_string();
|
||||
if !ctx.trim().is_empty() {
|
||||
let mut ctx_seen = session.seen();
|
||||
for line in ctx.lines() {
|
||||
if line.starts_with("--- ") && line.ends_with(" ---") {
|
||||
let inner = &line[4..line.len() - 4];
|
||||
if let Some(paren) = inner.rfind(" (") {
|
||||
let key = inner[..paren].trim();
|
||||
mark_seen(&session.state_dir, &session.session_id, key, &mut ctx_seen);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let chunks = chunk_context(&ctx, CHUNK_SIZE);
|
||||
|
||||
if let Some(first) = chunks.first() {
|
||||
out.push_str(first);
|
||||
}
|
||||
save_pending_chunks(&session.state_dir, &session.session_id, &chunks[1..]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(chunk) = pop_pending_chunk(&session.state_dir, &session.session_id) {
|
||||
out.push_str(&chunk);
|
||||
} else {
|
||||
let cfg = crate::config::get();
|
||||
if cfg.surface_hooks.iter().any(|h| h == &session.hook_event) {
|
||||
surface_agent_cycle(session, &mut out, &mut log_f);
|
||||
}
|
||||
}
|
||||
|
||||
cleanup_stale_files(&session.state_dir, Duration::from_secs(86400));
|
||||
|
||||
let _ = write!(log_f, "{}", out);
|
||||
out
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue