consciousness/src/hippocampus/memory_search.rs

// memory-search: context loading + ambient memory retrieval
//
// Core hook logic lives here as a library module so poc-hook can call
// it directly (no subprocess). The memory-search binary is a thin CLI
// wrapper with --hook for debugging and show_seen for inspection.

use std::collections::HashSet;
use std::fs;
use std::fs::File;
use std::io::Write;
use std::path::Path;
use std::process::Command;
use std::time::{Duration, SystemTime, UNIX_EPOCH};

fn now_secs() -> u64 {
    SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs()
}

/// Max bytes per context chunk (hook output limit is ~10K chars)
const CHUNK_SIZE: usize = 9000;

pub use crate::session::Session;

/// Run the hook logic on parsed JSON input. Returns output to inject.
pub fn run_hook(input: &str) -> String {
    // Daemon agent calls set POC_AGENT=1 — skip memory search.
    if std::env::var("POC_AGENT").is_ok() { return String::new(); }

    let Some(session) = Session::from_json(input) else { return String::new() };
    hook(&session)
}

/// Split context output into chunks of approximately `max_bytes`, breaking
/// at section boundaries ("--- KEY (group) ---" lines).
fn chunk_context(ctx: &str, max_bytes: usize) -> Vec<String> {
    let mut sections: Vec<String> = Vec::new();
    let mut current = String::new();

    for line in ctx.lines() {
        if line.starts_with("--- ") && line.ends_with(" ---") && !current.is_empty() {
            sections.push(std::mem::take(&mut current));
        }
        if !current.is_empty() {
            current.push('\n');
        }
        current.push_str(line);
    }
    if !current.is_empty() {
        sections.push(current);
    }

    let mut chunks: Vec<String> = Vec::new();
    let mut chunk = String::new();
    for section in sections {
        if !chunk.is_empty() && chunk.len() + section.len() + 1 > max_bytes {
            chunks.push(std::mem::take(&mut chunk));
        }
        if !chunk.is_empty() {
            chunk.push('\n');
        }
        chunk.push_str(&section);
    }
    if !chunk.is_empty() {
        chunks.push(chunk);
    }
    chunks
}

fn save_pending_chunks(dir: &Path, session_id: &str, chunks: &[String]) {
    let chunks_dir = dir.join(format!("chunks-{}", session_id));
    let _ = fs::remove_dir_all(&chunks_dir);
    if chunks.is_empty() { return; }
    fs::create_dir_all(&chunks_dir).ok();
    for (i, chunk) in chunks.iter().enumerate() {
        let path = chunks_dir.join(format!("{:04}", i));
        fs::write(path, chunk).ok();
    }
}

fn pop_pending_chunk(dir: &Path, session_id: &str) -> Option<String> {
    let chunks_dir = dir.join(format!("chunks-{}", session_id));
    if !chunks_dir.exists() { return None; }

    let mut entries: Vec<_> = fs::read_dir(&chunks_dir).ok()?
        .flatten()
        .filter(|e| e.file_type().map(|t| t.is_file()).unwrap_or(false))
        .collect();
    entries.sort_by_key(|e| e.file_name());

    let first = entries.first()?;
    let content = fs::read_to_string(first.path()).ok()?;
    fs::remove_file(first.path()).ok();

    if fs::read_dir(&chunks_dir).ok().map(|mut d| d.next().is_none()).unwrap_or(true) {
        fs::remove_dir(&chunks_dir).ok();
    }

    Some(content)
}

fn generate_cookie() -> String {
    uuid::Uuid::new_v4().as_simple().to_string()[..12].to_string()
}

fn parse_seen_line(line: &str) -> &str {
    line.split_once('\t').map(|(_, key)| key).unwrap_or(line)
}

pub fn load_seen(dir: &Path, session_id: &str) -> HashSet<String> {
    let path = dir.join(format!("seen-{}", session_id));
    if path.exists() {
        fs::read_to_string(&path)
            .unwrap_or_default()
            .lines()
            .filter(|s| !s.is_empty())
            .map(|s| parse_seen_line(s).to_string())
            .collect()
    } else {
        HashSet::new()
    }
}

fn mark_seen(dir: &Path, session_id: &str, key: &str, seen: &mut HashSet<String>) {
    if !seen.insert(key.to_string()) { return; }
    let path = dir.join(format!("seen-{}", session_id));
    if let Ok(mut f) = fs::OpenOptions::new().create(true).append(true).open(path) {
        let ts = chrono::Local::now().format("%Y-%m-%dT%H:%M:%S");
        writeln!(f, "{}\t{}", ts, key).ok();
    }
}

/// Unified agent cycle — runs surface-observe agent with state dir.
/// Reads output files for surface results, spawns new agent when ready.
///
/// Pipelining: if a running agent is past the surface phase, start
/// a new one so surface stays fresh.
fn surface_observe_cycle(session: &Session, out: &mut String, log_f: &mut File) {
    let state_dir = crate::store::memory_dir()
        .join("agent-output")
        .join("surface-observe");
    fs::create_dir_all(&state_dir).ok();

    let timeout = crate::config::get()
        .surface_timeout_secs
        .unwrap_or(300) as u64;

    // Scan pid files — find live agents and their phases
    let mut any_in_surface = false;
    let mut any_alive = false;
    if let Ok(entries) = fs::read_dir(&state_dir) {
        for entry in entries.flatten() {
            let name = entry.file_name();
            let name_str = name.to_string_lossy();
            if !name_str.starts_with("pid-") { continue; }
            let pid: u32 = name_str.strip_prefix("pid-")
                .and_then(|s| s.parse().ok())
                .unwrap_or(0);
            if pid == 0 { continue; }

            let alive = unsafe { libc::kill(pid as i32, 0) == 0 };
            if !alive {
                let _ = writeln!(log_f, "cleanup stale pid-{}", pid);
                fs::remove_file(entry.path()).ok();
                continue;
            }

            // Check for timeout
            let phase_json = fs::read_to_string(entry.path()).unwrap_or_default();
            let started: u64 = phase_json.split("\"started\":")
                .nth(1)
                .and_then(|s| s.trim_start().split(|c: char| !c.is_ascii_digit()).next())
                .and_then(|s| s.parse().ok())
                .unwrap_or(0);
            if started > 0 && now_secs().saturating_sub(started) > timeout {
                let _ = writeln!(log_f, "killing timed-out pid-{} ({}s)", pid, timeout);
                unsafe { libc::kill(pid as i32, libc::SIGTERM); }
                fs::remove_file(entry.path()).ok();
                continue;
            }

            any_alive = true;

            let in_surface = phase_json.contains("\"phase\":\"surface\"")
                || phase_json.contains("\"phase\":\"step-0\"");
            if in_surface {
                any_in_surface = true;
            }
            let _ = writeln!(log_f, "alive pid-{}: {}", pid, phase_json.trim());
        }
    }

    // Read surface output and inject into context
    let surface_path = state_dir.join("surface");
    if let Ok(content) = fs::read_to_string(&surface_path) {
        let Ok(store) = crate::store::Store::load() else { return; };
        let mut seen = session.seen();
        let seen_path = session.path("seen");
        for key in content.lines().map(|l| l.trim()).filter(|l| !l.is_empty()) {
            if !seen.insert(key.to_string()) {
                let _ = writeln!(log_f, "  skip (seen): {}", key);
                continue;
            }
            if let Some(rendered) = crate::cli::node::render_node(&store, key) {
                if !rendered.trim().is_empty() {
                    use std::fmt::Write as _;
                    writeln!(out, "--- {} (surfaced) ---", key).ok();
                    write!(out, "{}", rendered).ok();
                    let _ = writeln!(log_f, "  rendered {}: {} bytes", key, rendered.len());
                    if let Ok(mut f) = fs::OpenOptions::new()
                        .create(true).append(true).open(&seen_path) {
                        let ts = chrono::Local::now().format("%Y-%m-%dT%H:%M:%S");
                        writeln!(f, "{}\t{}", ts, key).ok();
                    }
                }
            }
        }
        // Clear surface output after consuming
        fs::remove_file(&surface_path).ok();
    }

    // Start a new agent if:
    // - nothing running, OR
    // - something running but past surface phase (pipelining)
    if any_in_surface {
        let _ = writeln!(log_f, "agent in surface phase, waiting");
        return;
    }

    if any_alive {
        let _ = writeln!(log_f, "agent past surface, starting new (pipeline)");
    }

    let log_dir = crate::store::memory_dir().join("logs");
    fs::create_dir_all(&log_dir).ok();
    let agent_log = fs::File::create(log_dir.join("surface-observe.log"))
        .unwrap_or_else(|_| fs::File::create("/dev/null").unwrap());

    if let Ok(child) = Command::new("poc-memory")
        .args(["agent", "run", "surface-observe", "--count", "1", "--local",
               "--state-dir", &state_dir.to_string_lossy()])
        .env("POC_SESSION_ID", &session.session_id)
        .stdout(agent_log.try_clone().unwrap_or_else(|_| fs::File::create("/dev/null").unwrap()))
        .stderr(agent_log)
        .spawn()
    {
        let _ = writeln!(log_f, "spawned pid {}", child.id());
    }
}

fn cleanup_stale_files(dir: &Path, max_age: Duration) {
    let entries = match fs::read_dir(dir) {
        Ok(e) => e,
        Err(_) => return,
    };
    let cutoff = SystemTime::now() - max_age;
    for entry in entries.flatten() {
        if let Ok(meta) = entry.metadata() {
            if let Ok(modified) = meta.modified() {
                if modified < cutoff {
                    fs::remove_file(entry.path()).ok();
                }
            }
        }
    }
}

fn hook(session: &Session) -> String {
    let mut out = String::new();
    let is_compaction = crate::transcript::detect_new_compaction(
        &session.state_dir, &session.session_id, &session.transcript_path,
    );
    let cookie_path = session.path("cookie");
    let is_first = !cookie_path.exists();

    let log_path = session.state_dir.join(format!("hook-log-{}", session.session_id));
    let Ok(mut log_f) = fs::OpenOptions::new().create(true).append(true).open(log_path) else { return Default::default(); };
    let ts = chrono::Local::now().format("%Y-%m-%dT%H:%M:%S");
    let _ = writeln!(log_f, "\n=== {} ({}) {} bytes ===", ts, session.hook_event, out.len());

    let _ = writeln!(log_f, "is_first {is_first} is_compaction {is_compaction}");

    if is_first || is_compaction {
        if is_compaction {
            fs::rename(&session.path("seen"), &session.path("seen-prev")).ok();
        } else {
            fs::remove_file(&session.path("seen")).ok();
            fs::remove_file(&session.path("seen-prev")).ok();
        }
        fs::remove_file(&session.path("returned")).ok();

        if is_first {
            fs::write(&cookie_path, generate_cookie()).ok();
        }

        if let Ok(output) = Command::new("poc-memory").args(["admin", "load-context"]).output() {
            if output.status.success() {
                let ctx = String::from_utf8_lossy(&output.stdout).to_string();
                if !ctx.trim().is_empty() {
                    let mut ctx_seen = session.seen();
                    for line in ctx.lines() {
                        if line.starts_with("--- ") && line.ends_with(" ---") {
                            let inner = &line[4..line.len() - 4];
                            if let Some(paren) = inner.rfind(" (") {
                                let key = inner[..paren].trim();
                                mark_seen(&session.state_dir, &session.session_id, key, &mut ctx_seen);
                            }
                        }
                    }

                    let chunks = chunk_context(&ctx, CHUNK_SIZE);

                    if let Some(first) = chunks.first() {
                        out.push_str(first);
                    }
                    save_pending_chunks(&session.state_dir, &session.session_id, &chunks[1..]);
                }
            }
        }
    }

    if let Some(chunk) = pop_pending_chunk(&session.state_dir, &session.session_id) {
        out.push_str(&chunk);
    } else {
        let cfg = crate::config::get();
        if cfg.surface_hooks.iter().any(|h| h == &session.hook_event) {
            surface_observe_cycle(session, &mut out, &mut log_f);
        }
    }

    cleanup_stale_files(&session.state_dir, Duration::from_secs(86400));

    let _ = write!(log_f, "{}", out);
    out
}