// Neuroscience-inspired memory algorithms // // Systematic replay (hippocampal replay), schema assimilation, // interference detection, emotional gating, consolidation priority // scoring, and the agent consolidation harness. use crate::capnp_store::Store; use crate::graph::{self, Graph}; use crate::similarity; use std::time::{SystemTime, UNIX_EPOCH}; fn now_epoch() -> f64 { SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() .as_secs_f64() } const SECS_PER_DAY: f64 = 86400.0; /// Consolidation priority: how urgently a node needs attention /// /// priority = (1 - schema_fit) × spaced_repetition_due × emotion × (1 + interference) pub fn consolidation_priority(store: &Store, key: &str, graph: &Graph) -> f64 { let node = match store.nodes.get(key) { Some(n) => n, None => return 0.0, }; // Schema fit: 0 = poorly integrated, 1 = well integrated let fit = graph::schema_fit(graph, key) as f64; let fit_factor = 1.0 - fit; // Spaced repetition: how overdue is this node for replay? let interval_secs = node.spaced_repetition_interval as f64 * SECS_PER_DAY; let time_since_replay = if node.last_replayed > 0.0 { (now_epoch() - node.last_replayed).max(0.0) } else { // Never replayed — treat as very overdue interval_secs * 3.0 }; let overdue_ratio = (time_since_replay / interval_secs).min(5.0); // Emotional intensity: higher emotion = higher priority let emotion_factor = 1.0 + (node.emotion as f64 / 10.0); fit_factor * overdue_ratio * emotion_factor } /// Item in the replay queue pub struct ReplayItem { pub key: String, pub priority: f64, pub interval_days: u32, pub emotion: f32, pub schema_fit: f32, } /// Generate the replay queue: nodes ordered by consolidation priority pub fn replay_queue(store: &Store, count: usize) -> Vec { let graph = store.build_graph(); let fits = graph::schema_fit_all(&graph); let mut items: Vec = store.nodes.iter() .map(|(key, node)| { let priority = consolidation_priority(store, key, &graph); let fit = fits.get(key).copied().unwrap_or(0.0); ReplayItem { key: key.clone(), priority, interval_days: node.spaced_repetition_interval, emotion: node.emotion, schema_fit: fit, } }) .collect(); items.sort_by(|a, b| b.priority.partial_cmp(&a.priority).unwrap()); items.truncate(count); items } /// Detect interfering memory pairs: high text similarity but different communities pub fn detect_interference( store: &Store, graph: &Graph, threshold: f32, ) -> Vec<(String, String, f32)> { let communities = graph.communities(); // Only compare nodes within a reasonable set — take the most active ones let mut docs: Vec<(String, String)> = store.nodes.iter() .filter(|(_, n)| n.content.len() > 50) // skip tiny nodes .map(|(k, n)| (k.clone(), n.content.clone())) .collect(); // For large stores, sample to keep pairwise comparison feasible if docs.len() > 200 { docs.sort_by(|a, b| b.1.len().cmp(&a.1.len())); docs.truncate(200); } let similar = similarity::pairwise_similar(&docs, threshold); // Filter to pairs in different communities similar.into_iter() .filter(|(a, b, _)| { let ca = communities.get(a); let cb = communities.get(b); match (ca, cb) { (Some(a), Some(b)) => a != b, _ => true, // if community unknown, flag it } }) .collect() } /// Schema assimilation scoring for a new node. /// Returns how easily the node integrates into existing structure. /// /// High fit (>0.5): auto-link, done /// Medium fit (0.2-0.5): agent reviews, proposes links /// Low fit (<0.2): deep examination needed — new schema seed, bridge, or noise? pub fn schema_assimilation(store: &Store, key: &str) -> (f32, &'static str) { let graph = store.build_graph(); let fit = graph::schema_fit(&graph, key); let recommendation = if fit > 0.5 { "auto-integrate" } else if fit > 0.2 { "agent-review" } else if graph.degree(key) > 0 { "deep-examine-bridge" } else { "deep-examine-orphan" }; (fit, recommendation) } /// Prompt template directory fn prompts_dir() -> std::path::PathBuf { // Check for prompts relative to binary, then fall back to ~/poc/memory/prompts/ let home = std::env::var("HOME").unwrap_or_default(); std::path::PathBuf::from(home).join("poc/memory/prompts") } /// Load a prompt template, replacing {{PLACEHOLDER}} with data fn load_prompt(name: &str, replacements: &[(&str, &str)]) -> Result { let path = prompts_dir().join(format!("{}.md", name)); let mut content = std::fs::read_to_string(&path) .map_err(|e| format!("load prompt {}: {}", path.display(), e))?; for (placeholder, data) in replacements { content = content.replace(placeholder, data); } Ok(content) } /// Format topology header for agent prompts — current graph health metrics fn format_topology_header(graph: &Graph) -> String { let sigma = graph.small_world_sigma(); let alpha = graph.degree_power_law_exponent(); let gini = graph.degree_gini(); let avg_cc = graph.avg_clustering_coefficient(); let n = graph.nodes().len(); let e = graph.edge_count(); format!( "## Current graph topology\n\ Nodes: {} Edges: {} Communities: {}\n\ Small-world σ: {:.1} Power-law α: {:.2} Degree Gini: {:.3}\n\ Avg clustering coefficient: {:.4}\n\n\ Each node below shows its hub-link ratio (fraction of edges to top-5% degree nodes).\n\ Use `poc-memory link-impact SOURCE TARGET` to evaluate proposed links.\n\n", n, e, graph.community_count(), sigma, alpha, gini, avg_cc) } /// Compute the hub degree threshold (top 5% by degree) fn hub_threshold(graph: &Graph) -> usize { let mut degrees: Vec = graph.nodes().iter() .map(|k| graph.degree(k)) .collect(); degrees.sort_unstable(); if degrees.len() >= 20 { degrees[degrees.len() * 95 / 100] } else { usize::MAX } } /// Format node data section for prompt templates fn format_nodes_section(store: &Store, items: &[ReplayItem], graph: &Graph) -> String { let hub_thresh = hub_threshold(graph); let mut out = String::new(); for item in items { let node = match store.nodes.get(&item.key) { Some(n) => n, None => continue, }; out.push_str(&format!("## {} \n", item.key)); out.push_str(&format!("Priority: {:.3} Schema fit: {:.3} Emotion: {:.1} ", item.priority, item.schema_fit, item.emotion)); out.push_str(&format!("Category: {} Interval: {}d\n", node.category.label(), node.spaced_repetition_interval)); if let Some(community) = node.community_id { out.push_str(&format!("Community: {} ", community)); } let deg = graph.degree(&item.key); let cc = graph.clustering_coefficient(&item.key); // Hub-link ratio: what fraction of this node's edges go to hubs? let neighbors = graph.neighbors(&item.key); let hub_links = neighbors.iter() .filter(|(n, _)| graph.degree(n) >= hub_thresh) .count(); let hub_ratio = if deg > 0 { hub_links as f32 / deg as f32 } else { 0.0 }; let is_hub = deg >= hub_thresh; out.push_str(&format!("Degree: {} CC: {:.3} Hub-link ratio: {:.0}% ({}/{})", deg, cc, hub_ratio * 100.0, hub_links, deg)); if is_hub { out.push_str(" ← THIS IS A HUB"); } else if hub_ratio > 0.6 { out.push_str(" ← mostly hub-connected, needs lateral links"); } out.push('\n'); // Content (truncated for large nodes) let content = &node.content; if content.len() > 1500 { let end = content.floor_char_boundary(1500); out.push_str(&format!("\nContent ({} chars, truncated):\n{}\n[...]\n\n", content.len(), &content[..end])); } else { out.push_str(&format!("\nContent:\n{}\n\n", content)); } // Neighbors let neighbors = graph.neighbors(&item.key); if !neighbors.is_empty() { out.push_str("Neighbors:\n"); for (n, strength) in neighbors.iter().take(15) { let n_cc = graph.clustering_coefficient(n); let n_community = store.nodes.get(n.as_str()) .and_then(|n| n.community_id); out.push_str(&format!(" - {} (str={:.2}, cc={:.3}", n, strength, n_cc)); if let Some(c) = n_community { out.push_str(&format!(", c{}", c)); } out.push_str(")\n"); } } out.push_str("\n---\n\n"); } out } /// Format health data for the health agent prompt fn format_health_section(store: &Store, graph: &Graph) -> String { let health = graph::health_report(graph, store); let mut out = health; out.push_str("\n\n## Weight distribution\n"); // Weight histogram let mut buckets = [0u32; 10]; // 0.0-0.1, 0.1-0.2, ..., 0.9-1.0 for node in store.nodes.values() { let bucket = ((node.weight * 10.0) as usize).min(9); buckets[bucket] += 1; } for (i, &count) in buckets.iter().enumerate() { let lo = i as f32 / 10.0; let hi = (i + 1) as f32 / 10.0; let bar = "█".repeat((count as usize) / 10); out.push_str(&format!(" {:.1}-{:.1}: {:4} {}\n", lo, hi, count, bar)); } // Near-prune nodes let near_prune: Vec<_> = store.nodes.iter() .filter(|(_, n)| n.weight < 0.15) .map(|(k, n)| (k.clone(), n.weight)) .collect(); if !near_prune.is_empty() { out.push_str(&format!("\n## Near-prune nodes ({} total)\n", near_prune.len())); for (k, w) in near_prune.iter().take(20) { out.push_str(&format!(" [{:.3}] {}\n", w, k)); } } // Community sizes let communities = graph.communities(); let mut comm_sizes: std::collections::HashMap> = std::collections::HashMap::new(); for (key, &label) in communities { comm_sizes.entry(label).or_default().push(key.clone()); } let mut sizes: Vec<_> = comm_sizes.iter() .map(|(id, members)| (*id, members.len(), members.clone())) .collect(); sizes.sort_by(|a, b| b.1.cmp(&a.1)); out.push_str("\n## Largest communities\n"); for (id, size, members) in sizes.iter().take(10) { out.push_str(&format!(" Community {} ({} nodes): ", id, size)); let sample: Vec<_> = members.iter().take(5).map(|s| s.as_str()).collect(); out.push_str(&sample.join(", ")); if *size > 5 { out.push_str(", ..."); } out.push('\n'); } out } /// Format interference pairs for the separator agent prompt fn format_pairs_section( pairs: &[(String, String, f32)], store: &Store, graph: &Graph, ) -> String { let mut out = String::new(); let communities = graph.communities(); for (a, b, sim) in pairs { out.push_str(&format!("## Pair: similarity={:.3}\n", sim)); let ca = communities.get(a).map(|c| format!("c{}", c)).unwrap_or_else(|| "?".into()); let cb = communities.get(b).map(|c| format!("c{}", c)).unwrap_or_else(|| "?".into()); // Node A out.push_str(&format!("\n### {} ({})\n", a, ca)); if let Some(node) = store.nodes.get(a) { let content = if node.content.len() > 500 { let end = node.content.floor_char_boundary(500); format!("{}...", &node.content[..end]) } else { node.content.clone() }; out.push_str(&format!("Category: {} Weight: {:.2}\n{}\n", node.category.label(), node.weight, content)); } // Node B out.push_str(&format!("\n### {} ({})\n", b, cb)); if let Some(node) = store.nodes.get(b) { let content = if node.content.len() > 500 { let end = node.content.floor_char_boundary(500); format!("{}...", &node.content[..end]) } else { node.content.clone() }; out.push_str(&format!("Category: {} Weight: {:.2}\n{}\n", node.category.label(), node.weight, content)); } out.push_str("\n---\n\n"); } out } /// Run agent consolidation on top-priority nodes pub fn consolidation_batch(store: &Store, count: usize, auto: bool) -> Result<(), String> { let graph = store.build_graph(); let items = replay_queue(store, count); if items.is_empty() { println!("No nodes to consolidate."); return Ok(()); } let nodes_section = format_nodes_section(store, &items, &graph); if auto { // Generate the replay agent prompt with data filled in let prompt = load_prompt("replay", &[("{{NODES}}", &nodes_section)])?; println!("{}", prompt); } else { // Interactive: show what needs attention and available agent types println!("Consolidation batch ({} nodes):\n", items.len()); for item in &items { let node_type = store.nodes.get(&item.key) .map(|n| if n.key.contains("journal") { "episodic" } else { "semantic" }) .unwrap_or("?"); println!(" [{:.3}] {} (fit={:.3}, interval={}d, type={})", item.priority, item.key, item.schema_fit, item.interval_days, node_type); } // Also show interference pairs let pairs = detect_interference(store, &graph, 0.6); if !pairs.is_empty() { println!("\nInterfering pairs ({}):", pairs.len()); for (a, b, sim) in pairs.iter().take(5) { println!(" [{:.3}] {} ↔ {}", sim, a, b); } } println!("\nAgent prompts:"); println!(" --auto Generate replay agent prompt"); println!(" --agent replay Replay agent (schema assimilation)"); println!(" --agent linker Linker agent (relational binding)"); println!(" --agent separator Separator agent (pattern separation)"); println!(" --agent transfer Transfer agent (CLS episodic→semantic)"); println!(" --agent health Health agent (synaptic homeostasis)"); } Ok(()) } /// Generate a specific agent prompt with filled-in data pub fn agent_prompt(store: &Store, agent: &str, count: usize) -> Result { let graph = store.build_graph(); let topology = format_topology_header(&graph); match agent { "replay" => { let items = replay_queue(store, count); let nodes_section = format_nodes_section(store, &items, &graph); load_prompt("replay", &[("{{TOPOLOGY}}", &topology), ("{{NODES}}", &nodes_section)]) } "linker" => { // Filter to episodic entries let mut items = replay_queue(store, count * 2); items.retain(|item| { store.nodes.get(&item.key) .map(|n| matches!(n.node_type, crate::capnp_store::NodeType::EpisodicSession)) .unwrap_or(false) || item.key.contains("journal") || item.key.contains("session") }); items.truncate(count); let nodes_section = format_nodes_section(store, &items, &graph); load_prompt("linker", &[("{{TOPOLOGY}}", &topology), ("{{NODES}}", &nodes_section)]) } "separator" => { let pairs = detect_interference(store, &graph, 0.5); let pairs_section = format_pairs_section(&pairs, store, &graph); load_prompt("separator", &[("{{TOPOLOGY}}", &topology), ("{{PAIRS}}", &pairs_section)]) } "transfer" => { // Recent episodic entries let mut episodes: Vec<_> = store.nodes.iter() .filter(|(k, _)| k.contains("journal") || k.contains("session")) .map(|(k, n)| (k.clone(), n.timestamp)) .collect(); episodes.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap()); episodes.truncate(count); let episode_keys: Vec<_> = episodes.iter().map(|(k, _)| k.clone()).collect(); let items: Vec = episode_keys.iter() .filter_map(|k| { let node = store.nodes.get(k)?; let fit = graph::schema_fit(&graph, k); Some(ReplayItem { key: k.clone(), priority: consolidation_priority(store, k, &graph), interval_days: node.spaced_repetition_interval, emotion: node.emotion, schema_fit: fit, }) }) .collect(); let episodes_section = format_nodes_section(store, &items, &graph); load_prompt("transfer", &[("{{TOPOLOGY}}", &topology), ("{{EPISODES}}", &episodes_section)]) } "health" => { let health_section = format_health_section(store, &graph); load_prompt("health", &[("{{TOPOLOGY}}", &topology), ("{{HEALTH}}", &health_section)]) } _ => Err(format!("Unknown agent: {}. Use: replay, linker, separator, transfer, health", agent)), } } /// Agent allocation from the control loop pub struct ConsolidationPlan { pub replay_count: usize, pub linker_count: usize, pub separator_count: usize, pub transfer_count: usize, pub run_health: bool, pub rationale: Vec, } /// Analyze metrics and decide how much each agent needs to run. /// /// This is the control loop: metrics → error signal → agent allocation. /// Target values are based on healthy small-world networks. pub fn consolidation_plan(store: &Store) -> ConsolidationPlan { let graph = store.build_graph(); let alpha = graph.degree_power_law_exponent(); let gini = graph.degree_gini(); let avg_fit = { let fits = graph::schema_fit_all(&graph); if fits.is_empty() { 0.0 } else { fits.values().sum::() / fits.len() as f32 } }; let interference_pairs = detect_interference(store, &graph, 0.5); let interference_count = interference_pairs.len(); // Count episodic vs semantic nodes let episodic_count = store.nodes.iter() .filter(|(k, _)| k.contains("journal") || k.contains("session")) .count(); let _semantic_count = store.nodes.len() - episodic_count; let episodic_ratio = if store.nodes.is_empty() { 0.0 } else { episodic_count as f32 / store.nodes.len() as f32 }; let mut plan = ConsolidationPlan { replay_count: 0, linker_count: 0, separator_count: 0, transfer_count: 0, run_health: true, // always run health first rationale: Vec::new(), }; // Target: α ≥ 2.5 (healthy scale-free) // Current distance determines replay + linker allocation if alpha < 2.0 { plan.replay_count += 10; plan.linker_count += 5; plan.rationale.push(format!( "α={:.2} (target ≥2.5): extreme hub dominance → 10 replay + 5 linker for lateral links", alpha)); } else if alpha < 2.5 { plan.replay_count += 5; plan.linker_count += 3; plan.rationale.push(format!( "α={:.2} (target ≥2.5): moderate hub dominance → 5 replay + 3 linker", alpha)); } else { plan.replay_count += 3; plan.rationale.push(format!( "α={:.2}: healthy — 3 replay for maintenance", alpha)); } // Target: Gini ≤ 0.4 if gini > 0.5 { plan.replay_count += 3; plan.rationale.push(format!( "Gini={:.3} (target ≤0.4): high inequality → +3 replay (lateral focus)", gini)); } // Target: avg schema fit ≥ 0.2 if avg_fit < 0.1 { plan.replay_count += 5; plan.rationale.push(format!( "Schema fit={:.3} (target ≥0.2): very poor integration → +5 replay", avg_fit)); } else if avg_fit < 0.2 { plan.replay_count += 2; plan.rationale.push(format!( "Schema fit={:.3} (target ≥0.2): low integration → +2 replay", avg_fit)); } // Interference: >100 pairs is a lot, <10 is clean if interference_count > 100 { plan.separator_count += 10; plan.rationale.push(format!( "Interference: {} pairs (target <50) → 10 separator", interference_count)); } else if interference_count > 20 { plan.separator_count += 5; plan.rationale.push(format!( "Interference: {} pairs (target <50) → 5 separator", interference_count)); } else if interference_count > 0 { plan.separator_count += interference_count.min(3); plan.rationale.push(format!( "Interference: {} pairs → {} separator", interference_count, plan.separator_count)); } // Episodic → semantic transfer // If >60% of nodes are episodic, knowledge isn't being extracted if episodic_ratio > 0.6 { plan.transfer_count += 10; plan.rationale.push(format!( "Episodic ratio: {:.0}% ({}/{}) → 10 transfer (knowledge extraction needed)", episodic_ratio * 100.0, episodic_count, store.nodes.len())); } else if episodic_ratio > 0.4 { plan.transfer_count += 5; plan.rationale.push(format!( "Episodic ratio: {:.0}% → 5 transfer", episodic_ratio * 100.0)); } plan } /// Format the consolidation plan for display pub fn format_plan(plan: &ConsolidationPlan) -> String { let mut out = String::from("Consolidation Plan\n==================\n\n"); out.push_str("Analysis:\n"); for r in &plan.rationale { out.push_str(&format!(" • {}\n", r)); } out.push_str("\nAgent allocation:\n"); if plan.run_health { out.push_str(" 1. health — system audit\n"); } let mut step = 2; if plan.replay_count > 0 { out.push_str(&format!(" {}. replay ×{:2} — schema assimilation + lateral linking\n", step, plan.replay_count)); step += 1; } if plan.linker_count > 0 { out.push_str(&format!(" {}. linker ×{:2} — relational binding from episodes\n", step, plan.linker_count)); step += 1; } if plan.separator_count > 0 { out.push_str(&format!(" {}. separator ×{} — pattern separation\n", step, plan.separator_count)); step += 1; } if plan.transfer_count > 0 { out.push_str(&format!(" {}. transfer ×{:2} — episodic→semantic extraction\n", step, plan.transfer_count)); } let total = plan.replay_count + plan.linker_count + plan.separator_count + plan.transfer_count + if plan.run_health { 1 } else { 0 }; out.push_str(&format!("\nTotal agent runs: {}\n", total)); out } /// Brief daily check: compare current metrics to last snapshot pub fn daily_check(store: &Store) -> String { let graph = store.build_graph(); let alpha = graph.degree_power_law_exponent(); let gini = graph.degree_gini(); let sigma = graph.small_world_sigma(); let avg_cc = graph.avg_clustering_coefficient(); let avg_fit = { let fits = graph::schema_fit_all(&graph); if fits.is_empty() { 0.0 } else { fits.values().sum::() / fits.len() as f32 } }; let history = graph::load_metrics_history(); let prev = history.last(); let mut out = String::from("Memory daily check\n"); // Current state out.push_str(&format!(" σ={:.1} α={:.2} gini={:.3} cc={:.4} fit={:.3}\n", sigma, alpha, gini, avg_cc, avg_fit)); // Trend if let Some(p) = prev { let d_sigma = sigma - p.sigma; let d_alpha = alpha - p.alpha; let d_gini = gini - p.gini; out.push_str(&format!(" Δσ={:+.1} Δα={:+.2} Δgini={:+.3}\n", d_sigma, d_alpha, d_gini)); // Assessment let mut issues = Vec::new(); if alpha < 2.0 { issues.push("hub dominance critical"); } if gini > 0.5 { issues.push("high inequality"); } if avg_fit < 0.1 { issues.push("poor integration"); } if d_sigma < -5.0 { issues.push("σ declining"); } if d_alpha < -0.1 { issues.push("α declining"); } if d_gini > 0.02 { issues.push("inequality increasing"); } if issues.is_empty() { out.push_str(" Status: healthy\n"); } else { out.push_str(&format!(" Status: needs attention — {}\n", issues.join(", "))); out.push_str(" Run: poc-memory consolidate-session\n"); } } else { out.push_str(" (first snapshot, no trend data yet)\n"); } // Log this snapshot too let now = crate::capnp_store::now_epoch(); let date = crate::capnp_store::format_datetime_space(now); graph::save_metrics_snapshot(&graph::MetricsSnapshot { timestamp: now, date, nodes: graph.nodes().len(), edges: graph.edge_count(), communities: graph.community_count(), sigma, alpha, gini, avg_cc, avg_path_length: graph.avg_path_length(), avg_schema_fit: avg_fit, }); out }