spectral decomposition, search improvements, char boundary fix
- New spectral module: Laplacian eigendecomposition of the memory graph. Commands: spectral, spectral-save, spectral-neighbors, spectral-positions, spectral-suggest. Spectral neighbors expand search results beyond keyword matching to structural proximity. - Search: use StoreView trait to avoid 6MB state.bin rewrite on every query. Append-only retrieval logging. Spectral expansion shows structurally nearby nodes after text results. - Fix panic in journal-tail: string truncation at byte 67 could land inside a multi-byte character (em dash). Now walks back to char boundary. - Replay queue: show classification and spectral outlier score. - Knowledge agents: extractor, challenger, connector prompts and runner scripts for automated graph enrichment. - memory-search hook: stale state file cleanup (24h expiry).
This commit is contained in:
ProofOfConcept
parent
94dbca6018
commit
71e6f15d82
16 changed files with 3600 additions and 103 deletions
240
src/main.rs
240
src/main.rs
|
|
@ -20,6 +20,7 @@ mod search;
|
|||
mod similarity;
|
||||
mod migrate;
|
||||
mod neuro;
|
||||
mod spectral;
|
||||
|
||||
pub mod memory_capnp {
|
||||
include!(concat!(env!("OUT_DIR"), "/schema/memory_capnp.rs"));
|
||||
|
|
@ -101,6 +102,11 @@ fn main() {
|
|||
"differentiate" => cmd_differentiate(&args[2..]),
|
||||
"link-audit" => cmd_link_audit(&args[2..]),
|
||||
"trace" => cmd_trace(&args[2..]),
|
||||
"spectral" => cmd_spectral(&args[2..]),
|
||||
"spectral-save" => cmd_spectral_save(&args[2..]),
|
||||
"spectral-neighbors" => cmd_spectral_neighbors(&args[2..]),
|
||||
"spectral-positions" => cmd_spectral_positions(&args[2..]),
|
||||
"spectral-suggest" => cmd_spectral_suggest(&args[2..]),
|
||||
"list-keys" => cmd_list_keys(),
|
||||
"list-edges" => cmd_list_edges(),
|
||||
"dump-json" => cmd_dump_json(),
|
||||
|
|
@ -171,6 +177,11 @@ Commands:
|
|||
Redistribute hub links to section-level children
|
||||
link-audit [--apply] Walk every link, send to Sonnet for quality review
|
||||
trace KEY Walk temporal links: semantic ↔ episodic ↔ conversation
|
||||
spectral [K] Spectral decomposition of the memory graph (default K=30)
|
||||
spectral-save [K] Compute and save spectral embedding (default K=20)
|
||||
spectral-neighbors KEY [N] Find N spectrally nearest nodes (default N=15)
|
||||
spectral-positions [N] Show N nodes ranked by outlier/bridge score (default 30)
|
||||
spectral-suggest [N] Find N spectrally close but unlinked pairs (default 20)
|
||||
list-keys List all node keys (one per line)
|
||||
list-edges List all edges (tsv: source target strength type)
|
||||
dump-json Dump entire store as JSON
|
||||
|
|
@ -185,34 +196,76 @@ Commands:
|
|||
}
|
||||
|
||||
fn cmd_search(args: &[String]) -> Result<(), String> {
|
||||
use capnp_store::StoreView;
|
||||
|
||||
if args.is_empty() {
|
||||
return Err("Usage: poc-memory search QUERY [QUERY...]".into());
|
||||
}
|
||||
let query = args.join(" ");
|
||||
let mut store = capnp_store::Store::load()?;
|
||||
let results = search::search(&query, &store);
|
||||
|
||||
let view = capnp_store::AnyView::load()?;
|
||||
let results = search::search(&query, &view);
|
||||
|
||||
if results.is_empty() {
|
||||
eprintln!("No results for '{}'", query);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Log retrieval
|
||||
store.log_retrieval(&query, &results.iter().map(|r| r.key.clone()).collect::<Vec<_>>());
|
||||
store.save()?;
|
||||
// Log retrieval to a small append-only file (avoid 6MB state.bin rewrite)
|
||||
capnp_store::Store::log_retrieval_static(&query,
|
||||
&results.iter().map(|r| r.key.clone()).collect::<Vec<_>>());
|
||||
|
||||
// Show text results
|
||||
let text_keys: std::collections::HashSet<String> = results.iter()
|
||||
.take(15).map(|r| r.key.clone()).collect();
|
||||
|
||||
for (i, r) in results.iter().enumerate().take(15) {
|
||||
let marker = if r.is_direct { "→" } else { " " };
|
||||
let weight = store.node_weight(&r.key).unwrap_or(0.0);
|
||||
let weight = view.node_weight(&r.key);
|
||||
print!("{}{:2}. [{:.2}/{:.2}] {}", marker, i + 1, r.activation, weight, r.key);
|
||||
if let Some(community) = store.node_community(&r.key) {
|
||||
print!(" (c{})", community);
|
||||
}
|
||||
println!();
|
||||
if let Some(ref snippet) = r.snippet {
|
||||
println!(" {}", snippet);
|
||||
}
|
||||
}
|
||||
|
||||
// Spectral expansion: find neighbors of top text hits
|
||||
if let Ok(emb) = spectral::load_embedding() {
|
||||
let seeds: Vec<&str> = results.iter()
|
||||
.take(5)
|
||||
.map(|r| r.key.as_str())
|
||||
.filter(|k| emb.coords.contains_key(*k))
|
||||
.collect();
|
||||
|
||||
if !seeds.is_empty() {
|
||||
let spectral_hits = spectral::nearest_to_seeds(&emb, &seeds, 10);
|
||||
// Filter to nodes not already in text results
|
||||
let new_hits: Vec<_> = spectral_hits.into_iter()
|
||||
.filter(|(k, _)| !text_keys.contains(k))
|
||||
.take(5)
|
||||
.collect();
|
||||
|
||||
if !new_hits.is_empty() {
|
||||
println!("\nSpectral neighbors (structural, not keyword):");
|
||||
for (k, _dist) in &new_hits {
|
||||
let weight = view.node_weight(k);
|
||||
print!(" ~ [{:.2}] {}", weight, k);
|
||||
println!();
|
||||
// Show first line of content as snippet
|
||||
if let Some(content) = view.node_content(k) {
|
||||
let snippet: String = content.lines()
|
||||
.find(|l| !l.trim().is_empty() && !l.starts_with('#'))
|
||||
.unwrap_or("")
|
||||
.chars().take(100).collect();
|
||||
if !snippet.is_empty() {
|
||||
println!(" {}", snippet);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -457,8 +510,9 @@ fn cmd_replay_queue(args: &[String]) -> Result<(), String> {
|
|||
let queue = neuro::replay_queue(&store, count);
|
||||
println!("Replay queue ({} items):", queue.len());
|
||||
for (i, item) in queue.iter().enumerate() {
|
||||
println!(" {:2}. [{:.3}] {} (interval={}d, emotion={:.1})",
|
||||
i + 1, item.priority, item.key, item.interval_days, item.emotion);
|
||||
println!(" {:2}. [{:.3}] {:>10} {} (interval={}d, emotion={:.1}, spectral={:.1})",
|
||||
i + 1, item.priority, item.classification, item.key,
|
||||
item.interval_days, item.emotion, item.outlier_score);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1003,6 +1057,166 @@ fn cmd_trace(args: &[String]) -> Result<(), String> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_spectral(args: &[String]) -> Result<(), String> {
|
||||
let k: usize = args.first()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(30);
|
||||
let store = capnp_store::Store::load()?;
|
||||
let g = graph::build_graph(&store);
|
||||
let result = spectral::decompose(&g, k);
|
||||
spectral::print_summary(&result, &g);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_spectral_save(args: &[String]) -> Result<(), String> {
|
||||
let k: usize = args.first()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(20);
|
||||
let store = capnp_store::Store::load()?;
|
||||
let g = graph::build_graph(&store);
|
||||
let result = spectral::decompose(&g, k);
|
||||
let emb = spectral::to_embedding(&result);
|
||||
spectral::save_embedding(&emb)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_spectral_neighbors(args: &[String]) -> Result<(), String> {
|
||||
if args.is_empty() {
|
||||
return Err("usage: spectral-neighbors KEY [N]".to_string());
|
||||
}
|
||||
let key = &args[0];
|
||||
let n: usize = args.get(1)
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(15);
|
||||
|
||||
let emb = spectral::load_embedding()?;
|
||||
|
||||
// Show which dimensions this node loads on
|
||||
let dims = spectral::dominant_dimensions(&emb, &[key.as_str()]);
|
||||
println!("Node: {} (embedding: {} dims)", key, emb.dims);
|
||||
println!("Top spectral axes:");
|
||||
for &(d, loading) in dims.iter().take(5) {
|
||||
println!(" axis {:<2} (λ={:.4}): loading={:.5}", d, emb.eigenvalues[d], loading);
|
||||
}
|
||||
|
||||
println!("\nNearest neighbors in spectral space:");
|
||||
let neighbors = spectral::nearest_neighbors(&emb, key, n);
|
||||
for (i, (k, dist)) in neighbors.iter().enumerate() {
|
||||
println!(" {:>2}. {:.5} {}", i + 1, dist, k);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_spectral_positions(args: &[String]) -> Result<(), String> {
|
||||
let n: usize = args.first()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(30);
|
||||
|
||||
let store = capnp_store::Store::load()?;
|
||||
let emb = spectral::load_embedding()?;
|
||||
|
||||
// Build communities fresh from graph (don't rely on cached node fields)
|
||||
let g = store.build_graph();
|
||||
let communities = g.communities().clone();
|
||||
|
||||
let positions = spectral::analyze_positions(&emb, &communities);
|
||||
|
||||
// Show outliers first
|
||||
println!("Spectral position analysis — {} nodes", positions.len());
|
||||
println!(" outlier: dist_to_center / median (>1 = unusual position)");
|
||||
println!(" bridge: dist_to_center / dist_to_nearest_other_community");
|
||||
println!();
|
||||
|
||||
// Group by classification
|
||||
let mut bridges: Vec<&spectral::SpectralPosition> = Vec::new();
|
||||
let mut outliers: Vec<&spectral::SpectralPosition> = Vec::new();
|
||||
let mut core: Vec<&spectral::SpectralPosition> = Vec::new();
|
||||
|
||||
for pos in positions.iter().take(n) {
|
||||
match spectral::classify_position(pos) {
|
||||
"bridge" => bridges.push(pos),
|
||||
"outlier" => outliers.push(pos),
|
||||
"core" => core.push(pos),
|
||||
_ => outliers.push(pos), // peripheral goes with outliers for display
|
||||
}
|
||||
}
|
||||
|
||||
if !bridges.is_empty() {
|
||||
println!("=== Bridges (between communities) ===");
|
||||
for pos in &bridges {
|
||||
println!(" [{:.2}/{:.2}] c{} → c{} {}",
|
||||
pos.outlier_score, pos.bridge_score,
|
||||
pos.community, pos.nearest_community, pos.key);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
println!("=== Top outliers (far from own community center) ===");
|
||||
for pos in positions.iter().take(n) {
|
||||
let class = spectral::classify_position(pos);
|
||||
println!(" {:>10} outlier={:.2} bridge={:.2} c{:<3} {}",
|
||||
class, pos.outlier_score, pos.bridge_score,
|
||||
pos.community, pos.key);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_spectral_suggest(args: &[String]) -> Result<(), String> {
|
||||
let n: usize = args.first()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(20);
|
||||
|
||||
let store = capnp_store::Store::load()?;
|
||||
let emb = spectral::load_embedding()?;
|
||||
let g = store.build_graph();
|
||||
let communities = g.communities();
|
||||
|
||||
// Only consider nodes with enough edges for meaningful spectral position
|
||||
let min_degree = 3;
|
||||
let well_connected: std::collections::HashSet<&str> = emb.coords.keys()
|
||||
.filter(|k| g.degree(k) >= min_degree)
|
||||
.map(|k| k.as_str())
|
||||
.collect();
|
||||
|
||||
// Filter embedding to well-connected nodes
|
||||
let filtered_emb = spectral::SpectralEmbedding {
|
||||
dims: emb.dims,
|
||||
eigenvalues: emb.eigenvalues.clone(),
|
||||
coords: emb.coords.iter()
|
||||
.filter(|(k, _)| well_connected.contains(k.as_str()))
|
||||
.map(|(k, v)| (k.clone(), v.clone()))
|
||||
.collect(),
|
||||
};
|
||||
|
||||
// Build set of existing linked pairs
|
||||
let mut linked: std::collections::HashSet<(String, String)> =
|
||||
std::collections::HashSet::new();
|
||||
for rel in &store.relations {
|
||||
linked.insert((rel.source_key.clone(), rel.target_key.clone()));
|
||||
linked.insert((rel.target_key.clone(), rel.source_key.clone()));
|
||||
}
|
||||
|
||||
eprintln!("Searching {} well-connected nodes (degree >= {})...",
|
||||
filtered_emb.coords.len(), min_degree);
|
||||
let pairs = spectral::unlinked_neighbors(&filtered_emb, &linked, n);
|
||||
|
||||
println!("{} closest unlinked pairs (candidates for extractor agents):", pairs.len());
|
||||
for (i, (k1, k2, dist)) in pairs.iter().enumerate() {
|
||||
let c1 = communities.get(k1)
|
||||
.map(|c| format!("c{}", c))
|
||||
.unwrap_or_else(|| "?".into());
|
||||
let c2 = communities.get(k2)
|
||||
.map(|c| format!("c{}", c))
|
||||
.unwrap_or_else(|| "?".into());
|
||||
let cross = if c1 != c2 { " [cross-community]" } else { "" };
|
||||
println!(" {:>2}. dist={:.4} {} ({}) ↔ {} ({}){}",
|
||||
i + 1, dist, k1, c1, k2, c2, cross);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_list_keys() -> Result<(), String> {
|
||||
let store = capnp_store::Store::load()?;
|
||||
let mut keys: Vec<_> = store.nodes.keys().collect();
|
||||
|
|
@ -1353,7 +1567,9 @@ fn cmd_journal_tail(args: &[String]) -> Result<(), String> {
|
|||
} else {
|
||||
// Use first content line, truncated
|
||||
title = if stripped.len() > 70 {
|
||||
format!("{}...", &stripped[..67])
|
||||
let mut end = 67;
|
||||
while !stripped.is_char_boundary(end) { end -= 1; }
|
||||
format!("{}...", &stripped[..end])
|
||||
} else {
|
||||
stripped.to_string()
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue