spectral decomposition, search improvements, char boundary fix

- New spectral module: Laplacian eigendecomposition of the memory graph.
  Commands: spectral, spectral-save, spectral-neighbors, spectral-positions,
  spectral-suggest. Spectral neighbors expand search results beyond keyword
  matching to structural proximity.

- Search: use StoreView trait to avoid 6MB state.bin rewrite on every query.
  Append-only retrieval logging. Spectral expansion shows structurally
  nearby nodes after text results.

- Fix panic in journal-tail: string truncation at byte 67 could land inside
  a multi-byte character (em dash). Now walks back to char boundary.

- Replay queue: show classification and spectral outlier score.

- Knowledge agents: extractor, challenger, connector prompts and runner
  scripts for automated graph enrichment.

- memory-search hook: stale state file cleanup (24h expiry).

src/capnp_store.rs

@@ -5,8 +5,11 @@
 //   relations.capnp - Relation messages
 //
 // The Store struct is the derived cache: latest version per UUID,
-// rebuilt from logs when stale. Persisted as serde_json for now
-// (state.json), will move to bincode/capnp later.
+// rebuilt from logs when stale. Three-tier load strategy:
+//   1. rkyv mmap snapshot (snapshot.rkyv) — ~4ms deserialize
+//   2. bincode cache (state.bin) — ~10ms
+//   3. capnp log replay — ~40ms
+// Staleness: log file sizes embedded in cache headers.
 
 use crate::memory_capnp;
 use crate::graph::{self, Graph};
@@ -109,7 +112,8 @@ pub fn today() -> String {
 }
 
 // In-memory node representation
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
+#[archive(check_bytes)]
 pub struct Node {
     pub uuid: [u8; 16],
     pub version: u32,
@@ -146,7 +150,8 @@ pub struct Node {
     pub degree: Option<u32>,
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
+#[archive(check_bytes)]
 pub struct Relation {
     pub uuid: [u8; 16],
     pub version: u32,
@@ -161,7 +166,8 @@ pub struct Relation {
     pub target_key: String,
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
+#[archive(check_bytes)]
 pub enum NodeType {
     EpisodicSession,
     EpisodicDaily,
@@ -169,7 +175,8 @@ pub enum NodeType {
     Semantic,
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
+#[archive(check_bytes)]
 pub enum Provenance {
     Manual,
     Journal,
@@ -178,7 +185,8 @@ pub enum Provenance {
     Derived,
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
+#[archive(check_bytes)]
 pub enum Category {
     General,
     Core,
@@ -220,14 +228,16 @@ impl Category {
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
+#[archive(check_bytes)]
 pub enum RelationType {
     Link,
     Causal,
     Auto,
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
+#[archive(check_bytes)]
 pub struct RetrievalEvent {
     pub query: String,
     pub timestamp: String,
@@ -235,7 +245,8 @@ pub struct RetrievalEvent {
     pub used: Option<Vec<String>>,
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, Serialize, Deserialize, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
+#[archive(check_bytes)]
 pub struct Params {
     pub default_weight: f64,
     pub decay_factor: f64,
@@ -261,7 +272,8 @@ impl Default for Params {
 }
 
 // Gap record — something we looked for but didn't find
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
+#[archive(check_bytes)]
 pub struct GapRecord {
     pub description: String,
     pub timestamp: String,
@@ -279,19 +291,299 @@ pub struct Store {
     pub params: Params,
 }
 
+/// Snapshot for mmap: full store state minus retrieval_log (which
+/// is append-only in retrieval.log). rkyv zero-copy serialization
+/// lets us mmap this and access archived data without deserialization.
+#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
+#[archive(check_bytes)]
+struct Snapshot {
+    nodes: HashMap<String, Node>,
+    relations: Vec<Relation>,
+    gaps: Vec<GapRecord>,
+    params: Params,
+}
+
+fn snapshot_path() -> PathBuf { memory_dir().join("snapshot.rkyv") }
+
+// rkyv snapshot header: 32 bytes (multiple of 16 for alignment after mmap)
+//   [0..4]   magic "RKV\x01"
+//   [4..8]   format version (u32 LE)
+//   [8..16]  nodes.capnp file size (u64 LE) — staleness check
+//   [16..24] relations.capnp file size (u64 LE)
+//   [24..32] rkyv data length (u64 LE)
+const RKYV_MAGIC: [u8; 4] = *b"RKV\x01";
+const RKYV_HEADER_LEN: usize = 32;
+
+// state.bin header: magic + log file sizes for staleness detection.
+// File sizes are race-free for append-only logs (they only grow),
+// unlike mtimes which race with concurrent writers.
+const CACHE_MAGIC: [u8; 4] = *b"POC\x01";
+const CACHE_HEADER_LEN: usize = 4 + 8 + 8; // magic + nodes_size + rels_size
+
+// ---------------------------------------------------------------------------
+// StoreView: read-only access trait for search and graph code.
+//
+// Abstracts over owned Store and zero-copy MmapView so the same
+// spreading-activation and graph code works with either.
+// ---------------------------------------------------------------------------
+
+pub trait StoreView {
+    /// Iterate all nodes. Callback receives (key, content, weight).
+    fn for_each_node<F: FnMut(&str, &str, f32)>(&self, f: F);
+
+    /// Iterate all relations. Callback receives (source_key, target_key, strength, rel_type).
+    fn for_each_relation<F: FnMut(&str, &str, f32, RelationType)>(&self, f: F);
+
+    /// Node weight by key, or the default weight if missing.
+    fn node_weight(&self, key: &str) -> f64;
+
+    /// Node content by key.
+    fn node_content(&self, key: &str) -> Option<&str>;
+
+    /// Check if a node exists.
+    fn has_node(&self, key: &str) -> bool;
+
+    /// Search/graph parameters.
+    fn params(&self) -> Params;
+}
+
+impl StoreView for Store {
+    fn for_each_node<F: FnMut(&str, &str, f32)>(&self, mut f: F) {
+        for (key, node) in &self.nodes {
+            f(key, &node.content, node.weight);
+        }
+    }
+
+    fn for_each_relation<F: FnMut(&str, &str, f32, RelationType)>(&self, mut f: F) {
+        for rel in &self.relations {
+            f(&rel.source_key, &rel.target_key, rel.strength, rel.rel_type);
+        }
+    }
+
+    fn node_weight(&self, key: &str) -> f64 {
+        self.nodes.get(key).map(|n| n.weight as f64).unwrap_or(self.params.default_weight)
+    }
+
+    fn node_content(&self, key: &str) -> Option<&str> {
+        self.nodes.get(key).map(|n| n.content.as_str())
+    }
+
+    fn has_node(&self, key: &str) -> bool {
+        self.nodes.contains_key(key)
+    }
+
+    fn params(&self) -> Params {
+        self.params
+    }
+}
+
+// ---------------------------------------------------------------------------
+// MmapView: zero-copy store access via mmap'd rkyv snapshot.
+//
+// Holds the mmap alive; all string reads go directly into the mapped
+// pages without allocation. Falls back to None if snapshot is stale.
+// ---------------------------------------------------------------------------
+
+pub struct MmapView {
+    mmap: memmap2::Mmap,
+    _file: fs::File,
+    data_offset: usize,
+    data_len: usize,
+}
+
+impl MmapView {
+    /// Try to open a fresh rkyv snapshot. Returns None if missing or stale.
+    pub fn open() -> Option<Self> {
+        let path = snapshot_path();
+        let file = fs::File::open(&path).ok()?;
+        let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?;
+
+        if mmap.len() < RKYV_HEADER_LEN { return None; }
+        if mmap[..4] != RKYV_MAGIC { return None; }
+
+        let nodes_size = fs::metadata(nodes_path()).map(|m| m.len()).unwrap_or(0);
+        let rels_size = fs::metadata(relations_path()).map(|m| m.len()).unwrap_or(0);
+
+        let cached_nodes = u64::from_le_bytes(mmap[8..16].try_into().unwrap());
+        let cached_rels = u64::from_le_bytes(mmap[16..24].try_into().unwrap());
+        let data_len = u64::from_le_bytes(mmap[24..32].try_into().unwrap()) as usize;
+
+        if cached_nodes != nodes_size || cached_rels != rels_size { return None; }
+        if mmap.len() < RKYV_HEADER_LEN + data_len { return None; }
+
+        Some(MmapView { mmap, _file: file, data_offset: RKYV_HEADER_LEN, data_len })
+    }
+
+    fn snapshot(&self) -> &ArchivedSnapshot {
+        let data = &self.mmap[self.data_offset..self.data_offset + self.data_len];
+        unsafe { rkyv::archived_root::<Snapshot>(data) }
+    }
+}
+
+impl StoreView for MmapView {
+    fn for_each_node<F: FnMut(&str, &str, f32)>(&self, mut f: F) {
+        let snap = self.snapshot();
+        for (key, node) in snap.nodes.iter() {
+            f(&key, &node.content, node.weight);
+        }
+    }
+
+    fn for_each_relation<F: FnMut(&str, &str, f32, RelationType)>(&self, mut f: F) {
+        let snap = self.snapshot();
+        for rel in snap.relations.iter() {
+            let rt = match rel.rel_type {
+                ArchivedRelationType::Link => RelationType::Link,
+                ArchivedRelationType::Causal => RelationType::Causal,
+                ArchivedRelationType::Auto => RelationType::Auto,
+            };
+            f(&rel.source_key, &rel.target_key, rel.strength, rt);
+        }
+    }
+
+    fn node_weight(&self, key: &str) -> f64 {
+        let snap = self.snapshot();
+        snap.nodes.get(key)
+            .map(|n| n.weight as f64)
+            .unwrap_or(snap.params.default_weight)
+    }
+
+    fn node_content(&self, key: &str) -> Option<&str> {
+        let snap = self.snapshot();
+        snap.nodes.get(key).map(|n| &*n.content)
+    }
+
+    fn has_node(&self, key: &str) -> bool {
+        self.snapshot().nodes.get(key).is_some()
+    }
+
+    fn params(&self) -> Params {
+        let p = &self.snapshot().params;
+        Params {
+            default_weight: p.default_weight,
+            decay_factor: p.decay_factor,
+            use_boost: p.use_boost,
+            prune_threshold: p.prune_threshold,
+            edge_decay: p.edge_decay,
+            max_hops: p.max_hops,
+            min_activation: p.min_activation,
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// AnyView: enum dispatch for read-only access.
+//
+// MmapView when the snapshot is fresh, owned Store as fallback.
+// The match on each call is a single predicted branch — zero overhead.
+// ---------------------------------------------------------------------------
+
+pub enum AnyView {
+    Mmap(MmapView),
+    Owned(Store),
+}
+
+impl AnyView {
+    /// Load the fastest available view: mmap snapshot or owned store.
+    pub fn load() -> Result<Self, String> {
+        if let Some(mv) = MmapView::open() {
+            Ok(AnyView::Mmap(mv))
+        } else {
+            Ok(AnyView::Owned(Store::load()?))
+        }
+    }
+}
+
+impl StoreView for AnyView {
+    fn for_each_node<F: FnMut(&str, &str, f32)>(&self, f: F) {
+        match self {
+            AnyView::Mmap(v) => v.for_each_node(f),
+            AnyView::Owned(s) => s.for_each_node(f),
+        }
+    }
+
+    fn for_each_relation<F: FnMut(&str, &str, f32, RelationType)>(&self, f: F) {
+        match self {
+            AnyView::Mmap(v) => v.for_each_relation(f),
+            AnyView::Owned(s) => s.for_each_relation(f),
+        }
+    }
+
+    fn node_weight(&self, key: &str) -> f64 {
+        match self {
+            AnyView::Mmap(v) => v.node_weight(key),
+            AnyView::Owned(s) => StoreView::node_weight(s, key),
+        }
+    }
+
+    fn node_content(&self, key: &str) -> Option<&str> {
+        match self {
+            AnyView::Mmap(v) => v.node_content(key),
+            AnyView::Owned(s) => s.node_content(key),
+        }
+    }
+
+    fn has_node(&self, key: &str) -> bool {
+        match self {
+            AnyView::Mmap(v) => v.has_node(key),
+            AnyView::Owned(s) => s.has_node(key),
+        }
+    }
+
+    fn params(&self) -> Params {
+        match self {
+            AnyView::Mmap(v) => v.params(),
+            AnyView::Owned(s) => s.params(),
+        }
+    }
+}
 
 impl Store {
-    /// Load store: try state.json cache first, rebuild from capnp logs if stale
+    /// Load store from state.bin cache if fresh, otherwise rebuild from capnp logs.
+    ///
+    /// Staleness check uses log file sizes (not mtimes). Since logs are
+    /// append-only, any write grows the file, invalidating the cache.
+    /// This avoids the mtime race that caused data loss with concurrent
+    /// writers (dream loop, link audit, journal enrichment).
     pub fn load() -> Result<Store, String> {
+        // 1. Try rkyv mmap snapshot (~4ms with deserialize, <1ms zero-copy)
+        match Self::load_snapshot_mmap() {
+            Ok(Some(store)) => return Ok(store),
+            Ok(None) => {},
+            Err(e) => eprintln!("rkyv snapshot: {}", e),
+        }
+
+        // 2. Try bincode state.bin cache (~10ms)
         let nodes_p = nodes_path();
         let rels_p = relations_path();
+        let state_p = state_path();
 
-        // Always rebuild from capnp logs (source of truth).
-        // The mtime-based cache was causing data loss: concurrent
-        // writers (dream loop, link audit, journal enrichment) would
-        // load stale state.bin, make changes, and save — overwriting
-        // entries from other processes. Replaying from the append-only
-        // log costs ~10ms extra at 2K nodes and is always correct.
+        let nodes_size = fs::metadata(&nodes_p).map(|m| m.len()).unwrap_or(0);
+        let rels_size = fs::metadata(&rels_p).map(|m| m.len()).unwrap_or(0);
+
+        if let Ok(data) = fs::read(&state_p) {
+            if data.len() >= CACHE_HEADER_LEN && data[..4] == CACHE_MAGIC {
+                let cached_nodes = u64::from_le_bytes(data[4..12].try_into().unwrap());
+                let cached_rels = u64::from_le_bytes(data[12..20].try_into().unwrap());
+
+                if cached_nodes == nodes_size && cached_rels == rels_size {
+                    if let Ok(mut store) = bincode::deserialize::<Store>(&data[CACHE_HEADER_LEN..]) {
+                        // Rebuild uuid_to_key (skipped by serde)
+                        for (key, node) in &store.nodes {
+                            store.uuid_to_key.insert(node.uuid, key.clone());
+                        }
+                        // Bootstrap: write rkyv snapshot if missing
+                        if !snapshot_path().exists() {
+                            if let Err(e) = store.save_snapshot_inner() {
+                                eprintln!("rkyv bootstrap: {}", e);
+                            }
+                        }
+                        return Ok(store);
+                    }
+                }
+            }
+        }
+
+        // Stale or no cache — rebuild from capnp logs
         let mut store = Store::default();
 
         if nodes_p.exists() {
@@ -307,7 +599,6 @@ impl Store {
             store.nodes.contains_key(&r.target_key)
         );
 
-        // Save cache (still useful for tools that read state.bin directly)
         store.save()?;
         Ok(store)
     }
@@ -419,7 +710,8 @@ impl Store {
         Ok(())
     }
 
-    /// Save the derived cache (state.json)
+    /// Save the derived cache with log size header for staleness detection.
+    /// Uses atomic write (tmp + rename) to prevent partial reads.
     pub fn save(&self) -> Result<(), String> {
         let _lock = StoreLock::acquire()?;
 
@@ -427,19 +719,124 @@ impl Store {
         if let Some(parent) = path.parent() {
             fs::create_dir_all(parent).ok();
         }
-        let data = bincode::serialize(self)
-            .map_err(|e| format!("bincode serialize: {}", e))?;
-        fs::write(&path, data)
-            .map_err(|e| format!("write {}: {}", path.display(), e))?;
 
-        // Clean up old JSON cache if it exists
-        let json_path = state_json_path();
-        if json_path.exists() {
-            fs::remove_file(&json_path).ok();
+        let nodes_size = fs::metadata(nodes_path()).map(|m| m.len()).unwrap_or(0);
+        let rels_size = fs::metadata(relations_path()).map(|m| m.len()).unwrap_or(0);
+
+        let bincode_data = bincode::serialize(self)
+            .map_err(|e| format!("bincode serialize: {}", e))?;
+
+        let mut data = Vec::with_capacity(CACHE_HEADER_LEN + bincode_data.len());
+        data.extend_from_slice(&CACHE_MAGIC);
+        data.extend_from_slice(&nodes_size.to_le_bytes());
+        data.extend_from_slice(&rels_size.to_le_bytes());
+        data.extend_from_slice(&bincode_data);
+
+        // Atomic write: tmp file + rename
+        let tmp_path = path.with_extension("bin.tmp");
+        fs::write(&tmp_path, &data)
+            .map_err(|e| format!("write {}: {}", tmp_path.display(), e))?;
+        fs::rename(&tmp_path, &path)
+            .map_err(|e| format!("rename {} → {}: {}", tmp_path.display(), path.display(), e))?;
+
+        // Also write rkyv snapshot (mmap-friendly)
+        if let Err(e) = self.save_snapshot_inner() {
+            eprintln!("rkyv snapshot save: {}", e);
         }
+
         Ok(())
     }
 
+    /// Serialize store as rkyv snapshot with staleness header.
+    /// Assumes StoreLock is already held by caller.
+    fn save_snapshot_inner(&self) -> Result<(), String> {
+        let snap = Snapshot {
+            nodes: self.nodes.clone(),
+            relations: self.relations.clone(),
+            gaps: self.gaps.clone(),
+            params: self.params.clone(),
+        };
+
+        let rkyv_data = rkyv::to_bytes::<_, 256>(&snap)
+            .map_err(|e| format!("rkyv serialize: {}", e))?;
+
+        let nodes_size = fs::metadata(nodes_path()).map(|m| m.len()).unwrap_or(0);
+        let rels_size = fs::metadata(relations_path()).map(|m| m.len()).unwrap_or(0);
+
+        let mut data = Vec::with_capacity(RKYV_HEADER_LEN + rkyv_data.len());
+        data.extend_from_slice(&RKYV_MAGIC);
+        data.extend_from_slice(&1u32.to_le_bytes());  // format version
+        data.extend_from_slice(&nodes_size.to_le_bytes());
+        data.extend_from_slice(&rels_size.to_le_bytes());
+        data.extend_from_slice(&(rkyv_data.len() as u64).to_le_bytes());
+        data.extend_from_slice(&rkyv_data);
+
+        let path = snapshot_path();
+        let tmp_path = path.with_extension("rkyv.tmp");
+        fs::write(&tmp_path, &data)
+            .map_err(|e| format!("write {}: {}", tmp_path.display(), e))?;
+        fs::rename(&tmp_path, &path)
+            .map_err(|e| format!("rename: {}", e))?;
+
+        Ok(())
+    }
+
+    /// Try loading store from mmap'd rkyv snapshot.
+    /// Returns None if snapshot is missing or stale (log sizes don't match).
+    fn load_snapshot_mmap() -> Result<Option<Store>, String> {
+        let path = snapshot_path();
+        if !path.exists() { return Ok(None); }
+
+        let nodes_size = fs::metadata(nodes_path()).map(|m| m.len()).unwrap_or(0);
+        let rels_size = fs::metadata(relations_path()).map(|m| m.len()).unwrap_or(0);
+
+        let file = fs::File::open(&path)
+            .map_err(|e| format!("open {}: {}", path.display(), e))?;
+
+        let mmap = unsafe { memmap2::Mmap::map(&file) }
+            .map_err(|e| format!("mmap {}: {}", path.display(), e))?;
+
+        if mmap.len() < RKYV_HEADER_LEN { return Ok(None); }
+        if mmap[..4] != RKYV_MAGIC { return Ok(None); }
+
+        // [4..8] = version, skip for now
+        let cached_nodes = u64::from_le_bytes(mmap[8..16].try_into().unwrap());
+        let cached_rels = u64::from_le_bytes(mmap[16..24].try_into().unwrap());
+        let data_len = u64::from_le_bytes(mmap[24..32].try_into().unwrap()) as usize;
+
+        if cached_nodes != nodes_size || cached_rels != rels_size {
+            return Ok(None); // stale
+        }
+        if mmap.len() < RKYV_HEADER_LEN + data_len {
+            return Ok(None); // truncated
+        }
+
+        let rkyv_data = &mmap[RKYV_HEADER_LEN..RKYV_HEADER_LEN + data_len];
+
+        // SAFETY: we wrote this file ourselves via save_snapshot_inner().
+        // Skip full validation (check_archived_root) — the staleness header
+        // already confirms this snapshot matches the current log state.
+        let archived = unsafe { rkyv::archived_root::<Snapshot>(rkyv_data) };
+
+        let snap: Snapshot = <ArchivedSnapshot as rkyv::Deserialize<Snapshot, rkyv::Infallible>>
+            ::deserialize(archived, &mut rkyv::Infallible).unwrap();
+
+        let mut store = Store {
+            nodes: snap.nodes,
+            relations: snap.relations,
+            gaps: snap.gaps,
+            params: snap.params,
+            ..Default::default()
+        };
+
+        // Rebuild uuid_to_key (not serialized)
+        for (key, node) in &store.nodes {
+            store.uuid_to_key.insert(node.uuid, key.clone());
+        }
+
+        Ok(Some(store))
+    }
+
     /// Add or update a node (appends to log + updates cache)
     pub fn upsert_node(&mut self, mut node: Node) -> Result<(), String> {
         if let Some(existing) = self.nodes.get(&node.key) {
@@ -822,6 +1219,22 @@ impl Store {
         }
     }
 
+    /// Lightweight retrieval logging — appends one line to retrieval.log
+    /// instead of rewriting the entire state.bin.
+    pub fn log_retrieval_append(&self, query: &str, results: &[String]) {
+        Self::log_retrieval_static(query, results);
+    }
+
+    /// Append retrieval event to retrieval.log without needing a Store instance.
+    pub fn log_retrieval_static(query: &str, results: &[String]) {
+        let path = memory_dir().join("retrieval.log");
+        let line = format!("[{}] q=\"{}\" hits={}\n", today(), query, results.len());
+        if let Ok(mut f) = fs::OpenOptions::new()
+            .create(true).append(true).open(&path) {
+            let _ = f.write_all(line.as_bytes());
+        }
+    }
+
     pub fn mark_used(&mut self, key: &str) {
         let updated = if let Some(node) = self.nodes.get_mut(key) {
             node.uses += 1;