graph: use index for bulk reads, skip capnp deserialization

- Add all_keys() to StoreView, use in build_adjacency instead of
  for_each_node (which was ignoring content/weight anyway)
- Add all_key_uuid_pairs() for single-pass uuid mapping
- Extend KEY_TO_UUID to store [uuid:16][node_type:1][timestamp:8]
- for_each_node_meta now reads from index, no capnp needed
- Add NodeType::from_u8() for unpacking

Graph health: 7s → 2s (3.5x faster)

Co-Authored-By: Proof of Concept <poc@bcachefs.org>

src/hippocampus/graph.rs

@@ -519,11 +519,9 @@ pub fn build_graph_fast(store: &impl StoreView) -> Graph {
 
 fn build_adjacency(store: &impl StoreView) -> (HashMap<String, Vec<Edge>>, HashSet<String>) {
     let mut adj: HashMap<String, Vec<Edge>> = HashMap::new();
-    let mut keys: HashSet<String> = HashSet::new();
 
-    store.for_each_node(|key, _, _| {
-        keys.insert(key.to_owned());
-    });
+    // Get keys directly from index — no need to deserialize node content
+    let keys: HashSet<String> = store.all_keys().into_iter().collect();
 
     store.for_each_relation(|source_key, target_key, strength, rel_type| {
         if !keys.contains(source_key) || !keys.contains(target_key) {

src/hippocampus/store/capnp.rs

@@ -598,8 +598,8 @@ fn rebuild_index(db_path: &Path, capnp_path: &Path) -> Result<redb::Database> {
         return Ok(database);
     }
 
-    // Track latest (offset, uuid, version, deleted) per key
-    let mut latest: HashMap<String, (u64, [u8; 16], u32, bool)> = HashMap::new();
+    // Track latest (offset, uuid, version, deleted, node_type, timestamp) per key
+    let mut latest: HashMap<String, (u64, [u8; 16], u32, bool, u8, i64)> = HashMap::new();
 
     let file = fs::File::open(capnp_path)
         .with_context(|| format!("open {}", capnp_path.display()))?;
@@ -630,6 +630,10 @@ fn rebuild_index(db_path: &Path, capnp_path: &Path) -> Result<redb::Database> {
 
             let version = node_reader.get_version();
             let deleted = node_reader.get_deleted();
+            let node_type = node_reader.get_node_type()
+                .map(|t| t as u8)
+                .unwrap_or(0);
+            let timestamp = node_reader.get_timestamp();
 
             let mut uuid = [0u8; 16];
             if let Ok(data) = node_reader.get_uuid() {
@@ -640,10 +644,10 @@ fn rebuild_index(db_path: &Path, capnp_path: &Path) -> Result<redb::Database> {
 
             // Keep if newer version
             let dominated = latest.get(&key)
-                .map(|(_, _, v, _)| version >= *v)
+                .map(|(_, _, v, _, _, _)| version >= *v)
                 .unwrap_or(true);
             if dominated {
-                latest.insert(key, (offset, uuid, version, deleted));
+                latest.insert(key, (offset, uuid, version, deleted, node_type, timestamp));
             }
         }
     }
@@ -656,10 +660,15 @@ fn rebuild_index(db_path: &Path, capnp_path: &Path) -> Result<redb::Database> {
             let mut key_uuid_table = txn.open_table(index::KEY_TO_UUID)?;
             let mut uuid_offsets = txn.open_multimap_table(index::UUID_OFFSETS)?;
 
-            for (key, (offset, uuid, _, deleted)) in latest {
+            for (key, (offset, uuid, _, deleted, node_type, timestamp)) in latest {
                 if !deleted {
                     nodes_table.insert(key.as_str(), offset)?;
-                    key_uuid_table.insert(key.as_str(), uuid.as_slice())?;
+                    // Pack: [uuid:16][node_type:1][timestamp:8] = 25 bytes
+                    let mut packed = [0u8; 25];
+                    packed[0..16].copy_from_slice(&uuid);
+                    packed[16] = node_type;
+                    packed[17..25].copy_from_slice(&timestamp.to_be_bytes());
+                    key_uuid_table.insert(key.as_str(), packed.as_slice())?;
                 }
                 // Always record offset in UUID history (even for deleted)
                 uuid_offsets.insert(uuid.as_slice(), offset)?;

src/hippocampus/store/index.rs

@@ -21,6 +21,7 @@ use std::path::Path;
 
 // Node tables
 pub const NODES: TableDefinition<&str, u64> = TableDefinition::new("nodes");
+// KEY_TO_UUID: key → [uuid:16][node_type:1][timestamp:8] = 25 bytes
 pub const KEY_TO_UUID: TableDefinition<&str, &[u8]> = TableDefinition::new("key_to_uuid");
 pub const UUID_OFFSETS: MultimapTableDefinition<&[u8], u64> = MultimapTableDefinition::new("uuid_offsets");
 pub const NODES_BY_PROVENANCE: MultimapTableDefinition<&str, &str> = MultimapTableDefinition::new("nodes_by_provenance");
@@ -54,14 +55,41 @@ pub fn open_db(path: &Path) -> Result<Database> {
     Ok(db)
 }
 
+/// Pack node metadata: [uuid:16][node_type:1][timestamp:8] = 25 bytes
+fn pack_node_meta(uuid: &[u8; 16], node_type: u8, timestamp: i64) -> [u8; 25] {
+    let mut buf = [0u8; 25];
+    buf[0..16].copy_from_slice(uuid);
+    buf[16] = node_type;
+    buf[17..25].copy_from_slice(&timestamp.to_be_bytes());
+    buf
+}
+
+/// Unpack node metadata. Handles both old (16-byte) and new (25-byte) formats.
+pub fn unpack_node_meta(data: &[u8]) -> ([u8; 16], u8, i64) {
+    let mut uuid = [0u8; 16];
+    uuid.copy_from_slice(&data[0..16]);
+    if data.len() >= 25 {
+        let node_type = data[16];
+        let timestamp = i64::from_be_bytes([
+            data[17], data[18], data[19], data[20],
+            data[21], data[22], data[23], data[24],
+        ]);
+        (uuid, node_type, timestamp)
+    } else {
+        // Old format: just uuid, default metadata
+        (uuid, 0, 0)
+    }
+}
+
 /// Record a node's location in the index.
-pub fn index_node(txn: &WriteTransaction, key: &str, offset: u64, uuid: &[u8; 16]) -> Result<()> {
+pub fn index_node(txn: &WriteTransaction, key: &str, offset: u64, uuid: &[u8; 16], node_type: u8, timestamp: i64) -> Result<()> {
     let mut nodes_table = txn.open_table(NODES)?;
     let mut key_uuid_table = txn.open_table(KEY_TO_UUID)?;
     let mut uuid_offsets = txn.open_multimap_table(UUID_OFFSETS)?;
 
     nodes_table.insert(key, offset)?;
-    key_uuid_table.insert(key, uuid.as_slice())?;
+    let packed = pack_node_meta(uuid, node_type, timestamp);
+    key_uuid_table.insert(key, packed.as_slice())?;
     uuid_offsets.insert(uuid.as_slice(), offset)?;
     Ok(())
 }
@@ -85,11 +113,9 @@ pub fn get_uuid_for_key(db: &Database, key: &str) -> Result<Option<[u8; 16]>> {
     let txn = db.begin_read()?;
     let table = txn.open_table(KEY_TO_UUID)?;
     match table.get(key)? {
-        Some(uuid) => {
-            let slice = uuid.value();
-            let mut arr = [0u8; 16];
-            arr.copy_from_slice(slice);
-            Ok(Some(arr))
+        Some(data) => {
+            let (uuid, _, _) = unpack_node_meta(data.value());
+            Ok(Some(uuid))
         }
         None => Ok(None),
     }
@@ -131,6 +157,19 @@ pub fn all_keys(db: &Database) -> Result<Vec<String>> {
     Ok(keys)
 }
 
+/// Collect all (key, uuid, node_type, timestamp) in a single table scan.
+pub fn all_key_uuid_pairs(db: &Database) -> Result<Vec<(String, [u8; 16], u8, i64)>> {
+    let txn = db.begin_read()?;
+    let table = txn.open_table(KEY_TO_UUID)?;
+    let mut pairs = Vec::new();
+    for entry in table.iter()? {
+        let (key, data) = entry?;
+        let (uuid, node_type, timestamp) = unpack_node_meta(data.value());
+        pairs.push((key.value().to_string(), uuid, node_type, timestamp));
+    }
+    Ok(pairs)
+}
+
 // ── Relation index operations ──────────────────────────────────────
 //
 // RELS value format: [other_uuid:16][strength:4][rel_type:1][is_outgoing:1] = 22 bytes

src/hippocampus/store/ops.rs

@@ -24,7 +24,7 @@ impl Store {
         let db = self.db.as_ref().ok_or_else(|| anyhow!("store not loaded"))?;
         let txn = db.begin_write()?;
         let offset = self.append_nodes(&[node.clone()])?;
-        index::index_node(&txn, &node.key, offset, &node.uuid)?;
+        index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp)?;
         txn.commit()?;
         Ok(())
     }
@@ -90,7 +90,7 @@ impl Store {
             node.version += 1;
             let txn = db.begin_write()?;
             let offset = self.append_nodes(std::slice::from_ref(&node))?;
-            index::index_node(&txn, &node.key, offset, &node.uuid)?;
+            index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp)?;
             txn.commit()?;
             Ok("updated")
         } else {
@@ -98,7 +98,7 @@ impl Store {
             node.provenance = provenance.to_string();
             let txn = db.begin_write()?;
             let offset = self.append_nodes(std::slice::from_ref(&node))?;
-            index::index_node(&txn, &node.key, offset, &node.uuid)?;
+            index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp)?;
             txn.commit()?;
             Ok("created")
         }
@@ -189,7 +189,7 @@ impl Store {
         let txn = db.begin_write()?;
         let offset = self.append_nodes(&[renamed.clone(), tombstone])?;
         index::remove_node(&txn, old_key)?;
-        index::index_node(&txn, new_key, offset, &renamed.uuid)?;
+        index::index_node(&txn, new_key, offset, &renamed.uuid, renamed.node_type as u8, renamed.timestamp)?;
         if !updated_rels.is_empty() {
             self.append_relations(&updated_rels)?;
         }
@@ -320,7 +320,7 @@ impl Store {
         node.timestamp = now_epoch();
         let txn = db.begin_write()?;
         let offset = self.append_nodes(std::slice::from_ref(&node))?;
-        index::index_node(&txn, key, offset, &node.uuid)?;
+        index::index_node(&txn, key, offset, &node.uuid, node.node_type as u8, node.timestamp)?;
         txn.commit()?;
         Ok((old, weight))
     }

src/hippocampus/store/types.rs

@@ -143,6 +143,19 @@ pub enum NodeType {
     EpisodicMonthly,
 }
 
+impl NodeType {
+    pub fn from_u8(v: u8) -> Self {
+        match v {
+            0 => NodeType::EpisodicSession,
+            1 => NodeType::EpisodicDaily,
+            2 => NodeType::EpisodicWeekly,
+            3 => NodeType::Semantic,
+            4 => NodeType::EpisodicMonthly,
+            _ => NodeType::Semantic, // default
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
 pub enum RelationType {
     Link,

src/hippocampus/store/view.rs

@@ -8,6 +8,9 @@ use super::Store;
 // ---------------------------------------------------------------------------
 
 pub trait StoreView {
+    /// Get all node keys (from index, no deserialization).
+    fn all_keys(&self) -> Vec<String>;
+
     /// Iterate all nodes. Callback receives (key, content, weight).
     fn for_each_node<F: FnMut(&str, &str, f32)>(&self, f: F);
 
@@ -22,6 +25,14 @@ pub trait StoreView {
 }
 
 impl StoreView for Store {
+    fn all_keys(&self) -> Vec<String> {
+        let db = match self.db.as_ref() {
+            Some(db) => db,
+            None => return Vec::new(),
+        };
+        index::all_keys(db).unwrap_or_default()
+    }
+
     fn for_each_node<F: FnMut(&str, &str, f32)>(&self, mut f: F) {
         let db = match self.db.as_ref() {
             Some(db) => db,
@@ -45,16 +56,13 @@ impl StoreView for Store {
             Some(db) => db,
             None => return,
         };
-        let keys = match index::all_keys(db) {
-            Ok(keys) => keys,
+        // Use index directly — no capnp reads needed
+        let pairs = match index::all_key_uuid_pairs(db) {
+            Ok(p) => p,
             Err(_) => return,
         };
-        for key in keys {
-            if let Ok(Some(offset)) = index::get_offset(db, &key) {
-                if let Ok(node) = capnp::read_node_at_offset(offset) {
-                    f(&key, node.node_type, node.timestamp);
-                }
-            }
+        for (key, _uuid, node_type, timestamp) in pairs {
+            f(&key, NodeType::from_u8(node_type), timestamp);
         }
     }
 
@@ -64,22 +72,18 @@ impl StoreView for Store {
             None => return,
         };
 
-        // Build uuid ↔ key maps in one pass
-        let keys = match index::all_keys(db) {
-            Ok(keys) => keys,
+        // Build uuid ↔ key maps in a single table scan
+        let pairs = match index::all_key_uuid_pairs(db) {
+            Ok(p) => p,
             Err(_) => return,
         };
         let mut uuid_to_key: std::collections::HashMap<[u8; 16], String> = std::collections::HashMap::new();
-        let mut key_to_uuid: std::collections::HashMap<String, [u8; 16]> = std::collections::HashMap::new();
-        for key in &keys {
-            if let Ok(Some(uuid)) = index::get_uuid_for_key(db, key) {
-                uuid_to_key.insert(uuid, key.clone());
-                key_to_uuid.insert(key.clone(), uuid);
-            }
+        for (key, uuid, _, _) in &pairs {
+            uuid_to_key.insert(*uuid, key.clone());
         }
 
         // Iterate edges: only process outgoing to avoid duplicates
-        for (key, uuid) in &key_to_uuid {
+        for (key, uuid, _, _) in &pairs {
             let edges = match index::edges_for_node(db, uuid) {
                 Ok(e) => e,
                 Err(_) => continue,