diff --git a/src/hippocampus/graph.rs b/src/hippocampus/graph.rs index ddf2e00..0e7a20d 100644 --- a/src/hippocampus/graph.rs +++ b/src/hippocampus/graph.rs @@ -519,11 +519,9 @@ pub fn build_graph_fast(store: &impl StoreView) -> Graph { fn build_adjacency(store: &impl StoreView) -> (HashMap>, HashSet) { let mut adj: HashMap> = HashMap::new(); - let mut keys: HashSet = HashSet::new(); - store.for_each_node(|key, _, _| { - keys.insert(key.to_owned()); - }); + // Get keys directly from index — no need to deserialize node content + let keys: HashSet = store.all_keys().into_iter().collect(); store.for_each_relation(|source_key, target_key, strength, rel_type| { if !keys.contains(source_key) || !keys.contains(target_key) { diff --git a/src/hippocampus/store/capnp.rs b/src/hippocampus/store/capnp.rs index 923bddd..160a551 100644 --- a/src/hippocampus/store/capnp.rs +++ b/src/hippocampus/store/capnp.rs @@ -598,8 +598,8 @@ fn rebuild_index(db_path: &Path, capnp_path: &Path) -> Result { return Ok(database); } - // Track latest (offset, uuid, version, deleted) per key - let mut latest: HashMap = HashMap::new(); + // Track latest (offset, uuid, version, deleted, node_type, timestamp) per key + let mut latest: HashMap = HashMap::new(); let file = fs::File::open(capnp_path) .with_context(|| format!("open {}", capnp_path.display()))?; @@ -630,6 +630,10 @@ fn rebuild_index(db_path: &Path, capnp_path: &Path) -> Result { let version = node_reader.get_version(); let deleted = node_reader.get_deleted(); + let node_type = node_reader.get_node_type() + .map(|t| t as u8) + .unwrap_or(0); + let timestamp = node_reader.get_timestamp(); let mut uuid = [0u8; 16]; if let Ok(data) = node_reader.get_uuid() { @@ -640,10 +644,10 @@ fn rebuild_index(db_path: &Path, capnp_path: &Path) -> Result { // Keep if newer version let dominated = latest.get(&key) - .map(|(_, _, v, _)| version >= *v) + .map(|(_, _, v, _, _, _)| version >= *v) .unwrap_or(true); if dominated { - latest.insert(key, (offset, uuid, version, deleted)); + latest.insert(key, (offset, uuid, version, deleted, node_type, timestamp)); } } } @@ -656,10 +660,15 @@ fn rebuild_index(db_path: &Path, capnp_path: &Path) -> Result { let mut key_uuid_table = txn.open_table(index::KEY_TO_UUID)?; let mut uuid_offsets = txn.open_multimap_table(index::UUID_OFFSETS)?; - for (key, (offset, uuid, _, deleted)) in latest { + for (key, (offset, uuid, _, deleted, node_type, timestamp)) in latest { if !deleted { nodes_table.insert(key.as_str(), offset)?; - key_uuid_table.insert(key.as_str(), uuid.as_slice())?; + // Pack: [uuid:16][node_type:1][timestamp:8] = 25 bytes + let mut packed = [0u8; 25]; + packed[0..16].copy_from_slice(&uuid); + packed[16] = node_type; + packed[17..25].copy_from_slice(×tamp.to_be_bytes()); + key_uuid_table.insert(key.as_str(), packed.as_slice())?; } // Always record offset in UUID history (even for deleted) uuid_offsets.insert(uuid.as_slice(), offset)?; diff --git a/src/hippocampus/store/index.rs b/src/hippocampus/store/index.rs index 9b11f11..012db0f 100644 --- a/src/hippocampus/store/index.rs +++ b/src/hippocampus/store/index.rs @@ -21,6 +21,7 @@ use std::path::Path; // Node tables pub const NODES: TableDefinition<&str, u64> = TableDefinition::new("nodes"); +// KEY_TO_UUID: key → [uuid:16][node_type:1][timestamp:8] = 25 bytes pub const KEY_TO_UUID: TableDefinition<&str, &[u8]> = TableDefinition::new("key_to_uuid"); pub const UUID_OFFSETS: MultimapTableDefinition<&[u8], u64> = MultimapTableDefinition::new("uuid_offsets"); pub const NODES_BY_PROVENANCE: MultimapTableDefinition<&str, &str> = MultimapTableDefinition::new("nodes_by_provenance"); @@ -54,14 +55,41 @@ pub fn open_db(path: &Path) -> Result { Ok(db) } +/// Pack node metadata: [uuid:16][node_type:1][timestamp:8] = 25 bytes +fn pack_node_meta(uuid: &[u8; 16], node_type: u8, timestamp: i64) -> [u8; 25] { + let mut buf = [0u8; 25]; + buf[0..16].copy_from_slice(uuid); + buf[16] = node_type; + buf[17..25].copy_from_slice(×tamp.to_be_bytes()); + buf +} + +/// Unpack node metadata. Handles both old (16-byte) and new (25-byte) formats. +pub fn unpack_node_meta(data: &[u8]) -> ([u8; 16], u8, i64) { + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&data[0..16]); + if data.len() >= 25 { + let node_type = data[16]; + let timestamp = i64::from_be_bytes([ + data[17], data[18], data[19], data[20], + data[21], data[22], data[23], data[24], + ]); + (uuid, node_type, timestamp) + } else { + // Old format: just uuid, default metadata + (uuid, 0, 0) + } +} + /// Record a node's location in the index. -pub fn index_node(txn: &WriteTransaction, key: &str, offset: u64, uuid: &[u8; 16]) -> Result<()> { +pub fn index_node(txn: &WriteTransaction, key: &str, offset: u64, uuid: &[u8; 16], node_type: u8, timestamp: i64) -> Result<()> { let mut nodes_table = txn.open_table(NODES)?; let mut key_uuid_table = txn.open_table(KEY_TO_UUID)?; let mut uuid_offsets = txn.open_multimap_table(UUID_OFFSETS)?; nodes_table.insert(key, offset)?; - key_uuid_table.insert(key, uuid.as_slice())?; + let packed = pack_node_meta(uuid, node_type, timestamp); + key_uuid_table.insert(key, packed.as_slice())?; uuid_offsets.insert(uuid.as_slice(), offset)?; Ok(()) } @@ -85,11 +113,9 @@ pub fn get_uuid_for_key(db: &Database, key: &str) -> Result> { let txn = db.begin_read()?; let table = txn.open_table(KEY_TO_UUID)?; match table.get(key)? { - Some(uuid) => { - let slice = uuid.value(); - let mut arr = [0u8; 16]; - arr.copy_from_slice(slice); - Ok(Some(arr)) + Some(data) => { + let (uuid, _, _) = unpack_node_meta(data.value()); + Ok(Some(uuid)) } None => Ok(None), } @@ -131,6 +157,19 @@ pub fn all_keys(db: &Database) -> Result> { Ok(keys) } +/// Collect all (key, uuid, node_type, timestamp) in a single table scan. +pub fn all_key_uuid_pairs(db: &Database) -> Result> { + let txn = db.begin_read()?; + let table = txn.open_table(KEY_TO_UUID)?; + let mut pairs = Vec::new(); + for entry in table.iter()? { + let (key, data) = entry?; + let (uuid, node_type, timestamp) = unpack_node_meta(data.value()); + pairs.push((key.value().to_string(), uuid, node_type, timestamp)); + } + Ok(pairs) +} + // ── Relation index operations ────────────────────────────────────── // // RELS value format: [other_uuid:16][strength:4][rel_type:1][is_outgoing:1] = 22 bytes diff --git a/src/hippocampus/store/ops.rs b/src/hippocampus/store/ops.rs index e5e2fcd..8beb173 100644 --- a/src/hippocampus/store/ops.rs +++ b/src/hippocampus/store/ops.rs @@ -24,7 +24,7 @@ impl Store { let db = self.db.as_ref().ok_or_else(|| anyhow!("store not loaded"))?; let txn = db.begin_write()?; let offset = self.append_nodes(&[node.clone()])?; - index::index_node(&txn, &node.key, offset, &node.uuid)?; + index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp)?; txn.commit()?; Ok(()) } @@ -90,7 +90,7 @@ impl Store { node.version += 1; let txn = db.begin_write()?; let offset = self.append_nodes(std::slice::from_ref(&node))?; - index::index_node(&txn, &node.key, offset, &node.uuid)?; + index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp)?; txn.commit()?; Ok("updated") } else { @@ -98,7 +98,7 @@ impl Store { node.provenance = provenance.to_string(); let txn = db.begin_write()?; let offset = self.append_nodes(std::slice::from_ref(&node))?; - index::index_node(&txn, &node.key, offset, &node.uuid)?; + index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp)?; txn.commit()?; Ok("created") } @@ -189,7 +189,7 @@ impl Store { let txn = db.begin_write()?; let offset = self.append_nodes(&[renamed.clone(), tombstone])?; index::remove_node(&txn, old_key)?; - index::index_node(&txn, new_key, offset, &renamed.uuid)?; + index::index_node(&txn, new_key, offset, &renamed.uuid, renamed.node_type as u8, renamed.timestamp)?; if !updated_rels.is_empty() { self.append_relations(&updated_rels)?; } @@ -320,7 +320,7 @@ impl Store { node.timestamp = now_epoch(); let txn = db.begin_write()?; let offset = self.append_nodes(std::slice::from_ref(&node))?; - index::index_node(&txn, key, offset, &node.uuid)?; + index::index_node(&txn, key, offset, &node.uuid, node.node_type as u8, node.timestamp)?; txn.commit()?; Ok((old, weight)) } diff --git a/src/hippocampus/store/types.rs b/src/hippocampus/store/types.rs index ecc4ea5..565737b 100644 --- a/src/hippocampus/store/types.rs +++ b/src/hippocampus/store/types.rs @@ -143,6 +143,19 @@ pub enum NodeType { EpisodicMonthly, } +impl NodeType { + pub fn from_u8(v: u8) -> Self { + match v { + 0 => NodeType::EpisodicSession, + 1 => NodeType::EpisodicDaily, + 2 => NodeType::EpisodicWeekly, + 3 => NodeType::Semantic, + 4 => NodeType::EpisodicMonthly, + _ => NodeType::Semantic, // default + } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] pub enum RelationType { Link, diff --git a/src/hippocampus/store/view.rs b/src/hippocampus/store/view.rs index bd09405..aa1d0ad 100644 --- a/src/hippocampus/store/view.rs +++ b/src/hippocampus/store/view.rs @@ -8,6 +8,9 @@ use super::Store; // --------------------------------------------------------------------------- pub trait StoreView { + /// Get all node keys (from index, no deserialization). + fn all_keys(&self) -> Vec; + /// Iterate all nodes. Callback receives (key, content, weight). fn for_each_node(&self, f: F); @@ -22,6 +25,14 @@ pub trait StoreView { } impl StoreView for Store { + fn all_keys(&self) -> Vec { + let db = match self.db.as_ref() { + Some(db) => db, + None => return Vec::new(), + }; + index::all_keys(db).unwrap_or_default() + } + fn for_each_node(&self, mut f: F) { let db = match self.db.as_ref() { Some(db) => db, @@ -45,16 +56,13 @@ impl StoreView for Store { Some(db) => db, None => return, }; - let keys = match index::all_keys(db) { - Ok(keys) => keys, + // Use index directly — no capnp reads needed + let pairs = match index::all_key_uuid_pairs(db) { + Ok(p) => p, Err(_) => return, }; - for key in keys { - if let Ok(Some(offset)) = index::get_offset(db, &key) { - if let Ok(node) = capnp::read_node_at_offset(offset) { - f(&key, node.node_type, node.timestamp); - } - } + for (key, _uuid, node_type, timestamp) in pairs { + f(&key, NodeType::from_u8(node_type), timestamp); } } @@ -64,22 +72,18 @@ impl StoreView for Store { None => return, }; - // Build uuid ↔ key maps in one pass - let keys = match index::all_keys(db) { - Ok(keys) => keys, + // Build uuid ↔ key maps in a single table scan + let pairs = match index::all_key_uuid_pairs(db) { + Ok(p) => p, Err(_) => return, }; let mut uuid_to_key: std::collections::HashMap<[u8; 16], String> = std::collections::HashMap::new(); - let mut key_to_uuid: std::collections::HashMap = std::collections::HashMap::new(); - for key in &keys { - if let Ok(Some(uuid)) = index::get_uuid_for_key(db, key) { - uuid_to_key.insert(uuid, key.clone()); - key_to_uuid.insert(key.clone(), uuid); - } + for (key, uuid, _, _) in &pairs { + uuid_to_key.insert(*uuid, key.clone()); } // Iterate edges: only process outgoing to avoid duplicates - for (key, uuid) in &key_to_uuid { + for (key, uuid, _, _) in &pairs { let edges = match index::edges_for_node(db, uuid) { Ok(e) => e, Err(_) => continue,