kent/consciousness
1
0
Fork 1
Code Issues 1 Pull requests 5 Projects Releases Packages Wiki Activity Actions

store: read nodes via index instead of HashMap

Browse source 
- Add get_node() and contains_key() methods that read via redb index
- Migrate all store/ reads to use index lookup
- Remove HashMap cache updates from mutations (write-through to capnp+index only)
- Remove replay_nodes() - load no longer builds HashMap
- Update db_is_healthy to validate by spot-checking offsets
- Fix set_weight bug: now persists weight changes to capnp

Store.nodes HashMap still exists for code outside store/ module,
but store/ itself no longer uses it.

Co-Authored-By: Proof of Concept <poc@bcachefs.org>
This commit is contained in:
Kent Overstreet 2026-04-13 19:31:28 -04:00
parent ba53597cf2
commit 7eb86656d4
4 changed files with 167 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

66
src/hippocampus/store/ops.rs
View file

@ -2,7 +2,7 @@
//
// CRUD (upsert, delete), maintenance (decay, cap_degree), and graph metrics.
use super::{index, types::*, Store};
use super::{capnp, index, types::*, Store};
use anyhow::{anyhow, bail, Result};
use std::collections::{HashMap, HashSet};
@ -17,7 +17,7 @@ pub fn current_provenance() -> String {
impl Store {
/// Add or update a node (appends to log + updates index).
pub fn upsert_node(&mut self, mut node: Node) -> Result<()> {
if let Some(existing) = self.nodes.get(&node.key) {
if let Some(existing) = self.get_node(&node.key)? {
node.uuid = existing.uuid;
node.version = existing.version + 1;
}
@ -25,8 +25,6 @@ impl Store {
if let Some(ref database) = self.db {
index::index_node(database, &node.key, offset, &node.uuid)?;
}
self.uuid_to_key.insert(node.uuid, node.key.clone());
self.nodes.insert(node.key.clone(), node);
Ok(())
}
@ -39,9 +37,24 @@ impl Store {
/// Recent nodes by provenance, sorted newest-first. Returns (key, timestamp).
pub fn recent_by_provenance(&self, provenance: &str, limit: usize) -> Vec<(String, i64)> {
let mut nodes: Vec<_> = self.nodes.values()
.filter(|n| !n.deleted && n.provenance == provenance)
.map(|n| (n.key.clone(), n.timestamp))
let db = match self.db.as_ref() {
Some(db) => db,
None => return Vec::new(),
};
let keys = match index::all_keys(db) {
Ok(keys) => keys,
Err(_) => return Vec::new(),
};
let mut nodes: Vec<_> = keys.iter()
.filter_map(|key| {
let offset = index::get_offset(db, key).ok()??;
let node = capnp::read_node_at_offset(offset).ok()?;
if !node.deleted && node.provenance == provenance {
Some((key.clone(), node.timestamp))
} else {
None
}
})
.collect();
nodes.sort_by(|a, b| b.1.cmp(&a.1));
nodes.truncate(limit);
@ -60,11 +73,11 @@ impl Store {
/// Upsert with explicit provenance (for agent-created nodes).
pub fn upsert_provenance(&mut self, key: &str, content: &str, provenance: &str) -> Result<&'static str> {
if let Some(existing) = self.nodes.get(key) {
if let Some(existing) = self.get_node(key)? {
if existing.content == content {
return Ok("unchanged");
}
let mut node = existing.clone();
let mut node = existing;
node.content = content.to_string();
node.provenance = provenance.to_string();
node.timestamp = now_epoch();
@ -73,7 +86,6 @@ impl Store {
if let Some(ref database) = self.db {
index::index_node(database, &node.key, offset, &node.uuid)?;
}
self.nodes.insert(key.to_string(), node);
Ok("updated")
} else {
let mut node = new_node(key, content);
@ -82,8 +94,6 @@ impl Store {
if let Some(ref database) = self.db {
index::index_node(database, &node.key, offset, &node.uuid)?;
}
self.uuid_to_key.insert(node.uuid, node.key.clone());
self.nodes.insert(key.to_string(), node);
Ok("created")
}
}
@ -92,10 +102,10 @@ impl Store {
pub fn delete_node(&mut self, key: &str) -> Result<()> {
let prov = current_provenance();
let node = self.nodes.get(key)
let node = self.get_node(key)?
.ok_or_else(|| anyhow!("No node '{}'", key))?;
let uuid = node.uuid;
let mut deleted = node.clone();
let mut deleted = node;
deleted.deleted = true;
deleted.version += 1;
deleted.provenance = prov;
@ -104,7 +114,6 @@ impl Store {
if let Some(ref database) = self.db {
index::remove_node(database, key, &uuid)?;
}
self.nodes.remove(key);
Ok(())
}
@ -117,12 +126,11 @@ impl Store {
if old_key == new_key {
return Ok(());
}
if self.nodes.contains_key(new_key) {
if self.contains_key(new_key)? {
bail!("Key '{}' already exists", new_key);
}
let node = self.nodes.get(old_key)
.ok_or_else(|| anyhow!("No node '{}'", old_key))?
.clone();
let node = self.get_node(old_key)?
.ok_or_else(|| anyhow!("No node '{}'", old_key))?;
let prov = current_provenance();
@ -164,10 +172,7 @@ impl Store {
index::index_node(database, new_key, offset, &renamed.uuid)?;
}
// Update in-memory cache
self.nodes.remove(old_key);
self.uuid_to_key.insert(renamed.uuid, new_key.to_string());
self.nodes.insert(new_key.to_string(), renamed);
// Update in-memory relations cache
for updated in &updated_rels {
if let Some(r) = self.relations.iter_mut().find(|r| r.uuid == updated.uuid) {
r.source_key = updated.source_key.clone();
@ -261,10 +266,19 @@ impl Store {
/// Set a node's weight directly. Returns (old, new).
pub fn set_weight(&mut self, key: &str, weight: f32) -> Result<(f32, f32)> {
let weight = weight.clamp(0.01, 1.0);
let node = self.nodes.get_mut(key)
let mut node = self.get_node(key)?
.ok_or_else(|| anyhow!("node not found: {}", key))?;
let old = node.weight;
if (old - weight).abs() < 0.001 {
return Ok((old, weight)); // unchanged
}
node.weight = weight;
node.version += 1;
node.timestamp = now_epoch();
let offset = self.append_nodes(std::slice::from_ref(&node))?;
if let Some(ref database) = self.db {
index::index_node(database, key, offset, &node.uuid)?;
}
Ok((old, weight))
}
@ -317,10 +331,10 @@ impl Store {
bail!("link already exists: {} ↔ {}", source, target);
}
let source_uuid = self.nodes.get(source)
let source_uuid = self.get_node(source)?
.map(|n| n.uuid)
.ok_or_else(|| anyhow!("source not found: {}", source))?;
let target_uuid = self.nodes.get(target)
let target_uuid = self.get_node(target)?
.map(|n| n.uuid)
.ok_or_else(|| anyhow!("target not found: {}", target))?;