kent/consciousness
1
0
Fork 2
Code Issues 1 Pull requests 5 Projects Releases Packages Wiki Activity Actions

index: add NODES_BY_PROVENANCE with timestamp-sorted values

Browse source 
- Store [negated_timestamp:8][key] as value for descending sort
- recent_by_provenance uses index directly, no capnp reads
- Eliminates 24k×5 capnp reads from subconscious snapshots

Co-Authored-By: Proof of Concept <poc@bcachefs.org>
This commit is contained in:
Kent Overstreet 2026-04-13 22:25:12 -04:00
commit 19789b7e74
3 changed files with 70 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

32
src/hippocampus/store/ops.rs
View file

@ -2,7 +2,7 @@
//
// CRUD (upsert, delete), maintenance (decay, cap_degree), and graph metrics.
use super::{capnp, index, types::*, Store};
use super::{index, types::*, Store};
use anyhow::{anyhow, bail, Result};
use std::collections::{HashMap, HashSet};
@ -24,7 +24,7 @@ impl Store {
let db = self.db.as_ref().ok_or_else(|| anyhow!("store not loaded"))?;
let txn = db.begin_write()?;
let offset = self.append_nodes(&[node.clone()])?;
index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp)?;
index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp, &node.provenance)?;
txn.commit()?;
Ok(())
}
@ -45,24 +45,8 @@ impl Store {
Some(db) => db,
None => return Vec::new(),
};
let keys = match index::all_keys(db) {
Ok(keys) => keys,
Err(_) => return Vec::new(),
};
let mut nodes: Vec<_> = keys.iter()
.filter_map(|key| {
let offset = index::get_offset(db, key).ok()??;
let node = capnp::read_node_at_offset(offset).ok()?;
if !node.deleted && node.provenance == provenance {
Some((key.clone(), node.timestamp))
} else {
None
}
})
.collect();
nodes.sort_by(|a, b| b.1.cmp(&a.1));
nodes.truncate(limit);
nodes
// Index stores entries sorted by timestamp descending, so just take first N
index::recent_by_provenance(db, provenance, limit).unwrap_or_default()
}
/// Upsert a node: update if exists (and content changed), create if not.
@ -90,7 +74,7 @@ impl Store {
node.version += 1;
let txn = db.begin_write()?;
let offset = self.append_nodes(std::slice::from_ref(&node))?;
index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp)?;
index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp, &node.provenance)?;
txn.commit()?;
Ok("updated")
} else {
@ -98,7 +82,7 @@ impl Store {
node.provenance = provenance.to_string();
let txn = db.begin_write()?;
let offset = self.append_nodes(std::slice::from_ref(&node))?;
index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp)?;
index::index_node(&txn, &node.key, offset, &node.uuid, node.node_type as u8, node.timestamp, &node.provenance)?;
txn.commit()?;
Ok("created")
}
@ -189,7 +173,7 @@ impl Store {
let txn = db.begin_write()?;
let offset = self.append_nodes(&[renamed.clone(), tombstone])?;
index::remove_node(&txn, old_key)?;
index::index_node(&txn, new_key, offset, &renamed.uuid, renamed.node_type as u8, renamed.timestamp)?;
index::index_node(&txn, new_key, offset, &renamed.uuid, renamed.node_type as u8, renamed.timestamp, &renamed.provenance)?;
if !updated_rels.is_empty() {
self.append_relations(&updated_rels)?;
}
@ -320,7 +304,7 @@ impl Store {
node.timestamp = now_epoch();
let txn = db.begin_write()?;
let offset = self.append_nodes(std::slice::from_ref(&node))?;
index::index_node(&txn, key, offset, &node.uuid, node.node_type as u8, node.timestamp)?;
index::index_node(&txn, key, offset, &node.uuid, node.node_type as u8, node.timestamp, &node.provenance)?;
txn.commit()?;
Ok((old, weight))
}