forked from kent/consciousness
store: wire up RELS index for relations
Complete redb schema with bidirectional relation indexing: - RELS multimap: uuid → packed(other_uuid, strength, rel_type, is_outgoing) - Each edge stored twice (once per endpoint) with direction bit - pack_rel/unpack_rel for 22-byte packed format Wired up: - replay_relations indexes all relations on load - add_relation indexes new relations - for_each_relation reads from index (graph building) - add_link uses index for existence check - set_link_strength finds/updates edges via index - cap_degree uses index for degree counting and pruning - rename_node finds edges by uuid Vec<Relation> still maintained for remaining uses (normalize_strengths, graph_health diagnostics). To be removed in follow-up. Co-Authored-By: Kent Overstreet <kent.overstreet@linux.dev>
This commit is contained in:
Kent Overstreet
parent
8cfe9a4d70
commit
5fe51fbfda
5 changed files with 365 additions and 136 deletions
|
|
@ -1,19 +1,36 @@
|
|||
// redb index tables
|
||||
//
|
||||
// capnp logs are source of truth; redb provides indexed access.
|
||||
// Tables:
|
||||
// nodes: key → offset in capnp log (u64)
|
||||
// uuid_to_key: [u8;16] → key
|
||||
//
|
||||
// To read a node: lookup offset in redb, seek in capnp file, deserialize.
|
||||
// Node tables:
|
||||
// NODES: key → offset (current version)
|
||||
// KEY_TO_UUID: key → uuid
|
||||
// UUID_OFFSETS: uuid → offsets (multimap, all versions)
|
||||
// NODES_BY_PROVENANCE: provenance → keys (multimap)
|
||||
// NODES_BY_TYPE: [type_byte][timestamp_be] → key (for range queries by type+date)
|
||||
//
|
||||
// Relation tables:
|
||||
// RELS: node_uuid → (other_uuid, strength, rel_type, is_outgoing) packed (multimap)
|
||||
// Each relation stored twice — once per endpoint with direction bit.
|
||||
//
|
||||
// To get key from uuid: UUID_OFFSETS → read_node_at_offset() → node.key
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition};
|
||||
use redb::{Database, MultimapTableDefinition, ReadableDatabase, ReadableTable, TableDefinition};
|
||||
use std::path::Path;
|
||||
|
||||
// Table definitions - nodes maps key to byte offset in capnp log
|
||||
// Node tables
|
||||
pub const NODES: TableDefinition<&str, u64> = TableDefinition::new("nodes");
|
||||
pub const UUID_TO_KEY: TableDefinition<&[u8], &str> = TableDefinition::new("uuid_to_key");
|
||||
pub const KEY_TO_UUID: TableDefinition<&str, &[u8]> = TableDefinition::new("key_to_uuid");
|
||||
pub const UUID_OFFSETS: MultimapTableDefinition<&[u8], u64> = MultimapTableDefinition::new("uuid_offsets");
|
||||
pub const NODES_BY_PROVENANCE: MultimapTableDefinition<&str, &str> = MultimapTableDefinition::new("nodes_by_provenance");
|
||||
// Composite key: [node_type: u8][timestamp: i64 BE] for range queries
|
||||
pub const NODES_BY_TYPE: TableDefinition<&[u8], &str> = TableDefinition::new("nodes_by_type");
|
||||
|
||||
// Relations table - each relation stored twice (once per endpoint)
|
||||
// Value: (other_uuid: [u8;16], strength: f32, rel_type: u8, is_outgoing: bool)
|
||||
// Packed as 22 bytes: [other_uuid:16][strength:4][rel_type:1][is_outgoing:1]
|
||||
pub const RELS: MultimapTableDefinition<&[u8], &[u8]> = MultimapTableDefinition::new("rels");
|
||||
|
||||
/// Open or create the redb database, ensuring all tables exist.
|
||||
pub fn open_db(path: &Path) -> Result<Database> {
|
||||
|
|
@ -23,8 +40,14 @@ pub fn open_db(path: &Path) -> Result<Database> {
|
|||
// Ensure tables exist by opening a write transaction
|
||||
let txn = db.begin_write()?;
|
||||
{
|
||||
// Node tables
|
||||
let _ = txn.open_table(NODES)?;
|
||||
let _ = txn.open_table(UUID_TO_KEY)?;
|
||||
let _ = txn.open_table(KEY_TO_UUID)?;
|
||||
let _ = txn.open_multimap_table(UUID_OFFSETS)?;
|
||||
let _ = txn.open_multimap_table(NODES_BY_PROVENANCE)?;
|
||||
let _ = txn.open_table(NODES_BY_TYPE)?;
|
||||
// Relations
|
||||
let _ = txn.open_multimap_table(RELS)?;
|
||||
}
|
||||
txn.commit()?;
|
||||
|
||||
|
|
@ -36,10 +59,12 @@ pub fn index_node(db: &Database, key: &str, offset: u64, uuid: &[u8; 16]) -> Res
|
|||
let txn = db.begin_write()?;
|
||||
{
|
||||
let mut nodes_table = txn.open_table(NODES)?;
|
||||
let mut uuid_table = txn.open_table(UUID_TO_KEY)?;
|
||||
let mut key_uuid_table = txn.open_table(KEY_TO_UUID)?;
|
||||
let mut uuid_offsets = txn.open_multimap_table(UUID_OFFSETS)?;
|
||||
|
||||
nodes_table.insert(key, offset)?;
|
||||
uuid_table.insert(uuid.as_slice(), key)?;
|
||||
key_uuid_table.insert(key, uuid.as_slice())?;
|
||||
uuid_offsets.insert(uuid.as_slice(), offset)?;
|
||||
}
|
||||
txn.commit()?;
|
||||
Ok(())
|
||||
|
|
@ -59,15 +84,31 @@ pub fn contains_key(db: &Database, key: &str) -> Result<bool> {
|
|||
Ok(table.get(key)?.is_some())
|
||||
}
|
||||
|
||||
/// Remove a node from the index.
|
||||
pub fn remove_node(db: &Database, key: &str, uuid: &[u8; 16]) -> Result<()> {
|
||||
/// Get a node's UUID from its key.
|
||||
pub fn get_uuid_for_key(db: &Database, key: &str) -> Result<Option<[u8; 16]>> {
|
||||
let txn = db.begin_read()?;
|
||||
let table = txn.open_table(KEY_TO_UUID)?;
|
||||
match table.get(key)? {
|
||||
Some(uuid) => {
|
||||
let slice = uuid.value();
|
||||
let mut arr = [0u8; 16];
|
||||
arr.copy_from_slice(slice);
|
||||
Ok(Some(arr))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a node from the index (key mappings only; UUID history preserved).
|
||||
pub fn remove_node(db: &Database, key: &str, _uuid: &[u8; 16]) -> Result<()> {
|
||||
let txn = db.begin_write()?;
|
||||
{
|
||||
let mut nodes_table = txn.open_table(NODES)?;
|
||||
let mut uuid_table = txn.open_table(UUID_TO_KEY)?;
|
||||
let mut key_uuid_table = txn.open_table(KEY_TO_UUID)?;
|
||||
// Note: UUID_OFFSETS is not cleared - preserves version history
|
||||
|
||||
nodes_table.remove(key)?;
|
||||
uuid_table.remove(uuid.as_slice())?;
|
||||
key_uuid_table.remove(key)?;
|
||||
}
|
||||
txn.commit()?;
|
||||
Ok(())
|
||||
|
|
@ -84,3 +125,89 @@ pub fn all_keys(db: &Database) -> Result<Vec<String>> {
|
|||
}
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
// ── Relation index operations ──────────────────────────────────────
|
||||
//
|
||||
// RELS value format: [other_uuid:16][strength:4][rel_type:1][is_outgoing:1] = 22 bytes
|
||||
|
||||
/// Pack relation data into bytes for RELS table.
|
||||
fn pack_rel(other_uuid: &[u8; 16], strength: f32, rel_type: u8, is_outgoing: bool) -> [u8; 22] {
|
||||
let mut buf = [0u8; 22];
|
||||
buf[0..16].copy_from_slice(other_uuid);
|
||||
buf[16..20].copy_from_slice(&strength.to_be_bytes());
|
||||
buf[20] = rel_type;
|
||||
buf[21] = if is_outgoing { 1 } else { 0 };
|
||||
buf
|
||||
}
|
||||
|
||||
/// Unpack relation data from RELS table.
|
||||
pub fn unpack_rel(data: &[u8]) -> ([u8; 16], f32, u8, bool) {
|
||||
let mut other_uuid = [0u8; 16];
|
||||
other_uuid.copy_from_slice(&data[0..16]);
|
||||
let strength = f32::from_be_bytes([data[16], data[17], data[18], data[19]]);
|
||||
let rel_type = data[20];
|
||||
let is_outgoing = data[21] != 0;
|
||||
(other_uuid, strength, rel_type, is_outgoing)
|
||||
}
|
||||
|
||||
/// Index a relation: store twice (once per endpoint).
|
||||
pub fn index_relation(
|
||||
db: &Database,
|
||||
source_uuid: &[u8; 16],
|
||||
target_uuid: &[u8; 16],
|
||||
strength: f32,
|
||||
rel_type: u8,
|
||||
) -> Result<()> {
|
||||
let txn = db.begin_write()?;
|
||||
{
|
||||
let mut rels = txn.open_multimap_table(RELS)?;
|
||||
|
||||
// Store outgoing: source → (target, strength, type, true)
|
||||
let outgoing = pack_rel(target_uuid, strength, rel_type, true);
|
||||
rels.insert(source_uuid.as_slice(), outgoing.as_slice())?;
|
||||
|
||||
// Store incoming: target → (source, strength, type, false)
|
||||
let incoming = pack_rel(source_uuid, strength, rel_type, false);
|
||||
rels.insert(target_uuid.as_slice(), incoming.as_slice())?;
|
||||
}
|
||||
txn.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove a relation from the index.
|
||||
pub fn remove_relation(
|
||||
db: &Database,
|
||||
source_uuid: &[u8; 16],
|
||||
target_uuid: &[u8; 16],
|
||||
strength: f32,
|
||||
rel_type: u8,
|
||||
) -> Result<()> {
|
||||
let txn = db.begin_write()?;
|
||||
{
|
||||
let mut rels = txn.open_multimap_table(RELS)?;
|
||||
|
||||
let outgoing = pack_rel(target_uuid, strength, rel_type, true);
|
||||
rels.remove(source_uuid.as_slice(), outgoing.as_slice())?;
|
||||
|
||||
let incoming = pack_rel(source_uuid, strength, rel_type, false);
|
||||
rels.remove(target_uuid.as_slice(), incoming.as_slice())?;
|
||||
}
|
||||
txn.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get all edges for a node. Returns (other_uuid, strength, rel_type, is_outgoing).
|
||||
pub fn edges_for_node(db: &Database, node_uuid: &[u8; 16]) -> Result<Vec<([u8; 16], f32, u8, bool)>> {
|
||||
let txn = db.begin_read()?;
|
||||
let rels = txn.open_multimap_table(RELS)?;
|
||||
|
||||
let mut edges = Vec::new();
|
||||
for entry in rels.get(node_uuid.as_slice())? {
|
||||
let guard = entry?;
|
||||
let slice = guard.value();
|
||||
let mut data = [0u8; 22];
|
||||
data.copy_from_slice(slice);
|
||||
edges.push(unpack_rel(&data));
|
||||
}
|
||||
Ok(edges)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue