forked from kent/consciousness
Compute parent/child (session→daily→weekly→monthly) and prev/next (chronological ordering within each level) edges at graph build time from node metadata. Parse dates from keys for digest nodes (whose timestamps reflect creation, not covered date) and prefer key-parsed dates over timestamp-derived dates for sessions (timezone fix). Result: ~9185 implicit edges, communities halved, gini improved. Co-Authored-By: Kent Overstreet <kent.overstreet@linux.dev>
217 lines
7.9 KiB
Rust
217 lines
7.9 KiB
Rust
// Read-only access abstractions for the memory store
|
|
//
|
|
// StoreView: trait abstracting over owned Store and zero-copy MmapView.
|
|
// MmapView: mmap'd rkyv snapshot for sub-millisecond read-only access.
|
|
// AnyView: enum dispatch selecting fastest available view at runtime.
|
|
|
|
use super::types::*;
|
|
|
|
use std::fs;
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// StoreView: read-only access trait for search and graph code.
|
|
//
|
|
// Abstracts over owned Store and zero-copy MmapView so the same
|
|
// spreading-activation and graph code works with either.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
pub trait StoreView {
|
|
/// Iterate all nodes. Callback receives (key, content, weight).
|
|
fn for_each_node<F: FnMut(&str, &str, f32)>(&self, f: F);
|
|
|
|
/// Iterate all nodes with metadata. Callback receives (key, node_type, timestamp).
|
|
fn for_each_node_meta<F: FnMut(&str, NodeType, i64)>(&self, f: F);
|
|
|
|
/// Iterate all relations. Callback receives (source_key, target_key, strength, rel_type).
|
|
fn for_each_relation<F: FnMut(&str, &str, f32, RelationType)>(&self, f: F);
|
|
|
|
/// Node weight by key, or the default weight if missing.
|
|
fn node_weight(&self, key: &str) -> f64;
|
|
|
|
/// Node content by key.
|
|
fn node_content(&self, key: &str) -> Option<&str>;
|
|
|
|
/// Search/graph parameters.
|
|
fn params(&self) -> Params;
|
|
}
|
|
|
|
impl StoreView for Store {
|
|
fn for_each_node<F: FnMut(&str, &str, f32)>(&self, mut f: F) {
|
|
for (key, node) in &self.nodes {
|
|
f(key, &node.content, node.weight);
|
|
}
|
|
}
|
|
|
|
fn for_each_node_meta<F: FnMut(&str, NodeType, i64)>(&self, mut f: F) {
|
|
for (key, node) in &self.nodes {
|
|
f(key, node.node_type, node.timestamp);
|
|
}
|
|
}
|
|
|
|
fn for_each_relation<F: FnMut(&str, &str, f32, RelationType)>(&self, mut f: F) {
|
|
for rel in &self.relations {
|
|
if rel.deleted { continue; }
|
|
f(&rel.source_key, &rel.target_key, rel.strength, rel.rel_type);
|
|
}
|
|
}
|
|
|
|
fn node_weight(&self, key: &str) -> f64 {
|
|
self.nodes.get(key).map(|n| n.weight as f64).unwrap_or(self.params.default_weight)
|
|
}
|
|
|
|
fn node_content(&self, key: &str) -> Option<&str> {
|
|
self.nodes.get(key).map(|n| n.content.as_str())
|
|
}
|
|
|
|
fn params(&self) -> Params {
|
|
self.params
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// MmapView: zero-copy store access via mmap'd rkyv snapshot.
|
|
//
|
|
// Holds the mmap alive; all string reads go directly into the mapped
|
|
// pages without allocation. Falls back to None if snapshot is stale.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
pub struct MmapView {
|
|
mmap: memmap2::Mmap,
|
|
_file: fs::File,
|
|
data_offset: usize,
|
|
data_len: usize,
|
|
}
|
|
|
|
impl MmapView {
|
|
/// Try to open a fresh rkyv snapshot. Returns None if missing or stale.
|
|
pub fn open() -> Option<Self> {
|
|
let path = snapshot_path();
|
|
let file = fs::File::open(&path).ok()?;
|
|
let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?;
|
|
|
|
if mmap.len() < RKYV_HEADER_LEN { return None; }
|
|
if mmap[..4] != RKYV_MAGIC { return None; }
|
|
|
|
let nodes_size = fs::metadata(nodes_path()).map(|m| m.len()).unwrap_or(0);
|
|
let rels_size = fs::metadata(relations_path()).map(|m| m.len()).unwrap_or(0);
|
|
|
|
let cached_nodes = u64::from_le_bytes(mmap[8..16].try_into().unwrap());
|
|
let cached_rels = u64::from_le_bytes(mmap[16..24].try_into().unwrap());
|
|
let data_len = u64::from_le_bytes(mmap[24..32].try_into().unwrap()) as usize;
|
|
|
|
if cached_nodes != nodes_size || cached_rels != rels_size { return None; }
|
|
if mmap.len() < RKYV_HEADER_LEN + data_len { return None; }
|
|
|
|
Some(MmapView { mmap, _file: file, data_offset: RKYV_HEADER_LEN, data_len })
|
|
}
|
|
|
|
fn snapshot(&self) -> &ArchivedSnapshot {
|
|
let data = &self.mmap[self.data_offset..self.data_offset + self.data_len];
|
|
unsafe { rkyv::archived_root::<Snapshot>(data) }
|
|
}
|
|
}
|
|
|
|
impl StoreView for MmapView {
|
|
fn for_each_node<F: FnMut(&str, &str, f32)>(&self, mut f: F) {
|
|
let snap = self.snapshot();
|
|
for (key, node) in snap.nodes.iter() {
|
|
f(key, &node.content, node.weight);
|
|
}
|
|
}
|
|
|
|
fn for_each_node_meta<F: FnMut(&str, NodeType, i64)>(&self, mut f: F) {
|
|
let snap = self.snapshot();
|
|
for (key, node) in snap.nodes.iter() {
|
|
let nt = match node.node_type {
|
|
ArchivedNodeType::EpisodicSession => NodeType::EpisodicSession,
|
|
ArchivedNodeType::EpisodicDaily => NodeType::EpisodicDaily,
|
|
ArchivedNodeType::EpisodicWeekly => NodeType::EpisodicWeekly,
|
|
ArchivedNodeType::EpisodicMonthly => NodeType::EpisodicMonthly,
|
|
ArchivedNodeType::Semantic => NodeType::Semantic,
|
|
};
|
|
f(key, nt, node.timestamp);
|
|
}
|
|
}
|
|
|
|
fn for_each_relation<F: FnMut(&str, &str, f32, RelationType)>(&self, mut f: F) {
|
|
let snap = self.snapshot();
|
|
for rel in snap.relations.iter() {
|
|
if rel.deleted { continue; }
|
|
let rt = match rel.rel_type {
|
|
ArchivedRelationType::Link => RelationType::Link,
|
|
ArchivedRelationType::Causal => RelationType::Causal,
|
|
ArchivedRelationType::Auto => RelationType::Auto,
|
|
};
|
|
f(&rel.source_key, &rel.target_key, rel.strength, rt);
|
|
}
|
|
}
|
|
|
|
fn node_weight(&self, key: &str) -> f64 {
|
|
let snap = self.snapshot();
|
|
snap.nodes.get(key)
|
|
.map(|n| n.weight as f64)
|
|
.unwrap_or(snap.params.default_weight)
|
|
}
|
|
|
|
fn node_content(&self, key: &str) -> Option<&str> {
|
|
let snap = self.snapshot();
|
|
snap.nodes.get(key).map(|n| &*n.content)
|
|
}
|
|
|
|
fn params(&self) -> Params {
|
|
let p = &self.snapshot().params;
|
|
Params {
|
|
default_weight: p.default_weight,
|
|
decay_factor: p.decay_factor,
|
|
use_boost: p.use_boost,
|
|
prune_threshold: p.prune_threshold,
|
|
edge_decay: p.edge_decay,
|
|
max_hops: p.max_hops,
|
|
min_activation: p.min_activation,
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// AnyView: enum dispatch for read-only access.
|
|
//
|
|
// MmapView when the snapshot is fresh, owned Store as fallback.
|
|
// The match on each call is a single predicted branch — zero overhead.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
pub enum AnyView {
|
|
Mmap(MmapView),
|
|
Owned(Store),
|
|
}
|
|
|
|
impl AnyView {
|
|
/// Load the fastest available view: mmap snapshot or owned store.
|
|
pub fn load() -> Result<Self, String> {
|
|
if let Some(mv) = MmapView::open() {
|
|
Ok(AnyView::Mmap(mv))
|
|
} else {
|
|
Ok(AnyView::Owned(Store::load()?))
|
|
}
|
|
}
|
|
}
|
|
|
|
impl StoreView for AnyView {
|
|
fn for_each_node<F: FnMut(&str, &str, f32)>(&self, f: F) {
|
|
match self { AnyView::Mmap(v) => v.for_each_node(f), AnyView::Owned(s) => s.for_each_node(f) }
|
|
}
|
|
fn for_each_node_meta<F: FnMut(&str, NodeType, i64)>(&self, f: F) {
|
|
match self { AnyView::Mmap(v) => v.for_each_node_meta(f), AnyView::Owned(s) => s.for_each_node_meta(f) }
|
|
}
|
|
fn for_each_relation<F: FnMut(&str, &str, f32, RelationType)>(&self, f: F) {
|
|
match self { AnyView::Mmap(v) => v.for_each_relation(f), AnyView::Owned(s) => s.for_each_relation(f) }
|
|
}
|
|
fn node_weight(&self, key: &str) -> f64 {
|
|
match self { AnyView::Mmap(v) => v.node_weight(key), AnyView::Owned(s) => s.node_weight(key) }
|
|
}
|
|
fn node_content(&self, key: &str) -> Option<&str> {
|
|
match self { AnyView::Mmap(v) => v.node_content(key), AnyView::Owned(s) => s.node_content(key) }
|
|
}
|
|
fn params(&self) -> Params {
|
|
match self { AnyView::Mmap(v) => v.params(), AnyView::Owned(s) => s.params() }
|
|
}
|
|
}
|