From ea0d631051622982e265afa4574daf9d960b37f3 Mon Sep 17 00:00:00 2001 From: ProofOfConcept Date: Tue, 3 Mar 2026 12:25:10 -0500 Subject: [PATCH] capnp_store: declarative serialization via macros MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace 130 lines of manual field-by-field capnp serialization with two declarative macros: capnp_enum! — generates to_capnp/from_capnp for enum types capnp_message! — generates from_capnp/to_capnp for structs Adding a field to the capnp schema now means adding it in one place; both read and write directions are generated from the same declaration. Eliminates: read_content_node, write_content_node, read_relation, write_relation, read_provenance (5 functions → 2 macro invocations). Callers updated to method syntax: Node::from_capnp() / node.to_capnp(). Co-Authored-By: Kent Overstreet --- Cargo.lock | 1 + Cargo.toml | 1 + src/capnp_store.rs | 249 ++++++++++++++++++++------------------------- 3 files changed, 115 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76f950b..4e5e1d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1018,6 +1018,7 @@ dependencies = [ "faer", "libc", "memmap2", + "paste", "peg", "rayon", "regex", diff --git a/Cargo.toml b/Cargo.toml index 968c02d..6d4b065 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ rkyv = { version = "0.7", features = ["validation", "std"] } memmap2 = "0.9" rayon = "1" peg = "0.8" +paste = "1" [build-dependencies] capnpc = "0.20" diff --git a/src/capnp_store.rs b/src/capnp_store.rs index d67eddc..6bab353 100644 --- a/src/capnp_store.rs +++ b/src/capnp_store.rs @@ -29,6 +29,77 @@ use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; +// --------------------------------------------------------------------------- +// Capnp serialization macros +// +// Declarative mapping between Rust types and capnp generated types. +// Adding a field to the schema means adding it in one place below; +// both read and write are generated from the same declaration. +// --------------------------------------------------------------------------- + +/// Generate to_capnp/from_capnp conversion methods for an enum. +macro_rules! capnp_enum { + ($rust_type:ident, $capnp_type:path, [$($variant:ident),+ $(,)?]) => { + impl $rust_type { + fn to_capnp(&self) -> $capnp_type { + match self { + $(Self::$variant => <$capnp_type>::$variant,)+ + } + } + fn from_capnp(v: $capnp_type) -> Self { + match v { + $(<$capnp_type>::$variant => Self::$variant,)+ + } + } + } + }; +} + +/// Generate from_capnp/to_capnp methods for a struct with capnp serialization. +/// Fields are grouped by serialization kind: +/// text - capnp Text fields (String in Rust) +/// uuid - capnp Data fields ([u8; 16] in Rust) +/// prim - copy types (u32, f32, f64, bool) +/// enm - enums with to_capnp/from_capnp methods +/// skip - Rust-only fields not in capnp (set to Default on read) +macro_rules! capnp_message { + ( + $struct:ident, + reader: $reader:ty, + builder: $builder:ty, + text: [$($tf:ident),* $(,)?], + uuid: [$($uf:ident),* $(,)?], + prim: [$($pf:ident),* $(,)?], + enm: [$($ef:ident: $et:ident),* $(,)?], + skip: [$($sf:ident),* $(,)?] $(,)? + ) => { + impl $struct { + fn from_capnp(r: $reader) -> Result { + paste::paste! { + Ok(Self { + $($tf: read_text(r.[]()),)* + $($uf: read_uuid(r.[]()),)* + $($pf: r.[](),)* + $($ef: $et::from_capnp( + r.[]().map_err(|_| concat!("bad ", stringify!($ef)))? + ),)* + $($sf: Default::default(),)* + }) + } + } + + fn to_capnp(&self, mut b: $builder) { + paste::paste! { + $(b.[](&self.$tf);)* + $(b.[](&self.$uf);)* + $(b.[](self.$pf);)* + $(b.[](self.$ef.to_capnp());)* + } + } + } + }; +} + // Data dir: ~/.claude/memory/ fn memory_dir() -> PathBuf { PathBuf::from(env::var("HOME").expect("HOME not set")) @@ -234,6 +305,40 @@ pub enum RelationType { Auto, } +capnp_enum!(NodeType, memory_capnp::NodeType, + [EpisodicSession, EpisodicDaily, EpisodicWeekly, Semantic]); + +capnp_enum!(Provenance, memory_capnp::Provenance, + [Manual, Journal, Agent, Dream, Derived]); + +capnp_enum!(Category, memory_capnp::Category, + [General, Core, Technical, Observation, Task]); + +capnp_enum!(RelationType, memory_capnp::RelationType, + [Link, Causal, Auto]); + +capnp_message!(Node, + reader: memory_capnp::content_node::Reader<'_>, + builder: memory_capnp::content_node::Builder<'_>, + text: [key, content, source_ref, created, state_tag], + uuid: [uuid], + prim: [version, timestamp, weight, emotion, deleted, + retrievals, uses, wrongs, last_replayed, + spaced_repetition_interval, position], + enm: [node_type: NodeType, provenance: Provenance, category: Category], + skip: [community_id, clustering_coefficient, degree], +); + +capnp_message!(Relation, + reader: memory_capnp::relation::Reader<'_>, + builder: memory_capnp::relation::Builder<'_>, + text: [source_key, target_key], + uuid: [uuid, source, target], + prim: [version, timestamp, strength, deleted], + enm: [rel_type: RelationType, provenance: Provenance], + skip: [], +); + #[derive(Clone, Debug, Serialize, Deserialize, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] #[archive(check_bytes)] pub struct RetrievalEvent { @@ -614,7 +719,7 @@ impl Store { .map_err(|e| format!("read node log: {}", e))?; for node_reader in log.get_nodes() .map_err(|e| format!("get nodes: {}", e))? { - let node = read_content_node(node_reader)?; + let node = Node::from_capnp(node_reader)?; let existing_version = self.nodes.get(&node.key) .map(|n| n.version) .unwrap_or(0); @@ -646,7 +751,7 @@ impl Store { .map_err(|e| format!("read relation log: {}", e))?; for rel_reader in log.get_relations() .map_err(|e| format!("get relations: {}", e))? { - let rel = read_relation(rel_reader)?; + let rel = Relation::from_capnp(rel_reader)?; let existing_version = by_uuid.get(&rel.uuid) .map(|r| r.version) .unwrap_or(0); @@ -677,7 +782,7 @@ impl Store { let log = msg.init_root::(); let mut list = log.init_nodes(nodes.len() as u32); for (i, node) in nodes.iter().enumerate() { - write_content_node(list.reborrow().get(i as u32), node); + node.to_capnp(list.reborrow().get(i as u32)); } } serialize::write_message(&mut writer, &msg) @@ -701,7 +806,7 @@ impl Store { let log = msg.init_root::(); let mut list = log.init_relations(relations.len() as u32); for (i, rel) in relations.iter().enumerate() { - write_relation(list.reborrow().get(i as u32), rel); + rel.to_capnp(list.reborrow().get(i as u32)); } } serialize::write_message(&mut writer, &msg) @@ -1834,135 +1939,7 @@ fn read_uuid(result: capnp::Result<&[u8]>) -> [u8; 16] { out } -fn read_content_node(r: memory_capnp::content_node::Reader) -> Result { - Ok(Node { - uuid: read_uuid(r.get_uuid()), - version: r.get_version(), - timestamp: r.get_timestamp(), - node_type: match r.get_node_type().map_err(|_| "bad node_type")? { - memory_capnp::NodeType::EpisodicSession => NodeType::EpisodicSession, - memory_capnp::NodeType::EpisodicDaily => NodeType::EpisodicDaily, - memory_capnp::NodeType::EpisodicWeekly => NodeType::EpisodicWeekly, - memory_capnp::NodeType::Semantic => NodeType::Semantic, - }, - provenance: read_provenance(r.get_provenance().map_err(|_| "bad provenance")?)?, - key: read_text(r.get_key()), - content: read_text(r.get_content()), - weight: r.get_weight(), - category: match r.get_category().map_err(|_| "bad category")? { - memory_capnp::Category::General => Category::General, - memory_capnp::Category::Core => Category::Core, - memory_capnp::Category::Technical => Category::Technical, - memory_capnp::Category::Observation => Category::Observation, - memory_capnp::Category::Task => Category::Task, - }, - emotion: r.get_emotion(), - deleted: r.get_deleted(), - source_ref: read_text(r.get_source_ref()), - created: read_text(r.get_created()), - retrievals: r.get_retrievals(), - uses: r.get_uses(), - wrongs: r.get_wrongs(), - state_tag: read_text(r.get_state_tag()), - last_replayed: r.get_last_replayed(), - spaced_repetition_interval: r.get_spaced_repetition_interval(), - position: r.get_position(), - community_id: None, - clustering_coefficient: None, - degree: None, - }) -} - -fn read_provenance(p: memory_capnp::Provenance) -> Result { - Ok(match p { - memory_capnp::Provenance::Manual => Provenance::Manual, - memory_capnp::Provenance::Journal => Provenance::Journal, - memory_capnp::Provenance::Agent => Provenance::Agent, - memory_capnp::Provenance::Dream => Provenance::Dream, - memory_capnp::Provenance::Derived => Provenance::Derived, - }) -} - -fn write_content_node(mut b: memory_capnp::content_node::Builder, node: &Node) { - b.set_uuid(&node.uuid); - b.set_version(node.version); - b.set_timestamp(node.timestamp); - b.set_node_type(match node.node_type { - NodeType::EpisodicSession => memory_capnp::NodeType::EpisodicSession, - NodeType::EpisodicDaily => memory_capnp::NodeType::EpisodicDaily, - NodeType::EpisodicWeekly => memory_capnp::NodeType::EpisodicWeekly, - NodeType::Semantic => memory_capnp::NodeType::Semantic, - }); - b.set_provenance(match node.provenance { - Provenance::Manual => memory_capnp::Provenance::Manual, - Provenance::Journal => memory_capnp::Provenance::Journal, - Provenance::Agent => memory_capnp::Provenance::Agent, - Provenance::Dream => memory_capnp::Provenance::Dream, - Provenance::Derived => memory_capnp::Provenance::Derived, - }); - b.set_key(&node.key); - b.set_content(&node.content); - b.set_weight(node.weight); - b.set_category(match node.category { - Category::General => memory_capnp::Category::General, - Category::Core => memory_capnp::Category::Core, - Category::Technical => memory_capnp::Category::Technical, - Category::Observation => memory_capnp::Category::Observation, - Category::Task => memory_capnp::Category::Task, - }); - b.set_emotion(node.emotion); - b.set_deleted(node.deleted); - b.set_source_ref(&node.source_ref); - b.set_created(&node.created); - b.set_retrievals(node.retrievals); - b.set_uses(node.uses); - b.set_wrongs(node.wrongs); - b.set_state_tag(&node.state_tag); - b.set_last_replayed(node.last_replayed); - b.set_spaced_repetition_interval(node.spaced_repetition_interval); - b.set_position(node.position); -} - -fn read_relation(r: memory_capnp::relation::Reader) -> Result { - Ok(Relation { - uuid: read_uuid(r.get_uuid()), - version: r.get_version(), - timestamp: r.get_timestamp(), - source: read_uuid(r.get_source()), - target: read_uuid(r.get_target()), - rel_type: match r.get_rel_type().map_err(|_| "bad rel_type")? { - memory_capnp::RelationType::Link => RelationType::Link, - memory_capnp::RelationType::Causal => RelationType::Causal, - memory_capnp::RelationType::Auto => RelationType::Auto, - }, - strength: r.get_strength(), - provenance: read_provenance(r.get_provenance().map_err(|_| "bad provenance")?)?, - deleted: r.get_deleted(), - source_key: read_text(r.get_source_key()), - target_key: read_text(r.get_target_key()), - }) -} - -fn write_relation(mut b: memory_capnp::relation::Builder, rel: &Relation) { - b.set_uuid(&rel.uuid); - b.set_version(rel.version); - b.set_timestamp(rel.timestamp); - b.set_source(&rel.source); - b.set_target(&rel.target); - b.set_rel_type(match rel.rel_type { - RelationType::Link => memory_capnp::RelationType::Link, - RelationType::Causal => memory_capnp::RelationType::Causal, - RelationType::Auto => memory_capnp::RelationType::Auto, - }); - b.set_strength(rel.strength); - b.set_provenance(match rel.provenance { - Provenance::Manual => memory_capnp::Provenance::Manual, - Provenance::Journal => memory_capnp::Provenance::Journal, - Provenance::Agent => memory_capnp::Provenance::Agent, - Provenance::Dream => memory_capnp::Provenance::Dream, - Provenance::Derived => memory_capnp::Provenance::Derived, - }); - b.set_deleted(rel.deleted); - b.set_source_key(&rel.source_key); - b.set_target_key(&rel.target_key); -} +// Serialization functions (read_content_node, write_content_node, read_relation, +// write_relation, read_provenance) replaced by capnp_enum! + capnp_message! +// macro invocations above. Node::from_capnp/to_capnp and Relation::from_capnp/to_capnp +// are generated declaratively.