replace state.json cache with bincode state.bin
Faster serialization/deserialization, smaller on disk (4.2MB vs 5.9MB). Automatic migration from state.json on first load — reads the JSON, writes state.bin, deletes the old file. Added list-keys, list-edges, dump-json commands so Python scripts no longer need to parse the cache directly. Updated bulk-categorize.py and consolidation-loop.py to use the new CLI commands.
This commit is contained in:
parent
c4d1675128
commit
4b0bba7c56
6 changed files with 88 additions and 24 deletions
|
|
@ -34,7 +34,8 @@ fn memory_dir() -> PathBuf {
|
|||
|
||||
fn nodes_path() -> PathBuf { memory_dir().join("nodes.capnp") }
|
||||
fn relations_path() -> PathBuf { memory_dir().join("relations.capnp") }
|
||||
fn state_path() -> PathBuf { memory_dir().join("state.json") }
|
||||
fn state_path() -> PathBuf { memory_dir().join("state.bin") }
|
||||
fn state_json_path() -> PathBuf { memory_dir().join("state.json") }
|
||||
fn lock_path() -> PathBuf { memory_dir().join(".store.lock") }
|
||||
|
||||
/// RAII file lock using flock(2). Dropped when scope exits.
|
||||
|
|
@ -274,14 +275,27 @@ impl Store {
|
|||
};
|
||||
|
||||
if cache_fresh {
|
||||
let data = fs::read_to_string(&state)
|
||||
.map_err(|e| format!("read state.json: {}", e))?;
|
||||
let mut store: Store = serde_json::from_str(&data)
|
||||
.map_err(|e| format!("parse state.json: {}", e))?;
|
||||
let data = fs::read(&state)
|
||||
.map_err(|e| format!("read state.bin: {}", e))?;
|
||||
let mut store: Store = bincode::deserialize(&data)
|
||||
.map_err(|e| format!("parse state.bin: {}", e))?;
|
||||
store.rebuild_uuid_index();
|
||||
return Ok(store);
|
||||
}
|
||||
|
||||
// Try legacy JSON cache for migration
|
||||
let json_state = state_json_path();
|
||||
if json_state.exists() {
|
||||
let data = fs::read_to_string(&json_state)
|
||||
.map_err(|e| format!("read state.json: {}", e))?;
|
||||
if let Ok(mut store) = serde_json::from_str::<Store>(&data) {
|
||||
store.rebuild_uuid_index();
|
||||
// Migrate to bincode
|
||||
store.save()?;
|
||||
return Ok(store);
|
||||
}
|
||||
}
|
||||
|
||||
// Rebuild from capnp logs
|
||||
let mut store = Store::default();
|
||||
|
||||
|
|
@ -412,10 +426,16 @@ impl Store {
|
|||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).ok();
|
||||
}
|
||||
let json = serde_json::to_string(self)
|
||||
.map_err(|e| format!("serialize: {}", e))?;
|
||||
fs::write(&path, json)
|
||||
let data = bincode::serialize(self)
|
||||
.map_err(|e| format!("bincode serialize: {}", e))?;
|
||||
fs::write(&path, data)
|
||||
.map_err(|e| format!("write {}: {}", path.display(), e))?;
|
||||
|
||||
// Clean up old JSON cache if it exists
|
||||
let json_path = state_json_path();
|
||||
if json_path.exists() {
|
||||
fs::remove_file(&json_path).ok();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
35
src/main.rs
35
src/main.rs
|
|
@ -59,6 +59,9 @@ fn main() {
|
|||
"apply-agent" => cmd_apply_agent(&args[2..]),
|
||||
"digest" => cmd_digest(&args[2..]),
|
||||
"trace" => cmd_trace(&args[2..]),
|
||||
"list-keys" => cmd_list_keys(),
|
||||
"list-edges" => cmd_list_edges(),
|
||||
"dump-json" => cmd_dump_json(),
|
||||
_ => {
|
||||
eprintln!("Unknown command: {}", args[1]);
|
||||
usage();
|
||||
|
|
@ -103,7 +106,10 @@ Commands:
|
|||
apply-agent [--all] Import pending agent results into the graph
|
||||
digest daily [DATE] Generate daily episodic digest (default: today)
|
||||
digest weekly [DATE] Generate weekly digest (any date in target week)
|
||||
trace KEY Walk temporal links: semantic ↔ episodic ↔ conversation");
|
||||
trace KEY Walk temporal links: semantic ↔ episodic ↔ conversation
|
||||
list-keys List all node keys (one per line)
|
||||
list-edges List all edges (tsv: source target strength type)
|
||||
dump-json Dump entire store as JSON");
|
||||
}
|
||||
|
||||
fn cmd_search(args: &[String]) -> Result<(), String> {
|
||||
|
|
@ -738,6 +744,33 @@ fn cmd_trace(args: &[String]) -> Result<(), String> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_list_keys() -> Result<(), String> {
|
||||
let store = capnp_store::Store::load()?;
|
||||
let mut keys: Vec<_> = store.nodes.keys().collect();
|
||||
keys.sort();
|
||||
for key in keys {
|
||||
println!("{}", key);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_list_edges() -> Result<(), String> {
|
||||
let store = capnp_store::Store::load()?;
|
||||
for rel in &store.relations {
|
||||
println!("{}\t{}\t{:.2}\t{:?}",
|
||||
rel.source_key, rel.target_key, rel.strength, rel.rel_type);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_dump_json() -> Result<(), String> {
|
||||
let store = capnp_store::Store::load()?;
|
||||
let json = serde_json::to_string_pretty(&store)
|
||||
.map_err(|e| format!("serialize: {}", e))?;
|
||||
println!("{}", json);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_interference(args: &[String]) -> Result<(), String> {
|
||||
let mut threshold = 0.4f32;
|
||||
let mut i = 0;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue