From 5526a26d4c5cbf696def2adfa7b1b7dd4e26bbb4 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 2 Apr 2026 02:21:45 -0400 Subject: [PATCH] Journal: store as structured Vec, not String MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep journal entries as structured data in ContextState. Render to text only when building the context message. Debug screen reads the structured entries directly — no parsing ## headers back out. Compaction paths temporarily parse the string from build_context_window back to entries (to be cleaned up when compaction is reworked). Co-Authored-By: Proof of Concept --- src/agent/journal.rs | 2 +- src/agent/runner.rs | 98 ++++++++++++++++++++++---------------------- src/agent/types.rs | 2 +- 3 files changed, 52 insertions(+), 50 deletions(-) diff --git a/src/agent/journal.rs b/src/agent/journal.rs index ff0824b..73437f1 100644 --- a/src/agent/journal.rs +++ b/src/agent/journal.rs @@ -66,7 +66,7 @@ pub fn parse_journal_tail(path: &Path, max_bytes: u64) -> Vec { } /// Parse journal entries from text (separated for testing). -fn parse_journal_text(text: &str) -> Vec { +pub fn parse_journal_text(text: &str) -> Vec { let mut entries = Vec::new(); let mut current_timestamp: Option> = None; let mut current_content = String::new(); diff --git a/src/agent/runner.rs b/src/agent/runner.rs index 8823b95..fd027ea 100644 --- a/src/agent/runner.rs +++ b/src/agent/runner.rs @@ -78,6 +78,15 @@ pub struct Agent { pub agent_cycles: crate::subconscious::subconscious::AgentCycleState, } +fn render_journal(entries: &[journal::JournalEntry]) -> String { + let mut text = String::from("[Earlier — from your journal]\n\n"); + for entry in entries { + use std::fmt::Write; + writeln!(text, "## {}\n{}\n", entry.timestamp.format("%Y-%m-%dT%H:%M"), entry.content).ok(); + } + text +} + impl Agent { pub fn new( client: ApiClient, @@ -93,7 +102,7 @@ impl Agent { let context = ContextState { system_prompt: system_prompt.clone(), personality, - journal: String::new(), + journal: Vec::new(), working_stack: Vec::new(), loaded_nodes: Vec::new(), }; @@ -125,7 +134,7 @@ impl Agent { agent.push_context(Message::user(rendered)); } if !agent.context.journal.is_empty() { - agent.push_context(Message::user(agent.context.journal.clone())); + agent.push_context(Message::user(render_journal(&agent.context.journal))); } agent.measure_budget(); agent.publish_context_state(); @@ -638,41 +647,21 @@ impl Agent { children: personality_children, }); - // Journal — split into per-entry children + // Journal { - let mut journal_children = Vec::new(); - let mut current_header = String::new(); - let mut current_body = String::new(); - for line in self.context.journal.lines() { - if line.starts_with("## ") { - if !current_header.is_empty() { - let body = std::mem::take(&mut current_body); - let preview: String = body.lines().next().unwrap_or("").chars().take(60).collect(); - journal_children.push(ContextSection { - name: format!("{}: {}", current_header, preview), - tokens: count(&body), - content: body, - children: Vec::new(), - }); + let journal_children: Vec = self.context.journal.iter() + .map(|entry| { + let preview: String = entry.content.lines() + .find(|l| !l.trim().is_empty()) + .unwrap_or("").chars().take(60).collect(); + ContextSection { + name: format!("{}: {}", entry.timestamp.format("%Y-%m-%dT%H:%M"), preview), + tokens: count(&entry.content), + content: entry.content.clone(), + children: Vec::new(), } - current_header = line.trim_start_matches("## ").to_string(); - current_body.clear(); - } else { - if !current_body.is_empty() || !line.is_empty() { - current_body.push_str(line); - current_body.push('\n'); - } - } - } - if !current_header.is_empty() { - let preview: String = current_body.lines().next().unwrap_or("").chars().take(60).collect(); - journal_children.push(ContextSection { - name: format!("{}: {}", current_header, preview), - tokens: count(¤t_body), - content: current_body, - children: Vec::new(), - }); - } + }) + .collect(); let journal_tokens: usize = journal_children.iter().map(|c| c.tokens).sum(); sections.push(ContextSection { name: format!("Journal ({} entries)", journal_children.len()), @@ -798,16 +787,31 @@ impl Agent { let mut journal_nodes: Vec<_> = store.nodes.values() .filter(|n| n.node_type == crate::store::NodeType::EpisodicSession) .collect(); + let mut dbg = std::fs::OpenOptions::new().create(true).append(true) + .open("/tmp/poc-journal-debug.log").ok(); + macro_rules! dbg_log { + ($($arg:tt)*) => { + if let Some(ref mut f) = dbg { use std::io::Write; let _ = writeln!(f, $($arg)*); } + } + } + dbg_log!("[journal] {} nodes, oldest_msg={:?}", journal_nodes.len(), oldest_msg_ts); + journal_nodes.sort_by_key(|n| n.created_at); + if let Some(first) = journal_nodes.first() { + dbg_log!("[journal] first created_at={}", first.created_at); + } + if let Some(last) = journal_nodes.last() { + dbg_log!("[journal] last created_at={}", last.created_at); + } // Find the cutoff index — entries older than conversation, plus one overlap let cutoff_idx = if let Some(cutoff) = oldest_msg_ts { + let cutoff_ts = cutoff.timestamp(); + dbg_log!("[journal] cutoff timestamp={}", cutoff_ts); let mut idx = journal_nodes.len(); for (i, node) in journal_nodes.iter().enumerate() { - let ts = chrono::DateTime::from_timestamp(node.created_at, 0) - .unwrap_or_default(); - if ts >= cutoff { - idx = i + 1; // include this overlapping entry + if node.created_at >= cutoff_ts { + idx = i + 1; break; } } @@ -815,11 +819,13 @@ impl Agent { } else { journal_nodes.len() }; + dbg_log!("[journal] cutoff_idx={}", cutoff_idx); // Walk backwards from cutoff, accumulating entries within 5% of context let count = |s: &str| self.tokenizer.encode_with_special_tokens(s).len(); let context_window = crate::agent::context::model_context_window(&self.client.model); let journal_budget = context_window * 5 / 100; + dbg_log!("[journal] budget={} tokens ({}*5%)", journal_budget, context_window); let mut entries = Vec::new(); let mut total_tokens = 0; @@ -837,18 +843,14 @@ impl Agent { total_tokens += tokens; } entries.reverse(); + dbg_log!("[journal] loaded {} entries, {} tokens", entries.len(), total_tokens); if entries.is_empty() { + dbg_log!("[journal] no entries!"); return; } - // Render directly — no plan_context needed - let mut text = String::from("[Earlier — from your journal]\n\n"); - for entry in &entries { - use std::fmt::Write; - writeln!(text, "## {}\n{}\n", entry.timestamp.format("%Y-%m-%dT%H:%M"), entry.content).ok(); - } - self.context.journal = text; + self.context.journal = entries; } /// Re-render the context message in self.messages from live ContextState. @@ -1014,7 +1016,7 @@ impl Agent { &self.client.model, &self.tokenizer, ); - self.context.journal = journal; + self.context.journal = journal::parse_journal_text(&journal); self.messages = messages; self.last_prompt_tokens = 0; self.measure_budget(); @@ -1076,7 +1078,7 @@ impl Agent { ); dbglog!("[restore] journal text: {} chars, {} lines", journal.len(), journal.lines().count()); - self.context.journal = journal; + self.context.journal = journal::parse_journal_text(&journal); self.messages = messages; dbglog!("[restore] built context window: {} messages", self.messages.len()); self.last_prompt_tokens = 0; diff --git a/src/agent/types.rs b/src/agent/types.rs index ad1df93..09384c3 100644 --- a/src/agent/types.rs +++ b/src/agent/types.rs @@ -326,7 +326,7 @@ impl ToolDef { pub struct ContextState { pub system_prompt: String, pub personality: Vec<(String, String)>, - pub journal: String, + pub journal: Vec, pub working_stack: Vec, /// Memory nodes currently loaded in the context window. /// Tracked so the agent knows what it's "seeing" and can