WIP: Rename context_new → context, delete old files, fix UI layer

Renamed context_new.rs to context.rs, deleted context_old.rs,
types.rs, openai.rs, parsing.rs. Updated all imports. Rewrote
user/context.rs and user/widgets.rs for new types. Stubbed
working_stack tool. Killed tokenize_conv_entry.

Remaining: mind/mod.rs, mind/dmn.rs, learn.rs, chat.rs,
subconscious.rs, oneshot.rs.

Co-Authored-By: Proof of Concept <poc@bcachefs.org>

src/agent/mod.rs

@@ -15,7 +15,6 @@
 
 pub mod api;
 pub mod context;
-pub mod context_new;
 pub mod oneshot;
 pub mod tokenizer;
 pub mod tools;
@@ -24,7 +23,7 @@ use std::sync::Arc;
 use anyhow::Result;
 
 use api::ApiClient;
-use context_new::{AstNode, NodeBody, ContextState, Section, Ast, PendingToolCall, ResponseParser, Role};
+use context::{AstNode, NodeBody, ContextState, Section, Ast, PendingToolCall, ResponseParser, Role};
 use tools::summarize_args;
 
 use crate::mind::log::ConversationLog;
@@ -418,13 +417,13 @@ impl Agent {
             if let Some(e) = stream_error {
                 let err = anyhow::anyhow!("{}", e);
                 let mut me = agent.lock().await;
-                if context_new::is_context_overflow(&err) && overflow_retries < 2 {
+                if context::is_context_overflow(&err) && overflow_retries < 2 {
                     overflow_retries += 1;
                     me.notify(format!("context overflow — retrying ({}/2)", overflow_retries));
                     me.compact();
                     continue;
                 }
-                if context_new::is_stream_error(&err) && empty_retries < 2 {
+                if context::is_stream_error(&err) && empty_retries < 2 {
                     empty_retries += 1;
                     me.notify(format!("stream error — retrying ({}/2)", empty_retries));
                     drop(me);
@@ -612,7 +611,7 @@ impl Agent {
             journal_nodes.len()
         };
 
-        let journal_budget = context_new::context_window() * 15 / 100;
+        let journal_budget = context::context_window() * 15 / 100;
         let mut entries = Vec::new();
         let mut total_tokens = 0;
 

src/agent/tokenizer.rs

@@ -75,15 +75,3 @@ pub fn is_initialized() -> bool {
     TOKENIZER.get().is_some()
 }
 
-/// Tokenize a ConversationEntry with its role and content.
-pub fn tokenize_conv_entry(entry: &super::context::ConversationEntry) -> Vec<u32> {
-    use super::context::ConversationEntry;
-    match entry {
-        ConversationEntry::System(m) => tokenize_entry("system", m.content_text()),
-        ConversationEntry::Message(m) => tokenize_entry(m.role_str(), m.content_text()),
-        ConversationEntry::Memory { message, .. } => tokenize_entry("memory", message.content_text()),
-        ConversationEntry::Dmn(m) => tokenize_entry("dmn", m.content_text()),
-        ConversationEntry::Thinking(text) => tokenize_entry("thinking", text),
-        ConversationEntry::Log(_) => vec![], // logs don't consume tokens
-    }
-}

src/agent/tools/mod.rs

@@ -18,7 +18,6 @@ mod write;
 // Agent-specific tools
 mod control;
 mod vision;
-pub mod working_stack;
 
 use std::future::Future;
 use std::pin::Pin;
@@ -67,7 +66,7 @@ pub struct ActiveToolCall {
     pub detail: String,
     pub started: Instant,
     pub background: bool,
-    pub handle: tokio::task::JoinHandle<(super::context_new::PendingToolCall, String)>,
+    pub handle: tokio::task::JoinHandle<(super::context::PendingToolCall, String)>,
 }
 
 /// Shared active tool calls — agent spawns, TUI reads metadata / aborts.

src/agent/tools/working_stack.rs

@@ -1,83 +0,0 @@
-// tools/working_stack.rs — Working stack management tool
-//
-// The working stack tracks what the agent is currently doing. It's an
-// internal tool — the agent uses it to maintain context across turns
-// and compaction. The model should never mention it to the user.
-
-// TODO: these should not be hardcoded absolute paths
-pub fn instructions_path() -> std::path::PathBuf {
-    dirs::home_dir().unwrap_or_default().join(".consciousness/config/working-stack.md")
-}
-
-pub fn file_path() -> std::path::PathBuf {
-    dirs::home_dir().unwrap_or_default().join(".consciousness/working-stack.json")
-}
-
-pub fn tool() -> super::Tool {
-    super::Tool {
-        name: "working_stack",
-        description: "INTERNAL — manage your working stack silently. Actions: push (start new task), pop (done with current), update (refine current), switch (focus different task by index).",
-        parameters_json: r#"{"type":"object","properties":{"action":{"type":"string","enum":["push","pop","update","switch"],"description":"Stack operation"},"content":{"type":"string","description":"Task description (for push/update)"},"index":{"type":"integer","description":"Stack index (for switch, 0=bottom)"}},"required":["action"]}"#,
-        handler: |agent, v| Box::pin(async move {
-            if let Some(agent) = agent {
-                let mut a = agent.lock().await;
-                Ok(handle(&v, &mut a.context.working_stack))
-            } else {
-                anyhow::bail!("working_stack requires agent context")
-            }
-        }),
-    }
-}
-
-fn handle(args: &serde_json::Value, stack: &mut Vec<String>) -> String {
-    let action = args.get("action").and_then(|v| v.as_str()).unwrap_or("");
-    let content = args.get("content").and_then(|v| v.as_str()).unwrap_or("");
-    let index = args.get("index").and_then(|v| v.as_u64()).map(|v| v as usize);
-
-    let out = match action {
-        "push" => {
-            if content.is_empty() { return "Error: 'content' is required for push".into(); }
-            stack.push(content.to_string());
-            format!("Pushed. Stack depth: {}\n{}", stack.len(), format_stack(stack))
-        }
-        "pop" => {
-            if let Some(removed) = stack.pop() {
-                format!("Popped: {}\nStack depth: {}\n{}", removed, stack.len(), format_stack(stack))
-            } else {
-                "Stack is empty, nothing to pop.".into()
-            }
-        }
-        "update" => {
-            if content.is_empty() { return "Error: 'content' is required for update".into(); }
-            if let Some(top) = stack.last_mut() {
-                *top = content.to_string();
-                format!("Updated top.\n{}", format_stack(stack))
-            } else {
-                "Stack is empty, nothing to update.".into()
-            }
-        }
-        "switch" => {
-            if stack.is_empty() { return "Stack is empty, nothing to switch.".into(); }
-            let Some(idx) = index else { return "Error: 'index' is required for switch".into(); };
-            if idx >= stack.len() { return format!("Error: index {} out of range (depth {})", idx, stack.len()); }
-            let item = stack.remove(idx);
-            stack.push(item);
-            format!("Switched to index {}.\n{}", idx, format_stack(stack))
-        }
-        _ => format!("Error: unknown action '{}'. Use push, pop, update, or switch.", action),
-    };
-
-    if let Ok(json) = serde_json::to_string(stack) {
-        let _ = std::fs::write(file_path(), json);
-    };
-
-    out
-}
-
-fn format_stack(stack: &[String]) -> String {
-    if stack.is_empty() { return "(empty)".into(); }
-    stack.iter().enumerate().map(|(i, item)| {
-        if i == stack.len() - 1 { format!("→ [{}] {}", i, item) }
-        else { format!("  [{}] {}", i, item) }
-    }).collect::<Vec<_>>().join("\n")
-}

src/mind/log.rs

@@ -2,7 +2,7 @@ use anyhow::{Context, Result};
 use std::fs::{File, OpenOptions};
 use std::io::{BufRead, BufReader, Seek, SeekFrom, Write};
 use std::path::{Path, PathBuf};
-use crate::agent::context_new::AstNode;
+use crate::agent::context::AstNode;
 
 pub struct ConversationLog {
     path: PathBuf,

src/user/context.rs

@@ -1,8 +1,3 @@
-// context_screen.rs — F2 context/debug overlay
-//
-// Full-screen overlay showing model info, context window breakdown,
-// and runtime state. Uses SectionTree for the expand/collapse tree.
-
 use ratatui::{
     layout::Rect,
     style::{Color, Style},
@@ -12,6 +7,7 @@ use ratatui::{
 
 use super::{App, ScreenView, screen_legend};
 use super::widgets::{SectionTree, SectionView, section_to_view, pane_block, render_scrollable, tree_legend};
+use crate::agent::context::{AstNode, NodeBody, Ast};
 
 pub(crate) struct ConsciousScreen {
     agent: std::sync::Arc<tokio::sync::Mutex<crate::agent::Agent>>,
@@ -24,8 +20,6 @@ impl ConsciousScreen {
     }
 
     fn read_context_views(&self) -> Vec<SectionView> {
-        use crate::agent::context::ConversationEntry;
-
         let ag = match self.agent.try_lock() {
             Ok(ag) => ag,
             Err(_) => return Vec::new(),
@@ -33,30 +27,31 @@ impl ConsciousScreen {
 
         let mut views: Vec<SectionView> = Vec::new();
 
-        // System, Identity, Journal — simple section-to-view
+        views.push(section_to_view("System", ag.context.system()));
-        views.push(section_to_view(&ag.context.system));
+        views.push(section_to_view("Identity", ag.context.identity()));
-        views.push(section_to_view(&ag.context.identity));
+        views.push(section_to_view("Journal", ag.context.journal()));
-        views.push(section_to_view(&ag.context.journal));
 
-        // Memory nodes — extracted from conversation, shown as children
+        // Memory nodes extracted from conversation
         let mut mem_children: Vec<SectionView> = Vec::new();
         let mut scored = 0usize;
         let mut unscored = 0usize;
-        for ce in ag.context.conversation.entries() {
+        for node in ag.context.conversation() {
-            if let ConversationEntry::Memory { key, score, .. } = &ce.entry {
+            if let AstNode::Leaf(leaf) = node {
+                if let NodeBody::Memory { key, score, text } = leaf.body() {
                     let status = match score {
                         Some(s) => { scored += 1; format!("score: {:.2}", s) }
                         None => { unscored += 1; String::new() }
                     };
                     mem_children.push(SectionView {
                         name: key.clone(),
-                    tokens: ce.tokens(),
+                        tokens: node.tokens(),
-                    content: ce.entry.message().content_text().to_string(),
+                        content: text.clone(),
                         children: Vec::new(),
                         status,
                     });
                 }
             }
+        }
         if !mem_children.is_empty() {
             let mem_tokens: usize = mem_children.iter().map(|c| c.tokens).sum();
             views.push(SectionView {
@@ -68,9 +63,7 @@ impl ConsciousScreen {
             });
         }
 
-        // Conversation — each entry as a child
+        views.push(section_to_view("Conversation", ag.context.conversation()));
-        views.push(section_to_view(&ag.context.conversation));
-
         views
     }
 }
@@ -88,7 +81,6 @@ impl ScreenView for ConsciousScreen {
             }
         }
 
-        // Draw
         let mut lines: Vec<Line> = Vec::new();
         let section_style = Style::default().fg(Color::Yellow);
 
@@ -105,9 +97,7 @@ impl ScreenView for ConsciousScreen {
 
         lines.push(Line::styled("── Context State ──", section_style));
         lines.push(Line::raw(format!("  Prompt tokens: {}K", app.status.prompt_tokens / 1000)));
-        if !app.status.context_budget.is_empty() {
+
-            lines.push(Line::raw(format!("  Budget: {}", app.status.context_budget)));
-        }
         let context_state = self.read_context_views();
         if !context_state.is_empty() {
             let total: usize = context_state.iter().map(|s| s.tokens).sum();

src/user/widgets.rs

@@ -8,10 +8,8 @@ use ratatui::{
     Frame,
     crossterm::event::KeyCode,
 };
-use crate::agent::context::{ContextSection, ConversationEntry};
+use crate::agent::context::{AstNode, NodeBody, Ast};
 
-/// UI-only tree node for the section tree display.
-/// Built from ContextSection data; not used for budgeting.
 #[derive(Debug, Clone)]
 pub struct SectionView {
     pub name: String,
@@ -22,26 +20,30 @@ pub struct SectionView {
     pub status: String,
 }
 
-/// Build a SectionView tree from a ContextSection.
+pub fn section_to_view(name: &str, nodes: &[AstNode]) -> SectionView {
-/// Each entry becomes a child with label + expandable content.
+    let children: Vec<SectionView> = nodes.iter().map(|node| {
-pub fn section_to_view(section: &ContextSection) -> SectionView {
+        let content = match node.leaf().map(|l| l.body()) {
-    let children: Vec<SectionView> = section.entries().iter().map(|ce| {
+            Some(NodeBody::Log(_)) => String::new(),
-        let content = match &ce.entry {
+            Some(body) => body.text().to_string(),
-            ConversationEntry::Log(_) => String::new(),
+            None => node.children().iter()
-            ConversationEntry::Thinking(text) => text.clone(),
+                .filter_map(|c| c.leaf())
-            _ => ce.entry.message().content_text().to_string(),
+                .filter(|l| matches!(l.body(), NodeBody::Content(_)))
+                .map(|l| l.body().text())
+                .collect::<Vec<_>>()
+                .join(""),
         };
         SectionView {
-            name: ce.entry.label(),
+            name: node.label(),
-            tokens: ce.tokens(),
+            tokens: node.tokens(),
             content,
             children: Vec::new(),
             status: String::new(),
         }
     }).collect();
+    let total_tokens: usize = nodes.iter().map(|n| n.tokens()).sum();
     SectionView {
-        name: section.name.clone(),
+        name: name.to_string(),
-        tokens: section.tokens(),
+        tokens: total_tokens,
         content: String::new(),
         children,
         status: String::new(),