Add ConversationEntry::Thinking — 0 tokens, not sent to API

Thinking/reasoning content is now a first-class entry type:
- Serialized as {"thinking": "..."} in conversation log
- 0 tokens for budgeting (doesn't count against context window)
- Filtered from assemble_api_messages (not sent back to model)
- Displayed in UI with "thinking: ..." label and expandable content

Co-Authored-By: Proof of Concept <poc@bcachefs.org>
This commit is contained in:
Kent Overstreet 2026-04-07 22:46:06 -04:00
parent 7c5fddcb19
commit e0ee441aec
3 changed files with 32 additions and 13 deletions

View file

@ -22,6 +22,8 @@ pub enum ConversationEntry {
Memory { key: String, message: Message, score: Option<f64> }, Memory { key: String, message: Message, score: Option<f64> },
/// DMN heartbeat/autonomous prompt — evicted aggressively during compaction. /// DMN heartbeat/autonomous prompt — evicted aggressively during compaction.
Dmn(Message), Dmn(Message),
/// Model thinking/reasoning — not sent to the API, 0 tokens for budgeting.
Thinking(String),
/// Debug/status log line — written to conversation log for tracing, /// Debug/status log line — written to conversation log for tracing,
/// skipped on read-back. /// skipped on read-back.
Log(String), Log(String),
@ -291,8 +293,12 @@ impl Serialize for ConversationEntry {
} }
map.end() map.end()
} }
Self::Thinking(text) => {
let mut map = s.serialize_map(Some(1))?;
map.serialize_entry("thinking", text)?;
map.end()
}
Self::Log(text) => { Self::Log(text) => {
use serde::ser::SerializeMap;
let mut map = s.serialize_map(Some(1))?; let mut map = s.serialize_map(Some(1))?;
map.serialize_entry("log", text)?; map.serialize_entry("log", text)?;
map.end() map.end()
@ -304,7 +310,10 @@ impl Serialize for ConversationEntry {
impl<'de> Deserialize<'de> for ConversationEntry { impl<'de> Deserialize<'de> for ConversationEntry {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> { fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let mut json: serde_json::Value = serde_json::Value::deserialize(d)?; let mut json: serde_json::Value = serde_json::Value::deserialize(d)?;
// Log entries — skip on read-back if json.get("thinking").is_some() {
let text = json["thinking"].as_str().unwrap_or("").to_string();
return Ok(Self::Thinking(text));
}
if json.get("log").is_some() { if json.get("log").is_some() {
let text = json["log"].as_str().unwrap_or("").to_string(); let text = json["log"].as_str().unwrap_or("").to_string();
return Ok(Self::Log(text)); return Ok(Self::Log(text));
@ -330,10 +339,14 @@ impl ConversationEntry {
match self { match self {
Self::System(m) | Self::Message(m) | Self::Dmn(m) => m, Self::System(m) | Self::Message(m) | Self::Dmn(m) => m,
Self::Memory { message, .. } => message, Self::Memory { message, .. } => message,
Self::Log(_) => panic!("Log entries have no API message"), Self::Thinking(_) | Self::Log(_) => panic!("Thinking/Log entries have no API message"),
} }
} }
pub fn is_thinking(&self) -> bool {
matches!(self, Self::Thinking(_))
}
pub fn is_memory(&self) -> bool { pub fn is_memory(&self) -> bool {
matches!(self, Self::Memory { .. }) matches!(self, Self::Memory { .. })
} }
@ -352,6 +365,12 @@ impl ConversationEntry {
match self { match self {
Self::System(_) => "system: [system prompt]".to_string(), Self::System(_) => "system: [system prompt]".to_string(),
Self::Dmn(_) => "dmn: [heartbeat]".to_string(), Self::Dmn(_) => "dmn: [heartbeat]".to_string(),
Self::Thinking(text) => {
let preview: String = text.chars().take(60).collect();
let preview = preview.replace('\n', " ");
if text.len() > 60 { format!("thinking: {}...", preview) }
else { format!("thinking: {}", preview) }
}
Self::Log(text) => { Self::Log(text) => {
let preview: String = text.chars().take(60).collect(); let preview: String = text.chars().take(60).collect();
format!("log: {}", preview.replace('\n', " ")) format!("log: {}", preview.replace('\n', " "))
@ -390,17 +409,17 @@ impl ConversationEntry {
match self { match self {
Self::System(m) | Self::Message(m) | Self::Dmn(m) => m, Self::System(m) | Self::Message(m) | Self::Dmn(m) => m,
Self::Memory { message, .. } => message, Self::Memory { message, .. } => message,
Self::Log(_) => panic!("Log entries have no message"), Self::Thinking(_) | Self::Log(_) => panic!("Thinking/Log entries have no message"),
} }
} }
/// Get a mutable reference to the inner message. /// Get a mutable reference to the inner message.
/// Panics on Log entries. /// Panics on Thinking/Log entries.
pub fn message_mut(&mut self) -> &mut Message { pub fn message_mut(&mut self) -> &mut Message {
match self { match self {
Self::System(m) | Self::Message(m) | Self::Dmn(m) => m, Self::System(m) | Self::Message(m) | Self::Dmn(m) => m,
Self::Memory { message, .. } => message, Self::Memory { message, .. } => message,
Self::Log(_) => panic!("Log entries have no message"), Self::Thinking(_) | Self::Log(_) => panic!("Thinking/Log entries have no message"),
} }
} }
} }

View file

@ -305,7 +305,7 @@ impl Agent {
} }
// Conversation entries // Conversation entries
msgs.extend(self.context.conversation.entries().iter() msgs.extend(self.context.conversation.entries().iter()
.filter(|e| !e.entry.is_log()) .filter(|e| !e.entry.is_log() && !e.entry.is_thinking())
.map(|e| e.entry.api_message().clone())); .map(|e| e.entry.api_message().clone()));
msgs msgs
} }
@ -324,7 +324,7 @@ impl Agent {
eprintln!("warning: failed to log entry: {:#}", e); eprintln!("warning: failed to log entry: {:#}", e);
} }
} }
let tokens = if entry.is_log() { 0 } else { let tokens = if entry.is_log() || entry.is_thinking() { 0 } else {
context::msg_token_count(&self.tokenizer, entry.api_message()) context::msg_token_count(&self.tokenizer, entry.api_message())
}; };
self.context.conversation.push(ContextEntry { self.context.conversation.push(ContextEntry {

View file

@ -8,7 +8,7 @@ use ratatui::{
Frame, Frame,
crossterm::event::KeyCode, crossterm::event::KeyCode,
}; };
use crate::agent::context::ContextSection; use crate::agent::context::{ContextSection, ConversationEntry};
/// UI-only tree node for the section tree display. /// UI-only tree node for the section tree display.
/// Built from ContextSection data; not used for budgeting. /// Built from ContextSection data; not used for budgeting.
@ -26,10 +26,10 @@ pub struct SectionView {
/// Each entry becomes a child with label + expandable content. /// Each entry becomes a child with label + expandable content.
pub fn section_to_view(section: &ContextSection) -> SectionView { pub fn section_to_view(section: &ContextSection) -> SectionView {
let children: Vec<SectionView> = section.entries().iter().map(|ce| { let children: Vec<SectionView> = section.entries().iter().map(|ce| {
let content = if ce.entry.is_log() { let content = match &ce.entry {
String::new() ConversationEntry::Log(_) => String::new(),
} else { ConversationEntry::Thinking(text) => text.clone(),
ce.entry.message().content_text().to_string() _ => ce.entry.message().content_text().to_string(),
}; };
SectionView { SectionView {
name: ce.entry.label(), name: ce.entry.label(),