Add ConversationEntry::Thinking — 0 tokens, not sent to API

Thinking/reasoning content is now a first-class entry type:
- Serialized as {"thinking": "..."} in conversation log
- 0 tokens for budgeting (doesn't count against context window)
- Filtered from assemble_api_messages (not sent back to model)
- Displayed in UI with "thinking: ..." label and expandable content

Co-Authored-By: Proof of Concept <poc@bcachefs.org>
This commit is contained in:
Kent Overstreet 2026-04-07 22:46:06 -04:00
parent 7c5fddcb19
commit e0ee441aec
3 changed files with 32 additions and 13 deletions

View file

@ -22,6 +22,8 @@ pub enum ConversationEntry {
Memory { key: String, message: Message, score: Option<f64> },
/// DMN heartbeat/autonomous prompt — evicted aggressively during compaction.
Dmn(Message),
/// Model thinking/reasoning — not sent to the API, 0 tokens for budgeting.
Thinking(String),
/// Debug/status log line — written to conversation log for tracing,
/// skipped on read-back.
Log(String),
@ -291,8 +293,12 @@ impl Serialize for ConversationEntry {
}
map.end()
}
Self::Thinking(text) => {
let mut map = s.serialize_map(Some(1))?;
map.serialize_entry("thinking", text)?;
map.end()
}
Self::Log(text) => {
use serde::ser::SerializeMap;
let mut map = s.serialize_map(Some(1))?;
map.serialize_entry("log", text)?;
map.end()
@ -304,7 +310,10 @@ impl Serialize for ConversationEntry {
impl<'de> Deserialize<'de> for ConversationEntry {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let mut json: serde_json::Value = serde_json::Value::deserialize(d)?;
// Log entries — skip on read-back
if json.get("thinking").is_some() {
let text = json["thinking"].as_str().unwrap_or("").to_string();
return Ok(Self::Thinking(text));
}
if json.get("log").is_some() {
let text = json["log"].as_str().unwrap_or("").to_string();
return Ok(Self::Log(text));
@ -330,10 +339,14 @@ impl ConversationEntry {
match self {
Self::System(m) | Self::Message(m) | Self::Dmn(m) => m,
Self::Memory { message, .. } => message,
Self::Log(_) => panic!("Log entries have no API message"),
Self::Thinking(_) | Self::Log(_) => panic!("Thinking/Log entries have no API message"),
}
}
pub fn is_thinking(&self) -> bool {
matches!(self, Self::Thinking(_))
}
pub fn is_memory(&self) -> bool {
matches!(self, Self::Memory { .. })
}
@ -352,6 +365,12 @@ impl ConversationEntry {
match self {
Self::System(_) => "system: [system prompt]".to_string(),
Self::Dmn(_) => "dmn: [heartbeat]".to_string(),
Self::Thinking(text) => {
let preview: String = text.chars().take(60).collect();
let preview = preview.replace('\n', " ");
if text.len() > 60 { format!("thinking: {}...", preview) }
else { format!("thinking: {}", preview) }
}
Self::Log(text) => {
let preview: String = text.chars().take(60).collect();
format!("log: {}", preview.replace('\n', " "))
@ -390,17 +409,17 @@ impl ConversationEntry {
match self {
Self::System(m) | Self::Message(m) | Self::Dmn(m) => m,
Self::Memory { message, .. } => message,
Self::Log(_) => panic!("Log entries have no message"),
Self::Thinking(_) | Self::Log(_) => panic!("Thinking/Log entries have no message"),
}
}
/// Get a mutable reference to the inner message.
/// Panics on Log entries.
/// Panics on Thinking/Log entries.
pub fn message_mut(&mut self) -> &mut Message {
match self {
Self::System(m) | Self::Message(m) | Self::Dmn(m) => m,
Self::Memory { message, .. } => message,
Self::Log(_) => panic!("Log entries have no message"),
Self::Thinking(_) | Self::Log(_) => panic!("Thinking/Log entries have no message"),
}
}
}