consciousness/src/user/widgets.rs
Kent Overstreet 5e4067c04f Replace token counting with token generation via HuggingFace tokenizer
Add agent/tokenizer.rs with global Qwen 3.5 tokenizer that generates
actual token IDs including chat template wrapping. ContextEntry now
stores token_ids: Vec<u32> instead of tokens: usize — the count is
derived from the length.

ContextEntry::new() tokenizes automatically via the global tokenizer.
ContextSection::push_entry() takes a raw ConversationEntry and
tokenizes it. set_message() re-tokenizes without needing an external
tokenizer parameter.

Token IDs include the full chat template: <|im_start|>role\ncontent
<|im_end|>\n — so concatenating token_ids across entries produces a
ready-to-send prompt for vLLM's /v1/completions endpoint.

The old tiktoken CoreBPE is now unused on Agent (will be removed in
a followup). Token counts are now exact for Qwen 3.5 instead of the
~85-90% approximation from cl100k_base.

Co-Authored-By: Proof of Concept <poc@bcachefs.org>
2026-04-08 11:20:03 -04:00

290 lines
10 KiB
Rust

// widgets.rs — Shared TUI helpers and reusable components
use ratatui::{
layout::{Margin, Rect},
style::{Color, Modifier, Style},
text::Line,
widgets::{Block, Borders, Paragraph, Scrollbar, ScrollbarOrientation, ScrollbarState, Wrap},
Frame,
crossterm::event::KeyCode,
};
use crate::agent::context::{ContextSection, ConversationEntry};
/// UI-only tree node for the section tree display.
/// Built from ContextSection data; not used for budgeting.
#[derive(Debug, Clone)]
pub struct SectionView {
pub name: String,
pub tokens: usize,
pub content: String,
pub children: Vec<SectionView>,
/// Extra status text shown after the token count.
pub status: String,
}
/// Build a SectionView tree from a ContextSection.
/// Each entry becomes a child with label + expandable content.
pub fn section_to_view(section: &ContextSection) -> SectionView {
let children: Vec<SectionView> = section.entries().iter().map(|ce| {
let content = match &ce.entry {
ConversationEntry::Log(_) => String::new(),
ConversationEntry::Thinking(text) => text.clone(),
_ => ce.entry.message().content_text().to_string(),
};
SectionView {
name: ce.entry.label(),
tokens: ce.tokens(),
content,
children: Vec::new(),
status: String::new(),
}
}).collect();
SectionView {
name: section.name.clone(),
tokens: section.tokens(),
content: String::new(),
children,
status: String::new(),
}
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
/// Standard pane block — cyan border, right-aligned title.
pub fn pane_block(title: &str) -> Block<'_> {
Block::default()
.title_top(Line::from(format!(" {} ", title)).right_aligned())
.borders(Borders::ALL)
.border_style(Style::default().fg(Color::Cyan))
}
/// Focused pane block — brighter border to indicate focus.
pub fn pane_block_focused(title: &str, focused: bool) -> Block<'_> {
let color = if focused { Color::White } else { Color::Cyan };
Block::default()
.title_top(Line::from(format!(" {} ", title)).right_aligned())
.borders(Borders::ALL)
.border_style(Style::default().fg(color))
}
/// Format a duration in seconds as a compact human-readable string.
pub fn format_age(secs: f64) -> String {
if secs < 60.0 { format!("{:.0}s", secs) }
else if secs < 3600.0 { format!("{:.0}m", secs / 60.0) }
else if secs < 86400.0 { format!("{:.1}h", secs / 3600.0) }
else { format!("{:.0}d", secs / 86400.0) }
}
/// Format a unix epoch timestamp as age from now.
pub fn format_ts_age(ts: i64) -> String {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs() as i64;
format_age((now - ts).max(0) as f64)
}
/// Key legend for SectionTree panes.
pub fn tree_legend() -> Line<'static> {
Line::styled(
" ↑↓:nav →/Enter:expand ←:collapse e:expand all c:collapse all PgUp/Dn Home/End ",
Style::default().fg(Color::DarkGray),
)
}
/// Render a paragraph with a vertical scrollbar.
pub fn render_scrollable(
frame: &mut Frame,
area: Rect,
lines: Vec<Line<'_>>,
block: Block<'_>,
scroll: u16,
) {
let content_len = lines.len();
let para = Paragraph::new(lines)
.block(block)
.wrap(Wrap { trim: false })
.scroll((scroll, 0));
frame.render_widget(para, area);
let visible = area.height.saturating_sub(2) as usize;
if content_len > visible {
let mut sb_state = ScrollbarState::new(content_len)
.position(scroll as usize);
frame.render_stateful_widget(
Scrollbar::new(ScrollbarOrientation::VerticalRight),
area.inner(Margin { vertical: 1, horizontal: 0 }),
&mut sb_state,
);
}
}
// ---------------------------------------------------------------------------
// SectionTree — expand/collapse tree renderer for ContextSection
// ---------------------------------------------------------------------------
pub struct SectionTree {
pub selected: Option<usize>,
pub expanded: std::collections::HashSet<usize>,
pub scroll: u16,
}
impl SectionTree {
pub fn new() -> Self {
Self { selected: None, expanded: std::collections::HashSet::new(), scroll: 0 }
}
fn total_nodes(&self, sections: &[SectionView]) -> usize {
fn count_all(s: &SectionView) -> usize {
1 + s.children.iter().map(|c| count_all(c)).sum::<usize>()
}
sections.iter().map(|s| count_all(s)).sum()
}
pub fn item_count(&self, sections: &[SectionView]) -> usize {
fn count(section: &SectionView, expanded: &std::collections::HashSet<usize>, idx: &mut usize) -> usize {
let my_idx = *idx;
*idx += 1;
let mut total = 1;
if expanded.contains(&my_idx) {
for child in &section.children {
total += count(child, expanded, idx);
}
}
total
}
let mut idx = 0;
sections.iter().map(|s| count(s, &self.expanded, &mut idx)).sum()
}
pub fn handle_nav(&mut self, code: KeyCode, sections: &[SectionView], height: u16) {
let item_count = self.item_count(sections);
let page = height.saturating_sub(2) as usize;
match code {
KeyCode::Up => {
self.selected = Some(self.selected.unwrap_or(0).saturating_sub(1));
}
KeyCode::Down => {
let max = item_count.saturating_sub(1);
self.selected = Some(self.selected.map_or(0, |s| (s + 1).min(max)));
}
KeyCode::PageUp => {
let sel = self.selected.unwrap_or(0);
self.selected = Some(sel.saturating_sub(page));
self.scroll = self.scroll.saturating_sub(page as u16);
return;
}
KeyCode::PageDown => {
let max = item_count.saturating_sub(1);
let sel = self.selected.map_or(0, |s| (s + page).min(max));
self.selected = Some(sel);
self.scroll += page as u16;
return;
}
KeyCode::Home => {
self.selected = Some(0);
}
KeyCode::End => {
self.selected = Some(item_count.saturating_sub(1));
}
KeyCode::Right | KeyCode::Enter => {
if let Some(sel) = self.selected {
self.expanded.insert(sel);
}
}
KeyCode::Left => {
if let Some(sel) = self.selected {
self.expanded.remove(&sel);
}
}
KeyCode::Char('e') => {
let total = self.total_nodes(sections);
for i in 0..total {
self.expanded.insert(i);
}
}
KeyCode::Char('c') => {
self.expanded.clear();
}
_ => {}
}
self.scroll_to_selected(height);
}
fn scroll_to_selected(&mut self, height: u16) {
if let Some(sel) = self.selected {
let sel_line = sel as u16;
let visible = height.saturating_sub(2);
if sel_line < self.scroll {
self.scroll = sel_line;
} else if sel_line >= self.scroll + visible {
self.scroll = sel_line.saturating_sub(visible.saturating_sub(1));
}
}
}
pub fn render_sections(&self, sections: &[SectionView], lines: &mut Vec<Line>) {
let mut idx = 0;
for section in sections {
self.render_one(section, 0, lines, &mut idx);
}
}
fn render_one(
&self,
section: &SectionView,
depth: usize,
lines: &mut Vec<Line>,
idx: &mut usize,
) {
let my_idx = *idx;
let selected = self.selected == Some(my_idx);
let expanded = self.expanded.contains(&my_idx);
let has_children = !section.children.is_empty();
let has_content = !section.content.is_empty();
let expandable = has_children || has_content;
let indent = " ".repeat(depth + 1);
let marker = if !expandable { " " } else if expanded { "" } else { "" };
let name_col = format!("{}{} {}", indent, marker, section.name);
let tokens_col = format!("{:>6} tokens", section.tokens);
let label = if section.status.is_empty() {
format!("{:40} {}", name_col, tokens_col)
} else {
format!("{:40} {:16} {}", name_col, tokens_col, section.status)
};
let style = if selected {
Style::default().fg(Color::Yellow).add_modifier(Modifier::BOLD)
} else {
Style::default()
};
lines.push(Line::styled(label, style));
*idx += 1;
if expanded {
if has_children {
for child in &section.children {
self.render_one(child, depth + 1, lines, idx);
}
} else if has_content {
let content_indent = format!("{}", " ".repeat(depth + 1));
let content_lines: Vec<&str> = section.content.lines().collect();
let show = content_lines.len().min(50);
for line in &content_lines[..show] {
lines.push(Line::styled(
format!("{}{}", content_indent, line),
Style::default().fg(Color::DarkGray),
));
}
if content_lines.len() > 50 {
lines.push(Line::styled(
format!("{}... ({} more lines)", content_indent, content_lines.len() - 50),
Style::default().fg(Color::DarkGray),
));
}
}
}
}
}