Add agent/tokenizer.rs with global Qwen 3.5 tokenizer that generates actual token IDs including chat template wrapping. ContextEntry now stores token_ids: Vec<u32> instead of tokens: usize — the count is derived from the length. ContextEntry::new() tokenizes automatically via the global tokenizer. ContextSection::push_entry() takes a raw ConversationEntry and tokenizes it. set_message() re-tokenizes without needing an external tokenizer parameter. Token IDs include the full chat template: <|im_start|>role\ncontent <|im_end|>\n — so concatenating token_ids across entries produces a ready-to-send prompt for vLLM's /v1/completions endpoint. The old tiktoken CoreBPE is now unused on Agent (will be removed in a followup). Token counts are now exact for Qwen 3.5 instead of the ~85-90% approximation from cl100k_base. Co-Authored-By: Proof of Concept <poc@bcachefs.org>
290 lines
10 KiB
Rust
290 lines
10 KiB
Rust
// widgets.rs — Shared TUI helpers and reusable components
|
|
|
|
use ratatui::{
|
|
layout::{Margin, Rect},
|
|
style::{Color, Modifier, Style},
|
|
text::Line,
|
|
widgets::{Block, Borders, Paragraph, Scrollbar, ScrollbarOrientation, ScrollbarState, Wrap},
|
|
Frame,
|
|
crossterm::event::KeyCode,
|
|
};
|
|
use crate::agent::context::{ContextSection, ConversationEntry};
|
|
|
|
/// UI-only tree node for the section tree display.
|
|
/// Built from ContextSection data; not used for budgeting.
|
|
#[derive(Debug, Clone)]
|
|
pub struct SectionView {
|
|
pub name: String,
|
|
pub tokens: usize,
|
|
pub content: String,
|
|
pub children: Vec<SectionView>,
|
|
/// Extra status text shown after the token count.
|
|
pub status: String,
|
|
}
|
|
|
|
/// Build a SectionView tree from a ContextSection.
|
|
/// Each entry becomes a child with label + expandable content.
|
|
pub fn section_to_view(section: &ContextSection) -> SectionView {
|
|
let children: Vec<SectionView> = section.entries().iter().map(|ce| {
|
|
let content = match &ce.entry {
|
|
ConversationEntry::Log(_) => String::new(),
|
|
ConversationEntry::Thinking(text) => text.clone(),
|
|
_ => ce.entry.message().content_text().to_string(),
|
|
};
|
|
SectionView {
|
|
name: ce.entry.label(),
|
|
tokens: ce.tokens(),
|
|
content,
|
|
children: Vec::new(),
|
|
status: String::new(),
|
|
}
|
|
}).collect();
|
|
SectionView {
|
|
name: section.name.clone(),
|
|
tokens: section.tokens(),
|
|
content: String::new(),
|
|
children,
|
|
status: String::new(),
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Standard pane block — cyan border, right-aligned title.
|
|
pub fn pane_block(title: &str) -> Block<'_> {
|
|
Block::default()
|
|
.title_top(Line::from(format!(" {} ", title)).right_aligned())
|
|
.borders(Borders::ALL)
|
|
.border_style(Style::default().fg(Color::Cyan))
|
|
}
|
|
|
|
/// Focused pane block — brighter border to indicate focus.
|
|
pub fn pane_block_focused(title: &str, focused: bool) -> Block<'_> {
|
|
let color = if focused { Color::White } else { Color::Cyan };
|
|
Block::default()
|
|
.title_top(Line::from(format!(" {} ", title)).right_aligned())
|
|
.borders(Borders::ALL)
|
|
.border_style(Style::default().fg(color))
|
|
}
|
|
|
|
/// Format a duration in seconds as a compact human-readable string.
|
|
pub fn format_age(secs: f64) -> String {
|
|
if secs < 60.0 { format!("{:.0}s", secs) }
|
|
else if secs < 3600.0 { format!("{:.0}m", secs / 60.0) }
|
|
else if secs < 86400.0 { format!("{:.1}h", secs / 3600.0) }
|
|
else { format!("{:.0}d", secs / 86400.0) }
|
|
}
|
|
|
|
/// Format a unix epoch timestamp as age from now.
|
|
pub fn format_ts_age(ts: i64) -> String {
|
|
let now = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_secs() as i64;
|
|
format_age((now - ts).max(0) as f64)
|
|
}
|
|
|
|
/// Key legend for SectionTree panes.
|
|
pub fn tree_legend() -> Line<'static> {
|
|
Line::styled(
|
|
" ↑↓:nav →/Enter:expand ←:collapse e:expand all c:collapse all PgUp/Dn Home/End ",
|
|
Style::default().fg(Color::DarkGray),
|
|
)
|
|
}
|
|
|
|
/// Render a paragraph with a vertical scrollbar.
|
|
pub fn render_scrollable(
|
|
frame: &mut Frame,
|
|
area: Rect,
|
|
lines: Vec<Line<'_>>,
|
|
block: Block<'_>,
|
|
scroll: u16,
|
|
) {
|
|
let content_len = lines.len();
|
|
let para = Paragraph::new(lines)
|
|
.block(block)
|
|
.wrap(Wrap { trim: false })
|
|
.scroll((scroll, 0));
|
|
frame.render_widget(para, area);
|
|
|
|
let visible = area.height.saturating_sub(2) as usize;
|
|
if content_len > visible {
|
|
let mut sb_state = ScrollbarState::new(content_len)
|
|
.position(scroll as usize);
|
|
frame.render_stateful_widget(
|
|
Scrollbar::new(ScrollbarOrientation::VerticalRight),
|
|
area.inner(Margin { vertical: 1, horizontal: 0 }),
|
|
&mut sb_state,
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// SectionTree — expand/collapse tree renderer for ContextSection
|
|
// ---------------------------------------------------------------------------
|
|
|
|
pub struct SectionTree {
|
|
pub selected: Option<usize>,
|
|
pub expanded: std::collections::HashSet<usize>,
|
|
pub scroll: u16,
|
|
}
|
|
|
|
impl SectionTree {
|
|
pub fn new() -> Self {
|
|
Self { selected: None, expanded: std::collections::HashSet::new(), scroll: 0 }
|
|
}
|
|
|
|
fn total_nodes(&self, sections: &[SectionView]) -> usize {
|
|
fn count_all(s: &SectionView) -> usize {
|
|
1 + s.children.iter().map(|c| count_all(c)).sum::<usize>()
|
|
}
|
|
sections.iter().map(|s| count_all(s)).sum()
|
|
}
|
|
|
|
pub fn item_count(&self, sections: &[SectionView]) -> usize {
|
|
fn count(section: &SectionView, expanded: &std::collections::HashSet<usize>, idx: &mut usize) -> usize {
|
|
let my_idx = *idx;
|
|
*idx += 1;
|
|
let mut total = 1;
|
|
if expanded.contains(&my_idx) {
|
|
for child in §ion.children {
|
|
total += count(child, expanded, idx);
|
|
}
|
|
}
|
|
total
|
|
}
|
|
let mut idx = 0;
|
|
sections.iter().map(|s| count(s, &self.expanded, &mut idx)).sum()
|
|
}
|
|
|
|
pub fn handle_nav(&mut self, code: KeyCode, sections: &[SectionView], height: u16) {
|
|
let item_count = self.item_count(sections);
|
|
let page = height.saturating_sub(2) as usize;
|
|
match code {
|
|
KeyCode::Up => {
|
|
self.selected = Some(self.selected.unwrap_or(0).saturating_sub(1));
|
|
}
|
|
KeyCode::Down => {
|
|
let max = item_count.saturating_sub(1);
|
|
self.selected = Some(self.selected.map_or(0, |s| (s + 1).min(max)));
|
|
}
|
|
KeyCode::PageUp => {
|
|
let sel = self.selected.unwrap_or(0);
|
|
self.selected = Some(sel.saturating_sub(page));
|
|
self.scroll = self.scroll.saturating_sub(page as u16);
|
|
return;
|
|
}
|
|
KeyCode::PageDown => {
|
|
let max = item_count.saturating_sub(1);
|
|
let sel = self.selected.map_or(0, |s| (s + page).min(max));
|
|
self.selected = Some(sel);
|
|
self.scroll += page as u16;
|
|
return;
|
|
}
|
|
KeyCode::Home => {
|
|
self.selected = Some(0);
|
|
}
|
|
KeyCode::End => {
|
|
self.selected = Some(item_count.saturating_sub(1));
|
|
}
|
|
KeyCode::Right | KeyCode::Enter => {
|
|
if let Some(sel) = self.selected {
|
|
self.expanded.insert(sel);
|
|
}
|
|
}
|
|
KeyCode::Left => {
|
|
if let Some(sel) = self.selected {
|
|
self.expanded.remove(&sel);
|
|
}
|
|
}
|
|
KeyCode::Char('e') => {
|
|
let total = self.total_nodes(sections);
|
|
for i in 0..total {
|
|
self.expanded.insert(i);
|
|
}
|
|
}
|
|
KeyCode::Char('c') => {
|
|
self.expanded.clear();
|
|
}
|
|
_ => {}
|
|
}
|
|
self.scroll_to_selected(height);
|
|
}
|
|
|
|
fn scroll_to_selected(&mut self, height: u16) {
|
|
if let Some(sel) = self.selected {
|
|
let sel_line = sel as u16;
|
|
let visible = height.saturating_sub(2);
|
|
if sel_line < self.scroll {
|
|
self.scroll = sel_line;
|
|
} else if sel_line >= self.scroll + visible {
|
|
self.scroll = sel_line.saturating_sub(visible.saturating_sub(1));
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn render_sections(&self, sections: &[SectionView], lines: &mut Vec<Line>) {
|
|
let mut idx = 0;
|
|
for section in sections {
|
|
self.render_one(section, 0, lines, &mut idx);
|
|
}
|
|
}
|
|
|
|
fn render_one(
|
|
&self,
|
|
section: &SectionView,
|
|
depth: usize,
|
|
lines: &mut Vec<Line>,
|
|
idx: &mut usize,
|
|
) {
|
|
let my_idx = *idx;
|
|
let selected = self.selected == Some(my_idx);
|
|
let expanded = self.expanded.contains(&my_idx);
|
|
let has_children = !section.children.is_empty();
|
|
let has_content = !section.content.is_empty();
|
|
let expandable = has_children || has_content;
|
|
|
|
let indent = " ".repeat(depth + 1);
|
|
let marker = if !expandable { " " } else if expanded { "▼" } else { "▶" };
|
|
let name_col = format!("{}{} {}", indent, marker, section.name);
|
|
let tokens_col = format!("{:>6} tokens", section.tokens);
|
|
let label = if section.status.is_empty() {
|
|
format!("{:40} {}", name_col, tokens_col)
|
|
} else {
|
|
format!("{:40} {:16} {}", name_col, tokens_col, section.status)
|
|
};
|
|
let style = if selected {
|
|
Style::default().fg(Color::Yellow).add_modifier(Modifier::BOLD)
|
|
} else {
|
|
Style::default()
|
|
};
|
|
lines.push(Line::styled(label, style));
|
|
*idx += 1;
|
|
|
|
if expanded {
|
|
if has_children {
|
|
for child in §ion.children {
|
|
self.render_one(child, depth + 1, lines, idx);
|
|
}
|
|
} else if has_content {
|
|
let content_indent = format!("{} │ ", " ".repeat(depth + 1));
|
|
let content_lines: Vec<&str> = section.content.lines().collect();
|
|
let show = content_lines.len().min(50);
|
|
for line in &content_lines[..show] {
|
|
lines.push(Line::styled(
|
|
format!("{}{}", content_indent, line),
|
|
Style::default().fg(Color::DarkGray),
|
|
));
|
|
}
|
|
if content_lines.len() > 50 {
|
|
lines.push(Line::styled(
|
|
format!("{}... ({} more lines)", content_indent, content_lines.len() - 50),
|
|
Style::default().fg(Color::DarkGray),
|
|
));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|