Compare commits
No commits in common. "d2c0ef61a1e5bf1a589d3415465181ca12193cb0" and "bf503b571eca2f063f36e393c34c703b0a6525f8" have entirely different histories.
d2c0ef61a1
...
bf503b571e
4 changed files with 52 additions and 43 deletions
|
|
@ -527,9 +527,12 @@ impl Mind {
|
||||||
}
|
}
|
||||||
|
|
||||||
cmds.push(MindCommand::Compact);
|
cmds.push(MindCommand::Compact);
|
||||||
|
/*
|
||||||
|
* Broken since the AST context window conversion:
|
||||||
if !self.config.no_agents {
|
if !self.config.no_agents {
|
||||||
cmds.push(MindCommand::Score);
|
cmds.push(MindCommand::Score);
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
_ = tokio::time::sleep(timeout), if !has_input => dmn_expired = true,
|
_ = tokio::time::sleep(timeout), if !has_input => dmn_expired = true,
|
||||||
|
|
|
||||||
|
|
@ -48,25 +48,41 @@ fn is_assistant(node: &AstNode) -> bool {
|
||||||
matches!(node, AstNode::Branch { role: Role::Assistant, .. })
|
matches!(node, AstNode::Branch { role: Role::Assistant, .. })
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build a token ID array for a scoring call.
|
/// Push an AstNode as one or more JSON messages for the scoring API.
|
||||||
|
fn push_api_message(node: &AstNode, msgs: &mut Vec<serde_json::Value>) {
|
||||||
|
match node {
|
||||||
|
AstNode::Branch { role, children } => {
|
||||||
|
let content: String = children.iter().map(|c| c.render()).collect();
|
||||||
|
msgs.push(serde_json::json!({
|
||||||
|
"role": role.as_str(),
|
||||||
|
"content": content,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
AstNode::Leaf(leaf) => {
|
||||||
|
let role = match leaf.body() {
|
||||||
|
NodeBody::ToolResult(_) => "tool",
|
||||||
|
_ => "user",
|
||||||
|
};
|
||||||
|
msgs.push(serde_json::json!({
|
||||||
|
"role": role,
|
||||||
|
"content": leaf.body().text(),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the messages array for a scoring call.
|
||||||
///
|
///
|
||||||
/// Includes all sections up to and including conversation entries in
|
/// Always includes system prompt as prefix, then entries from `range`
|
||||||
/// `range`, with `filter` applied to conversation entries.
|
/// filtered by `filter`.
|
||||||
fn build_token_ids(
|
fn build_messages(
|
||||||
context: &ContextState,
|
context: &ContextState,
|
||||||
range: std::ops::Range<usize>,
|
range: std::ops::Range<usize>,
|
||||||
filter: Filter,
|
filter: Filter,
|
||||||
) -> Vec<u32> {
|
) -> Vec<serde_json::Value> {
|
||||||
use crate::agent::context::Ast;
|
let mut msgs = Vec::new();
|
||||||
let mut ids = Vec::new();
|
|
||||||
for node in context.system() {
|
for node in context.system() {
|
||||||
ids.extend(node.token_ids());
|
push_api_message(node, &mut msgs);
|
||||||
}
|
|
||||||
for node in context.identity() {
|
|
||||||
ids.extend(node.token_ids());
|
|
||||||
}
|
|
||||||
for node in context.journal() {
|
|
||||||
ids.extend(node.token_ids());
|
|
||||||
}
|
}
|
||||||
let entries = context.conversation();
|
let entries = context.conversation();
|
||||||
for i in range {
|
for i in range {
|
||||||
|
|
@ -78,9 +94,9 @@ fn build_token_ids(
|
||||||
Filter::SkipAllMemories => is_memory(node),
|
Filter::SkipAllMemories => is_memory(node),
|
||||||
};
|
};
|
||||||
if skip { continue; }
|
if skip { continue; }
|
||||||
ids.extend(node.token_ids());
|
push_api_message(node, &mut msgs);
|
||||||
}
|
}
|
||||||
ids
|
msgs
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Score API ───────────────────────────────────────────────────
|
// ── Score API ───────────────────────────────────────────────────
|
||||||
|
|
@ -104,19 +120,15 @@ fn http_client() -> crate::agent::api::http::HttpClient {
|
||||||
async fn call_score(
|
async fn call_score(
|
||||||
http: &crate::agent::api::http::HttpClient,
|
http: &crate::agent::api::http::HttpClient,
|
||||||
client: &ApiClient,
|
client: &ApiClient,
|
||||||
prompt: &[u32],
|
messages: &[serde_json::Value],
|
||||||
priority: Option<i32>,
|
|
||||||
) -> anyhow::Result<Vec<ScoreResult>> {
|
) -> anyhow::Result<Vec<ScoreResult>> {
|
||||||
let url = format!("{}/score", client.base_url());
|
let url = format!("{}/score", client.base_url());
|
||||||
let auth = format!("Bearer {}", client.api_key());
|
let auth = format!("Bearer {}", client.api_key());
|
||||||
let mut body = serde_json::json!({
|
let body = serde_json::json!({
|
||||||
"model": client.model,
|
"model": client.model,
|
||||||
"prompt": prompt,
|
"messages": messages,
|
||||||
"logprobs": 1,
|
"logprobs": 1,
|
||||||
});
|
});
|
||||||
if let Some(p) = priority {
|
|
||||||
body["priority"] = serde_json::json!(p);
|
|
||||||
}
|
|
||||||
let response = http
|
let response = http
|
||||||
.send_json("POST", &url, &[
|
.send_json("POST", &url, &[
|
||||||
("authorization", &auth),
|
("authorization", &auth),
|
||||||
|
|
@ -157,10 +169,9 @@ async fn score_divergence(
|
||||||
context: &ContextState,
|
context: &ContextState,
|
||||||
range: std::ops::Range<usize>,
|
range: std::ops::Range<usize>,
|
||||||
filter: Filter<'_>,
|
filter: Filter<'_>,
|
||||||
priority: Option<i32>,
|
|
||||||
) -> anyhow::Result<(Vec<f64>, Vec<ScoreResult>)> {
|
) -> anyhow::Result<(Vec<f64>, Vec<ScoreResult>)> {
|
||||||
let baseline = call_score(http, client, &build_token_ids(context, range.clone(), Filter::None), priority).await?;
|
let baseline = call_score(http, client, &build_messages(context, range.clone(), Filter::None)).await?;
|
||||||
let without = call_score(http, client, &build_token_ids(context, range, filter), priority).await?;
|
let without = call_score(http, client, &build_messages(context, range, filter)).await?;
|
||||||
let divs = divergence(&baseline, &without);
|
let divs = divergence(&baseline, &without);
|
||||||
Ok((divs, baseline))
|
Ok((divs, baseline))
|
||||||
}
|
}
|
||||||
|
|
@ -221,7 +232,7 @@ pub async fn score_memories(
|
||||||
let http = http_client();
|
let http = http_client();
|
||||||
let range = 0..context.conversation().len();
|
let range = 0..context.conversation().len();
|
||||||
|
|
||||||
let baseline = call_score(&http, client, &build_token_ids(context, range.clone(), Filter::None), Some(5)).await?;
|
let baseline = call_score(&http, client, &build_messages(context, range.clone(), Filter::None)).await?;
|
||||||
|
|
||||||
let total = memory_keys.len();
|
let total = memory_keys.len();
|
||||||
let mut matrix: Vec<Vec<f64>> = Vec::new();
|
let mut matrix: Vec<Vec<f64>> = Vec::new();
|
||||||
|
|
@ -230,8 +241,8 @@ pub async fn score_memories(
|
||||||
dbglog!(
|
dbglog!(
|
||||||
"scoring {}/{}: {}...", mem_idx + 1, total, key,
|
"scoring {}/{}: {}...", mem_idx + 1, total, key,
|
||||||
);
|
);
|
||||||
let msgs = build_token_ids(context, range.clone(), Filter::SkipKey(key));
|
let msgs = build_messages(context, range.clone(), Filter::SkipKey(key));
|
||||||
match call_score(&http, client, &msgs, Some(5)).await {
|
match call_score(&http, client, &msgs).await {
|
||||||
Ok(without) => matrix.push(divergence(&baseline, &without)),
|
Ok(without) => matrix.push(divergence(&baseline, &without)),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
dbglog!(
|
dbglog!(
|
||||||
|
|
@ -301,7 +312,7 @@ pub async fn score_memory(
|
||||||
}
|
}
|
||||||
|
|
||||||
let http = http_client();
|
let http = http_client();
|
||||||
let (divs, _) = score_divergence(&http, client, context, range, Filter::SkipKey(key), Some(5)).await?;
|
let (divs, _) = score_divergence(&http, client, context, range, Filter::SkipKey(key)).await?;
|
||||||
|
|
||||||
Ok(divs.iter().sum())
|
Ok(divs.iter().sum())
|
||||||
}
|
}
|
||||||
|
|
@ -365,25 +376,20 @@ where
|
||||||
cumulative.push(running);
|
cumulative.push(running);
|
||||||
}
|
}
|
||||||
|
|
||||||
dbglog!("[scoring] total_tokens={}, cutoff={}, {} candidates", total_tokens, token_cutoff, candidates.len());
|
|
||||||
|
|
||||||
for (pos, key, _) in &candidates {
|
for (pos, key, _) in &candidates {
|
||||||
// Only score memories in the first 60% of the conversation by tokens —
|
// Only score memories in the first 70% of the conversation by tokens —
|
||||||
// recent memories don't have enough responses to evaluate yet.
|
// recent memories don't have enough responses to evaluate yet.
|
||||||
let cum = cumulative.get(*pos).copied().unwrap_or(total_tokens);
|
if cumulative.get(*pos).copied().unwrap_or(total_tokens) > token_cutoff {
|
||||||
if cum > token_cutoff {
|
|
||||||
dbglog!("[scoring] skip {} (tokens {}/{} past cutoff)", key, cum, token_cutoff);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let (end, _) = nth_response_end(context.conversation(), *pos, response_window);
|
let (end, _) = nth_response_end(context.conversation(), *pos, response_window);
|
||||||
let range = *pos..end;
|
let range = *pos..end;
|
||||||
if !context.conversation()[range.clone()].iter().any(|node| is_assistant(node)) {
|
if !context.conversation()[range.clone()].iter().any(|node| is_assistant(node)) {
|
||||||
dbglog!("[scoring] skip {} (no assistant response in range {}..{})", key, pos, end);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let _scoring = crate::agent::start_activity(agent, format!("scoring: {}", key)).await;
|
let _scoring = crate::agent::start_activity(agent, format!("scoring: {}", key)).await;
|
||||||
match score_divergence(&http, client, context, range, Filter::SkipKey(key), Some(5)).await {
|
match score_divergence(&http, client, context, range, Filter::SkipKey(key)).await {
|
||||||
Ok((divs, _)) => {
|
Ok((divs, _)) => {
|
||||||
let n_responses = divs.len();
|
let n_responses = divs.len();
|
||||||
let max_div = divs.iter().cloned().fold(0.0f64, f64::max);
|
let max_div = divs.iter().cloned().fold(0.0f64, f64::max);
|
||||||
|
|
@ -429,7 +435,7 @@ pub async fn score_finetune(
|
||||||
}
|
}
|
||||||
|
|
||||||
let http = http_client();
|
let http = http_client();
|
||||||
let (divs, _) = score_divergence(&http, client, context, range, Filter::SkipAllMemories, Some(5)).await?;
|
let (divs, _) = score_divergence(&http, client, context, range, Filter::SkipAllMemories).await?;
|
||||||
|
|
||||||
let mut results: Vec<(usize, f64)> = response_positions.iter()
|
let mut results: Vec<(usize, f64)> = response_positions.iter()
|
||||||
.enumerate()
|
.enumerate()
|
||||||
|
|
|
||||||
|
|
@ -110,8 +110,8 @@ impl ScreenView for ConsciousScreen {
|
||||||
|
|
||||||
self.tree.render_sections(&context_state, &mut lines);
|
self.tree.render_sections(&context_state, &mut lines);
|
||||||
|
|
||||||
lines.push(Line::raw(format!(" {:53} {:>6} tokens", "────────", "──────")));
|
lines.push(Line::raw(format!(" {:23} {:>6} tokens", "────────", "──────")));
|
||||||
lines.push(Line::raw(format!(" {:53} {:>6} tokens", "Total", total)));
|
lines.push(Line::raw(format!(" {:23} {:>6} tokens", "Total", total)));
|
||||||
} else if let Some(ref info) = app.context_info {
|
} else if let Some(ref info) = app.context_info {
|
||||||
lines.push(Line::raw(format!(" System prompt: {:>6} chars", info.system_prompt_chars)));
|
lines.push(Line::raw(format!(" System prompt: {:>6} chars", info.system_prompt_chars)));
|
||||||
lines.push(Line::raw(format!(" Context message: {:>6} chars", info.context_message_chars)));
|
lines.push(Line::raw(format!(" Context message: {:>6} chars", info.context_message_chars)));
|
||||||
|
|
|
||||||
|
|
@ -259,9 +259,9 @@ impl SectionTree {
|
||||||
let name_col = format!("{}{} {}", indent, marker, section.name);
|
let name_col = format!("{}{} {}", indent, marker, section.name);
|
||||||
let tokens_col = format!("{:>6} tokens", section.tokens);
|
let tokens_col = format!("{:>6} tokens", section.tokens);
|
||||||
let label = if section.status.is_empty() {
|
let label = if section.status.is_empty() {
|
||||||
format!("{:70} {}", name_col, tokens_col)
|
format!("{:40} {}", name_col, tokens_col)
|
||||||
} else {
|
} else {
|
||||||
format!("{:70} {:16} {}", name_col, tokens_col, section.status)
|
format!("{:40} {:16} {}", name_col, tokens_col, section.status)
|
||||||
};
|
};
|
||||||
let style = if selected {
|
let style = if selected {
|
||||||
Style::default().fg(Color::Yellow).add_modifier(Modifier::BOLD)
|
Style::default().fg(Color::Yellow).add_modifier(Modifier::BOLD)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue