Restore full N×M memory scoring matrix (/score command)
The full matrix scorer was deleted during the AST conversion. Restore it: /score runs score_memories() which computes divergence for every memory × response pair, stores the MemoryScore on MindState, and displays per-memory weights with bar charts on the F2 screen. Both scoring paths now use ActivityGuard::update() for live progress in the status bar instead of creating a new activity per iteration. Also bumps score API timeout from 120s to 300s and adds progress logging throughout. Co-Authored-By: Proof of Concept <poc@bcachefs.org> Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
This commit is contained in:
ProofOfConcept
Kent Overstreet
parent
f6a6c37435
commit
58cec97e57
6 changed files with 187 additions and 98 deletions
|
|
@ -17,7 +17,7 @@
|
|||
use crate::agent::api::ApiClient;
|
||||
use crate::agent::context::{AstNode, Ast, NodeBody, ContextState, Role};
|
||||
|
||||
const SCORE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120);
|
||||
const SCORE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300);
|
||||
|
||||
// ── Message building ────────────────────────────────────────────
|
||||
|
||||
|
|
@ -167,98 +167,92 @@ async fn score_divergence(
|
|||
|
||||
// ── Full matrix scoring (debug screen) ──────────────────────────
|
||||
|
||||
/// Result of scoring one conversation's memory usage.
|
||||
pub struct MemoryScore {
|
||||
pub memory_weights: Vec<(String, f64)>,
|
||||
pub response_scores: Vec<f64>,
|
||||
/// Full matrix: divergence[memory_idx][response_idx]
|
||||
pub matrix: Vec<Vec<f64>>,
|
||||
pub memory_keys: Vec<String>,
|
||||
pub response_entry_indices: Vec<usize>,
|
||||
}
|
||||
|
||||
impl MemoryScore {
|
||||
pub fn important_memories_for_entry(&self, entry_idx: usize) -> Vec<(&str, f64)> {
|
||||
let Some(resp_idx) = self.response_entry_indices.iter().position(|&i| i == entry_idx)
|
||||
else { return Vec::new() };
|
||||
|
||||
let mut result: Vec<(&str, f64)> = self.memory_keys.iter()
|
||||
.zip(self.matrix.iter())
|
||||
.filter_map(|(key, row)| {
|
||||
let score = row.get(resp_idx).copied().unwrap_or(0.0);
|
||||
if score > 0.01 { Some((key.as_str(), score)) } else { None }
|
||||
})
|
||||
.collect();
|
||||
result.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/// Score how important each memory is to the conversation (full matrix).
|
||||
pub async fn score_memories(
|
||||
context: &ContextState,
|
||||
client: &ApiClient,
|
||||
) -> anyhow::Result<MemoryScore> {
|
||||
let mut memory_keys: Vec<String> = context.conversation().iter()
|
||||
.filter_map(|node| memory_key(node).map(String::from))
|
||||
.collect();
|
||||
memory_keys.dedup();
|
||||
|
||||
let response_indices: Vec<usize> = context.conversation().iter().enumerate()
|
||||
.filter(|(_, node)| is_assistant(node))
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
agent: &std::sync::Arc<crate::agent::Agent>,
|
||||
) -> anyhow::Result<()> {
|
||||
// Collect memory keys and response indices under a brief lock
|
||||
let (memory_keys, response_indices) = {
|
||||
let ctx = agent.context.lock().await;
|
||||
let mut keys: Vec<String> = ctx.conversation().iter()
|
||||
.filter_map(|node| memory_key(node).map(String::from))
|
||||
.collect();
|
||||
keys.dedup();
|
||||
let resp: Vec<usize> = ctx.conversation().iter().enumerate()
|
||||
.filter(|(_, node)| is_assistant(node))
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
(keys, resp)
|
||||
};
|
||||
|
||||
if memory_keys.is_empty() || response_indices.is_empty() {
|
||||
return Ok(MemoryScore {
|
||||
memory_weights: Vec::new(), response_scores: Vec::new(),
|
||||
matrix: Vec::new(), memory_keys: Vec::new(),
|
||||
response_entry_indices: Vec::new(),
|
||||
});
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
||||
let http = http_client();
|
||||
let range = 0..context.conversation().len();
|
||||
|
||||
let baseline = call_score(&http, client, &build_token_ids(context, range.clone(), Filter::None), Some(5)).await?;
|
||||
|
||||
let total = memory_keys.len();
|
||||
let mut matrix: Vec<Vec<f64>> = Vec::new();
|
||||
dbglog!("[scoring-full] starting: {} memories × {} responses",
|
||||
total, response_indices.len());
|
||||
|
||||
let http = http_client();
|
||||
|
||||
let activity = crate::agent::start_activity(agent, "scoring: baseline").await;
|
||||
let baseline_tokens = {
|
||||
let ctx = agent.context.lock().await;
|
||||
build_token_ids(&ctx, 0..ctx.conversation().len(), Filter::None)
|
||||
};
|
||||
let baseline = call_score(&http, client, &baseline_tokens, Some(5)).await?;
|
||||
dbglog!("[scoring-full] baseline done ({} response scores)", baseline.len());
|
||||
|
||||
for (mem_idx, key) in memory_keys.iter().enumerate() {
|
||||
dbglog!(
|
||||
"scoring {}/{}: {}...", mem_idx + 1, total, key,
|
||||
);
|
||||
let msgs = build_token_ids(context, range.clone(), Filter::SkipKey(key));
|
||||
match call_score(&http, client, &msgs, Some(5)).await {
|
||||
Ok(without) => matrix.push(divergence(&baseline, &without)),
|
||||
Err(e) => {
|
||||
dbglog!(
|
||||
"[training] {} FAILED: {:#}", key, e,
|
||||
);
|
||||
matrix.push(vec![0.0; baseline.len()]);
|
||||
activity.update(format!("scoring: {}/{}", mem_idx + 1, total)).await;
|
||||
dbglog!("[scoring-full] {}/{}: {}", mem_idx + 1, total, key);
|
||||
let tokens = {
|
||||
let ctx = agent.context.lock().await;
|
||||
build_token_ids(&ctx, 0..ctx.conversation().len(), Filter::SkipKey(key))
|
||||
};
|
||||
let row = match call_score(&http, client, &tokens, Some(5)).await {
|
||||
Ok(without) => {
|
||||
let divs = divergence(&baseline, &without);
|
||||
let max_div = divs.iter().cloned().fold(0.0f64, f64::max);
|
||||
dbglog!("[scoring-full] {}/{}: {} max_div={:.3}",
|
||||
mem_idx + 1, total, key, max_div);
|
||||
divs
|
||||
}
|
||||
Err(e) => {
|
||||
dbglog!("[scoring-full] {}/{}: {} FAILED: {:#}",
|
||||
mem_idx + 1, total, key, e);
|
||||
vec![0.0; baseline.len()]
|
||||
}
|
||||
};
|
||||
// Write this memory's scores to the live AST nodes
|
||||
{
|
||||
let mut ctx = agent.context.lock().await;
|
||||
let mut set_count = 0;
|
||||
|
||||
for (resp_idx, &idx) in response_indices.iter().enumerate() {
|
||||
if idx >= ctx.conversation().len() { continue; }
|
||||
let node = &mut ctx.conversation_mut()[idx];
|
||||
if let AstNode::Branch {
|
||||
role: Role::Assistant, memory_scores, ..
|
||||
} = node {
|
||||
if let Some(&score) = row.get(resp_idx) {
|
||||
if score > 0.01 {
|
||||
memory_scores.insert(key.clone(), score);
|
||||
set_count += 1;
|
||||
} else {
|
||||
memory_scores.remove(key.as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dbglog!("[scoring-full] {}/{} AST: set={}", mem_idx + 1, total, set_count);
|
||||
}
|
||||
agent.state.lock().await.changed.notify_one();
|
||||
}
|
||||
|
||||
|
||||
let memory_weights: Vec<(String, f64)> = memory_keys.iter()
|
||||
.zip(matrix.iter())
|
||||
.map(|(key, row)| (key.clone(), row.iter().sum()))
|
||||
.collect();
|
||||
|
||||
let mut response_scores = vec![0.0; response_indices.len()];
|
||||
for row in &matrix {
|
||||
for (j, &v) in row.iter().enumerate() {
|
||||
if j < response_scores.len() { response_scores[j] += v; }
|
||||
}
|
||||
}
|
||||
|
||||
Ok(MemoryScore {
|
||||
memory_weights, response_scores, matrix, memory_keys,
|
||||
response_entry_indices: response_indices,
|
||||
})
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Find the entry index after `start` that contains the Nth assistant response.
|
||||
|
|
@ -365,7 +359,9 @@ where
|
|||
cumulative.push(running);
|
||||
}
|
||||
|
||||
dbglog!("[scoring] total_tokens={}, cutoff={}, {} candidates", total_tokens, token_cutoff, candidates.len());
|
||||
let total = candidates.len();
|
||||
dbglog!("[scoring] total_tokens={}, cutoff={}, {} candidates", total_tokens, token_cutoff, total);
|
||||
let activity = crate::agent::start_activity(agent, format!("scoring: 0/{}", total)).await;
|
||||
|
||||
for (pos, key, _) in &candidates {
|
||||
// Only score memories in the first 60% of the conversation by tokens —
|
||||
|
|
@ -382,7 +378,7 @@ where
|
|||
continue;
|
||||
}
|
||||
|
||||
let _scoring = crate::agent::start_activity(agent, format!("scoring: {}", key)).await;
|
||||
activity.update(format!("scoring: {}/{} {}", scored + 1, total, key)).await;
|
||||
match score_divergence(&http, client, context, range, Filter::SkipKey(key), Some(5)).await {
|
||||
Ok((divs, _)) => {
|
||||
let n_responses = divs.len();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue