Remove dead action pipeline: parsing, depth tracking, knowledge loop, fact miner
Agents now apply changes via tool calls (poc-memory write/link-add/etc) during the LLM call. The old pipeline — where agents output WRITE_NODE/ LINK/REFINE text, which was parsed and applied separately — is dead code. Removed: - Action/ActionKind/Confidence types and all parse_* functions - DepthDb, depth tracking, confidence gating - apply_action, stamp_content, has_edge - NamingResolution, resolve_naming and related naming agent code - KnowledgeLoopConfig, CycleResult, GraphMetrics, convergence checking - run_knowledge_loop, run_cycle, check_convergence - apply_consolidation (old report re-processing) - fact_mine.rs (folded into observation agent) - resolve_action_names Simplified: - AgentResult no longer carries actions/no_ops - run_and_apply_with_log just runs the agent - consolidate_full simplified action tracking -1364 lines.
This commit is contained in:
ProofOfConcept
parent
b709d58a4f
commit
6932e05b38
7 changed files with 43 additions and 1364 deletions
|
|
@ -1,16 +1,12 @@
|
|||
// Consolidation pipeline: plan → agents → apply → digests → links
|
||||
// Consolidation pipeline: plan → agents → maintenance → digests → links
|
||||
//
|
||||
// consolidate_full() runs the full autonomous consolidation:
|
||||
// 1. Plan: analyze metrics, allocate agents
|
||||
// 2. Execute: run each agent, parse + apply actions inline
|
||||
// 2. Execute: run each agent (agents apply changes via tool calls)
|
||||
// 3. Graph maintenance (orphans, degree cap)
|
||||
// 4. Digest: generate missing daily/weekly/monthly digests
|
||||
// 5. Links: apply links extracted from digests
|
||||
// 6. Summary: final metrics comparison
|
||||
//
|
||||
// Actions are parsed directly from agent output using the same parser
|
||||
// as the knowledge loop (WRITE_NODE, LINK, REFINE), eliminating the
|
||||
// second LLM call that was previously needed.
|
||||
|
||||
use super::digest;
|
||||
use super::knowledge;
|
||||
|
|
@ -25,7 +21,6 @@ fn log_line(buf: &mut String, line: &str) {
|
|||
}
|
||||
|
||||
/// Run the full autonomous consolidation pipeline with logging.
|
||||
/// If `on_progress` is provided, it's called at each significant step.
|
||||
pub fn consolidate_full(store: &mut Store) -> Result<(), String> {
|
||||
consolidate_full_with_progress(store, &|_| {})
|
||||
}
|
||||
|
|
@ -60,8 +55,6 @@ pub fn consolidate_full_with_progress(
|
|||
log_line(&mut log_buf, "\n--- Step 2: Execute agents ---");
|
||||
let mut agent_num = 0usize;
|
||||
let mut agent_errors = 0usize;
|
||||
let mut total_applied = 0usize;
|
||||
let mut total_actions = 0usize;
|
||||
|
||||
let batch_size = 5;
|
||||
let runs = plan.to_agent_runs(batch_size);
|
||||
|
|
@ -83,27 +76,24 @@ pub fn consolidate_full_with_progress(
|
|||
*store = Store::load()?;
|
||||
}
|
||||
|
||||
let (total, applied) = match knowledge::run_and_apply(store, agent_type, *count, "consolidate") {
|
||||
Ok(r) => r,
|
||||
match knowledge::run_and_apply(store, agent_type, *count, "consolidate") {
|
||||
Ok(()) => {
|
||||
let msg = format!(" Done");
|
||||
log_line(&mut log_buf, &msg);
|
||||
on_progress(&msg);
|
||||
println!("{}", msg);
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!(" ERROR: {}", e);
|
||||
log_line(&mut log_buf, &msg);
|
||||
eprintln!("{}", msg);
|
||||
agent_errors += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
total_actions += total;
|
||||
total_applied += applied;
|
||||
|
||||
let msg = format!(" Done: {} actions ({} applied)", total, applied);
|
||||
log_line(&mut log_buf, &msg);
|
||||
on_progress(&msg);
|
||||
println!("{}", msg);
|
||||
}
|
||||
}
|
||||
|
||||
log_line(&mut log_buf, &format!("\nAgents complete: {} run, {} errors, {} actions ({} applied)",
|
||||
agent_num - agent_errors, agent_errors, total_actions, total_applied));
|
||||
log_line(&mut log_buf, &format!("\nAgents complete: {} run, {} errors",
|
||||
agent_num - agent_errors, agent_errors));
|
||||
store.save()?;
|
||||
|
||||
// --- Step 3: Link orphans ---
|
||||
|
|
@ -183,76 +173,3 @@ pub fn consolidate_full_with_progress(
|
|||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Re-parse and apply actions from stored consolidation reports.
|
||||
/// This is for manually re-processing reports — during normal consolidation,
|
||||
/// actions are applied inline as each agent runs.
|
||||
pub fn apply_consolidation(store: &mut Store, do_apply: bool, report_key: Option<&str>) -> Result<(), String> {
|
||||
let reports: Vec<String> = if let Some(key) = report_key {
|
||||
vec![key.to_string()]
|
||||
} else {
|
||||
// Find the most recent batch of reports
|
||||
let mut keys: Vec<&String> = store.nodes.keys()
|
||||
.filter(|k| k.starts_with("_consolidation-") && !k.contains("-actions-") && !k.contains("-log-"))
|
||||
.collect();
|
||||
keys.sort();
|
||||
keys.reverse();
|
||||
|
||||
if keys.is_empty() { return Ok(()); }
|
||||
|
||||
let latest_ts = keys[0].rsplit('-').next().unwrap_or("").to_string();
|
||||
keys.into_iter()
|
||||
.filter(|k| k.ends_with(&latest_ts))
|
||||
.cloned()
|
||||
.collect()
|
||||
};
|
||||
|
||||
if reports.is_empty() {
|
||||
println!("No consolidation reports found.");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
println!("Found {} reports:", reports.len());
|
||||
let mut all_actions = Vec::new();
|
||||
for key in &reports {
|
||||
let content = store.nodes.get(key).map(|n| n.content.as_str()).unwrap_or("");
|
||||
let actions = knowledge::parse_all_actions(content);
|
||||
println!(" {} → {} actions", key, actions.len());
|
||||
all_actions.extend(actions);
|
||||
}
|
||||
|
||||
if !do_apply {
|
||||
println!("\nDRY RUN — {} actions parsed", all_actions.len());
|
||||
for action in &all_actions {
|
||||
match &action.kind {
|
||||
knowledge::ActionKind::Link { source, target } =>
|
||||
println!(" LINK {} → {}", source, target),
|
||||
knowledge::ActionKind::WriteNode { key, .. } =>
|
||||
println!(" WRITE {}", key),
|
||||
knowledge::ActionKind::Refine { key, .. } =>
|
||||
println!(" REFINE {}", key),
|
||||
knowledge::ActionKind::Demote { key } =>
|
||||
println!(" DEMOTE {}", key),
|
||||
knowledge::ActionKind::Delete { key } =>
|
||||
println!(" DELETE {}", key),
|
||||
}
|
||||
}
|
||||
println!("\nTo apply: poc-memory apply-consolidation --apply");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let ts = store::compact_timestamp();
|
||||
let mut applied = 0;
|
||||
for action in &all_actions {
|
||||
if knowledge::apply_action(store, action, "consolidate", &ts, 0) {
|
||||
applied += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if applied > 0 {
|
||||
store.save()?;
|
||||
}
|
||||
|
||||
println!("Applied: {}/{} actions", applied, all_actions.len());
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,8 +59,8 @@ fn job_consolidation_agent(
|
|||
ctx.log_line(msg);
|
||||
log_event(&job_name2, "progress", msg);
|
||||
};
|
||||
let (total, applied) = super::knowledge::run_and_apply_with_log(&mut store, &agent, batch, "consolidate", &log)?;
|
||||
ctx.log_line(&format!("done: {} actions ({} applied)", total, applied));
|
||||
super::knowledge::run_and_apply_with_log(&mut store, &agent, batch, "consolidate", &log)?;
|
||||
ctx.log_line("done");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
|
@ -377,20 +377,10 @@ fn job_digest_links(ctx: &ExecutionContext) -> Result<(), TaskError> {
|
|||
})
|
||||
}
|
||||
|
||||
fn job_knowledge_loop(ctx: &ExecutionContext) -> Result<(), TaskError> {
|
||||
run_job(ctx, "knowledge-loop", || {
|
||||
let config = super::knowledge::KnowledgeLoopConfig {
|
||||
max_cycles: 100,
|
||||
batch_size: 5,
|
||||
..Default::default()
|
||||
};
|
||||
ctx.log_line("running agents");
|
||||
let results = super::knowledge::run_knowledge_loop(&config)?;
|
||||
ctx.log_line(format!("{} cycles, {} actions",
|
||||
results.len(),
|
||||
results.iter().map(|r| r.total_applied).sum::<usize>()));
|
||||
Ok(())
|
||||
})
|
||||
fn job_knowledge_loop(_ctx: &ExecutionContext) -> Result<(), TaskError> {
|
||||
// Knowledge loop removed — agents now use tool calls directly.
|
||||
// Consolidation is handled by consolidate_full() in the consolidate job.
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn job_digest(ctx: &ExecutionContext) -> Result<(), TaskError> {
|
||||
|
|
|
|||
|
|
@ -1,303 +0,0 @@
|
|||
// fact_mine.rs — extract atomic factual claims from conversation transcripts
|
||||
//
|
||||
// Chunks conversation text into overlapping windows, sends each to Haiku
|
||||
// for extraction, deduplicates by claim text. Output: JSON array of facts.
|
||||
//
|
||||
// Uses Haiku (not Sonnet) for cost efficiency on high-volume extraction.
|
||||
|
||||
use crate::config;
|
||||
use super::llm;
|
||||
use super::transcript;
|
||||
use crate::store;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
|
||||
const CHARS_PER_TOKEN: usize = 4;
|
||||
const WINDOW_TOKENS: usize = 2000;
|
||||
const OVERLAP_TOKENS: usize = 200;
|
||||
const WINDOW_CHARS: usize = WINDOW_TOKENS * CHARS_PER_TOKEN;
|
||||
const OVERLAP_CHARS: usize = OVERLAP_TOKENS * CHARS_PER_TOKEN;
|
||||
|
||||
fn extraction_prompt() -> String {
|
||||
let cfg = config::get();
|
||||
format!(
|
||||
r#"Extract atomic factual claims from this conversation excerpt.
|
||||
|
||||
Speakers are labeled [{user}] and [{assistant}] in the transcript.
|
||||
Use their proper names in claims — not "the user" or "the assistant."
|
||||
|
||||
Each claim should be:
|
||||
- A single verifiable statement
|
||||
- Specific enough to be useful in isolation
|
||||
- Tagged with domain (e.g., bcachefs/btree, bcachefs/alloc, bcachefs/journal,
|
||||
bcachefs/ec, bcachefs/reconcile, rust/idioms, workflow/preferences,
|
||||
linux/kernel, memory/design, identity/personal)
|
||||
- Tagged with confidence: "stated" (explicitly said), "implied" (logically follows),
|
||||
or "speculative" (hypothesis, not confirmed)
|
||||
- Include which speaker said it ("{user}", "{assistant}", or "Unknown")
|
||||
|
||||
Do NOT extract:
|
||||
- Opinions or subjective assessments
|
||||
- Conversational filler or greetings
|
||||
- Things that are obviously common knowledge
|
||||
- Restatements of the same fact (pick the clearest version)
|
||||
- System messages, tool outputs, or error logs (extract what was LEARNED from them)
|
||||
- Anything about the conversation itself ("{user} and {assistant} discussed...")
|
||||
- Facts only relevant to this specific conversation (e.g. transient file paths, mid-debug state)
|
||||
|
||||
Output as a JSON array. Each element:
|
||||
{{
|
||||
"claim": "the exact factual statement",
|
||||
"domain": "category/subcategory",
|
||||
"confidence": "stated|implied|speculative",
|
||||
"speaker": "{user}|{assistant}|Unknown"
|
||||
}}
|
||||
|
||||
If the excerpt contains no extractable facts, output an empty array: []
|
||||
|
||||
--- CONVERSATION EXCERPT ---
|
||||
"#, user = cfg.user_name, assistant = cfg.assistant_name)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Fact {
|
||||
pub claim: String,
|
||||
pub domain: String,
|
||||
pub confidence: String,
|
||||
pub speaker: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub source_file: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub source_chunk: Option<usize>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub source_offset: Option<usize>,
|
||||
}
|
||||
|
||||
/// Extract user/assistant text messages from a JSONL transcript.
|
||||
fn extract_messages(path: &Path) -> Vec<transcript::TranscriptMessage> {
|
||||
transcript::parse_transcript(path)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter(|m| m.text.len() >= 20)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Format messages into a single text for chunking.
|
||||
fn format_for_extraction(messages: &[transcript::TranscriptMessage]) -> String {
|
||||
let cfg = config::get();
|
||||
messages.iter()
|
||||
.map(|msg| {
|
||||
let role = if msg.role == "user" { &cfg.user_name } else { &cfg.assistant_name };
|
||||
let text = crate::util::truncate(&msg.text, 2800, "\n[...truncated...]");
|
||||
let ts = if msg.timestamp.len() >= 19 { &msg.timestamp[..19] } else { "" };
|
||||
if ts.is_empty() {
|
||||
format!("[{}] {}", role, text)
|
||||
} else {
|
||||
format!("[{} {}] {}", role, ts, text)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n")
|
||||
}
|
||||
|
||||
/// Split text into overlapping windows, breaking at paragraph boundaries.
|
||||
fn chunk_text(text: &str) -> Vec<(usize, &str)> {
|
||||
let mut chunks = Vec::new();
|
||||
let mut start = 0;
|
||||
|
||||
while start < text.len() {
|
||||
let mut end = text.floor_char_boundary((start + WINDOW_CHARS).min(text.len()));
|
||||
|
||||
// Try to break at a paragraph boundary
|
||||
if end < text.len() {
|
||||
if let Some(para) = text[start..end].rfind("\n\n") {
|
||||
if para > WINDOW_CHARS / 2 {
|
||||
end = start + para;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
chunks.push((start, &text[start..end]));
|
||||
|
||||
let next = text.floor_char_boundary(end.saturating_sub(OVERLAP_CHARS));
|
||||
if next <= start {
|
||||
start = end;
|
||||
} else {
|
||||
start = next;
|
||||
}
|
||||
}
|
||||
|
||||
chunks
|
||||
}
|
||||
|
||||
/// Parse JSON facts from model response.
|
||||
fn parse_facts(response: &str) -> Vec<Fact> {
|
||||
let cleaned = response.trim();
|
||||
// Strip markdown code block
|
||||
let cleaned = if cleaned.starts_with("```") {
|
||||
cleaned.lines()
|
||||
.filter(|l| !l.starts_with("```"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
} else {
|
||||
cleaned.to_string()
|
||||
};
|
||||
|
||||
// Find JSON array
|
||||
let start = cleaned.find('[');
|
||||
let end = cleaned.rfind(']');
|
||||
let (Some(start), Some(end)) = (start, end) else { return Vec::new() };
|
||||
|
||||
serde_json::from_str(&cleaned[start..=end]).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Mine a single transcript for atomic facts.
|
||||
/// The optional `progress` callback receives status strings (e.g. "chunk 3/47").
|
||||
pub fn mine_transcript(
|
||||
path: &Path,
|
||||
dry_run: bool,
|
||||
progress: Option<&dyn Fn(&str)>,
|
||||
) -> Result<Vec<Fact>, String> {
|
||||
let filename = path.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| "unknown".into());
|
||||
let log = |msg: &str| {
|
||||
eprintln!("{}", msg);
|
||||
if let Some(cb) = progress { cb(msg); }
|
||||
};
|
||||
|
||||
log(&format!("Mining: {}", filename));
|
||||
|
||||
let messages = extract_messages(path);
|
||||
if messages.is_empty() {
|
||||
log("No messages found");
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
log(&format!("{} messages extracted", messages.len()));
|
||||
|
||||
let text = format_for_extraction(&messages);
|
||||
let chunks = chunk_text(&text);
|
||||
log(&format!("{} chunks ({} chars)", chunks.len(), text.len()));
|
||||
|
||||
if dry_run {
|
||||
for (i, (offset, chunk)) in chunks.iter().enumerate() {
|
||||
eprintln!("\n--- Chunk {} (offset {}, {} chars) ---", i + 1, offset, chunk.len());
|
||||
eprintln!("{}", crate::util::truncate(chunk, 500, ""));
|
||||
if chunk.len() > 500 {
|
||||
eprintln!(" ... ({} more chars)", chunk.len() - 500);
|
||||
}
|
||||
}
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let prompt_prefix = extraction_prompt();
|
||||
let mut all_facts = Vec::new();
|
||||
for (i, (_offset, chunk)) in chunks.iter().enumerate() {
|
||||
let status = format!("chunk {}/{} ({} chars)", i + 1, chunks.len(), chunk.len());
|
||||
eprint!(" {}...", status);
|
||||
if let Some(cb) = progress { cb(&status); }
|
||||
|
||||
let prompt = format!("{}{}\n\n--- END OF EXCERPT ---\n\nReturn ONLY a JSON array of factual claims, or [] if none.", prompt_prefix, chunk);
|
||||
let response = match llm::call_haiku("fact-mine", &prompt) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
eprintln!(" error: {}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let mut facts = parse_facts(&response);
|
||||
for fact in &mut facts {
|
||||
fact.source_file = Some(filename.clone());
|
||||
fact.source_chunk = Some(i + 1);
|
||||
fact.source_offset = Some(*_offset);
|
||||
}
|
||||
|
||||
eprintln!(" {} facts", facts.len());
|
||||
all_facts.extend(facts);
|
||||
}
|
||||
|
||||
// Deduplicate by claim text
|
||||
let mut seen = HashSet::new();
|
||||
let before = all_facts.len();
|
||||
all_facts.retain(|f| seen.insert(f.claim.to_lowercase()));
|
||||
let dupes = before - all_facts.len();
|
||||
if dupes > 0 {
|
||||
log(&format!("{} duplicates removed", dupes));
|
||||
}
|
||||
|
||||
log(&format!("Total: {} unique facts", all_facts.len()));
|
||||
Ok(all_facts)
|
||||
}
|
||||
|
||||
/// Mine a transcript and store facts in the capnp store.
|
||||
/// Returns the number of facts stored.
|
||||
/// The optional `progress` callback receives status strings for daemon display.
|
||||
pub fn mine_and_store(
|
||||
path: &Path,
|
||||
progress: Option<&dyn Fn(&str)>,
|
||||
) -> Result<usize, String> {
|
||||
let facts = mine_transcript(path, false, progress)?;
|
||||
|
||||
let filename = path.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| "unknown".into());
|
||||
|
||||
let proposed_key = format!("_facts-{}", filename.trim_end_matches(".jsonl"));
|
||||
|
||||
// Always write a marker so we don't re-queue empty transcripts
|
||||
let json = if facts.is_empty() {
|
||||
"[]".to_string()
|
||||
} else {
|
||||
serde_json::to_string_pretty(&facts)
|
||||
.map_err(|e| format!("serialize facts: {}", e))?
|
||||
};
|
||||
|
||||
let mut store = store::Store::load()?;
|
||||
|
||||
// Run naming resolution to get a good key (and possibly merge into existing)
|
||||
let resolution = super::knowledge::resolve_naming(&store, &proposed_key, &json);
|
||||
let key = match resolution {
|
||||
super::knowledge::NamingResolution::Create(k) => k,
|
||||
super::knowledge::NamingResolution::MergeInto(existing_key) => {
|
||||
// Merge: append facts to existing node's content
|
||||
eprintln!(" Merging facts into existing node: {}", existing_key);
|
||||
if let Some(node) = store.nodes.get(existing_key.as_str()) {
|
||||
let merged = format!("{}\n\n{}", node.content, json);
|
||||
store.upsert_provenance(&existing_key, &merged, "fact-mine:write")?;
|
||||
store.save()?;
|
||||
return Ok(facts.len());
|
||||
}
|
||||
// Fallback if existing node disappeared
|
||||
proposed_key
|
||||
}
|
||||
};
|
||||
|
||||
store.upsert_provenance(&key, &json, "fact-mine:write")?;
|
||||
store.save()?;
|
||||
|
||||
eprintln!(" Stored {} facts as {}", facts.len(), key);
|
||||
Ok(facts.len())
|
||||
}
|
||||
|
||||
/// Mine transcripts, returning all facts. Skips files with fewer than min_messages.
|
||||
pub fn mine_batch(paths: &[&Path], min_messages: usize, dry_run: bool) -> Result<Vec<Fact>, String> {
|
||||
let mut all_facts = Vec::new();
|
||||
|
||||
for path in paths {
|
||||
let messages = extract_messages(path);
|
||||
if messages.len() < min_messages {
|
||||
eprintln!("Skipping {} ({} messages < {})",
|
||||
path.file_name().map(|n| n.to_string_lossy()).unwrap_or_default(),
|
||||
messages.len(), min_messages);
|
||||
continue;
|
||||
}
|
||||
|
||||
let facts = mine_transcript(path, dry_run, None)?;
|
||||
all_facts.extend(facts);
|
||||
}
|
||||
|
||||
Ok(all_facts)
|
||||
}
|
||||
|
|
@ -1,385 +1,33 @@
|
|||
// knowledge.rs — knowledge agent action parsing, depth tracking, and convergence loop
|
||||
// knowledge.rs — agent execution and conversation fragment selection
|
||||
//
|
||||
// Agent prompts live in agents/*.agent files, dispatched via defs.rs.
|
||||
// This module handles:
|
||||
// - Action parsing (WRITE_NODE, LINK, REFINE from LLM output)
|
||||
// - Inference depth tracking (prevents runaway abstraction)
|
||||
// - Action application (write to store with provenance)
|
||||
// - Convergence loop (sequences agents, measures graph stability)
|
||||
// - Agent execution (build prompt → call LLM with tools → log)
|
||||
// - Conversation fragment selection (for observation agent)
|
||||
//
|
||||
// Agents apply changes via tool calls (poc-memory write/link-add/etc)
|
||||
// during the LLM call — no action parsing needed.
|
||||
|
||||
use crate::graph::Graph;
|
||||
use crate::spectral;
|
||||
use super::llm;
|
||||
use crate::store::{self, Store, new_relation, RelationType};
|
||||
use crate::store::{self, Store};
|
||||
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::path::PathBuf;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Action types
|
||||
// Agent execution
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Action {
|
||||
pub kind: ActionKind,
|
||||
pub confidence: Confidence,
|
||||
pub weight: f64,
|
||||
pub depth: i32,
|
||||
pub applied: Option<bool>,
|
||||
pub rejected_reason: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum ActionKind {
|
||||
WriteNode {
|
||||
key: String,
|
||||
content: String,
|
||||
covers: Vec<String>,
|
||||
},
|
||||
Link {
|
||||
source: String,
|
||||
target: String,
|
||||
},
|
||||
Refine {
|
||||
key: String,
|
||||
content: String,
|
||||
},
|
||||
Demote {
|
||||
key: String,
|
||||
},
|
||||
Delete {
|
||||
key: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Confidence {
|
||||
High,
|
||||
Medium,
|
||||
Low,
|
||||
}
|
||||
|
||||
impl Confidence {
|
||||
/// Weight for delta metrics — how much this action contributes to change measurement.
|
||||
fn delta_weight(self) -> f64 {
|
||||
match self {
|
||||
Self::High => 1.0,
|
||||
Self::Medium => 0.6,
|
||||
Self::Low => 0.3,
|
||||
}
|
||||
}
|
||||
|
||||
/// Confidence value for depth gating — capped below 1.0 so even "high" must clear thresholds.
|
||||
fn gate_value(self) -> f64 {
|
||||
match self {
|
||||
Self::High => 0.9,
|
||||
Self::Medium => 0.6,
|
||||
Self::Low => 0.3,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse(s: &str) -> Self {
|
||||
match s.to_lowercase().as_str() {
|
||||
"high" => Self::High,
|
||||
"low" => Self::Low,
|
||||
_ => Self::Medium,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Action parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn parse_write_nodes(text: &str) -> Vec<Action> {
|
||||
// Match WRITE_NODE or **WRITE_NODE** with optional backtick-wrapped key
|
||||
let re = Regex::new(r"(?s)\*{0,2}WRITE_NODE\*{0,2}\s+`?(\S+?)`?\s*\n(.*?)\*{0,2}END_NODE\*{0,2}").unwrap();
|
||||
let conf_re = Regex::new(r"(?i)CONFIDENCE:\s*(high|medium|low)").unwrap();
|
||||
let covers_re = Regex::new(r"COVERS:\s*(.+)").unwrap();
|
||||
|
||||
re.captures_iter(text)
|
||||
.map(|cap| {
|
||||
let key = cap[1].to_string();
|
||||
let mut content = cap[2].trim().to_string();
|
||||
|
||||
let confidence = conf_re
|
||||
.captures(&content)
|
||||
.map(|c| Confidence::parse(&c[1]))
|
||||
.unwrap_or(Confidence::Medium);
|
||||
content = conf_re.replace(&content, "").trim().to_string();
|
||||
|
||||
let covers: Vec<String> = covers_re
|
||||
.captures(&content)
|
||||
.map(|c| c[1].split(',').map(|s| s.trim().to_string()).collect())
|
||||
.unwrap_or_default();
|
||||
content = covers_re.replace(&content, "").trim().to_string();
|
||||
|
||||
Action {
|
||||
weight: confidence.delta_weight(),
|
||||
kind: ActionKind::WriteNode { key, content, covers },
|
||||
confidence,
|
||||
depth: 0,
|
||||
applied: None,
|
||||
rejected_reason: None,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn parse_links(text: &str) -> Vec<Action> {
|
||||
// Match LINK or **LINK** with optional backtick-wrapped keys
|
||||
let re = Regex::new(r"(?m)^\*{0,2}LINK\*{0,2}\s+`?([^\s`]+)`?\s+`?([^\s`]+)`?").unwrap();
|
||||
re.captures_iter(text)
|
||||
.map(|cap| Action {
|
||||
kind: ActionKind::Link {
|
||||
source: cap[1].to_string(),
|
||||
target: cap[2].to_string(),
|
||||
},
|
||||
confidence: Confidence::Low,
|
||||
weight: 0.3,
|
||||
depth: -1,
|
||||
applied: None,
|
||||
rejected_reason: None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn parse_refines(text: &str) -> Vec<Action> {
|
||||
let re = Regex::new(r"(?s)REFINE\s+(\S+)\s*\n(.*?)END_REFINE").unwrap();
|
||||
re.captures_iter(text)
|
||||
.map(|cap| {
|
||||
let key = cap[1].trim_matches('*').trim().to_string();
|
||||
Action {
|
||||
kind: ActionKind::Refine {
|
||||
key,
|
||||
content: cap[2].trim().to_string(),
|
||||
},
|
||||
confidence: Confidence::Medium,
|
||||
weight: 0.7,
|
||||
depth: 0,
|
||||
applied: None,
|
||||
rejected_reason: None,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn parse_demotes(text: &str) -> Vec<Action> {
|
||||
let re = Regex::new(r"(?m)^DEMOTE\s+(\S+)").unwrap();
|
||||
re.captures_iter(text)
|
||||
.map(|cap| Action {
|
||||
kind: ActionKind::Demote {
|
||||
key: cap[1].to_string(),
|
||||
},
|
||||
confidence: Confidence::Medium,
|
||||
weight: 0.5,
|
||||
depth: -1,
|
||||
applied: None,
|
||||
rejected_reason: None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn parse_deletes(text: &str) -> Vec<Action> {
|
||||
let re = Regex::new(r"(?m)^DELETE\s+(\S+)").unwrap();
|
||||
re.captures_iter(text)
|
||||
.map(|cap| Action {
|
||||
kind: ActionKind::Delete {
|
||||
key: cap[1].to_string(),
|
||||
},
|
||||
confidence: Confidence::High,
|
||||
weight: 1.0,
|
||||
depth: 0,
|
||||
applied: None,
|
||||
rejected_reason: None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn parse_all_actions(text: &str) -> Vec<Action> {
|
||||
let mut actions = parse_write_nodes(text);
|
||||
actions.extend(parse_links(text));
|
||||
actions.extend(parse_refines(text));
|
||||
actions.extend(parse_demotes(text));
|
||||
actions.extend(parse_deletes(text));
|
||||
actions
|
||||
}
|
||||
|
||||
pub fn count_no_ops(text: &str) -> usize {
|
||||
let no_conn = Regex::new(r"\bNO_CONNECTION\b").unwrap().find_iter(text).count();
|
||||
let affirm = Regex::new(r"\bAFFIRM\b").unwrap().find_iter(text).count();
|
||||
let no_extract = Regex::new(r"\bNO_EXTRACTION\b").unwrap().find_iter(text).count();
|
||||
no_conn + affirm + no_extract
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Inference depth tracking
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const DEPTH_DB_KEY: &str = "_knowledge-depths";
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct DepthDb {
|
||||
depths: HashMap<String, i32>,
|
||||
}
|
||||
|
||||
impl DepthDb {
|
||||
pub fn load(store: &Store) -> Self {
|
||||
let depths = store.nodes.get(DEPTH_DB_KEY)
|
||||
.and_then(|n| serde_json::from_str(&n.content).ok())
|
||||
.unwrap_or_default();
|
||||
Self { depths }
|
||||
}
|
||||
|
||||
pub fn save(&self, store: &mut Store) {
|
||||
if let Ok(json) = serde_json::to_string(&self.depths) {
|
||||
store.upsert_provenance(DEPTH_DB_KEY, &json,
|
||||
"observation:write").ok();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get(&self, key: &str) -> i32 {
|
||||
self.depths.get(key).copied().unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn set(&mut self, key: String, depth: i32) {
|
||||
self.depths.insert(key, depth);
|
||||
}
|
||||
}
|
||||
|
||||
/// Agent base depths: observation=1, extractor=2, connector=3
|
||||
fn agent_base_depth(agent: &str) -> Option<i32> {
|
||||
match agent {
|
||||
"observation" => Some(1),
|
||||
"extractor" => Some(2),
|
||||
"connector" => Some(3),
|
||||
"challenger" => None,
|
||||
_ => Some(2),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn compute_action_depth(db: &DepthDb, action: &Action, agent: &str) -> i32 {
|
||||
match &action.kind {
|
||||
ActionKind::Link { .. } | ActionKind::Demote { .. } | ActionKind::Delete { .. } => -1,
|
||||
ActionKind::Refine { key, .. } => db.get(key),
|
||||
ActionKind::WriteNode { covers, .. } => {
|
||||
if !covers.is_empty() {
|
||||
covers.iter().map(|k| db.get(k)).max().unwrap_or(0) + 1
|
||||
} else {
|
||||
agent_base_depth(agent).unwrap_or(2)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Confidence threshold that scales with inference depth.
|
||||
pub fn required_confidence(depth: i32, base: f64) -> f64 {
|
||||
if depth <= 0 {
|
||||
return 0.0;
|
||||
}
|
||||
1.0 - (1.0 - base).powi(depth)
|
||||
}
|
||||
|
||||
/// Confidence bonus from real-world use.
|
||||
pub fn use_bonus(use_count: u32) -> f64 {
|
||||
if use_count == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
1.0 - 1.0 / (1.0 + 0.15 * use_count as f64)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Action application
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn stamp_content(content: &str, agent: &str, timestamp: &str, depth: i32) -> String {
|
||||
format!("<!-- author: {} | created: {} | depth: {} -->\n{}", agent, timestamp, depth, content)
|
||||
}
|
||||
|
||||
/// Check if a link already exists between two keys.
|
||||
fn has_edge(store: &Store, source: &str, target: &str) -> bool {
|
||||
store.relations.iter().any(|r| {
|
||||
!r.deleted
|
||||
&& ((r.source_key == source && r.target_key == target)
|
||||
|| (r.source_key == target && r.target_key == source))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn apply_action(
|
||||
store: &mut Store,
|
||||
action: &Action,
|
||||
agent: &str,
|
||||
timestamp: &str,
|
||||
depth: i32,
|
||||
) -> bool {
|
||||
match &action.kind {
|
||||
ActionKind::WriteNode { key, content, .. } => {
|
||||
let stamped = stamp_content(content, agent, timestamp, depth);
|
||||
let prov = format!("{}:write", agent);
|
||||
store.upsert_provenance(key, &stamped, &prov).is_ok()
|
||||
}
|
||||
ActionKind::Link { source, target } => {
|
||||
if has_edge(store, source, target) {
|
||||
return false;
|
||||
}
|
||||
let source_uuid = match store.nodes.get(source.as_str()) {
|
||||
Some(n) => n.uuid,
|
||||
None => return false,
|
||||
};
|
||||
let target_uuid = match store.nodes.get(target.as_str()) {
|
||||
Some(n) => n.uuid,
|
||||
None => return false,
|
||||
};
|
||||
// Default strength 0.3 — caller should run Jaccard normalization
|
||||
// after batch apply if needed (building graph per-link is too expensive)
|
||||
let mut rel = new_relation(
|
||||
source_uuid, target_uuid,
|
||||
RelationType::Link,
|
||||
0.3,
|
||||
source, target,
|
||||
);
|
||||
rel.provenance = format!("{}:link", agent);
|
||||
store.add_relation(rel).is_ok()
|
||||
}
|
||||
ActionKind::Refine { key, content } => {
|
||||
let stamped = stamp_content(content, agent, timestamp, depth);
|
||||
let prov = format!("{}:refine", agent);
|
||||
store.upsert_provenance(key, &stamped, &prov).is_ok()
|
||||
}
|
||||
ActionKind::Demote { key } => {
|
||||
if let Some(node) = store.nodes.get_mut(key) {
|
||||
node.provenance = format!("{}:demote", agent);
|
||||
node.weight = (node.weight * 0.5).max(0.05);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
ActionKind::Delete { key } => {
|
||||
store.delete_node(key).is_ok()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract a short slug from agent output for human-readable report keys.
|
||||
/// Takes the first meaningful line, lowercases, keeps alphanum+hyphens, truncates.
|
||||
fn make_report_slug(output: &str) -> String {
|
||||
let line = output.lines()
|
||||
.map(|l| l.trim())
|
||||
.find(|l| !l.is_empty() && !l.starts_with('#') && !l.starts_with("```") && !l.starts_with("---"))
|
||||
.unwrap_or("");
|
||||
// Strip markdown bold/italic
|
||||
let clean: String = line.replace("**", "").replace('*', "");
|
||||
// Keep only alphanumeric, spaces, hyphens
|
||||
let filtered: String = clean.chars()
|
||||
.map(|c| if c.is_alphanumeric() || c == ' ' || c == '-' { c } else { ' ' })
|
||||
.collect();
|
||||
// Collapse whitespace, convert to kebab-case, truncate
|
||||
let slug: String = filtered.split_whitespace()
|
||||
.take(6)
|
||||
.collect::<Vec<_>>()
|
||||
|
|
@ -388,223 +36,19 @@ fn make_report_slug(output: &str) -> String {
|
|||
if slug.len() > 60 { slug[..60].to_string() } else { slug }
|
||||
}
|
||||
|
||||
fn agent_provenance(agent: &str) -> String {
|
||||
match agent {
|
||||
"observation" => "agent:knowledge-observation".to_string(),
|
||||
"extractor" | "pattern" => "agent:knowledge-pattern".to_string(),
|
||||
"connector" => "agent:knowledge-connector".to_string(),
|
||||
"challenger" => "agent:knowledge-challenger".to_string(),
|
||||
_ => format!("agent:{}", agent),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Naming resolution — called before creating any new node
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Resolution from the naming agent.
|
||||
#[derive(Debug)]
|
||||
pub enum NamingResolution {
|
||||
/// Create with the proposed key (or a better one).
|
||||
Create(String),
|
||||
/// Merge content into an existing node instead.
|
||||
MergeInto(String),
|
||||
}
|
||||
|
||||
/// Find existing nodes that might conflict with a proposed new node.
|
||||
/// Returns up to `limit` (key, content_preview) pairs.
|
||||
fn find_conflicts(
|
||||
store: &Store,
|
||||
proposed_key: &str,
|
||||
proposed_content: &str,
|
||||
limit: usize,
|
||||
) -> Vec<(String, String)> {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
// Extract search terms from the key (split on separators) and first ~200 chars of content
|
||||
let mut terms: BTreeMap<String, f64> = BTreeMap::new();
|
||||
for part in proposed_key.split(|c: char| c == '-' || c == '_' || c == '#' || c == '.') {
|
||||
let p = part.to_lowercase();
|
||||
if p.len() >= 3 {
|
||||
terms.insert(p, 1.0);
|
||||
}
|
||||
}
|
||||
// Add a few content terms
|
||||
let content_terms = crate::search::extract_query_terms(proposed_content, 5);
|
||||
for term in content_terms.split_whitespace() {
|
||||
terms.entry(term.to_string()).or_insert(0.5);
|
||||
}
|
||||
|
||||
if terms.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Use component matching to find related nodes
|
||||
let (seeds, _) = crate::search::match_seeds_opts(&terms, store, true, false);
|
||||
|
||||
let mut results: Vec<(String, f64)> = seeds.into_iter()
|
||||
.filter(|(k, _)| k != proposed_key)
|
||||
.collect();
|
||||
results.sort_by(|a, b| b.1.total_cmp(&a.1));
|
||||
|
||||
results.into_iter()
|
||||
.take(limit)
|
||||
.filter_map(|(key, _)| {
|
||||
let node = store.nodes.get(key.as_str())?;
|
||||
let preview: String = node.content.chars().take(200).collect();
|
||||
Some((key, preview))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Format the naming prompt for a proposed node.
|
||||
fn format_naming_prompt(
|
||||
proposed_key: &str,
|
||||
proposed_content: &str,
|
||||
conflicts: &[(String, String)],
|
||||
) -> String {
|
||||
let conflict_section = if conflicts.is_empty() {
|
||||
"(no existing nodes found with overlapping content)".to_string()
|
||||
} else {
|
||||
conflicts.iter()
|
||||
.map(|(key, preview)| format!("### `{}`\n\n{}", key, preview))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n")
|
||||
};
|
||||
|
||||
// Truncate content for the prompt (don't send huge nodes to Haiku)
|
||||
let content_preview: String = proposed_content.chars().take(1000).collect();
|
||||
|
||||
format!(
|
||||
"# Naming Agent — Node Key Resolution\n\n\
|
||||
You are given a proposed new node (key + content) and a list of existing\n\
|
||||
nodes that might overlap with it. Decide what to do:\n\n\
|
||||
1. **CREATE** — the proposed key is good and there's no meaningful overlap.\n\
|
||||
2. **RENAME** — the content is unique but the key is bad (UUID, truncated, generic).\n\
|
||||
3. **MERGE_INTO** — an existing node already covers this content.\n\n\
|
||||
Good keys: 2-5 words in kebab-case, optionally with `#` subtopic.\n\
|
||||
Bad keys: UUIDs, single generic words, truncated auto-slugs.\n\n\
|
||||
Respond with exactly ONE line: `CREATE key`, `RENAME better_key`, or `MERGE_INTO existing_key`.\n\n\
|
||||
## Proposed node\n\n\
|
||||
Key: `{}`\n\n\
|
||||
Content:\n```\n{}\n```\n\n\
|
||||
## Existing nodes that might overlap\n\n\
|
||||
{}",
|
||||
proposed_key, content_preview, conflict_section,
|
||||
)
|
||||
}
|
||||
|
||||
/// Parse naming agent response.
|
||||
fn parse_naming_response(response: &str) -> Option<NamingResolution> {
|
||||
for line in response.lines() {
|
||||
// Strip backticks — Haiku sometimes wraps the response line in them
|
||||
let trimmed = line.trim().trim_matches('`').trim();
|
||||
if let Some(key) = trimmed.strip_prefix("CREATE ") {
|
||||
return Some(NamingResolution::Create(key.trim().trim_matches('`').to_string()));
|
||||
}
|
||||
if let Some(key) = trimmed.strip_prefix("RENAME ") {
|
||||
return Some(NamingResolution::Create(key.trim().trim_matches('`').to_string()));
|
||||
}
|
||||
if let Some(key) = trimmed.strip_prefix("MERGE_INTO ") {
|
||||
return Some(NamingResolution::MergeInto(key.trim().trim_matches('`').to_string()));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Resolve naming for a proposed WriteNode action.
|
||||
///
|
||||
/// Searches for conflicts, calls the naming LLM (Haiku), and returns
|
||||
/// either a Create (possibly with a better key) or MergeInto resolution.
|
||||
/// On LLM failure, falls through to using the proposed key as-is.
|
||||
pub fn resolve_naming(
|
||||
store: &Store,
|
||||
proposed_key: &str,
|
||||
proposed_content: &str,
|
||||
) -> NamingResolution {
|
||||
let conflicts = find_conflicts(store, proposed_key, proposed_content, 5);
|
||||
let prompt = format_naming_prompt(proposed_key, proposed_content, &conflicts);
|
||||
|
||||
match llm::call_haiku("naming", &prompt) {
|
||||
Ok(response) => {
|
||||
match parse_naming_response(&response) {
|
||||
Some(resolution) => resolution,
|
||||
None => {
|
||||
eprintln!("naming: unparseable response, using proposed key");
|
||||
NamingResolution::Create(proposed_key.to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("naming: LLM error ({}), using proposed key", e);
|
||||
NamingResolution::Create(proposed_key.to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared agent execution
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Result of running a single agent through the common pipeline.
|
||||
/// Result of running a single agent.
|
||||
pub struct AgentResult {
|
||||
pub output: String,
|
||||
pub actions: Vec<Action>,
|
||||
pub no_ops: usize,
|
||||
pub node_keys: Vec<String>,
|
||||
}
|
||||
|
||||
/// Resolve naming for all WriteNode actions in a list.
|
||||
///
|
||||
/// For each WriteNode, calls the naming agent to check for conflicts and
|
||||
/// get a good key. May convert WriteNode → Refine (if MERGE_INTO) or
|
||||
/// update the key (if RENAME/CREATE with different key).
|
||||
pub fn resolve_action_names(store: &Store, actions: Vec<Action>) -> Vec<Action> {
|
||||
actions.into_iter().map(|action| {
|
||||
match &action.kind {
|
||||
ActionKind::WriteNode { key, content, covers } => {
|
||||
match resolve_naming(store, key, content) {
|
||||
NamingResolution::Create(new_key) => {
|
||||
if new_key == *key {
|
||||
action // keep as-is
|
||||
} else {
|
||||
eprintln!("naming: {} → {}", key, new_key);
|
||||
Action {
|
||||
kind: ActionKind::WriteNode {
|
||||
key: new_key,
|
||||
content: content.clone(),
|
||||
covers: covers.clone(),
|
||||
},
|
||||
..action
|
||||
}
|
||||
}
|
||||
}
|
||||
NamingResolution::MergeInto(existing_key) => {
|
||||
eprintln!("naming: {} → MERGE_INTO {}", key, existing_key);
|
||||
Action {
|
||||
kind: ActionKind::Refine {
|
||||
key: existing_key,
|
||||
content: content.clone(),
|
||||
},
|
||||
..action
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => action,
|
||||
}
|
||||
}).collect()
|
||||
}
|
||||
|
||||
/// Run a single agent and apply its actions (no depth tracking).
|
||||
///
|
||||
/// Returns (total_actions, applied_count) or an error.
|
||||
/// Run a single agent and return the result (no action application — tools handle that).
|
||||
pub fn run_and_apply(
|
||||
store: &mut Store,
|
||||
agent_name: &str,
|
||||
batch_size: usize,
|
||||
llm_tag: &str,
|
||||
) -> Result<(usize, usize), String> {
|
||||
) -> Result<(), String> {
|
||||
run_and_apply_with_log(store, agent_name, batch_size, llm_tag, &|_| {})
|
||||
}
|
||||
|
||||
|
|
@ -614,38 +58,17 @@ pub fn run_and_apply_with_log(
|
|||
batch_size: usize,
|
||||
llm_tag: &str,
|
||||
log: &dyn Fn(&str),
|
||||
) -> Result<(usize, usize), String> {
|
||||
) -> Result<(), String> {
|
||||
let result = run_one_agent(store, agent_name, batch_size, llm_tag, log, false)?;
|
||||
let actions = resolve_action_names(store, result.actions);
|
||||
let ts = store::compact_timestamp();
|
||||
let mut applied = 0;
|
||||
for action in &actions {
|
||||
let desc = match &action.kind {
|
||||
ActionKind::WriteNode { key, .. } => format!("WRITE {}", key),
|
||||
ActionKind::Refine { key, .. } => format!("REFINE {}", key),
|
||||
ActionKind::Link { source, target } => format!("LINK {} → {}", source, target),
|
||||
ActionKind::Demote { key } => format!("DEMOTE {}", key),
|
||||
ActionKind::Delete { key } => format!("DELETE {}", key),
|
||||
};
|
||||
if apply_action(store, action, agent_name, &ts, 0) {
|
||||
log(&format!("applied: {}", desc));
|
||||
applied += 1;
|
||||
} else {
|
||||
log(&format!("skipped: {}", desc));
|
||||
}
|
||||
}
|
||||
|
||||
// Mark conversation segments as mined after successful processing
|
||||
if agent_name == "observation" {
|
||||
mark_observation_done(&result.node_keys);
|
||||
}
|
||||
|
||||
Ok((actions.len(), applied))
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run a single agent: build prompt → call LLM → store output → parse actions → record visits.
|
||||
///
|
||||
/// This is the common pipeline shared by the knowledge loop, consolidation pipeline,
|
||||
/// and daemon. Callers handle action application (with or without depth tracking).
|
||||
/// Run an agent with explicit target keys, bypassing the agent's query.
|
||||
pub fn run_one_agent_with_keys(
|
||||
store: &mut Store,
|
||||
|
|
@ -727,11 +150,6 @@ fn run_one_agent_inner(
|
|||
if debug { print!("{}", response_section); }
|
||||
log(&format!("response {}KB", output.len() / 1024));
|
||||
|
||||
let actions = parse_all_actions(&output);
|
||||
let no_ops = count_no_ops(&output);
|
||||
|
||||
log(&format!("parsed {} actions, {} no-ops", actions.len(), no_ops));
|
||||
|
||||
// Record visits for processed nodes
|
||||
if !agent_batch.node_keys.is_empty() {
|
||||
store.record_agent_visits(&agent_batch.node_keys, agent_name).ok();
|
||||
|
|
@ -739,8 +157,6 @@ fn run_one_agent_inner(
|
|||
|
||||
Ok(AgentResult {
|
||||
output,
|
||||
actions,
|
||||
no_ops,
|
||||
node_keys: agent_batch.node_keys,
|
||||
})
|
||||
}
|
||||
|
|
@ -888,290 +304,3 @@ fn format_segment(messages: &[(usize, String, String, String)]) -> String {
|
|||
}
|
||||
fragments.join("\n\n")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Convergence metrics
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CycleResult {
|
||||
pub cycle: usize,
|
||||
pub timestamp: String,
|
||||
pub total_actions: usize,
|
||||
pub total_applied: usize,
|
||||
pub total_no_ops: usize,
|
||||
pub depth_rejected: usize,
|
||||
pub weighted_delta: f64,
|
||||
pub graph_metrics_before: GraphMetrics,
|
||||
pub graph_metrics_after: GraphMetrics,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct GraphMetrics {
|
||||
pub nodes: usize,
|
||||
pub edges: usize,
|
||||
pub cc: f64,
|
||||
pub sigma: f64,
|
||||
pub communities: usize,
|
||||
}
|
||||
|
||||
impl GraphMetrics {
|
||||
pub fn from_graph(store: &Store, graph: &Graph) -> Self {
|
||||
Self {
|
||||
nodes: store.nodes.len(),
|
||||
edges: graph.edge_count(),
|
||||
cc: graph.avg_clustering_coefficient() as f64,
|
||||
sigma: graph.small_world_sigma() as f64,
|
||||
communities: graph.community_count(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_stability(history: &[CycleResult], key: &str, window: usize) -> f64 {
|
||||
if history.len() < window { return f64::INFINITY; }
|
||||
|
||||
let values: Vec<f64> = history[history.len() - window..].iter()
|
||||
.map(|h| match key {
|
||||
"sigma" => h.graph_metrics_after.sigma,
|
||||
"cc" => h.graph_metrics_after.cc,
|
||||
"communities" => h.graph_metrics_after.communities as f64,
|
||||
_ => 0.0,
|
||||
})
|
||||
.collect();
|
||||
|
||||
if values.len() < 2 { return f64::INFINITY; }
|
||||
let mean = values.iter().sum::<f64>() / values.len() as f64;
|
||||
if mean == 0.0 { return 0.0; }
|
||||
let variance = values.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / values.len() as f64;
|
||||
variance.sqrt() / mean.abs()
|
||||
}
|
||||
|
||||
pub fn check_convergence(history: &[CycleResult], window: usize) -> bool {
|
||||
if history.len() < window { return false; }
|
||||
|
||||
let sigma_cv = metric_stability(history, "sigma", window);
|
||||
let cc_cv = metric_stability(history, "cc", window);
|
||||
let comm_cv = metric_stability(history, "communities", window);
|
||||
|
||||
let recent = &history[history.len() - window..];
|
||||
let avg_delta = recent.iter().map(|r| r.weighted_delta).sum::<f64>() / recent.len() as f64;
|
||||
|
||||
eprintln!("\n Convergence check (last {} cycles):", window);
|
||||
eprintln!(" sigma CV: {:.4} (< 0.05?)", sigma_cv);
|
||||
eprintln!(" CC CV: {:.4} (< 0.05?)", cc_cv);
|
||||
eprintln!(" community CV: {:.4} (< 0.10?)", comm_cv);
|
||||
eprintln!(" avg delta: {:.2} (< 1.00?)", avg_delta);
|
||||
|
||||
let structural = sigma_cv < 0.05 && cc_cv < 0.05 && comm_cv < 0.10;
|
||||
let behavioral = avg_delta < 1.0;
|
||||
|
||||
if structural && behavioral {
|
||||
eprintln!(" → CONVERGED");
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The knowledge loop
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct KnowledgeLoopConfig {
|
||||
pub max_cycles: usize,
|
||||
pub batch_size: usize,
|
||||
pub window: usize,
|
||||
pub max_depth: i32,
|
||||
pub confidence_base: f64,
|
||||
}
|
||||
|
||||
impl Default for KnowledgeLoopConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_cycles: 20,
|
||||
batch_size: 5,
|
||||
window: 5,
|
||||
max_depth: 4,
|
||||
confidence_base: 0.3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run_knowledge_loop(config: &KnowledgeLoopConfig) -> Result<Vec<CycleResult>, String> {
|
||||
let mut store = Store::load()?;
|
||||
let mut depth_db = DepthDb::load(&store);
|
||||
let mut history = Vec::new();
|
||||
|
||||
eprintln!("Knowledge Loop — fixed-point iteration");
|
||||
eprintln!(" max_cycles={} batch_size={}", config.max_cycles, config.batch_size);
|
||||
eprintln!(" window={} max_depth={}", config.window, config.max_depth);
|
||||
|
||||
for cycle in 1..=config.max_cycles {
|
||||
let result = run_cycle(cycle, config, &mut depth_db)?;
|
||||
history.push(result);
|
||||
|
||||
if check_convergence(&history, config.window) {
|
||||
eprintln!("\n CONVERGED after {} cycles", cycle);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Save loop summary as a store node
|
||||
if let Some(first) = history.first() {
|
||||
let key = format!("_knowledge-loop-{}", first.timestamp);
|
||||
if let Ok(json) = serde_json::to_string_pretty(&history) {
|
||||
store = Store::load()?;
|
||||
store.upsert_provenance(&key, &json,
|
||||
"observation:write").ok();
|
||||
depth_db.save(&mut store);
|
||||
store.save()?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(history)
|
||||
}
|
||||
|
||||
fn run_cycle(
|
||||
cycle_num: usize,
|
||||
config: &KnowledgeLoopConfig,
|
||||
depth_db: &mut DepthDb,
|
||||
) -> Result<CycleResult, String> {
|
||||
let timestamp = store::compact_timestamp();
|
||||
eprintln!("\n{}", "=".repeat(60));
|
||||
eprintln!("CYCLE {} — {}", cycle_num, timestamp);
|
||||
eprintln!("{}", "=".repeat(60));
|
||||
|
||||
let mut store = Store::load()?;
|
||||
let graph = store.build_graph();
|
||||
let metrics_before = GraphMetrics::from_graph(&store, &graph);
|
||||
eprintln!(" Before: nodes={} edges={} cc={:.3} sigma={:.3}",
|
||||
metrics_before.nodes, metrics_before.edges, metrics_before.cc, metrics_before.sigma);
|
||||
|
||||
let mut all_actions = Vec::new();
|
||||
let mut all_no_ops = 0;
|
||||
let mut depth_rejected = 0;
|
||||
let mut total_applied = 0;
|
||||
|
||||
// Run each agent via .agent file dispatch
|
||||
let agent_names = ["observation", "extractor", "connector", "challenger"];
|
||||
|
||||
for agent_name in &agent_names {
|
||||
eprintln!("\n --- {} (n={}) ---", agent_name, config.batch_size);
|
||||
|
||||
let result = match run_one_agent(&mut store, agent_name, config.batch_size, "knowledge", &|msg| eprintln!(" {}", msg), false) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
eprintln!(" ERROR: {}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let mut actions = result.actions;
|
||||
all_no_ops += result.no_ops;
|
||||
|
||||
eprintln!(" Actions: {} No-ops: {}", actions.len(), result.no_ops);
|
||||
|
||||
let mut applied = 0;
|
||||
for action in &mut actions {
|
||||
let depth = compute_action_depth(depth_db, action, agent_name);
|
||||
action.depth = depth;
|
||||
|
||||
match &action.kind {
|
||||
ActionKind::WriteNode { key, covers, .. } => {
|
||||
let conf_val = action.confidence.gate_value();
|
||||
let req = required_confidence(depth, config.confidence_base);
|
||||
|
||||
let source_uses: Vec<u32> = covers.iter()
|
||||
.filter_map(|k| store.nodes.get(k).map(|n| n.uses))
|
||||
.collect();
|
||||
let avg_uses = if source_uses.is_empty() { 0 }
|
||||
else { source_uses.iter().sum::<u32>() / source_uses.len() as u32 };
|
||||
let eff_conf = (conf_val + use_bonus(avg_uses)).min(1.0);
|
||||
|
||||
if eff_conf < req {
|
||||
action.applied = Some(false);
|
||||
action.rejected_reason = Some("depth_threshold".into());
|
||||
depth_rejected += 1;
|
||||
continue;
|
||||
}
|
||||
if depth > config.max_depth {
|
||||
action.applied = Some(false);
|
||||
action.rejected_reason = Some("max_depth".into());
|
||||
depth_rejected += 1;
|
||||
continue;
|
||||
}
|
||||
eprintln!(" WRITE {} depth={} conf={:.2} eff={:.2} req={:.2}",
|
||||
key, depth, conf_val, eff_conf, req);
|
||||
}
|
||||
ActionKind::Link { source, target } => {
|
||||
eprintln!(" LINK {} → {}", source, target);
|
||||
}
|
||||
ActionKind::Refine { key, .. } => {
|
||||
eprintln!(" REFINE {} depth={}", key, depth);
|
||||
}
|
||||
ActionKind::Demote { key } => {
|
||||
eprintln!(" DEMOTE {}", key);
|
||||
}
|
||||
ActionKind::Delete { key } => {
|
||||
eprintln!(" DELETE {}", key);
|
||||
}
|
||||
}
|
||||
|
||||
if apply_action(&mut store, action, agent_name, ×tamp, depth) {
|
||||
applied += 1;
|
||||
action.applied = Some(true);
|
||||
if let ActionKind::WriteNode { key, .. } | ActionKind::Refine { key, .. } = &action.kind {
|
||||
depth_db.set(key.clone(), depth);
|
||||
}
|
||||
} else {
|
||||
action.applied = Some(false);
|
||||
}
|
||||
}
|
||||
|
||||
eprintln!(" Applied: {}/{}", applied, actions.len());
|
||||
total_applied += applied;
|
||||
all_actions.extend(actions);
|
||||
}
|
||||
|
||||
depth_db.save(&mut store);
|
||||
|
||||
// Recompute spectral at most once per hour — O(n³) is expensive at 14k+ nodes
|
||||
if total_applied > 0 {
|
||||
let stale = spectral::embedding_path()
|
||||
.metadata()
|
||||
.and_then(|m| m.modified())
|
||||
.map(|t| t.elapsed().unwrap_or_default() > std::time::Duration::from_secs(3600))
|
||||
.unwrap_or(true);
|
||||
if stale {
|
||||
eprintln!("\n Recomputing spectral embedding (>1h stale)...");
|
||||
let graph = store.build_graph();
|
||||
let result = spectral::decompose(&graph, 8);
|
||||
let emb = spectral::to_embedding(&result);
|
||||
spectral::save_embedding(&emb).ok();
|
||||
}
|
||||
}
|
||||
|
||||
let graph = store.build_graph();
|
||||
let metrics_after = GraphMetrics::from_graph(&store, &graph);
|
||||
let weighted_delta: f64 = all_actions.iter()
|
||||
.filter(|a| a.applied == Some(true))
|
||||
.map(|a| a.weight)
|
||||
.sum();
|
||||
|
||||
eprintln!("\n CYCLE {} SUMMARY", cycle_num);
|
||||
eprintln!(" Applied: {}/{} depth-rejected: {} no-ops: {}",
|
||||
total_applied, all_actions.len(), depth_rejected, all_no_ops);
|
||||
eprintln!(" Weighted delta: {:.2}", weighted_delta);
|
||||
|
||||
Ok(CycleResult {
|
||||
cycle: cycle_num,
|
||||
timestamp,
|
||||
total_actions: all_actions.len(),
|
||||
total_applied,
|
||||
total_no_ops: all_no_ops,
|
||||
depth_rejected,
|
||||
weighted_delta,
|
||||
graph_metrics_before: metrics_before,
|
||||
graph_metrics_after: metrics_after,
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,11 +6,11 @@
|
|||
//
|
||||
// llm — model invocation, response parsing
|
||||
// prompts — prompt generation from store data
|
||||
// defs — agent file loading and placeholder resolution
|
||||
// audit — link quality review via Sonnet
|
||||
// consolidate — full consolidation pipeline
|
||||
// knowledge — knowledge production agents + convergence loop
|
||||
// knowledge — agent execution, conversation fragment selection
|
||||
// enrich — journal enrichment, experience mining
|
||||
// fact_mine — fact extraction from transcripts
|
||||
// digest — episodic digest generation (daily/weekly/monthly)
|
||||
// daemon — background job scheduler
|
||||
// transcript — shared JSONL transcript parsing
|
||||
|
|
@ -23,6 +23,5 @@ pub mod audit;
|
|||
pub mod consolidate;
|
||||
pub mod knowledge;
|
||||
pub mod enrich;
|
||||
pub mod fact_mine;
|
||||
pub mod digest;
|
||||
pub mod daemon;
|
||||
|
|
|
|||
|
|
@ -97,73 +97,20 @@ pub fn cmd_journal_enrich(jsonl_path: &str, entry_text: &str, grep_line: usize)
|
|||
crate::enrich::journal_enrich(&mut store, jsonl_path, entry_text, grep_line)
|
||||
}
|
||||
|
||||
pub fn cmd_apply_consolidation(do_apply: bool, report_file: Option<&str>) -> Result<(), String> {
|
||||
let mut store = store::Store::load()?;
|
||||
crate::consolidate::apply_consolidation(&mut store, do_apply, report_file)
|
||||
pub fn cmd_apply_consolidation(_do_apply: bool, _report_file: Option<&str>) -> Result<(), String> {
|
||||
Err("apply-consolidation has been removed — agents now apply changes via tool calls directly.".into())
|
||||
}
|
||||
|
||||
pub fn cmd_knowledge_loop(max_cycles: usize, batch_size: usize, window: usize, max_depth: i32) -> Result<(), String> {
|
||||
let config = crate::knowledge::KnowledgeLoopConfig {
|
||||
max_cycles,
|
||||
batch_size,
|
||||
window,
|
||||
max_depth,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let results = crate::knowledge::run_knowledge_loop(&config)?;
|
||||
eprintln!("\nCompleted {} cycles, {} total actions applied",
|
||||
results.len(),
|
||||
results.iter().map(|r| r.total_applied).sum::<usize>());
|
||||
Ok(())
|
||||
pub fn cmd_knowledge_loop(_max_cycles: usize, _batch_size: usize, _window: usize, _max_depth: i32) -> Result<(), String> {
|
||||
Err("knowledge-loop has been removed — agents now use tool calls directly. Use `poc-memory agent run` instead.".into())
|
||||
}
|
||||
|
||||
pub fn cmd_fact_mine(path: &str, batch: bool, dry_run: bool, output_file: Option<&str>, min_messages: usize) -> Result<(), String> {
|
||||
let p = std::path::Path::new(path);
|
||||
|
||||
let paths: Vec<std::path::PathBuf> = if batch {
|
||||
if !p.is_dir() {
|
||||
return Err(format!("Not a directory: {}", path));
|
||||
}
|
||||
let mut files: Vec<_> = std::fs::read_dir(p)
|
||||
.map_err(|e| format!("read dir: {}", e))?
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| p.extension().map(|x| x == "jsonl").unwrap_or(false))
|
||||
.collect();
|
||||
files.sort();
|
||||
eprintln!("Found {} transcripts", files.len());
|
||||
files
|
||||
} else {
|
||||
vec![p.to_path_buf()]
|
||||
};
|
||||
|
||||
let path_refs: Vec<&std::path::Path> = paths.iter().map(|p| p.as_path()).collect();
|
||||
let facts = crate::fact_mine::mine_batch(&path_refs, min_messages, dry_run)?;
|
||||
|
||||
if !dry_run {
|
||||
let json = serde_json::to_string_pretty(&facts)
|
||||
.map_err(|e| format!("serialize: {}", e))?;
|
||||
if let Some(out) = output_file {
|
||||
std::fs::write(out, &json).map_err(|e| format!("write: {}", e))?;
|
||||
eprintln!("\nWrote {} facts to {}", facts.len(), out);
|
||||
} else {
|
||||
println!("{}", json);
|
||||
}
|
||||
}
|
||||
|
||||
eprintln!("\nTotal: {} facts from {} transcripts", facts.len(), paths.len());
|
||||
Ok(())
|
||||
pub fn cmd_fact_mine(_path: &str, _batch: bool, _dry_run: bool, _output_file: Option<&str>, _min_messages: usize) -> Result<(), String> {
|
||||
Err("fact-mine has been removed — use the observation agent instead.".into())
|
||||
}
|
||||
|
||||
pub fn cmd_fact_mine_store(path: &str) -> Result<(), String> {
|
||||
let path = std::path::Path::new(path);
|
||||
if !path.exists() {
|
||||
return Err(format!("File not found: {}", path.display()));
|
||||
}
|
||||
let count = crate::fact_mine::mine_and_store(path, None)?;
|
||||
eprintln!("Stored {} facts", count);
|
||||
Ok(())
|
||||
pub fn cmd_fact_mine_store(_path: &str) -> Result<(), String> {
|
||||
Err("fact-mine-store has been removed — use the observation agent instead.".into())
|
||||
}
|
||||
|
||||
/// Sample recent actions from each agent type, sort by quality using
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ pub mod tui;
|
|||
// Re-export agent submodules at crate root for backwards compatibility
|
||||
pub use agents::{
|
||||
llm, audit, consolidate, knowledge,
|
||||
enrich, fact_mine, digest, daemon,
|
||||
enrich, digest, daemon,
|
||||
};
|
||||
|
||||
pub mod memory_capnp {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue