consciousness/src/digest.rs

// Episodic digest generation: daily, weekly, monthly, auto
//
// Temporal digest generation and digest link parsing. Each digest type
// gathers input from the store, builds a Sonnet prompt, calls Sonnet,
// writes results to the episodic dir, and extracts links.

use crate::llm::{call_sonnet, semantic_keys};
use crate::store::{self, Store, new_relation};
use crate::neuro;

use regex::Regex;
use std::fs;
use std::path::{Path, PathBuf};

use crate::util::memory_subdir;

/// Extract link proposals from digest text (backtick-arrow patterns)
fn extract_links(text: &str) -> Vec<(String, String)> {
    let re_left = Regex::new(r"`([^`]+)`\s*→").unwrap();
    let re_right = Regex::new(r"→\s*`([^`]+)`").unwrap();
    let mut links = Vec::new();

    for line in text.lines() {
        if let Some(cap) = re_left.captures(line) {
            links.push((cap[1].to_string(), line.trim().to_string()));
        }
        if let Some(cap) = re_right.captures(line) {
            links.push((cap[1].to_string(), line.trim().to_string()));
        }
    }
    links
}

// --- Daily digest ---

fn daily_journal_entries(store: &Store, target_date: &str) -> Vec<(String, String)> {
    // Collect journal nodes for the target date
    // Keys like: journal.md#j-2026-02-28t23-39-...
    let date_re = Regex::new(&format!(
        r"^journal\.md#j-{}", regex::escape(target_date)
    )).unwrap();

    let mut entries: Vec<_> = store.nodes.values()
        .filter(|n| date_re.is_match(&n.key))
        .map(|n| (n.key.clone(), n.content.clone()))
        .collect();
    entries.sort_by(|a, b| a.0.cmp(&b.0));
    entries
}

fn build_daily_prompt(date: &str, entries: &[(String, String)], keys: &[String]) -> Result<String, String> {
    let mut entries_text = String::new();
    for (key, content) in entries {
        let ts = key.strip_prefix("journal.md#j-").unwrap_or(key);
        entries_text.push_str(&format!("\n### {}\n\n{}\n", ts, content));
    }

    let keys_text: String = keys.iter()
        .map(|k| format!("  - {}", k))
        .collect::<Vec<_>>()
        .join("\n");

    neuro::load_prompt("daily-digest", &[
        ("{{DATE}}", date),
        ("{{ENTRIES}}", &entries_text),
        ("{{KEYS}}", &keys_text),
    ])
}

pub fn generate_daily(store: &mut Store, date: &str) -> Result<(), String> {
    println!("Generating daily digest for {}...", date);

    let entries = daily_journal_entries(store, date);
    if entries.is_empty() {
        println!("  No journal entries found for {}", date);
        return Ok(());
    }
    println!("  {} journal entries", entries.len());

    let keys = semantic_keys(store);
    println!("  {} semantic keys", keys.len());

    let prompt = build_daily_prompt(date, &entries, &keys)?;
    println!("  Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);

    println!("  Calling Sonnet...");
    let digest = call_sonnet(&prompt, 300)?;

    // Write to episodic dir
    let output_path = memory_subdir("episodic")?.join(format!("daily-{}.md", date));
    fs::write(&output_path, &digest)
        .map_err(|e| format!("write {}: {}", output_path.display(), e))?;
    println!("  Written: {}", output_path.display());

    // Import into store
    store.import_file(&output_path)?;
    store.save()?;

    // Extract and save links
    let links = extract_links(&digest);
    if !links.is_empty() {
        let links_json: Vec<serde_json::Value> = links.iter()
            .map(|(target, line)| serde_json::json!({"target": target, "line": line}))
            .collect();
        let result = serde_json::json!({
            "type": "daily-digest",
            "date": date,
            "digest_path": output_path.to_string_lossy(),
            "links": links_json,
        });
        let links_path = memory_subdir("agent-results")?.join(format!("daily-{}-links.json", date));
        let json = serde_json::to_string_pretty(&result)
            .map_err(|e| format!("serialize: {}", e))?;
        fs::write(&links_path, json)
            .map_err(|e| format!("write {}: {}", links_path.display(), e))?;
        println!("  {} links extracted → {}", links.len(), links_path.display());
    }

    let line_count = digest.lines().count();
    println!("  Done: {} lines", line_count);
    Ok(())
}

// --- Weekly digest ---

/// Get ISO week label and the 7 dates (Mon-Sun) for the week containing `date`.
fn week_dates(date: &str) -> Result<(String, Vec<String>), String> {
    use chrono::{Datelike, Duration, NaiveDate};

    let nd = NaiveDate::parse_from_str(date, "%Y-%m-%d")
        .map_err(|e| format!("bad date '{}': {}", date, e))?;
    let iso = nd.iso_week();
    let week_label = format!("{}-W{:02}", iso.year(), iso.week());

    // Find Monday of this week
    let days_since_monday = nd.weekday().num_days_from_monday() as i64;
    let monday = nd - Duration::days(days_since_monday);

    let dates = (0..7)
        .map(|i| (monday + Duration::days(i)).format("%Y-%m-%d").to_string())
        .collect();

    Ok((week_label, dates))
}

/// Returns (weekday 0=Sun, iso_year, iso_week) for a given date.
fn iso_week_info(y: i32, m: u32, d: u32) -> Result<(u32, i32, u32), String> {
    use chrono::{Datelike, NaiveDate};
    let date = NaiveDate::from_ymd_opt(y, m, d)
        .ok_or_else(|| format!("invalid date: {}-{}-{}", y, m, d))?;
    let wday = date.weekday().num_days_from_sunday();
    let iso = date.iso_week();
    Ok((wday, iso.year(), iso.week()))
}

fn load_digest_files(prefix: &str, labels: &[String]) -> Result<Vec<(String, String)>, String> {
    let dir = memory_subdir("episodic")?;
    let mut digests = Vec::new();
    for label in labels {
        let path = dir.join(format!("{}-{}.md", prefix, label));
        if let Ok(content) = fs::read_to_string(&path) {
            digests.push((label.clone(), content));
        }
    }
    Ok(digests)
}

fn build_weekly_prompt(week_label: &str, digests: &[(String, String)], keys: &[String]) -> Result<String, String> {
    let mut digests_text = String::new();
    for (date, content) in digests {
        digests_text.push_str(&format!("\n---\n## {}\n{}\n", date, content));
    }

    let keys_text: String = keys.iter()
        .map(|k| format!("  - {}", k))
        .collect::<Vec<_>>()
        .join("\n");

    let dates_covered: String = digests.iter()
        .map(|(d, _)| d.as_str())
        .collect::<Vec<_>>()
        .join(", ");

    neuro::load_prompt("weekly-digest", &[
        ("{{WEEK_LABEL}}", week_label),
        ("{{DATES_COVERED}}", &dates_covered),
        ("{{DIGESTS}}", &digests_text),
        ("{{KEYS}}", &keys_text),
    ])
}

pub fn generate_weekly(store: &mut Store, date: &str) -> Result<(), String> {
    let (week_label, dates) = week_dates(date)?;
    println!("Generating weekly digest for {}...", week_label);

    let digests = load_digest_files("daily", &dates)?;
    if digests.is_empty() {
        println!("  No daily digests found for {}", week_label);
        println!("  Run `poc-memory digest daily` first for relevant dates");
        return Ok(());
    }
    println!("  {} daily digests found", digests.len());

    let keys = semantic_keys(store);
    println!("  {} semantic keys", keys.len());

    let prompt = build_weekly_prompt(&week_label, &digests, &keys)?;
    println!("  Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);

    println!("  Calling Sonnet...");
    let digest = call_sonnet(&prompt, 300)?;

    let output_path = memory_subdir("episodic")?.join(format!("weekly-{}.md", week_label));
    fs::write(&output_path, &digest)
        .map_err(|e| format!("write {}: {}", output_path.display(), e))?;
    println!("  Written: {}", output_path.display());

    store.import_file(&output_path)?;
    store.save()?;

    // Save metadata
    let result = serde_json::json!({
        "type": "weekly-digest",
        "week": week_label,
        "digest_path": output_path.to_string_lossy(),
        "daily_digests": digests.iter().map(|(d, _)| d).collect::<Vec<_>>(),
    });
    let links_path = memory_subdir("agent-results")?.join(format!("weekly-{}-links.json", week_label));
    fs::write(&links_path, serde_json::to_string_pretty(&result).unwrap())
        .map_err(|e| format!("write {}: {}", links_path.display(), e))?;

    println!("  Done: {} lines", digest.lines().count());
    Ok(())
}

// --- Monthly digest ---

fn weeks_in_month(year: i32, month: u32) -> Vec<String> {
    use chrono::{Datelike, NaiveDate};
    let mut weeks = std::collections::BTreeSet::new();
    let mut d = 1u32;
    while let Some(date) = NaiveDate::from_ymd_opt(year, month, d) {
        if date.month() != month { break; }
        let iso = date.iso_week();
        weeks.insert(format!("{}-W{:02}", iso.year(), iso.week()));
        d += 1;
    }
    weeks.into_iter().collect()
}

fn build_monthly_prompt(month_label: &str, digests: &[(String, String)], keys: &[String]) -> Result<String, String> {
    let mut digests_text = String::new();
    for (week, content) in digests {
        digests_text.push_str(&format!("\n---\n## {}\n{}\n", week, content));
    }

    let keys_text: String = keys.iter()
        .map(|k| format!("  - {}", k))
        .collect::<Vec<_>>()
        .join("\n");

    let weeks_covered: String = digests.iter()
        .map(|(w, _)| w.as_str())
        .collect::<Vec<_>>()
        .join(", ");

    neuro::load_prompt("monthly-digest", &[
        ("{{MONTH_LABEL}}", month_label),
        ("{{WEEKS_COVERED}}", &weeks_covered),
        ("{{DIGESTS}}", &digests_text),
        ("{{KEYS}}", &keys_text),
    ])
}

pub fn generate_monthly(store: &mut Store, month_arg: &str) -> Result<(), String> {
    let (year, month) = if month_arg.is_empty() {
        let now = store::now_epoch();
        let (y, m, _, _, _, _) = store::epoch_to_local(now);
        (y, m)
    } else {
        let parts: Vec<&str> = month_arg.split('-').collect();
        if parts.len() != 2 {
            return Err(format!("bad month format: {} (expected YYYY-MM)", month_arg));
        }
        let y: i32 = parts[0].parse().map_err(|_| "bad year")?;
        let m: u32 = parts[1].parse().map_err(|_| "bad month")?;
        (y, m)
    };

    let month_label = format!("{}-{:02}", year, month);
    println!("Generating monthly digest for {}...", month_label);

    let week_labels = weeks_in_month(year, month);
    println!("  Weeks in month: {}", week_labels.join(", "));

    let digests = load_digest_files("weekly", &week_labels)?;
    if digests.is_empty() {
        println!("  No weekly digests found for {}", month_label);
        println!("  Run `poc-memory digest weekly` first for relevant weeks");
        return Ok(());
    }
    println!("  {} weekly digests found", digests.len());

    let keys = semantic_keys(store);
    println!("  {} semantic keys", keys.len());

    let prompt = build_monthly_prompt(&month_label, &digests, &keys)?;
    println!("  Prompt: {} chars (~{} tokens)", prompt.len(), prompt.len() / 4);

    println!("  Calling Sonnet...");
    let digest = call_sonnet(&prompt, 600)?;

    let output_path = memory_subdir("episodic")?.join(format!("monthly-{}.md", month_label));
    fs::write(&output_path, &digest)
        .map_err(|e| format!("write {}: {}", output_path.display(), e))?;
    println!("  Written: {}", output_path.display());

    store.import_file(&output_path)?;
    store.save()?;

    // Save metadata
    let result = serde_json::json!({
        "type": "monthly-digest",
        "month": month_label,
        "digest_path": output_path.to_string_lossy(),
        "weekly_digests": digests.iter().map(|(w, _)| w).collect::<Vec<_>>(),
    });
    let links_path = memory_subdir("agent-results")?.join(format!("monthly-{}-links.json", month_label));
    fs::write(&links_path, serde_json::to_string_pretty(&result).unwrap())
        .map_err(|e| format!("write {}: {}", links_path.display(), e))?;

    println!("  Done: {} lines", digest.lines().count());
    Ok(())
}

// --- Digest auto: freshness detection + bottom-up generation ---

/// Scan the store for dates/weeks/months that need digests and generate them.
/// Works bottom-up: daily first, then weekly (needs dailies), then monthly
/// (needs weeklies). Skips today (incomplete day). Skips already-existing
/// digests.
pub fn digest_auto(store: &mut Store) -> Result<(), String> {
    let today = store::today();
    let epi = memory_subdir("episodic")?;

    // --- Phase 1: find dates with journal entries but no daily digest ---
    let date_re = Regex::new(r"^\d{4}-\d{2}-\d{2}").unwrap();
    let mut dates: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
    for key in store.nodes.keys() {
        // Keys like: journal.md#j-2026-02-28t23-39-...
        if let Some(rest) = key.strip_prefix("journal.md#j-") {
            if rest.len() >= 10 && date_re.is_match(rest) {
                dates.insert(rest[..10].to_string());
            }
        }
    }

    let mut daily_generated = 0u32;
    let mut daily_skipped = 0u32;
    let mut daily_dates_done: Vec<String> = Vec::new();

    for date in &dates {
        if date == &today {
            continue; // don't digest an incomplete day
        }
        let path = epi.join(format!("daily-{}.md", date));
        if path.exists() {
            daily_skipped += 1;
            daily_dates_done.push(date.clone());
            continue;
        }
        println!("[auto] Missing daily digest for {}", date);
        generate_daily(store, date)?;
        daily_generated += 1;
        daily_dates_done.push(date.clone());
    }

    println!("[auto] Daily: {} generated, {} already existed",
        daily_generated, daily_skipped);

    // --- Phase 2: find complete weeks needing weekly digests ---
    // A week is "ready" if its Sunday is before today and at least one
    // daily digest exists for it.

    let mut weeks_seen: std::collections::BTreeMap<String, Vec<String>> = std::collections::BTreeMap::new();
    for date in &daily_dates_done {
        if let Ok((week_label, _week_dates)) = week_dates(date) {
            weeks_seen.entry(week_label).or_default().push(date.clone());
        }
    }

    let mut weekly_generated = 0u32;
    let mut weekly_skipped = 0u32;
    let mut weekly_labels_done: Vec<String> = Vec::new();

    for (week_label, example_dates) in &weeks_seen {
        // Check if this week is complete (Sunday has passed)
        if let Ok((_, week_day_list)) = week_dates(example_dates.first().unwrap()) {
            let sunday = week_day_list.last().unwrap();
            if sunday >= &today {
                continue; // week not over yet
            }
        }

        let path = epi.join(format!("weekly-{}.md", week_label));
        if path.exists() {
            weekly_skipped += 1;
            weekly_labels_done.push(week_label.clone());
            continue;
        }

        // Check that at least some dailies exist for this week
        let has_dailies = example_dates.iter().any(|d|
            epi.join(format!("daily-{}.md", d)).exists()
        );
        if !has_dailies {
            continue;
        }

        println!("[auto] Missing weekly digest for {}", week_label);
        generate_weekly(store, example_dates.first().unwrap())?;
        weekly_generated += 1;
        weekly_labels_done.push(week_label.clone());
    }

    println!("[auto] Weekly: {} generated, {} already existed",
        weekly_generated, weekly_skipped);

    // --- Phase 3: find complete months needing monthly digests ---
    // A month is "ready" if the month is before the current month and at
    // least one weekly digest exists for it.

    let (cur_y, cur_m, _, _, _, _) = store::epoch_to_local(store::now_epoch());
    let mut months_seen: std::collections::BTreeSet<(i32, u32)> = std::collections::BTreeSet::new();

    for date in &daily_dates_done {
        let parts: Vec<&str> = date.split('-').collect();
        if parts.len() >= 2 {
            if let (Ok(y), Ok(m)) = (parts[0].parse::<i32>(), parts[1].parse::<u32>()) {
                months_seen.insert((y, m));
            }
        }
    }

    let mut monthly_generated = 0u32;
    let mut monthly_skipped = 0u32;

    for (y, m) in &months_seen {
        // Skip current month (still in progress)
        if *y == cur_y && *m == cur_m {
            continue;
        }
        // Skip future months
        if *y > cur_y || (*y == cur_y && *m > cur_m) {
            continue;
        }

        let label = format!("{}-{:02}", y, m);
        let path = epi.join(format!("monthly-{}.md", label));
        if path.exists() {
            monthly_skipped += 1;
            continue;
        }

        // Check that at least one weekly exists for this month
        let week_labels = weeks_in_month(*y, *m);
        let has_weeklies = week_labels.iter().any(|w|
            epi.join(format!("weekly-{}.md", w)).exists()
        );
        if !has_weeklies {
            continue;
        }

        println!("[auto] Missing monthly digest for {}", label);
        generate_monthly(store, &label)?;
        monthly_generated += 1;
    }

    println!("[auto] Monthly: {} generated, {} already existed",
        monthly_generated, monthly_skipped);

    let total = daily_generated + weekly_generated + monthly_generated;
    if total == 0 {
        println!("[auto] All digests up to date.");
    } else {
        println!("[auto] Generated {} total digests.", total);
    }
    Ok(())
}


// --- Digest link parsing ---
// Replaces digest-link-parser.py: parses ## Links sections from digest
// files and applies them to the memory graph.

/// A parsed link from a digest's Links section.
pub struct DigestLink {
    pub source: String,
    pub target: String,
    pub reason: String,
    pub file: String,
}

/// Normalize a raw link target to a poc-memory key.
fn normalize_link_key(raw: &str) -> String {
    let key = raw.trim().trim_matches('`').trim();
    if key.is_empty() { return String::new(); }

    // Self-references
    let lower = key.to_lowercase();
    if lower.starts_with("this ") { return String::new(); }

    let mut key = key.to_string();

    // weekly/2026-W06 → weekly-2026-W06, etc.
    if let Some(pos) = key.find('/') {
        let prefix = &key[..pos];
        if prefix == "daily" || prefix == "weekly" || prefix == "monthly" {
            let rest = &key[pos + 1..];
            key = format!("{}-{}", prefix, rest);
        }
    }

    // daily-2026-02-04 → daily-2026-02-04.md
    let re = Regex::new(r"^(daily|weekly|monthly)-\d{4}").unwrap();
    if re.is_match(&key) && !key.ends_with(".md") {
        key.push_str(".md");
    }

    // Bare date → daily digest
    let date_re = Regex::new(r"^\d{4}-\d{2}-\d{2}$").unwrap();
    if date_re.is_match(key.strip_suffix(".md").unwrap_or(&key)) {
        let date = key.strip_suffix(".md").unwrap_or(&key);
        key = format!("daily-{}.md", date);
    }

    // Ensure .md extension
    if key.contains('#') {
        let (file, section) = key.split_once('#').unwrap();
        if !file.ends_with(".md") {
            key = format!("{}.md#{}", file, section);
        }
    } else if !key.ends_with(".md") && !key.contains('/') && !key.starts_with("NEW:") {
        key.push_str(".md");
    }

    key
}

/// Parse the Links section from a single digest file.
fn parse_digest_file_links(path: &Path) -> Vec<DigestLink> {
    let content = match fs::read_to_string(path) {
        Ok(c) => c,
        Err(_) => return Vec::new(),
    };

    let digest_name = path.file_stem()
        .and_then(|s| s.to_str())
        .unwrap_or("");
    let digest_key = format!("{}.md", digest_name);
    let filename = path.file_name()
        .and_then(|s| s.to_str())
        .unwrap_or("")
        .to_string();

    let link_re = Regex::new(r"^-\s+(.+?)\s*[→↔←]\s*(.+?)(?:\s*\((.+?)\))?\s*$").unwrap();
    let header_re = Regex::new(r"^##\s+Links").unwrap();
    let mut links = Vec::new();
    let mut in_links = false;

    for line in content.lines() {
        if header_re.is_match(line) {
            in_links = true;
            continue;
        }
        if in_links && line.starts_with("## ") {
            in_links = false;
            continue;
        }
        if !in_links { continue; }
        if line.starts_with("###") || line.starts_with("**") { continue; }

        if let Some(cap) = link_re.captures(line) {
            let raw_source = cap[1].trim();
            let raw_target = cap[2].trim();
            let reason = cap.get(3).map(|m| m.as_str().to_string()).unwrap_or_default();

            let mut source = normalize_link_key(raw_source);
            let mut target = normalize_link_key(raw_target);

            // Replace self-references with digest key
            if source.is_empty() { source = digest_key.clone(); }
            if target.is_empty() { target = digest_key.clone(); }

            // Handle "this daily/weekly/monthly" in raw text
            let raw_s_lower = raw_source.to_lowercase();
            let raw_t_lower = raw_target.to_lowercase();
            if raw_s_lower.contains("this daily") || raw_s_lower.contains("this weekly")
                || raw_s_lower.contains("this monthly")
            {
                source = digest_key.clone();
            }
            if raw_t_lower.contains("this daily") || raw_t_lower.contains("this weekly")
                || raw_t_lower.contains("this monthly")
            {
                target = digest_key.clone();
            }

            // Skip NEW: and self-links
            if source.starts_with("NEW:") || target.starts_with("NEW:") { continue; }
            if source == target { continue; }

            links.push(DigestLink { source, target, reason, file: filename.clone() });
        }
    }

    links
}

/// Parse links from all digest files in the episodic dir.
pub fn parse_all_digest_links() -> Result<Vec<DigestLink>, String> {
    let dir = memory_subdir("episodic")?;
    let mut all_links = Vec::new();

    for pattern in &["daily-*.md", "weekly-*.md", "monthly-*.md"] {
        if let Ok(entries) = fs::read_dir(&dir) {
            let mut files: Vec<PathBuf> = entries
                .filter_map(|e| e.ok())
                .map(|e| e.path())
                .filter(|p| {
                    p.file_name()
                        .and_then(|n| n.to_str())
                        .map(|n| {
                            let prefix = pattern.split('*').next().unwrap_or("");
                            n.starts_with(prefix) && n.ends_with(".md")
                        })
                        .unwrap_or(false)
                })
                .collect();
            files.sort();
            for path in files {
                all_links.extend(parse_digest_file_links(&path));
            }
        }
    }

    // Deduplicate by (source, target) pair
    let mut seen = std::collections::HashSet::new();
    all_links.retain(|link| seen.insert((link.source.clone(), link.target.clone())));

    Ok(all_links)
}

/// Apply parsed digest links to the store.
pub fn apply_digest_links(store: &mut Store, links: &[DigestLink]) -> (usize, usize, usize) {
    let mut applied = 0usize;
    let mut skipped = 0usize;
    let mut fallbacks = 0usize;

    for link in links {
        // Try resolving both keys
        let source = match store.resolve_key(&link.source) {
            Ok(s) => s,
            Err(_) => {
                // Try stripping section anchor as fallback
                if let Some(base) = link.source.split('#').next() {
                    match store.resolve_key(base) {
                        Ok(s) => { fallbacks += 1; s }
                        Err(_) => { skipped += 1; continue; }
                    }
                } else {
                    skipped += 1; continue;
                }
            }
        };
        let target = match store.resolve_key(&link.target) {
            Ok(t) => t,
            Err(_) => {
                if let Some(base) = link.target.split('#').next() {
                    match store.resolve_key(base) {
                        Ok(t) => { fallbacks += 1; t }
                        Err(_) => { skipped += 1; continue; }
                    }
                } else {
                    skipped += 1; continue;
                }
            }
        };

        // Refine target to best-matching section if available
        let source_content = store.nodes.get(&source)
            .map(|n| n.content.as_str()).unwrap_or("");
        let target = neuro::refine_target(store, source_content, &target);

        if source == target { skipped += 1; continue; }

        // Check if link already exists
        let exists = store.relations.iter().any(|r|
            r.source_key == source && r.target_key == target && !r.deleted
        );
        if exists { skipped += 1; continue; }

        let source_uuid = match store.nodes.get(&source) {
            Some(n) => n.uuid,
            None => { skipped += 1; continue; }
        };
        let target_uuid = match store.nodes.get(&target) {
            Some(n) => n.uuid,
            None => { skipped += 1; continue; }
        };

        let rel = new_relation(
            source_uuid, target_uuid,
            store::RelationType::Link,
            0.5,
            &source, &target,
        );
        if store.add_relation(rel).is_ok() {
            println!("  + {} → {}", source, target);
            applied += 1;
        }
    }

    (applied, skipped, fallbacks)
}