diff --git a/src/agent/context.rs b/src/agent/context.rs
index 0c98dd8..c25cdbe 100644
--- a/src/agent/context.rs
+++ b/src/agent/context.rs
@@ -223,12 +223,11 @@ fn trim_entries(entries: &[ContextEntry], fixed_tokens: usize) -> Vec<ContextEnt
 
 fn lowest_scored_memory(entries: &[ContextEntry]) -> Option<usize> {
     entries.iter().enumerate()
-        .filter(|(_, e)| e.entry.is_memory())
-        .min_by(|(_, a), (_, b)| {
-            let sa = match &a.entry { ConversationEntry::Memory { score, .. } => score.unwrap_or(0.0), _ => 0.0 };
-            let sb = match &b.entry { ConversationEntry::Memory { score, .. } => score.unwrap_or(0.0), _ => 0.0 };
-            sa.partial_cmp(&sb).unwrap_or(std::cmp::Ordering::Equal)
+        .filter_map(|(i, e)| match &e.entry {
+            ConversationEntry::Memory { score: Some(s), .. } => Some((i, *s)),
+            _ => None,
         })
+        .min_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
         .map(|(i, _)| i)
 }