From cb64cdf5fe02fee70655ccec343f55f7ddcd45a4 Mon Sep 17 00:00:00 2001
From: Kent Overstreet <kent.overstreet@linux.dev>
Date: Wed, 8 Apr 2026 11:55:32 -0400
Subject: [PATCH] Init tokenizer in consciousness binary main

The consciousness binary has its own main() separate from poc-memory.
Agent::new() creates ContextEntries which need the tokenizer, so it
must be initialized before Mind::new().

Co-Authored-By: Proof of Concept <poc@bcachefs.org>
---
 src/user/mod.rs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/user/mod.rs b/src/user/mod.rs
index 554cab9..a73954a 100644
--- a/src/user/mod.rs
+++ b/src/user/mod.rs
@@ -522,6 +522,13 @@ pub enum SubCmd {
 
 #[tokio::main]
 pub async fn main() {
+    // Initialize the Qwen tokenizer for direct token generation
+    let tokenizer_path = dirs::home_dir().unwrap_or_default()
+        .join(".consciousness/tokenizer-qwen35.json");
+    if tokenizer_path.exists() {
+        crate::agent::tokenizer::init(&tokenizer_path.to_string_lossy());
+    }
+
     let cli = CliArgs::parse();
 
     if cli.show_config {