Move tool definitions into ContextState as system entries

Tool definitions are now pushed as a ContextEntry in the system
section at Agent construction time, formatted in the Qwen chat
template style. They're tokenized, scored, and treated like any
other context entry.

assemble_prompt_tokens() no longer takes a tools parameter —
tools are already in the context. This prepares for the switch
to /v1/completions where tools aren't a separate API field.

Co-Authored-By: Proof of Concept <poc@bcachefs.org>

Cargo.lock

@@ -2,6 +2,12 @@
 # It is not intended for manual editing.
 version = 4
 
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
 [[package]]
 name = "ahash"
 version = "0.7.8"
@@ -51,6 +57,19 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "ansi-to-tui"
+version = "8.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e42366bb9d958f042bf58f0a85e1b2d091997c1257ca49bddd7e4827aadc65fd"
+dependencies = [
+ "nom 8.0.0",
+ "ratatui-core",
+ "simdutf8",
+ "smallvec",
+ "thiserror 2.0.18",
+]
+
 [[package]]
 name = "anstream"
 version = "1.0.0"
@@ -533,6 +552,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crc32fast"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crossbeam-deque"
 version = "0.8.6"
@@ -925,6 +953,16 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
 
+[[package]]
+name = "flate2"
+version = "1.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1458,6 +1496,12 @@ dependencies = [
  "bitflags 2.11.0",
 ]
 
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -1556,6 +1600,16 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
+[[package]]
+name = "miniz_oxide"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
 [[package]]
 name = "mio"
 version = "1.2.0"
@@ -1613,6 +1667,15 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "nom"
+version = "8.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "num-conv"
 version = "0.2.1"
@@ -1901,6 +1964,19 @@ version = "0.3.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
 
+[[package]]
+name = "plist"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07"
+dependencies = [
+ "base64 0.22.1",
+ "indexmap",
+ "quick-xml",
+ "serde",
+ "time",
+]
+
 [[package]]
 name = "poc-memory"
 version = "0.4.0"
@@ -2073,6 +2149,15 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
 
+[[package]]
+name = "quick-xml"
+version = "0.38.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.45"
@@ -2453,6 +2538,15 @@ version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "schannel"
 version = "0.1.29"
@@ -2606,6 +2700,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "simd-adler32"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
+
 [[package]]
 name = "simdutf8"
 version = "0.1.5"
@@ -2653,7 +2753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
 dependencies = [
  "base64 0.13.1",
- "nom",
+ "nom 7.1.3",
  "serde",
  "unicode-segmentation",
 ]
@@ -2719,6 +2819,27 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syntect"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "656b45c05d95a5704399aeef6bd0ddec7b2b3531b7c9e900abbf7c4d2190c925"
+dependencies = [
+ "bincode",
+ "flate2",
+ "fnv",
+ "once_cell",
+ "onig",
+ "plist",
+ "regex-syntax",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "thiserror 2.0.18",
+ "walkdir",
+ "yaml-rust",
+]
+
 [[package]]
 name = "tap"
 version = "1.0.1"
@@ -2732,7 +2853,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4ea810f0692f9f51b382fff5893887bb4580f5fa246fde546e0b13e7fcee662"
 dependencies = [
  "fnv",
- "nom",
+ "nom 7.1.3",
  "phf",
  "phf_codegen",
 ]
@@ -2835,12 +2956,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
 dependencies = [
  "deranged",
+ "itoa",
  "libc",
  "num-conv",
  "num_threads",
  "powerfmt",
  "serde_core",
  "time-core",
+ "time-macros",
 ]
 
 [[package]]
@@ -2849,6 +2972,16 @@ version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
 
+[[package]]
+name = "time-macros"
+version = "0.2.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
+dependencies = [
+ "num-conv",
+ "time-core",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.11.0"
@@ -2982,10 +3115,12 @@ name = "tui-markdown"
 version = "0.3.7"
 source = "git+https://github.com/koverstreet/tui-markdown#9cca280718c4e74f14f53e6df2843abf3ef3dccd"
 dependencies = [
+ "ansi-to-tui",
  "itertools",
  "log",
  "pulldown-cmark",
  "ratatui-core",
+ "syntect",
 ]
 
 [[package]]
@@ -3118,6 +3253,16 @@ dependencies = [
  "utf8parse",
 ]
 
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
 [[package]]
 name = "want"
 version = "0.3.1"
@@ -3337,6 +3482,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
@@ -3590,6 +3744,15 @@ dependencies = [
  "tap",
 ]
 
+[[package]]
+name = "yaml-rust"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
+dependencies = [
+ "linked-hash-map",
+]
+
 [[package]]
 name = "yansi"
 version = "1.0.1"

Cargo.toml

@@ -32,7 +32,7 @@ json5 = "1.3"
 
 crossterm = { version = "0.29", features = ["event-stream"] }
 ratatui = { version = "0.30", features = ["unstable-rendered-line-info"] }
-tui-markdown = { git = "https://github.com/koverstreet/tui-markdown", subdirectory = "tui-markdown", default-features = false }
+tui-markdown = { git = "https://github.com/koverstreet/tui-markdown", subdirectory = "tui-markdown" }
 tui-textarea = { version = "0.10.2", package = "tui-textarea-2" }
 
 uuid = { version = "1", features = ["v4"] }

src/agent/mod.rs

@@ -192,6 +192,24 @@ impl Agent {
         let mut system = ContextSection::new("System prompt");
         system.push(ContextEntry::new(
             ConversationEntry::System(Message::system(&system_prompt)), None));
+
+        // Tool definitions — part of the context, tokenized and scored
+        let tool_defs: Vec<String> = tools::tools().iter()
+            .map(|t| t.to_json()).collect();
+        if !tool_defs.is_empty() {
+            let tools_text = format!(
+                "# Tools\n\nYou have access to the following functions:\n\n<tools>\n{}\n</tools>\n\n\
+                If you choose to call a function ONLY reply in the following format with NO suffix:\n\n\
+                <tool_call>\n<function=example_function_name>\n\
+                <parameter=example_parameter_1>\nvalue_1\n</parameter>\n\
+                </function>\n</tool_call>\n\n\
+                IMPORTANT: Function calls MUST follow the specified format.",
+                tool_defs.join("\n"),
+            );
+            system.push(ContextEntry::new(
+                ConversationEntry::System(Message::system(&tools_text)), None));
+        }
+
         let mut identity = ContextSection::new("Identity");
         for (_name, content) in &personality {
             identity.push(ContextEntry::new(
@@ -293,6 +311,42 @@ impl Agent {
         msgs
     }
 
+    /// Assemble the full prompt as token IDs for the completions API.
+    /// System section (includes tools), identity, journal, conversation,
+    /// then the assistant prompt suffix.
+    pub fn assemble_prompt_tokens(&self) -> Vec<u32> {
+        let mut tokens = Vec::new();
+
+        // System section — includes system prompt + tool definitions
+        for e in self.context.system.entries() {
+            tokens.extend(&e.token_ids);
+        }
+
+        // Identity — rendered as one user message
+        let ctx = self.context.render_context_message();
+        if !ctx.is_empty() {
+            tokens.extend(tokenizer::tokenize_entry("user", &ctx));
+        }
+
+        // Journal — rendered as one user message
+        let jnl = self.context.render_journal();
+        if !jnl.is_empty() {
+            tokens.extend(tokenizer::tokenize_entry("user", &jnl));
+        }
+
+        // Conversation entries — use cached token_ids
+        for e in self.context.conversation.entries() {
+            if e.entry.is_log() || e.entry.is_thinking() { continue; }
+            tokens.extend(&e.token_ids);
+        }
+
+        // Prompt the assistant to respond
+        tokens.push(tokenizer::IM_START);
+        tokens.extend(tokenizer::encode("assistant\n"));
+
+        tokens
+    }
+
     /// Run agent orchestration cycle, returning structured output.
     /// Push a conversation message — stamped and logged.
     pub fn push_message(&mut self, mut msg: Message) {