From be909028a76af8acf2b02b2657beda2fc799fe23 Mon Sep 17 00:00:00 2001
From: Kent Overstreet <kent.overstreet@linux.dev>
Date: Mon, 13 Apr 2026 22:39:50 -0400
Subject: [PATCH] update tokenizers

---
 Cargo.lock                     | 38 +++++++++++++---------------------
 Cargo.toml                     |  2 +-
 src/hippocampus/store/index.rs | 10 ---------
 3 files changed, 15 insertions(+), 35 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index cb36deb..a005ed9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -592,15 +592,14 @@ dependencies = [
 
 [[package]]
 name = "console"
-version = "0.15.11"
+version = "0.16.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
+checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87"
 dependencies = [
  "encode_unicode",
  "libc",
- "once_cell",
  "unicode-width",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -1421,14 +1420,14 @@ dependencies = [
 
 [[package]]
 name = "indicatif"
-version = "0.17.11"
+version = "0.18.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
+checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb"
 dependencies = [
  "console",
- "number_prefix",
  "portable-atomic",
  "unicode-width",
+ "unit-prefix",
  "web-time",
 ]
 
@@ -1803,12 +1802,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "number_prefix"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
-
 [[package]]
 name = "once_cell"
 version = "1.21.4"
@@ -2947,9 +2940,9 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.21.4"
+version = "0.22.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476"
+checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223"
 dependencies = [
  "ahash",
  "aho-corasick",
@@ -3427,6 +3420,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
+[[package]]
+name = "unit-prefix"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3"
+
 [[package]]
 name = "untrusted"
 version = "0.9.0"
@@ -3778,15 +3777,6 @@ dependencies = [
  "windows-targets",
 ]
 
-[[package]]
-name = "windows-sys"
-version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
-dependencies = [
- "windows-targets",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.61.2"
diff --git a/Cargo.toml b/Cargo.toml
index 52f1a2b..caa283e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -58,7 +58,7 @@ futures = "0.3"
 capnp = "0.25"
 capnp-rpc = "0.25"
 
-tokenizers = "0.21"
+tokenizers = "0.22"
 
 http = "1"
 hyper = { version = "1", features = ["client", "http1"] }
diff --git a/src/hippocampus/store/index.rs b/src/hippocampus/store/index.rs
index a663366..95ebe19 100644
--- a/src/hippocampus/store/index.rs
+++ b/src/hippocampus/store/index.rs
@@ -128,16 +128,6 @@ pub fn recent_by_provenance(db: &Database, provenance: &str, limit: usize) -> Re
     Ok(results)
 }
 
-/// Get node metadata (uuid, node_type, timestamp) from KEY_TO_UUID.
-pub fn get_node_meta(db: &Database, key: &str) -> Result<Option<([u8; 16], u8, i64)>> {
-    let txn = db.begin_read()?;
-    let table = txn.open_table(KEY_TO_UUID)?;
-    match table.get(key)? {
-        Some(data) => Ok(Some(unpack_node_meta(data.value()))),
-        None => Ok(None),
-    }
-}
-
 /// Get offset for a node by key.
 pub fn get_offset(db: &Database, key: &str) -> Result<Option<u64>> {
     let txn = db.begin_read()?;