kent/consciousness
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Kill tiktoken — all token counting now uses Qwen 3.5 tokenizer

Browse source 
Remove tiktoken-rs dependency, CoreBPE field on Agent, and the
msg_token_count() function. All tokenization now goes through the
global HuggingFace tokenizer in agent/tokenizer.rs.

Co-Authored-By: Proof of Concept <poc@bcachefs.org>
This commit is contained in:
Kent Overstreet 2026-04-08 11:25:28 -04:00
parent 5e4067c04f
commit 67e3228c32
4 changed files with 1 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

46
Cargo.lock generated
View file

@ -219,17 +219,6 @@ dependencies = [
"generic-array",
]
[[package]]
name = "bstr"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab"
dependencies = [
"memchr",
"regex-automata",
"serde",
]
[[package]]
name = "bumpalo"
version = "3.20.2"
@ -894,17 +883,6 @@ dependencies = [
"regex",
]
[[package]]
name = "fancy-regex"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
dependencies = [
"bit-set",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "figment"
version = "0.10.19"
@ -1965,7 +1943,6 @@ dependencies = [
"serde_json",
"serde_urlencoded",
"skillratings",
"tiktoken-rs",
"tokenizers",
"tokio",
"tokio-rustls",
@ -2393,12 +2370,6 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc_version"
version = "0.4.1"
@ -2784,7 +2755,7 @@ dependencies = [
"anyhow",
"base64 0.22.1",
"bitflags 2.11.0",
"fancy-regex 0.11.0",
"fancy-regex",
"filedescriptor",
"finl_unicode",
"fixedbitset",
@ -2857,21 +2828,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "tiktoken-rs"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a19830747d9034cd9da43a60eaa8e552dfda7712424aebf187b7a60126bae0d"
dependencies = [
"anyhow",
"base64 0.22.1",
"bstr",
"fancy-regex 0.13.0",
"lazy_static",
"regex",
"rustc-hash",
]
[[package]]
name = "time"
version = "0.3.47"