kent/consciousness
1
0
Fork 1
Code Issues 1 Pull requests 5 Projects Releases Packages Wiki Activity Actions

update tokenizers

Browse source
This commit is contained in:
Kent Overstreet 2026-04-13 22:39:50 -04:00
parent 4d22a28794
commit be909028a7
3 changed files with 15 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

38
Cargo.lock generated
View file

@ -592,15 +592,14 @@ dependencies = [
[[package]]
name = "console"
version = "0.15.11"
version = "0.16.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87"
dependencies = [
"encode_unicode",
"libc",
"once_cell",
"unicode-width",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@ -1421,14 +1420,14 @@ dependencies = [
[[package]]
name = "indicatif"
version = "0.17.11"
version = "0.18.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb"
dependencies = [
"console",
"number_prefix",
"portable-atomic",
"unicode-width",
"unit-prefix",
"web-time",
]
@ -1803,12 +1802,6 @@ dependencies = [
"libc",
]
[[package]]
name = "number_prefix"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "once_cell"
version = "1.21.4"
@ -2947,9 +2940,9 @@ dependencies = [
[[package]]
name = "tokenizers"
version = "0.21.4"
version = "0.22.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476"
checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223"
dependencies = [
"ahash",
"aho-corasick",
@ -3427,6 +3420,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
[[package]]
name = "unit-prefix"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3"
[[package]]
name = "untrusted"
version = "0.9.0"
@ -3778,15 +3777,6 @@ dependencies = [
"windows-targets",
]
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-sys"
version = "0.61.2"

2
Cargo.toml
View file

@ -58,7 +58,7 @@ futures = "0.3"
capnp = "0.25"
capnp-rpc = "0.25"
tokenizers = "0.21"
tokenizers = "0.22"
http = "1"
hyper = { version = "1", features = ["client", "http1"] }

10
src/hippocampus/store/index.rs
View file

@ -128,16 +128,6 @@ pub fn recent_by_provenance(db: &Database, provenance: &str, limit: usize) -> Re
Ok(results)
}
/// Get node metadata (uuid, node_type, timestamp) from KEY_TO_UUID.
pub fn get_node_meta(db: &Database, key: &str) -> Result<Option<([u8; 16], u8, i64)>> {
let txn = db.begin_read()?;
let table = txn.open_table(KEY_TO_UUID)?;
match table.get(key)? {
Some(data) => Ok(Some(unpack_node_meta(data.value()))),
None => Ok(None),
}
}
/// Get offset for a node by key.
pub fn get_offset(db: &Database, key: &str) -> Result<Option<u64>> {
let txn = db.begin_read()?;