[package]
name = "llm-tokenizer"
version = "1.4.2"
edition = "2021"
description = "LLM tokenizer library with caching and chat template support"
license = "Apache-2.0"
repository = "https://github.com/lightseekorg/smg"
authors = [
    "Simo Lin <linsimo.mark@gmail.com>",
    "Chang Su <mckvtl@gmail.com>",
    "Keyang Ru <rukeyang@gmail.com>",
]
keywords = ["tokenizer", "llm", "huggingface", "tiktoken", "chat-template"]
categories = ["text-processing", "parsing"]

[lib]
name = "llm_tokenizer"

[dependencies]
anyhow.workspace = true
blake3.workspace = true
dashmap.workspace = true
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["sync", "rt-multi-thread", "macros"] }
tracing.workspace = true
uuid = { workspace = true, features = ["v4", "serde"] }
futures.workspace = true
# Only used by tokenizer
aho-corasick = "1.1"
base64 = "0.22"
rustc-hash = "2"
hf-hub = { version = "0.5.0", default-features = false, features = ["tokio", "rustls-tls"] }
minijinja = { version = "2.21", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
minijinja-contrib = { version = "2.21", features = ["pycompat"] }
rayon = "1.12"
tiktoken-rs = "0.12"
tokenizers = "0.23.1"

[dev-dependencies]
criterion = "0.8"
openai-protocol.workspace = true
reqwest = { workspace = true, features = ["blocking"] }
tempfile = "3.27"

[[bench]]
name = "stop_sequence_search"
harness = false

[[bench]]
name = "stop_decoder"
harness = false

[lints]
workspace = true
