[package]
name = "splintr"
version = "0.6.0"
edition = "2021"
description = "Fast Rust BPE tokenizer with Python bindings"
license = "MIT"
repository = "https://github.com/farhan/splintr"
homepage = "https://github.com/farhan/splintr"
readme = "README.md"
keywords = ["tokenizer", "bpe", "tiktoken", "gpt", "llm"]
categories = ["text-processing", "encoding"]

[lib]
name = "splintr"
crate-type = ["cdylib", "rlib"]

[dependencies]
# PCRE2 regex with JIT support (2-4x faster than fancy-regex)
pcre2 = "0.2"
# Rayon for internal parallelism
rayon = "1.10"
# Fast hashing (FxHashMap)
rustc-hash = "2.0"
# Error handling
thiserror = "2.0"
# Python bindings
pyo3 = { version = "0.23", features = ["extension-module"] }
# Base64 decoding for tiktoken vocab files
base64 = "0.22"
# Aho-Corasick for fast multi-pattern special token matching
aho-corasick = "1.1"
# LRU cache for frequent token sequences
lru = "0.12"

[profile.release]
opt-level = 3
lto = true
codegen-units = 1
