[package]
name = "splintr"
version = "0.9.0-beta.1"
edition = "2021"
description = "Fast Rust tokenizer (BPE + SentencePiece) with Python bindings"
license = "MIT"
repository = "https://github.com/ml-rust/splintr"
homepage = "https://github.com/ml-rust/splintr"
readme = "README.md"
keywords = ["tokenizer", "bpe", "sentencepiece", "tiktoken", "llm"]
categories = ["text-processing", "encoding"]

[lib]
name = "splintr"
crate-type = ["cdylib", "rlib"]

[features]
default = ["rayon", "regexr-jit"]
python = ["dep:pyo3"]
pcre2 = ["dep:pcre2"]
rayon = ["dep:rayon"]
regexr-jit = ["regexr/jit", "regexr/simd"]
wasm = []  # disables rayon, uses scalar regex — use with --no-default-features

[dependencies]
# PCRE2 regex with JIT support (optional, for benchmarking)
pcre2 = { version = "0.2", optional = true }
# Rayon for internal parallelism
rayon = { version = "1.10", optional = true }
# Fast hashing (FxHashMap)
rustc-hash = "2.0"
# Error handling
thiserror = "2.0"
# Python bindings
pyo3 = { version = "0.27", features = ["extension-module"], optional = true }
# Base64 decoding for tiktoken vocab files
base64 = "0.22"
# Aho-Corasick for fast multi-pattern special token matching
aho-corasick = "1.1"
# LRU cache for frequent token sequences
lru = "0.16"
# regexr regex engine (default backend)
regexr = { version = "0.1.0-beta.5", default-features = false }

[dev-dependencies]
# PCRE2 for benchmarking comparisons
pcre2 = "0.2"

[profile.release]
opt-level = 3
lto = true
codegen-units = 1
