[package]
name = "brainwires-datasets"
description = "Training data pipelines for the Brainwires Agent Framework — JSONL I/O, tokenization, deduplication, format conversion"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
homepage.workspace = true
authors.workspace = true
readme = "README.md"
documentation = "https://docs.rs/brainwires-datasets"
keywords = ["datasets", "training-data", "tokenization", "jsonl", "deduplication"]
categories = ["development-tools"]

[dependencies]
brainwires-core.workspace = true
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
anyhow.workspace = true
tracing.workspace = true
uuid.workspace = true

# Feature-gated
tokenizers = { version = "0.22", optional = true }
tiktoken-rs = { version = "0.9", optional = true }
sha2 = { workspace = true, optional = true }
rand = { workspace = true, optional = true }

[features]
default = ["hf-tokenizer"]
hf-tokenizer = ["dep:tokenizers"]
tiktoken = ["dep:tiktoken-rs"]
dedup = ["dep:sha2", "dep:rand"]
full = ["hf-tokenizer", "tiktoken", "dedup"]

[dev-dependencies]
serde_json.workspace = true
tempfile.workspace = true
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
