[package]
name = "chunkshop-rs"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
authors.workspace = true
description = "Standalone ingest-to-pgvector: source -> chunker -> embedder -> extractor -> table. int8 BGE by default; bakeoff matrix evaluator built in. Cross-language wire-format compatible with the Python `chunkshop` package."
readme = "../README.md"
keywords = ["rag", "pgvector", "embeddings", "chunking", "fastembed"]
categories = ["database", "science", "text-processing"]

# Keep the published .crate tarball lean. Without these excludes, cargo
# bundles all the parity-test JSON references — useful for repo CI but
# noise for users who install the crate.
exclude = [
    "tests/parity-fixtures/*",
]

[lib]
name = "chunkshop"
path = "src/lib.rs"

[[bin]]
name = "chunkshop-rs"
path = "src/main.rs"
# The CLI consumes load_config + run_cell + run_bakeoff_with_base, which
# transitively need every module. Gating to `full` ensures the binary only
# builds when the consumer opted into the heavy dep stack.
required-features = ["full"]

[[bin]]
# Thin CLI exposing build_fqn + code_symbol_node_id for the Python
# cross-port parity harness (RM-C SC-003). Depends only on the always-on
# `codeparse` module, so no `required-features` — builds against the slim
# library too.
name = "fqn-cli"
path = "src/bin/fqn-cli.rs"

[dependencies]
# --- Always-on core (small, used by the chunker module) ---
anyhow = "1"
regex = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_yaml_ng = "0.10"
# RM-B Task 5: SHA-256-keyed directory layout in LocalRawStore (path-
# traversal-safe doc_id mapping). Same hash Python uses, so a store written
# by Python is readable by Rust at byte-identical paths.
sha2 = "0.10"
thiserror = "1"
tracing = "0.1"

# --- Bin-only (kept always-on for diff minimality; clap + tracing-subscriber
# are tiny relative to the heavy deps below) ---
clap = { version = "4", features = ["derive"] }
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }

# --- `embedder` / `embedder-core` / `embedder-hub` features: ML inference stack ---
# fastembed pinned to >= 5.13.1 — earlier 5.x bumps (5.0.x..5.10.x) require an
# older `ort` (=2.0.0-rc.10 / =2.0.0-rc.11) and conflict with chunkshop's
# `ort = "=2.0.0-rc.12"` pin below. fastembed 5.13.1+ aligns on rc.12.
#
# Declared with `default-features = false` so its bundled `hf-hub-native-tls`
# feature does NOT pull `hf-hub` transitively. Consumers using `embedder-core`
# get fastembed's BYO (`try_new_from_user_defined`) path without `hf-hub` /
# `ureq` / native-tls deps. The `embedder-hub` feature re-enables fastembed's
# `hf-hub-native-tls` for the stock auto-download path.
fastembed = { version = "5.13.1", default-features = false, features = ["ort-download-binaries-native-tls", "image-models"], optional = true }
hf-hub = { version = "0.5", default-features = false, features = ["ureq"], optional = true }
ndarray = { version = "0.17.2", default-features = false, optional = true }
ort = { version = "=2.0.0-rc.12", default-features = false, features = ["ndarray", "std", "api-24"], optional = true }
tokenizers = { version = "0.22.2", default-features = false, features = ["onig"], optional = true }

# --- `extractor` feature ---
whatlang = { version = "0.16", optional = true }

# --- `source` feature: files (glob) / HTTP / S3 ---
futures = { version = "0.3", optional = true }
glob = { version = "0.3", optional = true }
object_store = { version = "0.11", default-features = false, features = ["aws"], optional = true }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"], optional = true }
sha1 = "0.10"

# --- `sink` feature: the full modular sink/backend layer ---
# The v4 modular-backends line moved every backend (PG/MariaDB/SQLite/
# ClickHouse) under the `sink` feature. So `sink` pulls the union of every
# crate `src/sinks/` + `src/backends/` use: blake2 (PG content hash), pgvector
# + sqlx (PG/MariaDB drivers), clickhouse, rusqlite + sqlite-vec (SQLite path),
# and url/urlencoding (used only by backends/clickhouse.rs to build the
# ClickHouse HTTP URL). default-features=false on sqlx is preserved from the
# v4 line so sqlx-sqlite's libsqlite3-sys does not clash with rusqlite's.
blake2 = { version = "0.10", default-features = false, optional = true }
# Pinned to =0.4.1: pgvector 0.4.2 widened its sqlx req to >=0.8,<0.10, which lets
# Cargo pull sqlx 0.9 alongside our pinned 0.8.6 -> two sqlx_core versions, trait mismatch.
pgvector = { version = "=0.4.1", features = ["sqlx"], optional = true }
sqlx = { version = "0.8.6", default-features = false, features = ["runtime-tokio", "postgres", "mysql", "json", "chrono"], optional = true }
clickhouse = { version = "0.15", optional = true }
rusqlite = { version = "0.32", features = ["bundled", "load_extension"], optional = true }
sqlite-vec = { version = "=0.1.6", optional = true }
url = { version = "2", optional = true }
urlencoding = { version = "2", optional = true }

# --- Shared runtime (gated by source + sink) ---
tokio = { version = "1", features = ["rt-multi-thread", "macros"], optional = true }

# --- Optional callable summarizer integration ---
lede = { version = "0.3", optional = true }

# --- `code-aware-*` features: tree-sitter parsers for SymbolAwareChunker (RM-C) ---
# Runtime + tags lib are version-locked together (both 0.26.x). Grammar crates
# are pinned to their latest crates.io release as of 2026-05-28; they expose
# C parsers via the parser-ABI (currently ABI 14/15) which is backward-
# compatible with tree-sitter 0.25/0.26 runtimes. All deps are `optional` so
# the default build is unaffected.
tree-sitter            = { version = "0.26", optional = true }
tree-sitter-tags       = { version = "0.26", optional = true }
tree-sitter-python     = { version = "0.25", optional = true }
tree-sitter-java       = { version = "0.23", optional = true }
# Should-haves — declared so the [features] block compiles. Extractors for
# these are deferred to a follow-up (gated on chunkshop#40).
tree-sitter-go         = { version = "0.25", optional = true }
tree-sitter-typescript = { version = "0.23", optional = true }
tree-sitter-javascript = { version = "0.25", optional = true }
tree-sitter-rust       = { version = "0.24", optional = true }

[dev-dependencies]
# `no-env-filter` lets `#[traced_test]` capture warn! events from any crate,
# not just the test executable. Required to assert on tracing events emitted
# by `chunkshop::semantic` and `chunkshop::oversize` from inside integration
# tests.
tracing-test = { version = "0.2", features = ["no-env-filter"] }
tempfile = "3"
# RM-B Task 1: tests/sync_primitives.rs drives the IncrementalSource and
# PrunableSource futures via futures::executor::block_on (no tokio runtime
# needed for the smoke path). Declared in dev-deps so the test compiles
# regardless of whether the `source` feature (which also pulls `futures`) is
# enabled.
futures = "0.3"
# RM-C Task 7: property-based invariants for build_fqn + code_symbol_node_id
# in tests/cross_port_proptest.rs (determinism, separator normalization,
# output shape, project-id divergence).
proptest = "1"

[features]
# Default = full feature set for backward compatibility. Existing consumers
# of `chunkshop = "0.3"` (and the `chunkshop-rs` binary) see no change.
default = ["full"]

# Everything — the historical compile target. Required by the CLI binary.
full = ["chunkers", "embedder", "extractor", "source", "sink", "pipeline", "bakeoff", "memory"]

# RM-A: agent-memory primitives (staging API, session_staging source,
# session_episode framer, consolidation chunker, MemorySink, consolidator
# trait + extractive default). Tracks chunkshop#9. Postgres-only, requires
# `source` + `sink` to share the existing backend layer.
memory = ["source", "sink"]

# Just the chunker module + its config types and supporting modules
# (sentence_split, summarizer, the `Document` struct). Pulls only the
# small always-on core deps. The `SemanticChunker` requires `embedder-hub`
# (or `embedder`) in addition; without that feature the semantic chunker is
# excluded.
chunkers = []

# ML inference stack (fastembed + ORT). Required for `FastembedEmbedder`
# and the `SemanticChunker`'s boundary embedder.
#
# Split into two sub-features so external consumers (e.g. embedded library
# users that load model bytes from their own storage) can avoid pulling
# `hf-hub`, `ureq`, and the native-tls crypto stack:
#
# - `embedder-core` — fastembed (BYO/`try_new_from_user_defined`) + ort +
#   ndarray + tokenizers. NO hf-hub. NO auto-download. Caller supplies model
#   bytes directly via `FastembedEmbedder::from_user_defined_files`.
#
# - `embedder-hub` — adds `hf-hub` and re-enables fastembed's
#   `hf-hub-native-tls` for the stock auto-download path
#   (`FastembedEmbedder::new` + the bit-near-exact Xenova int8 BGE path).
#
# - `embedder` = `embedder-core` + `embedder-hub` — historical alias,
#   preserves the dep tree existing consumers see today.
embedder = ["embedder-core", "embedder-hub"]
embedder-core = ["dep:fastembed", "dep:ndarray", "dep:ort", "dep:tokenizers"]
embedder-hub = ["embedder-core", "dep:hf-hub", "fastembed/hf-hub-native-tls"]

# Language-detection extractor (whatlang).
extractor = ["dep:whatlang"]

# Source loaders: files (glob), HTTP, S3. DB-table sources (PG/MariaDB/
# SQLite/ClickHouse) additionally require `sink` since they reuse the
# backend connection layer — they are gated on `all(source, sink)` in code.
source = ["dep:futures", "dep:glob", "dep:object_store", "dep:reqwest", "dep:tokio"]

# The full modular sink/backend layer: PG (pgvector), MariaDB (sqlx mysql),
# SQLite (rusqlite + sqlite-vec), ClickHouse (clickhouse + url/urlencoding).
# The v4 modular-backends line folded every backend under this one feature
# (no per-backend features — deliberate design decision).
sink = [
    "dep:blake2",
    "dep:pgvector",
    "dep:sqlx",
    "dep:clickhouse",
    "dep:rusqlite",
    "dep:sqlite-vec",
    "dep:url",
    "dep:urlencoding",
    "dep:tokio",
]

# High-level Pipeline + run_cell glue. Composes chunker + embedder +
# extractor + source + sink.
pipeline = ["chunkers", "embedder", "extractor", "source", "sink"]

# Matrix evaluator (chunkers x embedders → leaderboard).
bakeoff = ["pipeline"]

# Enables the `chunkshop.summarizers.lede` callable summarizer module by
# pulling the `lede` crate from crates.io. Build with: cargo build --features lede
lede = ["dep:lede"]

# Code-aware chunking (RM-C). Each grammar is opt-in to control binary size.
# See README.md "Code-aware chunking" for the size delta table.
code-aware = []  # Umbrella — implied by per-grammar features
code-aware-python = ["code-aware", "dep:tree-sitter", "dep:tree-sitter-python", "dep:tree-sitter-tags"]
code-aware-java   = ["code-aware", "dep:tree-sitter", "dep:tree-sitter-java",   "dep:tree-sitter-tags"]

# Should-have grammars (extractors not implemented in v1 — see chunkshop#40).
code-aware-go         = ["code-aware", "dep:tree-sitter", "dep:tree-sitter-go",         "dep:tree-sitter-tags"]
code-aware-typescript = ["code-aware", "dep:tree-sitter", "dep:tree-sitter-typescript", "dep:tree-sitter-tags"]
code-aware-javascript = ["code-aware", "dep:tree-sitter", "dep:tree-sitter-javascript", "dep:tree-sitter-tags"]
code-aware-rust       = ["code-aware", "dep:tree-sitter", "dep:tree-sitter-rust",       "dep:tree-sitter-tags"]
