[workspace]
members = [".", "kumo-derive", "benchmark/kumo"]
resolver = "2"

[package]
name = "kumo"
version = "0.3.9"
edition = "2024"
rust-version = "1.88"
authors = ["wihlarkop"]
description = "An async web crawling framework for Rust - Scrapy for Rust"
license = "MIT"
repository = "https://github.com/wihlarkop/kumo"
homepage = "https://github.com/wihlarkop/kumo"
documentation = "https://kumo.wihlarkop.com"
readme = "README.md"
keywords = ["web", "crawling", "scraping", "spider", "async"]
categories = ["web-programming", "asynchronous"]
exclude = [
  ".cargo/",
  ".github/",
  "benchmark/",
  "docs/",
  "lefthook.yml",
]

# Feature flags

[features]
default = []

# Database stores
postgres = ["dep:sqlx", "sqlx/postgres"]
sqlite   = ["dep:sqlx", "sqlx/sqlite"]
mysql    = ["dep:sqlx", "sqlx/mysql"]

# LLM extraction (base + per-provider)
llm    = ["dep:schemars", "dep:rig-core"]
claude = ["llm"]
openai = ["llm"]
gemini = ["llm"]
ollama = ["llm"]

# Selectors
jsonpath = ["dep:jsonpath-rust"]
xpath    = ["dep:sxd-xpath", "dep:sxd-document", "dep:sxd_html"]

# Fetchers
browser = ["dep:chromiumoxide", "dep:futures"]
stealth = ["dep:rquest", "dep:rquest-util"]   # TLS/HTTP2 fingerprint spoofing; requires cmake + nasm

# Frontiers
persistence    = []                      # FileFrontier - survives restarts
redis-frontier = ["dep:redis"]           # RedisFrontier - distributed crawling

# Derive macro
derive = ["dep:kumo-derive"]

# OpenTelemetry OTLP export
otel = [
    "dep:opentelemetry",
    "dep:opentelemetry_sdk",
    "dep:opentelemetry-otlp",
    "dep:tracing-opentelemetry",
]

# Cloud storage (backend-agnostic via object_store)
cloud       = ["dep:object_store"]
cloud-s3    = ["cloud", "object_store/aws"]
cloud-gcs   = ["cloud", "object_store/gcp"]
cloud-azure = ["cloud", "object_store/azure"]

# Dependencies

[dependencies]

# Async runtime
tokio = { version = "1.52.1", features = [
  "rt-multi-thread",
  "sync",
  "time",
  "signal",
  "macros",
  "net",
] }
tokio-stream = "0.1"
async-trait = "0.1.89"

# HTTP
reqwest = { version = "0.13.2", default-features = false, features = [
    "rustls",
    "http2",
    "gzip",
    "brotli",
    "deflate",
    "json",
    "cookies",
] }
url     = "2.5.8"
bytes   = "1.11.1"
httpdate = "1.0.3"

# HTML parsing & scraping
scraper = "0.26.0"
regex   = "1.12.3"

# Deduplication
bloomfilter = "3.0.1"

# Serialization
serde      = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.149"

# Error handling
thiserror = "2.0.18"

# Random (jitter in retry back-off)
rand = "0.9"

# Logging & tracing
tracing            = "0.1.44"
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }

# Optional - OpenTelemetry OTLP export
opentelemetry         = { version = "0.27", optional = true }
opentelemetry_sdk     = { version = "0.27", optional = true, features = ["rt-tokio"] }
opentelemetry-otlp    = { version = "0.27", optional = true, features = ["grpc-tonic"] }
tracing-opentelemetry = { version = "0.28", optional = true }

# Rate limiting
governor = "0.10.4"

# Robots.txt
texting_robots = "0.2.2"

# Optional - derive macro
kumo-derive = { version = "0.1.3", path = "kumo-derive", optional = true }

# Optional - selectors
jsonpath-rust = { version = "1.0.4", optional = true }
sxd-xpath    = { version = "0.4.2",  optional = true }
sxd-document = { version = "0.3.2",  optional = true }
sxd_html     = { version = "0.1.2",  optional = true }

# Optional - browser fetcher
chromiumoxide = { version = "0.9.1", optional = true }
futures       = { version = "0.3.32", optional = true, default-features = false, features = ["std"] }

# Optional - stealth fetcher (TLS/HTTP2 fingerprint spoofing via BoringSSL)
rquest      = { version = "5.1.0", optional = true, features = ["cookies"] }
rquest-util = { version = "2.2.1", optional = true }

# Optional - LLM extraction
schemars = { version = "1.2.1", optional = true, features = ["derive"] }
rig-core = { version = "0.35.0", optional = true }

# Optional - database stores
sqlx  = { version = "0.8.6", optional = true, features = ["runtime-tokio-rustls", "json"] }
redis = { version = "1.2.0", optional = true, features = ["tokio-rustls-comp"] }

# Optional - cloud storage (S3, GCS, Azure Blob, local, memory via object_store)
object_store = { version = "0.11", optional = true }

# Dev dependencies

[dev-dependencies]
criterion = { version = "0.7", features = ["async_tokio"] }
mockito  = "1.7.2"
tempfile = "3.27.0"
trybuild = { version = "1.0.116", features = ["diff"] }

# Examples

[[test]]
name = "derive_macro"
required-features = ["derive"]

[[example]]
name = "postgres"
required-features = ["postgres"]

[[example]]
name = "sqlite"
required-features = ["sqlite"]

[[example]]
name = "llm_extract"
required-features = ["claude"]

[[example]]
name = "llm_fallback"
required-features = ["derive", "claude"]

[[example]]
name = "browser"
required-features = ["browser"]

[[example]]
name = "books_derive"
required-features = ["derive"]

[[example]]
name = "link_extractor"

[[example]]
name = "http_cache"

[[example]]
name = "production_crawler"
required-features = ["persistence"]

[[example]]
name = "xpath"
required-features = ["xpath"]

[[example]]
name = "stealth"
required-features = ["stealth"]

[[example]]
name = "cloud"
required-features = ["cloud"]

[[bench]]
name = "fingerprint"
harness = false

[[bench]]
name = "frontier"
harness = false

