[package]
name = "spider"
version = "1.95.8"
authors = [
    "j-mendez <jeff@a11ywatch.com>"
]
description = "The fastest web crawler written in Rust."
repository = "https://github.com/spider-rs/spider"
readme = "README.md"
keywords = ["crawler", "spider"]
categories = ["web-programming", "command-line-utilities"]
license = "MIT"
documentation = "https://docs.rs/spider"
edition = "2018"

[badges]
maintenance = { status = "as-is" }

[dependencies]
reqwest = { version = "0.12", features = [
    "brotli",
    "gzip",
    "deflate",
    "stream",
] }
url = "2"
tokio = { version = "1", features = [
    "rt-multi-thread",
    "macros",
    "time",
    "parking_lot"
] }
tokio-stream = "0.1.14"
hashbrown = { version = "0.14.3", default-features = true }
log = "0.4"
percent-encoding = "2.3.1"
regex = { version = "1.10.4", optional = true }
ua_generator = { version = "^0.4.2", optional = true }
string_concat = "0.0.1"
lazy_static = "1.4.0"
compact_str = "0.7.1"
fast_html5ever = "0.26.6"
selectors = "0.25.0"
tendril = "0.4.3"
ahash = "0.8.11"
cssparser = "0.31.2"
smallvec = "1.11.2"
ego-tree = "0.6.2"
num_cpus = "1.16.0"
bytes = { version = "1.5.0", features = ["serde"] }
serde = { version = "1", optional = true, features = ["derive"] }
flexbuffers = { version = "2.0.0", optional = true }
itertools = { version = "0.12.0", optional = true }
case_insensitive_string = { version = "0.2.2", features = ["compact", "serde"] }
chromiumoxide = { version = "0.5.7", optional = true, features = [
    "tokio-runtime",
    "bytes",
], default-features = false }
sitemap = { version = "0.4.1", optional = true }
chrono = { version = "0.4.31", optional = true }
cron = { version = "0.12.0", optional = true }
async-trait = { version = "0.1.75", optional = true }
napi = { version = "2", features = [
    "async",
    "tokio_rt",
    "napi4",
], optional = true }
strum = { version = "0.26.2", features = ["derive"] }
async_job = { version = "0.1.4", optional = true }
reqwest-middleware = { version = "0.3.0", optional = true }
http-cache-reqwest = { version = "0.14.0", optional = true }
encoding_rs = { version = "0.8.33", optional = true }
const_format = { version = "0.2.32", optional = true }
async-openai = { version = "0.21.0", optional = true }
tiktoken-rs = { version = "0.5.8", optional = true }
lol_html = { version = "1.2.1", optional = true }
serde_json = { version = "1", optional = true }
quick-xml = { version = "0.31.0", features = ["serde", "serialize", "async-tokio"]}
moka = { version = "0.12.1", features = ["future"], optional = true }
fastrand = { version = "2", optional = true }
http-cache-semantics = { version = "2", optional = true }
http-cache = { version = "0.19.0", optional = true, default-features = false }
http = { version = "1", optional = true }

[target.'cfg(all(not(windows), not(target_os = "android"), not(target_env = "musl")))'.dependencies]
tikv-jemallocator = { version = "0.5.0", optional = true }

[features]
default = ["sync", "reqwest/native-tls-alpn", "cookies", "ua_generator"]
regex = ["dep:regex"]
glob = ["dep:regex", "dep:itertools"]
ua_generator = ["dep:ua_generator"]
jemalloc = ["tikv-jemallocator"]
decentralized = ["serde", "flexbuffers"]
control = []
time = []
sync = ["tokio/sync", "tokio/io-std"]
flexbuffers = ["dep:flexbuffers"]
serde = ["dep:serde", "hashbrown/serde", "compact_str/serde"]
fs = ["tokio/fs"]
full_resources = []
socks = ["reqwest/socks"]
reqwest_json = ["reqwest/json"]
sitemap = ["dep:sitemap"]
budget = []
cache = ["dep:reqwest-middleware", "dep:http-cache-reqwest"]
cache_mem = ["cache", "http-cache-reqwest/manager-moka"]
cache_openai = ["dep:moka"]
cache_chrome_hybrid = ["cache", "chrome", "dep:http-cache-semantics", "dep:http-cache", "dep:http"]
chrome = ["dep:chromiumoxide"]
chrome_headed = ["chrome"]
chrome_cpu = ["chrome"]
chrome_stealth = ["chrome"]
chrome_screenshot = ["chrome"]
chrome_store_page = ["chrome", "serde"]
chrome_intercept = ["chrome"]
chrome_headless_new = ["chrome"]
cookies = ["reqwest/cookies"]
cron = ["dep:async_job", "dep:chrono", "dep:cron", "dep:async-trait"]
napi = ["dep:napi"]
napi_rustls_tls = ["napi", "reqwest/rustls-tls"]
smart = ["chrome", "dep:regex"]
encoding = ["dep:encoding_rs"]
headers = []
real_browser = []
openai = ["chrome", "serde", "chrome_intercept", "dep:async-openai", "dep:tiktoken-rs", "dep:lol_html", "dep:serde_json"]
openai_slim_fit = []
decentralized_headers = ["dep:const_format", "dep:itertools"]
spoof = ["dep:fastrand"]

[package.metadata.docs.rs]
cargo-args = ["-Zunstable-options", "-Zrustdoc-scrape-examples"]
