# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.

[package]
edition = "2021"
name = "spider"
version = "2.48.10"
authors = ["j-mendez <jeff@spider.cloud>"]
build = false
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "A web crawler and scraper, building blocks for data curation workloads."
documentation = "https://docs.rs/spider"
readme = "README.md"
keywords = [
    "crawler",
    "spider",
    "scraper",
]
categories = [
    "web-programming",
    "command-line-utilities",
]
license = "MIT"
repository = "https://github.com/spider-rs/spider"

[package.metadata.docs.rs]
all-features = true
rustdoc-args = [
    "--cfg",
    "docsrs",
]
cargo-args = [
    "-Zunstable-options",
    "-Zrustdoc-scrape-examples",
]

[badges.maintenance]
status = "as-is"

[features]
__basic = [
    "sync",
    "cookies",
    "ua_generator",
    "encoding",
    "string_interner_buffer_backend",
    "balance",
    "real_browser",
    "disk_native_tls",
    "time",
    "adaptive_concurrency",
    "priority_frontier",
    "dns_cache",
    "rate_limit",
    "request_coalesce",
    "auto_throttle",
    "etag_cache",
    "warc",
]
adaptive_concurrency = []
adblock = [
    "chrome",
    "chromey/adblock",
]
agent = ["dep:spider_agent"]
agent_chrome = [
    "agent",
    "spider_agent/chrome",
    "chrome",
    "dep:dashmap",
]
agent_fs = [
    "agent",
    "spider_agent/fs",
]
agent_full = [
    "agent",
    "agent_skills",
    "spider_agent/full",
]
agent_openai = [
    "agent",
    "spider_agent/openai",
]
agent_search_bing = [
    "agent",
    "spider_agent/search_bing",
]
agent_search_brave = [
    "agent",
    "spider_agent/search_brave",
]
agent_search_serper = [
    "agent",
    "spider_agent/search_serper",
]
agent_search_tavily = [
    "agent",
    "spider_agent/search_tavily",
]
agent_skills = [
    "agent",
    "spider_agent/skills",
]
agent_skills_s3 = [
    "agent_skills",
    "spider_agent/skills_s3",
]
agent_webdriver = [
    "agent",
    "spider_agent/webdriver",
]
auto_throttle = [
    "dep:dashmap",
    "time",
]
balance = ["dep:sysinfo"]
basic = [
    "__basic",
    "basic_tls",
]
basic_tls = [
    "reqwest_native_tls_native_roots",
    "disk_native_tls",
]
bloom = ["dep:libc"]
cache = [
    "cache_request",
    "http-global-cache/cache",
    "http-cache-reqwest/manager-cacache",
]
cache_chrome_hybrid = [
    "cache",
    "chrome",
    "dep:http-cache-semantics",
    "dep:http-cache",
    "dep:http",
]
cache_chrome_hybrid_mem = [
    "cache_mem",
    "chrome",
    "dep:http-cache-semantics",
    "dep:http-cache",
    "dep:http",
]
cache_gemini = ["dep:moka"]
cache_mem = [
    "cache_request",
    "http-global-cache/cache_mem",
]
cache_openai = ["dep:moka"]
cache_request = [
    "dep:http-global-cache",
    "dep:reqwest-middleware",
    "dep:http-cache-reqwest",
    "dep:http",
    "dep:http-cache",
]
chrome = [
    "dep:chromey",
    "dep:base64",
    "dep:rand",
    "serde",
    "dep:fastrand",
    "dep:which",
    "dep:home",
    "cookies",
    "reqwest_json",
    "dep:dashmap",
]
chrome_cpu = ["chrome"]
chrome_headed = ["chrome"]
chrome_headless_new = ["chrome"]
chrome_intercept = ["chrome"]
chrome_remote_cache = [
    "chrome",
    "cache_mem",
    "chromey/_cache",
    "chromey/cache_mem",
]
chrome_remote_cache_disk = [
    "chrome_remote_cache",
    "chromey/cache",
]
chrome_remote_cache_mem = ["chrome_remote_cache"]
chrome_screenshot = ["chrome"]
chrome_serde_stacker = [
    "chrome",
    "chromey/serde_stacker",
]
chrome_simd = [
    "chrome",
    "chromey/simd",
    "simd",
]
chrome_stealth = ["chrome"]
chrome_store_page = [
    "chrome",
    "serde",
]
chrome_tls_connection = [
    "chrome",
    "chromey/chrome_tls_connection",
]
cmd = ["tokio/process"]
control = []
cookies = [
    "reqwest/cookies",
    "dep:cookie",
]
cowboy = []
cron = [
    "dep:async_job",
    "dep:chrono",
    "dep:cron",
    "dep:async-trait",
]
decentralized = [
    "serde",
    "flexbuffers",
]
decentralized_headers = [
    "dep:const_format",
    "dep:itertools",
]
default = [
    "basic",
    "io_uring",
    "tcp_fastopen",
    "splice",
    "numa",
    "zero_copy",
]
disk = ["dep:sqlx"]
disk_aws = [
    "disk",
    "sqlx/tls-rustls-aws-lc-rs",
]
disk_native_tls = [
    "disk",
    "sqlx/runtime-tokio-native-tls",
]
dns_cache = [
    "dep:dashmap",
    "dep:fastrand",
]
encoding = []
etag_cache = ["dep:dashmap"]
extra_information = []
firewall = [
    "dep:spider_firewall",
    "chromey/firewall",
]
flexbuffers = ["dep:flexbuffers"]
fs = ["tokio/fs"]
full_resources = []
gemini = [
    "chrome",
    "serde",
    "chrome_intercept",
    "dep:gemini-rust",
    "dep:serde_json",
]
glob = ["dep:itertools"]
h2_multiplex = [
    "dep:dashmap",
    "dep:http",
]
headers = ["dep:httpdate"]
hedge = []
inline-more = []
io_uring = ["dep:io-uring"]
lightpanda = [
    "parallel_backends",
    "chrome",
]
llm_json = ["dep:llm_json"]
numa = []
openai = [
    "chrome",
    "serde",
    "chrome_intercept",
    "dep:async-openai",
    "dep:tiktoken-rs",
    "dep:serde_json",
]
openai_slim_fit = []
page_error_status_details = []
parallel_backends = []
parallel_backends_full = [
    "lightpanda",
    "servo",
]
priority_frontier = []
rate_limit = ["dep:dashmap"]
real_browser = [
    "dep:statrs",
    "dep:rand",
    "dep:fastrand",
]
regex = []
remote_addr = []
request_coalesce = [
    "dep:dashmap",
    "sync",
]
reqwest_hickory_dns = ["reqwest/hickory-dns"]
reqwest_json = ["reqwest/json"]
reqwest_multipart = ["reqwest/multipart"]
reqwest_native_tls = ["reqwest/native-tls"]
reqwest_native_tls_alpn = ["reqwest/native-tls"]
reqwest_native_tls_manual_roots = ["reqwest/rustls"]
reqwest_native_tls_native_roots = ["reqwest/rustls"]
reqwest_native_tls_vendored = ["reqwest/native-tls-vendored"]
reqwest_native_tls_webpki_roots = ["reqwest/rustls"]
reqwest_rustls_tls = ["reqwest/rustls"]
robots_cache = ["dep:dashmap"]
rquest_hickory_dns = ["wreq/hickory-dns"]
search = ["serde"]
search_bing = ["search"]
search_brave = ["search"]
search_serper = ["search"]
search_tavily = ["search"]
serde = [
    "dep:serde",
    "dep:serde_json",
    "hashbrown/serde",
    "string-interner/serde",
    "dep:serde_regex",
    "smallvec/serde",
    "reqwest_json",
]
servo = [
    "parallel_backends",
    "webdriver",
]
simd = ["dep:sonic-rs"]
sitemap = ["dep:sitemap"]
smart = [
    "chrome",
    "chrome_intercept",
]
socks = ["reqwest/socks"]
spider_cloud = [
    "serde",
    "reqwest_json",
]
splice = []
spoof = ["dep:fastrand"]
string_interner_bucket_backend = ["dep:string-interner"]
string_interner_buffer_backend = ["dep:string-interner"]
string_interner_string_backend = ["dep:string-interner"]
sync = ["tokio/sync"]
tcp_fastopen = []
time = []
tokio_io_std = ["tokio/io-std"]
tracing = [
    "tokio/tracing",
    "dep:tracing",
]
ua_generator = ["dep:ua_generator"]
warc = ["sync"]
webdriver = [
    "dep:thirtyfour",
    "serde",
    "dep:rand",
    "dep:fastrand",
]
webdriver_chrome = ["webdriver"]
webdriver_edge = ["webdriver"]
webdriver_firefox = ["webdriver"]
webdriver_headed = ["webdriver"]
webdriver_screenshot = ["webdriver"]
webdriver_stealth = ["webdriver"]
wreq = [
    "dep:wreq",
    "dep:wreq-util",
]
zero_copy = ["dep:zerocopy"]

[lib]
name = "spider"
path = "src/lib.rs"

[[test]]
name = "chunk_idle_timeout"
path = "tests/chunk_idle_timeout.rs"

[[test]]
name = "crawler_test_com"
path = "tests/crawler_test_com/main.rs"

[[test]]
name = "live_spider_agent_cloud"
path = "tests/live_spider_agent_cloud.rs"

[[test]]
name = "parallel_backends"
path = "tests/parallel_backends.rs"

[[test]]
name = "smart_vs_chrome"
path = "tests/smart_vs_chrome.rs"

[dependencies.ahash]
version = "0.8"
features = ["std"]
default-features = false

[dependencies.aho-corasick]
version = "1"

[dependencies.async-openai]
version = "0.33"
features = ["chat-completion"]
optional = true

[dependencies.async-trait]
version = "0.1"
optional = true

[dependencies.async_job]
version = "0.1"
optional = true

[dependencies.auto_encoder]
version = "0.1"

[dependencies.base64]
version = "0.22"
optional = true

[dependencies.bytes]
version = "1"
features = ["serde"]

[dependencies.case_insensitive_string]
version = "0.2"
features = [
    "compact",
    "serde",
]

[dependencies.chromey]
version = "2"
features = [
    "bytes",
    "stream",
]
optional = true
default-features = false

[dependencies.chrono]
version = "0.4"
optional = true

[dependencies.const_format]
version = "0.2"
optional = true

[dependencies.cookie]
version = "0"
optional = true

[dependencies.cron]
version = "0.15"
optional = true

[dependencies.dashmap]
version = "6"
optional = true

[dependencies.fastrand]
version = "2"
optional = true

[dependencies.flexbuffers]
version = "25"
optional = true

[dependencies.gemini-rust]
version = "1.7"
optional = true

[dependencies.h2]
version = "0.4"

[dependencies.hashbrown]
version = "0.16"
default-features = true

[dependencies.home]
version = "0.5"
optional = true

[dependencies.http]
version = "1"
optional = true

[dependencies.http-cache]
version = "1.0.0-alpha.5"
features = ["url-standard"]
optional = true
default-features = false

[dependencies.http-cache-reqwest]
version = "1.0.0-alpha.5"
optional = true
default-features = false

[dependencies.http-cache-semantics]
version = "3"
optional = true

[dependencies.http-global-cache]
version = "0.2.0"
features = ["cache_request"]
optional = true
default-features = false

[dependencies.httpdate]
version = "1"
optional = true

[dependencies.itertools]
version = "0.14"
optional = true

[dependencies.lazy_static]
version = "1"

[dependencies.llm_json]
version = "1"
optional = true

[dependencies.log]
version = "0.4"

[dependencies.lol_html]
version = "2"

[dependencies.memchr]
version = "2"

[dependencies.moka]
version = "0.12"
features = ["future"]
optional = true

[dependencies.num_cpus]
version = "1"

[dependencies.percent-encoding]
version = "2"

[dependencies.phf]
version = "0.13"

[dependencies.pin-project-lite]
version = "0.2"

[dependencies.quick-xml]
version = "0.39"
features = [
    "serde",
    "serialize",
    "async-tokio",
]

[dependencies.rand]
version = "0.9"
optional = true

[dependencies.regex]
version = "1"

[dependencies.reqwest-middleware]
version = "0.5"
optional = true
default-features = false

[dependencies.serde]
version = "1"
features = [
    "derive",
    "rc",
]
optional = true

[dependencies.serde_json]
version = "1"
optional = true

[dependencies.serde_regex]
version = "1"
optional = true

[dependencies.sitemap]
version = "0.4"
optional = true

[dependencies.smallvec]
version = "1"

[dependencies.sonic-rs]
version = "0.5"
optional = true

[dependencies.spider_agent]
version = "2.48.10"
optional = true
default-features = false

[dependencies.spider_agent_types]
version = "2.48.10"

[dependencies.spider_fingerprint]
version = "2"
features = [
    "serde",
    "headers",
    "dynamic-versions",
]
default-features = false

[dependencies.spider_firewall]
version = "2"
optional = true

[dependencies.sqlx]
version = "0.8"
features = [
    "runtime-tokio",
    "sqlite",
]
optional = true

[dependencies.statrs]
version = "0.18"
optional = true

[dependencies.string-interner]
version = "0.19"
features = [
    "std",
    "inline-more",
    "backends",
]
optional = true
default-features = false

[dependencies.string_concat]
version = "0.0.1"

[dependencies.strum]
version = "0.28"
features = ["derive"]

[dependencies.sysinfo]
version = "0.38"
features = ["system"]
optional = true
default-features = false

[dependencies.thirtyfour]
version = "0.36"
optional = true

[dependencies.tiktoken-rs]
version = "0.9"
optional = true

[dependencies.tokio-stream]
version = "0.1"

[dependencies.tower]
version = "0.5"
features = ["limit"]

[dependencies.tracing]
version = "0.1"
features = [
    "std",
    "attributes",
]
optional = true
default-features = false

[dependencies.ua_generator]
version = "^0.5"
optional = true

[dependencies.url]
version = "2"

[dependencies.which]
version = "6.0"
optional = true

[dependencies.wreq]
version = "5"
features = [
    "json",
    "stream",
    "socks",
    "gzip",
    "brotli",
    "zstd",
    "deflate",
    "cookies",
]
optional = true

[dependencies.wreq-util]
version = "2"
features = ["emulation-serde"]
optional = true

[dependencies.zerocopy]
version = "0.8"
features = ["derive"]
optional = true

[dev-dependencies.env_logger]
version = "0.11"

[dev-dependencies.serde_json]
version = "1"

[target.'cfg(all(unix, not(target_os = "linux")))'.dependencies.libc]
version = "0.2"
optional = true

[target.'cfg(not(target_arch = "wasm32"))'.dependencies.fastrand]
version = "2"
optional = true

[target.'cfg(not(target_arch = "wasm32"))'.dependencies.reqwest]
version = "0.13"
features = [
    "brotli",
    "gzip",
    "deflate",
    "zstd",
    "stream",
    "http2",
    "query",
]

[target.'cfg(not(target_arch = "wasm32"))'.dependencies.tokio]
version = "1"
features = [
    "macros",
    "time",
    "rt-multi-thread",
]
default-features = false

[target.'cfg(target_arch = "wasm32")'.dependencies.fastrand]
version = "2"
features = ["js"]
optional = true

[target.'cfg(target_arch = "wasm32")'.dependencies.reqwest]
version = "0.13"
features = [
    "brotli",
    "gzip",
    "deflate",
    "stream",
    "http2",
    "query",
]

[target.'cfg(target_arch = "wasm32")'.dependencies.tokio]
version = "1"
features = [
    "macros",
    "time",
    "rt",
]
default-features = false

[target.'cfg(target_os = "linux")'.dependencies.io-uring]
version = "0.7"
optional = true

[target.'cfg(target_os = "linux")'.dependencies.libc]
version = "0.2"
