[package]
name = "crw-crawl"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
keywords.workspace = true
categories.workspace = true
description = "Async BFS web crawler with rate limiting and robots.txt support for CRW"

[dependencies]
crw-core = { path = "../crw-core", version = "0.8.2" }
crw-renderer = { path = "../crw-renderer", version = "0.8.2" }
crw-extract = { path = "../crw-extract", version = "0.8.2" }
reqwest = { workspace = true }
scraper = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
url = { workspace = true }
uuid = { workspace = true }
futures = { workspace = true }
dashmap = { workspace = true }
rand = { workspace = true }
flate2 = "1"
phf = { workspace = true }
publicsuffix = { workspace = true }

[dev-dependencies]
tokio = { workspace = true }
wiremock = { workspace = true }
proptest = { workspace = true }

[[bench]]
name = "map_url_filter"
harness = false
